Enable metadata persistence in all memory modes (#10742)

Makefile.am

@@ -376,14 +376,14 @@ RRD_PLUGIN_FILES = \
     database/rrdsetvar.h \
     database/rrdvar.c \
     database/rrdvar.h \
+    database/sqlite/sqlite_functions.c \
+    database/sqlite/sqlite_functions.h \
+    database/sqlite/sqlite3.c \
+    database/sqlite/sqlite3.h \
     $(NULL)
 
 if ENABLE_DBENGINE
     RRD_PLUGIN_FILES += \
-        database/sqlite/sqlite_functions.c \
-        database/sqlite/sqlite_functions.h \
-        database/sqlite/sqlite3.c \
-        database/sqlite/sqlite3.h \
         database/engine/rrdengine.c \
         database/engine/rrdengine.h \
         database/engine/rrddiskprotocol.h \

database/rrd.h

@@ -370,9 +370,9 @@ struct rrddim_query_handle {
 struct rrddim_volatile {
 #ifdef ENABLE_DBENGINE
     uuid_t *rrdeng_uuid;                 // database engine metric UUID
-    uuid_t *metric_uuid;                 // global UUID for this metric (unique_across hosts)
     struct pg_cache_page_index *page_index;
 #endif
+    uuid_t *metric_uuid;                 // global UUID for this metric (unique_across hosts)
     union rrddim_collect_handle handle;
     // ------------------------------------------------------------------------
     // function pointers that handle data collection
@@ -856,8 +856,8 @@ struct rrdhost {
 
 #ifdef ENABLE_DBENGINE
     struct rrdengine_instance *rrdeng_ctx;          // DB engine instance for this host
-    uuid_t  host_uuid;                              // Global GUID for this host
 #endif
+    uuid_t  host_uuid;                              // Global GUID for this host
 
 #ifdef ENABLE_HTTPS
     struct netdata_ssl ssl;                         //Structure used to encrypt the connection
@@ -1317,7 +1317,7 @@ extern void set_host_properties(
 
 #ifdef ENABLE_DBENGINE
 #include "database/engine/rrdengineapi.h"
-#include "sqlite/sqlite_functions.h"
 #endif
+#include "sqlite/sqlite_functions.h"
 
 #endif /* NETDATA_RRD_H */

database/rrddim.c

@@ -232,9 +232,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
         rc += rrddim_set_multiplier(st, rd, multiplier);
         rc += rrddim_set_divisor(st, rd, divisor);
         if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
-#ifdef ENABLE_DBENGINE
             store_active_dimension(rd->state->metric_uuid);
-#endif
             rd->state->collect_ops.init(rd);
             rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED);
             rrddimvar_create(rd, RRDVAR_TYPE_CALCULATED, NULL, NULL, &rd->last_stored_value, RRDVAR_OPTION_DEFAULT);
@@ -242,14 +240,11 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
             rrddimvar_create(rd, RRDVAR_TYPE_TIME_T, NULL, "_last_collected_t", &rd->last_collected_time.tv_sec, RRDVAR_OPTION_DEFAULT);
             calc_link_to_rrddim(rd);
         }
-        // DBENGINE available and activated?
-#ifdef ENABLE_DBENGINE
-        if (likely(rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && unlikely(rc)) {
+        if (unlikely(rc)) {
             debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rd->id);
             (void)sql_store_dimension(rd->state->metric_uuid, rd->rrdset->chart_uuid, rd->id, rd->name, rd->multiplier, rd->divisor,
                                       rd->algorithm);
         }
-#endif
         rrdset_unlock(st);
         return rd;
     }
@@ -396,7 +391,6 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
 #ifdef ENABLE_DBENGINE
         uuid_t *dim_uuid = find_dimension_uuid(st, rd);
         rrdeng_metric_init(rd, dim_uuid);
-        store_active_dimension(rd->state->metric_uuid);
         rd->state->collect_ops.init = rrdeng_store_metric_init;
         rd->state->collect_ops.store_metric = rrdeng_store_metric_next;
         rd->state->collect_ops.finalize = rrdeng_store_metric_finalize;
@@ -408,6 +402,9 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
         rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time;
 #endif
     } else {
+        rd->state->metric_uuid = find_dimension_uuid(st, rd);
+        if (unlikely(!rd->state->metric_uuid))
+            rd->state->metric_uuid = create_dimension_uuid(rd->rrdset, rd);
         rd->state->collect_ops.init         = rrddim_collect_init;
         rd->state->collect_ops.store_metric = rrddim_collect_store_metric;
         rd->state->collect_ops.finalize     = rrddim_collect_finalize;
@@ -418,6 +415,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
         rd->state->query_ops.latest_time    = rrddim_query_latest_time;
         rd->state->query_ops.oldest_time    = rrddim_query_oldest_time;
     }
+    store_active_dimension(rd->state->metric_uuid);
     rd->state->collect_ops.init(rd);
     // append this dimension
     if(!st->dimensions)
@@ -476,10 +474,8 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
     if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
         uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd);
         if (can_delete_metric && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
-#ifdef ENABLE_DBENGINE
             /* This metric has no data and no references */
             delete_dimension_uuid(rd->state->metric_uuid);
-#endif
         }
     }
 
@@ -503,6 +499,7 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
         error("RRDDIM: INTERNAL ERROR: attempt to remove from index dimension '%s' on chart '%s', removed a different dimension.", rd->id, st->id);
 
     // free(rd->annotations);
+    freez(rd->state->metric_uuid);
 
     RRD_MEMORY_MODE rrd_memory_mode = rd->rrd_memory_mode;
     switch(rrd_memory_mode) {
@@ -522,11 +519,6 @@ void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
             debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name);
             freez((void *)rd->id);
             freez(rd->cache_filename);
-#ifdef ENABLE_DBENGINE
-            if (rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
-                freez(rd->state->metric_uuid);
-            }
-#endif
             freez(rd->state);
             freez(rd);
             break;

database/rrdhost.c

@@ -298,15 +298,16 @@ RRDHOST *rrdhost_create(const char *hostname,
         return NULL;
     }
 
+    if (likely(!uuid_parse(host->machine_guid, host->host_uuid))) {
+        int rc = sql_store_host(&host->host_uuid, hostname, registry_hostname, update_every, os, timezone, tags);
+        if (unlikely(rc))
+            error_report("Failed to store machine GUID to the database");
+    }
+    else
+        error_report("Host machine GUID %s is not valid", host->machine_guid);
+
     if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
 #ifdef ENABLE_DBENGINE
-        if (likely(!uuid_parse(host->machine_guid, host->host_uuid))) {
-            int rc = sql_store_host(&host->host_uuid, hostname, registry_hostname, update_every, os, timezone, tags);
-            if (unlikely(rc))
-                error_report("Failed to store machine GUID to the database");
-        }
-        else
-            error_report("Host machine GUID %s is not valid", host->machine_guid);
         char dbenginepath[FILENAME_MAX + 1];
         int ret;
 
@@ -634,11 +635,11 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
     if (gap_when_lost_iterations_above < 1)
         gap_when_lost_iterations_above = 1;
 
-#ifdef ENABLE_DBENGINE
     if (unlikely(sql_init_database())) {
-        return 1;
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+            return 1;
+        info("Skipping SQLITE metadata initialization since memory mode is not db engine");
     }
-#endif
 
     health_init();
 

database/rrdset.c

@@ -385,6 +385,7 @@ void rrdset_free(RRDSET *st) {
     freez(st->state->old_context);
     free_label_list(st->state->labels.head);
     freez(st->state);
+    freez(st->chart_uuid);
 
     switch(st->rrd_memory_mode) {
         case RRD_MEMORY_MODE_SAVE:
@@ -397,10 +398,6 @@ void rrdset_free(RRDSET *st) {
         case RRD_MEMORY_MODE_ALLOC:
         case RRD_MEMORY_MODE_NONE:
         case RRD_MEMORY_MODE_DBENGINE:
-#ifdef ENABLE_DBENGINE
-            if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
-                freez(st->chart_uuid);
-#endif
             freez(st);
             break;
     }
@@ -660,15 +657,12 @@ RRDSET *rrdset_create_custom(
                 sched_yield();
             }
         }
-#ifdef ENABLE_DBENGINE
-        if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE &&
-            (mark_rebuild & (META_CHART_UPDATED | META_PLUGIN_UPDATED | META_MODULE_UPDATED))) {
+        if (mark_rebuild & (META_CHART_UPDATED | META_PLUGIN_UPDATED | META_MODULE_UPDATED)) {
             debug(D_METADATALOG, "CHART [%s] metadata updated", st->id);
             int rc = update_chart_metadata(st->chart_uuid, st, id, name);
             if (unlikely(rc))
                 error_report("Failed to update chart metadata in the database");
         }
-#endif
         /* Fall-through during switch from archived to active so that the host lock is taken and health is linked */
         if (!changed_from_archived_to_active)
             return st;
@@ -925,17 +919,14 @@ RRDSET *rrdset_create_custom(
 
     rrdsetcalc_link_matching(st);
     rrdcalctemplate_link_matching(st);
-#ifdef ENABLE_DBENGINE
-    if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
-        st->chart_uuid = find_chart_uuid(host, type, id, name);
-        if (unlikely(!st->chart_uuid))
-            st->chart_uuid = create_chart_uuid(st, id, name);
-        else
-            update_chart_metadata(st->chart_uuid, st, id, name);
 
-        store_active_chart(st->chart_uuid);
-    }
-#endif
+    st->chart_uuid = find_chart_uuid(host, type, id, name);
+    if (unlikely(!st->chart_uuid))
+        st->chart_uuid = create_chart_uuid(st, id, name);
+    else
+        update_chart_metadata(st->chart_uuid, st, id, name);
+
+    store_active_chart(st->chart_uuid);
 
     rrdhost_cleanup_obsolete_charts(host);
 
@@ -1934,17 +1925,14 @@ void rrdset_finalize_labels(RRDSET *st)
     } else {
         replace_label_list(labels, new_labels);
     }
-#ifdef ENABLE_DBENGINE
-    if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
-        netdata_rwlock_wrlock(&labels->labels_rwlock);
-        struct label *lbl = labels->head;
-        while (lbl) {
-            sql_store_chart_label(st->chart_uuid, (int)lbl->label_source, lbl->key, lbl->value);
-            lbl = lbl->next;
-        }
-        netdata_rwlock_unlock(&labels->labels_rwlock);
+
+    netdata_rwlock_wrlock(&labels->labels_rwlock);
+    struct label *lbl = labels->head;
+    while (lbl) {
+        sql_store_chart_label(st->chart_uuid, (int)lbl->label_source, lbl->key, lbl->value);
+        lbl = lbl->next;
     }
-#endif
+    netdata_rwlock_unlock(&labels->labels_rwlock);
 
     st->state->new_labels = NULL;
 }

database/sqlite/sqlite_functions.c

@@ -84,7 +84,8 @@ void store_active_chart(uuid_t *chart_uuid)
     int rc;
 
     if (unlikely(!db_meta)) {
-        error_report("Database has not been initialized");
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+            error_report("Database has not been initialized");
         return;
     }
 
@@ -111,7 +112,8 @@ void store_active_dimension(uuid_t *dimension_uuid)
     int rc;
 
     if (unlikely(!db_meta)) {
-        error_report("Database has not been initialized");
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+            error_report("Database has not been initialized");
         return;
     }
 
@@ -143,7 +145,9 @@ int sql_init_database(void)
     snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);
     rc = sqlite3_open(sqlite_database, &db_meta);
     if (rc != SQLITE_OK) {
-        error_report("Failed to initialize database at %s", sqlite_database);
+        error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc));
+        sqlite3_close(db_meta);
+        db_meta = NULL;
         return 1;
     }
 
@@ -220,6 +224,9 @@ uuid_t *find_dimension_uuid(RRDSET *st, RRDDIM *rd)
     uuid_t *uuid = NULL;
     int rc;
 
+    if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
+        return NULL;
+
     if (unlikely(!res)) {
         rc = sqlite3_prepare_v2(db_meta, SQL_FIND_DIMENSION_UUID, -1, &res, 0);
         if (rc != SQLITE_OK) {
@@ -333,6 +340,9 @@ uuid_t *find_chart_uuid(RRDHOST *host, const char *type, const char *id, const c
     uuid_t *uuid = NULL;
     int rc;
 
+    if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
+        return NULL;
+
     if (unlikely(!res)) {
         rc = sqlite3_prepare_v2(db_meta, SQL_FIND_CHART_UUID, -1, &res, 0);
         if (rc != SQLITE_OK) {
@@ -390,6 +400,9 @@ int update_chart_metadata(uuid_t *chart_uuid, RRDSET *st, const char *id, const
 {
     int rc;
 
+    if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
+        return 0;
+
     rc = sql_store_chart(
         chart_uuid, &st->rrdhost->host_uuid, st->type, id, name, st->family, st->context, st->title, st->units, st->plugin_name,
         st->module_name, st->priority, st->update_every, st->chart_type, st->rrd_memory_mode, st->entries);
@@ -429,6 +442,8 @@ int sql_store_host(
     int rc;
 
     if (unlikely(!db_meta)) {
+        if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
+            return 0;
         error_report("Database has not been initialized");
         return 1;
     }
@@ -499,6 +514,8 @@ int sql_store_chart(
     int rc, param = 0;
 
     if (unlikely(!db_meta)) {
+        if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
+            return 0;
         error_report("Database has not been initialized");
         return 1;
     }
@@ -623,6 +640,8 @@ int sql_store_dimension(
     int rc;
 
     if (unlikely(!db_meta)) {
+        if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
+            return 0;
         error_report("Database has not been initialized");
         return 1;
     }
@@ -1033,6 +1052,12 @@ void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, cha
     sqlite3_stmt *res = NULL;
     int rc;
 
+    if (unlikely(!db_meta)) {
+        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+            error_report("Database has not been initialized");
+        return;
+    }
+
     rc = sqlite3_prepare_v2(db_meta, SQL_INS_CHART_LABEL, -1, &res, 0);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to prepare statement store chart labels");