Migrate metadata log to SQLite (#10139)

CMakeLists.txt

@@ -597,6 +597,10 @@ set(RRD_PLUGIN_FILES
         database/rrdsetvar.h
         database/rrdvar.c
         database/rrdvar.h
+        database/sqlite/sqlite_functions.c
+        database/sqlite/sqlite_functions.h
+        database/sqlite/sqlite3.c
+        database/sqlite/sqlite3.h
         database/engine/rrdengine.c
         database/engine/rrdengine.h
         database/engine/rrddiskprotocol.h
@@ -612,7 +616,6 @@ set(RRD_PLUGIN_FILES
         database/engine/pagecache.h
         database/engine/rrdenglocking.c
         database/engine/rrdenglocking.h
-        database/engine/metadata_log/metadatalog.c
         database/engine/metadata_log/metadatalog.h
         database/engine/metadata_log/metadatalogapi.c
         database/engine/metadata_log/metadatalogapi.h
@@ -623,8 +626,6 @@ set(RRD_PLUGIN_FILES
         database/engine/metadata_log/metalogpluginsd.h
         database/engine/metadata_log/compaction.c
         database/engine/metadata_log/compaction.h
-        database/engine/global_uuid_map/global_uuid_map.c
-        database/engine/global_uuid_map/global_uuid_map.h
         )
 
 set(WEB_PLUGIN_FILES
@@ -1293,6 +1294,7 @@ endif()
         -Wl,--wrap=web_client_api_request_v1
         -Wl,--wrap=rrdhost_find_by_guid
         -Wl,--wrap=rrdset_find_byname
+        -Wl,--wrap=sql_create_host_by_uuid
         -Wl,--wrap=rrdset_find
         -Wl,--wrap=rrdpush_receiver_thread_spawn
         -Wl,--wrap=debug_int
@@ -1317,6 +1319,7 @@ endif()
         -Wl,--wrap=web_client_api_request_v1
         -Wl,--wrap=rrdhost_find_by_guid
         -Wl,--wrap=rrdset_find_byname
+        -Wl,--wrap=sql_create_host_by_uuid
         -Wl,--wrap=rrdset_find
         -Wl,--wrap=rrdpush_receiver_thread_spawn
         -Wl,--wrap=debug_int

Makefile.am

@@ -377,6 +377,10 @@ RRD_PLUGIN_FILES = \
 
 if ENABLE_DBENGINE
     RRD_PLUGIN_FILES += \
+        database/sqlite/sqlite_functions.c \
+        database/sqlite/sqlite_functions.h \
+        database/sqlite/sqlite3.c \
+        database/sqlite/sqlite3.h \
         database/engine/rrdengine.c \
         database/engine/rrdengine.h \
         database/engine/rrddiskprotocol.h \
@@ -392,7 +396,6 @@ if ENABLE_DBENGINE
         database/engine/pagecache.h \
         database/engine/rrdenglocking.c \
         database/engine/rrdenglocking.h \
-        database/engine/metadata_log/metadatalog.c \
         database/engine/metadata_log/metadatalog.h \
         database/engine/metadata_log/metadatalogapi.c \
         database/engine/metadata_log/metadatalogapi.h \
@@ -403,8 +406,6 @@ if ENABLE_DBENGINE
         database/engine/metadata_log/metalogpluginsd.h \
         database/engine/metadata_log/compaction.c \
         database/engine/metadata_log/compaction.h \
-        database/engine/global_uuid_map/global_uuid_map.c \
-        database/engine/global_uuid_map/global_uuid_map.h \
         $(NULL)
 endif
 

collectors/plugins.d/pluginsd_parser.c

@@ -274,7 +274,7 @@ PARSER_RC pluginsd_end(char **words, void *user, PLUGINSD_ACTION  *plugins_actio
 PARSER_RC pluginsd_chart(char **words, void *user, PLUGINSD_ACTION  *plugins_action)
 {
     RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
-    if (unlikely(!host)) {
+    if (unlikely(!host && !((PARSER_USER_OBJECT *) user)->host_exists)) {
         debug(D_PLUGINSD, "Ignoring chart belonging to missing or ignored host.");
         return PARSER_RC_OK;
     }
@@ -303,7 +303,10 @@ PARSER_RC pluginsd_chart(char **words, void *user, PLUGINSD_ACTION  *plugins_act
 
     // make sure we have the required variables
     if (unlikely((!type || !*type || !id || !*id))) {
-        error("requested a CHART, without a type.id, on host '%s'. Disabling it.", host->hostname);
+        if (likely(host))
+            error("requested a CHART, without a type.id, on host '%s'. Disabling it.", host->hostname);
+        else
+            error("requested a CHART, without a type.id. Disabling it.");
         ((PARSER_USER_OBJECT *) user)->enabled = 0;
         return PARSER_RC_ERROR;
     }
@@ -375,7 +378,7 @@ PARSER_RC pluginsd_dimension(char **words, void *user, PLUGINSD_ACTION  *plugins
 
     RRDSET *st = ((PARSER_USER_OBJECT *) user)->st;
     RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
-    if (unlikely(!host)) {
+    if (unlikely(!host && !((PARSER_USER_OBJECT *) user)->host_exists)) {
         debug(D_PLUGINSD, "Ignoring dimension belonging to missing or ignored host.");
         return PARSER_RC_OK;
     }
@@ -387,7 +390,7 @@ PARSER_RC pluginsd_dimension(char **words, void *user, PLUGINSD_ACTION  *plugins
         goto disable;
     }
 
-    if (unlikely(!st)) {
+    if (unlikely(!st && !((PARSER_USER_OBJECT *) user)->st_exists)) {
         error("requested a DIMENSION, without a CHART, on host '%s'. Disabling it.", host->hostname);
         goto disable;
     }
@@ -409,7 +412,7 @@ PARSER_RC pluginsd_dimension(char **words, void *user, PLUGINSD_ACTION  *plugins
     if (unlikely(!algorithm || !*algorithm))
         algorithm = "absolute";
 
-    if (unlikely(rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
+    if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_DEBUG)))
         debug(
             D_PLUGINSD,
             "creating dimension in chart %s, id='%s', name='%s', algorithm='%s', multiplier=%ld, divisor=%ld, hidden='%s'",

collectors/plugins.d/pluginsd_parser.h

@@ -16,6 +16,8 @@ typedef struct parser_user_object {
     struct label *new_labels;
     size_t count;
     int enabled;
+    uint8_t st_exists;
+    uint8_t host_exists;
     void *private; // the user can set this for private use
 } PARSER_USER_OBJECT;
 

collectors/statsd.plugin/statsd.c

@@ -1464,8 +1464,6 @@ static inline RRDSET *statsd_private_rrdset_create(
             , chart_type      // chart type
             , memory_mode     // memory mode
             , history         // history
-            , 0               // not archived
-            , NULL            // no known UUID
     );
     rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
 
@@ -2004,8 +2002,6 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
                 , chart->chart_type         // chart type
                 , app->rrd_memory_mode      // memory mode
                 , app->rrd_history_entries  // history
-                , 0                         // not archived
-                , NULL                      // no known UUID
         );
 
         rrdset_flag_set(chart->st, RRDSET_FLAG_STORE_FIRST);

configure.ac

@@ -1549,7 +1549,6 @@ AC_CONFIG_FILES([
     database/Makefile
     database/engine/Makefile
     database/engine/metadata_log/Makefile
-    database/engine/global_uuid_map/Makefile
     diagrams/Makefile
     exporting/Makefile
     exporting/graphite/Makefile

daemon/common.h

@@ -73,10 +73,6 @@
 // netdata agent spawn server
 #include "spawn/spawn.h"
 
-#ifdef ENABLE_DBENGINE
-#include "database/engine/global_uuid_map/global_uuid_map.h"
-#endif
-
 // the netdata deamon
 #include "daemon.h"
 #include "main.h"

daemon/main.c

@@ -60,9 +60,6 @@ void netdata_cleanup_and_exit(int ret) {
 
 #ifdef ENABLE_HTTPS
     security_clean_openssl();
-#endif
-#ifdef ENABLE_DBENGINE
-    free_global_guid_map();
 #endif
     info("EXIT: all done - netdata is now exiting - bye bye...");
     exit(ret);
@@ -1451,9 +1448,6 @@ int main(int argc, char **argv) {
     struct rrdhost_system_info *system_info = calloc(1, sizeof(struct rrdhost_system_info));
     get_system_info(system_info);
 
-#ifdef ENABLE_DBENGINE
-    init_global_guid_map();
-#endif
     if(rrd_init(netdata_configured_hostname, system_info))
         fatal("Cannot initialize localhost instance with name '%s'.", netdata_configured_hostname);
 
@@ -1471,10 +1465,6 @@ int main(int argc, char **argv) {
 
     // Load host labels
     reload_host_labels();
-#ifdef ENABLE_DBENGINE
-    if (localhost->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
-        metalog_commit_update_host(localhost);
-#endif
 
     // ------------------------------------------------------------------------
     // spawn the threads

database/engine/Makefile.am

@@ -5,7 +5,6 @@ MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
 
 SUBDIRS = \
     metadata_log \
-    global_uuid_map \
     $(NULL)
 
 dist_noinst_DATA = \

database/engine/global_uuid_map/Makefile.am

@@ -1,8 +0,0 @@
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-AUTOMAKE_OPTIONS = subdir-objects
-MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
-
-dist_noinst_DATA = \
-    README.md \
-    $(NULL)