Add support for persistent metadata (#9324)

* Implemented collector metadata logging 
* Added persistent GUIDs for charts and dimensions
* Added metadata log replay and automatic compaction
* Added detection of charts with no active collector (archived)
* Added new endpoint to report archived charts via `/api/v1/archivedcharts`
* Added support for collector metadata update

Co-authored-by: Markos Fountoulakis <44345837+mfundul@users.noreply.github.com>

CMakeLists.txt

@@ -556,6 +556,19 @@ set(RRD_PLUGIN_FILES
         database/engine/pagecache.h
         database/engine/rrdenglocking.c
         database/engine/rrdenglocking.h
+        database/engine/metadata_log/metadatalog.c
+        database/engine/metadata_log/metadatalog.h
+        database/engine/metadata_log/metadatalogapi.c
+        database/engine/metadata_log/metadatalogapi.h
+        database/engine/metadata_log/logfile.h
+        database/engine/metadata_log/logfile.c
+        database/engine/metadata_log/metadatalogprotocol.h
+        database/engine/metadata_log/metalogpluginsd.c
+        database/engine/metadata_log/metalogpluginsd.h
+        database/engine/metadata_log/compaction.c
+        database/engine/metadata_log/compaction.h
+        database/engine/global_uuid_map/global_uuid_map.c
+        database/engine/global_uuid_map/global_uuid_map.h
         )
 
 set(WEB_PLUGIN_FILES

Makefile.am

@@ -375,6 +375,19 @@ if ENABLE_DBENGINE
         database/engine/pagecache.h \
         database/engine/rrdenglocking.c \
         database/engine/rrdenglocking.h \
+        database/engine/metadata_log/metadatalog.c \
+        database/engine/metadata_log/metadatalog.h \
+        database/engine/metadata_log/metadatalogapi.c \
+        database/engine/metadata_log/metadatalogapi.h \
+        database/engine/metadata_log/logfile.h \
+        database/engine/metadata_log/logfile.c \
+        database/engine/metadata_log/metadatalogprotocol.h \
+        database/engine/metadata_log/metalogpluginsd.c \
+        database/engine/metadata_log/metalogpluginsd.h \
+        database/engine/metadata_log/compaction.c \
+        database/engine/metadata_log/compaction.h \
+        database/engine/global_uuid_map/global_uuid_map.c \
+        database/engine/global_uuid_map/global_uuid_map.h \
         $(NULL)
 endif
 

aclk/agent_cloud_link.c

@@ -680,6 +680,9 @@ static struct _collector *_add_collector(const char *hostname, const char *plugi
 void aclk_add_collector(const char *hostname, const char *plugin_name, const char *module_name)
 {
     struct _collector *tmp_collector;
+    if (unlikely(!netdata_ready)) {
+        return;
+    }
 
     COLLECTOR_LOCK;
 
@@ -711,6 +714,9 @@ void aclk_add_collector(const char *hostname, const char *plugin_name, const cha
 void aclk_del_collector(const char *hostname, const char *plugin_name, const char *module_name)
 {
     struct _collector *tmp_collector;
+    if (unlikely(!netdata_ready)) {
+        return;
+    }
 
     COLLECTOR_LOCK;
 
@@ -1752,7 +1758,7 @@ int aclk_send_info_metadata()
     debug(D_ACLK, "Metadata %s with info has %zu bytes", msg_id, local_buffer->len);
 
     buffer_sprintf(local_buffer, ", \n\t \"charts\" : ");
-    charts2json(localhost, local_buffer, 1);
+    charts2json(localhost, local_buffer, 1, 0);
     buffer_sprintf(local_buffer, "\n}\n}");
     debug(D_ACLK, "Metadata %s with chart has %zu bytes", msg_id, local_buffer->len);
 
@@ -1859,6 +1865,9 @@ int aclk_update_chart(RRDHOST *host, char *chart_name, ACLK_CMD aclk_cmd)
     UNUSED(chart_name);
     return 0;
 #else
+    if (unlikely(!netdata_ready))
+        return 0;
+
     if (!netdata_cloud_setting)
         return 0;
 
@@ -1886,6 +1895,9 @@ int aclk_update_alarm(RRDHOST *host, ALARM_ENTRY *ae)
 {
     BUFFER *local_buffer = NULL;
 
+    if (unlikely(!netdata_ready))
+        return 0;
+
     if (host != localhost)
         return 0;
 

collectors/diskspace.plugin/plugin_diskspace.c

@@ -254,7 +254,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
                                               netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
         if(unlikely(!m->st_space)) {
             m->do_space = CONFIG_BOOLEAN_YES;
-            m->st_space = rrdset_find_bytype_localhost("disk_space", disk);
+            m->st_space = rrdset_find_active_bytype_localhost("disk_space", disk);
             if(unlikely(!m->st_space)) {
                 char title[4096 + 1];
                 snprintfz(title, 4096, "Disk Space Usage for %s [%s]", family, mi->mount_source);
@@ -296,7 +296,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
                                                netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
         if(unlikely(!m->st_inodes)) {
             m->do_inodes = CONFIG_BOOLEAN_YES;
-            m->st_inodes = rrdset_find_bytype_localhost("disk_inodes", disk);
+            m->st_inodes = rrdset_find_active_bytype_localhost("disk_inodes", disk);
             if(unlikely(!m->st_inodes)) {
                 char title[4096 + 1];
                 snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", family, mi->mount_source);

collectors/freebsd.plugin/plugin_freebsd.c

@@ -129,7 +129,7 @@ void *freebsd_main(void *ptr) {
             static RRDSET *st = NULL;
 
             if(unlikely(!st)) {
-                st = rrdset_find_bytype_localhost("netdata", "plugin_freebsd_modules");
+                st = rrdset_find_active_bytype_localhost("netdata", "plugin_freebsd_modules");
 
                 if(!st) {
                     st = rrdset_create_localhost(

collectors/macos.plugin/macos_fw.c

@@ -145,7 +145,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         total_disk_writes += diskstat.bytes_write;
                     }
 
-                    st = rrdset_find_bytype_localhost("disk", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk"
@@ -183,7 +183,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.writes);
                     }
 
-                    st = rrdset_find_bytype_localhost("disk_ops", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk_ops", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk_ops"
@@ -222,7 +222,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.time_write);
                     }
 
-                    st = rrdset_find_bytype_localhost("disk_util", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk_util", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk_util"
@@ -260,7 +260,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                         CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.latency_write);
                     }
 
-                    st = rrdset_find_bytype_localhost("disk_iotime", diskstat.name);
+                    st = rrdset_find_active_bytype_localhost("disk_iotime", diskstat.name);
                     if (unlikely(!st)) {
                         st = rrdset_create_localhost(
                                 "disk_iotime"
@@ -297,7 +297,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                         // --------------------------------------------------------------------
 
-                        st = rrdset_find_bytype_localhost("disk_await", diskstat.name);
+                        st = rrdset_find_active_bytype_localhost("disk_await", diskstat.name);
                         if (unlikely(!st)) {
                             st = rrdset_create_localhost(
                                     "disk_await"
@@ -328,7 +328,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                         // --------------------------------------------------------------------
 
-                        st = rrdset_find_bytype_localhost("disk_avgsz", diskstat.name);
+                        st = rrdset_find_active_bytype_localhost("disk_avgsz", diskstat.name);
                         if (unlikely(!st)) {
                             st = rrdset_create_localhost(
                                     "disk_avgsz"
@@ -359,7 +359,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                         // --------------------------------------------------------------------
 
-                        st = rrdset_find_bytype_localhost("disk_svctm", diskstat.name);
+                        st = rrdset_find_active_bytype_localhost("disk_svctm", diskstat.name);
                         if (unlikely(!st)) {
                             st = rrdset_create_localhost(
                                     "disk_svctm"
@@ -401,7 +401,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
     }
 
     if (likely(do_io)) {
-        st = rrdset_find_bytype_localhost("system", "io");
+        st = rrdset_find_active_bytype_localhost("system", "io");
         if (unlikely(!st)) {
             st = rrdset_create_localhost(
                     "system"
@@ -453,7 +453,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                 // --------------------------------------------------------------------------
 
                 if (likely(do_space)) {
-                    st = rrdset_find_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
+                    st = rrdset_find_active_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
                     if (unlikely(!st)) {
                         snprintfz(title, 4096, "Disk Space Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
                         st = rrdset_create_localhost(
@@ -486,7 +486,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
                 // --------------------------------------------------------------------------
 
                 if (likely(do_inodes)) {
-                    st = rrdset_find_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
+                    st = rrdset_find_active_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
                     if (unlikely(!st)) {
                         snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
                         st = rrdset_create_localhost(
@@ -533,7 +533,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net"
@@ -561,7 +561,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_packets", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_packets", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_packets"
@@ -594,7 +594,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_errors", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_errors", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_errors"
@@ -623,7 +623,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_drops", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_drops", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_drops"
@@ -650,7 +650,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
 
                 // --------------------------------------------------------------------
 
-                st = rrdset_find_bytype_localhost("net_events", ifa->ifa_name);
+                st = rrdset_find_active_bytype_localhost("net_events", ifa->ifa_name);
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "net_events"

collectors/macos.plugin/macos_mach_smi.c

@@ -55,7 +55,7 @@ int do_macos_mach_smi(int update_every, usec_t dt) {
                 error("DISABLED: system.cpu");
             } else {
 
-                st = rrdset_find_bytype_localhost("system", "cpu");
+                st = rrdset_find_active_bytype_localhost("system", "cpu");
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "system"

collectors/macos.plugin/macos_sysctl.c

@@ -230,7 +230,7 @@ int do_macos_sysctl(int update_every, usec_t dt) {
                 error("DISABLED: system.load");
             } else {
 
-                st = rrdset_find_bytype_localhost("system", "load");
+                st = rrdset_find_active_bytype_localhost("system", "load");
                 if (unlikely(!st)) {
                     st = rrdset_create_localhost(
                             "system"

collectors/plugins.d/plugins_d.h

@@ -31,10 +31,10 @@
 #define PLUGINSD_KEYWORD_VARIABLE "VARIABLE"
 #define PLUGINSD_KEYWORD_LABEL "LABEL"
 #define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE"
-#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT"
 #define PLUGINSD_KEYWORD_GUID "GUID"
-#define PLUGINSD_KEYWORD_HOST "HOST"
+#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT"
 #define PLUGINSD_KEYWORD_TOMBSTONE "TOMBSTONE"
+#define PLUGINSD_KEYWORD_HOST "HOST"
 
 
 #define PLUGINSD_LINE_MAX 1024

collectors/plugins.d/pluginsd_parser.c

@@ -561,6 +561,71 @@ PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION  *plugins
     return PARSER_RC_OK;
 }
 
+PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+    char *uuid_str = words[1];
+    uuid_t uuid;
+
+    if (unlikely(!uuid_str)) {
+        error("requested a GUID, without a uuid.");
+        return PARSER_RC_ERROR;
+    }
+    if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+        error("requested a GUID, without a valid uuid string.");
+        return PARSER_RC_ERROR;
+    }
+
+    debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+    if (plugins_action->guid_action) {
+        return plugins_action->guid_action(user, &uuid);
+    }
+
+    return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_context(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+    char *uuid_str = words[1];
+    uuid_t uuid;
+
+    if (unlikely(!uuid_str)) {
+        error("requested a CONTEXT, without a uuid.");
+        return PARSER_RC_ERROR;
+    }
+    if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+        error("requested a CONTEXT, without a valid uuid string.");
+        return PARSER_RC_ERROR;
+    }
+
+    debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+    if (plugins_action->context_action) {
+        return plugins_action->context_action(user, &uuid);
+    }
+
+    return PARSER_RC_OK;
+}
+
+PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_action)
+{
+    char *uuid_str = words[1];
+    uuid_t uuid;
+
+    if (unlikely(!uuid_str)) {
+        error("requested a TOMBSTONE, without a uuid.");
+        return PARSER_RC_ERROR;
+    }
+    if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
+        error("requested a TOMBSTONE, without a valid uuid string.");
+        return PARSER_RC_ERROR;
+    }
+
+    debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
+    if (plugins_action->tombstone_action) {
+        return plugins_action->tombstone_action(user, &uuid);
+    }
+
+    return PARSER_RC_OK;
+}
 
 // New plugins.d parser