Implemented the HOST command in metadata log replay (#9489)

collectors/plugins.d/pluginsd_parser.c

@@ -627,6 +627,33 @@ PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_
     return PARSER_RC_OK;
 }
 
+PARSER_RC metalog_pluginsd_host(char **words, void *user, PLUGINSD_ACTION  *plugins_action)
+{
+    char *machine_guid = words[1];
+    char *hostname = words[2];
+    char *registry_hostname = words[3];
+    char *update_every_s = words[4];
+    char *os = words[5];
+    char *timezone = words[6];
+    char *tags = words[7];
+
+    int update_every = 1;
+    if (likely(update_every_s && *update_every_s))
+        update_every = str2i(update_every_s);
+    if (unlikely(!update_every))
+        update_every = 1;
+
+    debug(D_PLUGINSD, "HOST PARSED: guid=%s, hostname=%s, reg_host=%s, update=%d, os=%s, timezone=%s, tags=%s",
+         machine_guid, hostname, registry_hostname, update_every, os, timezone, tags);
+
+    if (plugins_action->host_action) {
+        return plugins_action->host_action(
+            user, machine_guid, hostname, registry_hostname, update_every, os, timezone, tags);
+    }
+
+    return PARSER_RC_OK;
+}
+
 // New plugins.d parser
 
 inline size_t pluginsd_process(RRDHOST *host, struct plugind *cd, FILE *fp, int trust_durations)

database/engine/global_uuid_map/global_uuid_map.c

@@ -155,6 +155,7 @@ GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes)
                     return GUID_TYPE_NOSPACE;
                 strncpyz(object, (char *) *PValue+1, max_bytes - 1);
                 break;
+            case GUID_TYPE_HOST:
             case GUID_TYPE_CHART:
             case GUID_TYPE_DIMENSION:
                 if (unlikely(max_bytes < (size_t) value_type * 16))
@@ -221,6 +222,11 @@ int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int
             memcpy(target_object + 1, (((RRDSET *)object))->rrdhost->host_uuid, 16);
             memcpy(target_object + 17, temp_uuid, 16);
             break;
+        case GUID_TYPE_HOST:
+            target_object = mallocz(17);
+            target_object[0] = object_type;
+            memcpy(target_object + 1, (((RRDHOST *)object))->host_uuid, 16);
+            break;
         case GUID_TYPE_CHAR:
             target_object = mallocz(strlen((char *) object)+2);
             target_object[0] = object_type;

database/engine/metadata_log/compaction.c

@@ -79,10 +79,10 @@ static void compaction_test_quota(struct metalog_worker_config *wc)
 static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
 {
     GUID_TYPE ret;
-    RRDHOST *host = ctx->rrdeng_ctx->host;
     RRDSET *st;
     RRDDIM *rd;
     BUFFER *buffer;
+    RRDHOST *host = ctx->rrdeng_ctx->host;
 
     ret = find_object_by_guid(uuid, NULL, 0);
     switch (ret) {
@@ -92,6 +92,11 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
         case GUID_TYPE_CHART:
             st = metalog_get_chart_from_uuid(ctx, uuid);
             if (st) {
+                if (ctx->current_compaction_id > st->rrdhost->compaction_id) {
+                    error("Forcing compaction of HOST %s from CHART %s", st->rrdhost->hostname, st->id);
+                    compact_record_by_uuid(ctx, &st->rrdhost->host_uuid);
+                }
+
                 if (ctx->current_compaction_id > st->compaction_id) {
                     st->compaction_id = ctx->current_compaction_id;
                     buffer = metalog_update_chart_buffer(st, ctx->current_compaction_id);
@@ -106,11 +111,13 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
         case GUID_TYPE_DIMENSION:
             rd = metalog_get_dimension_from_uuid(ctx, uuid);
             if (rd) {
+                if (ctx->current_compaction_id > rd->rrdset->rrdhost->compaction_id) {
+                    error("Forcing compaction of HOST %s", rd->rrdset->rrdhost->hostname);
+                    compact_record_by_uuid(ctx, &rd->rrdset->rrdhost->host_uuid);
+                }
                 if (ctx->current_compaction_id > rd->rrdset->compaction_id) {
-                    error("Forcing compaction of chart %s", rd->rrdset->id);
-                    rd->rrdset->compaction_id = ctx->current_compaction_id;
-                    buffer = metalog_update_chart_buffer(rd->rrdset, ctx->current_compaction_id);
-                    metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, rd->rrdset->chart_uuid, 1);
+                    error("Forcing compaction of CHART %s", rd->rrdset->id);
+                    compact_record_by_uuid(ctx, rd->rrdset->chart_uuid);
                 } else if (ctx->current_compaction_id > rd->state->compaction_id) {
                     rd->state->compaction_id = ctx->current_compaction_id;
                     buffer = metalog_update_dimension_buffer(rd);
@@ -123,6 +130,8 @@ static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
             }
             break;
         case GUID_TYPE_HOST:
+            //TODO: will be enabled when multidb is activated
+            //RRDHOST *host = metalog_get_host_from_uuid(ctx, uuid);
             if (ctx->current_compaction_id > host->compaction_id) {
                 host->compaction_id = ctx->current_compaction_id;
                 buffer = metalog_update_host_buffer(host);

database/engine/metadata_log/logfile.c

@@ -675,6 +675,7 @@ static int scan_metalog_files(struct metalog_instance *ctx)
         failed_to_load = matched_files;
         goto after_failed_to_parse;
     }
+    parser_add_keyword(parser, PLUGINSD_KEYWORD_HOST, metalog_pluginsd_host);
     parser_add_keyword(parser, PLUGINSD_KEYWORD_GUID, pluginsd_guid);
     parser_add_keyword(parser, PLUGINSD_KEYWORD_CONTEXT, pluginsd_context);
     parser_add_keyword(parser, PLUGINSD_KEYWORD_TOMBSTONE, pluginsd_tombstone);
@@ -683,6 +684,8 @@ static int scan_metalog_files(struct metalog_instance *ctx)
     parser->plugins_action->guid_action      = &metalog_pluginsd_guid_action;
     parser->plugins_action->context_action   = &metalog_pluginsd_context_action;
     parser->plugins_action->tombstone_action = &metalog_pluginsd_tombstone_action;
+    parser->plugins_action->host_action      = &metalog_pluginsd_host_action;
+
 
     metalog_parser_object.parser = parser;
     ctx->metalog_parser_object = &metalog_parser_object;

database/engine/metadata_log/metadatalogapi.c

@@ -266,6 +266,22 @@ void metalog_commit_delete_dimension(RRDDIM *rd)
     metalog_commit_deletion_record(ctx, buffer);
 }
 
+RRDHOST *metalog_get_host_from_uuid(struct metalog_instance *ctx, uuid_t *host_guid)
+{
+    UNUSED(ctx);
+    GUID_TYPE ret;
+    char machine_guid[37];
+
+    uuid_unparse_lower(*host_guid, machine_guid);
+    ret = find_object_by_guid(host_guid, NULL, 0);
+    if (unlikely(GUID_TYPE_HOST != ret)) {
+        error("Host with GUID %s not found in the global map", machine_guid);
+        return NULL;
+    }
+    RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
+    return host;
+}
+
 RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid)
 {
     GUID_TYPE ret;

database/engine/metadata_log/metadatalogapi.h

@@ -16,6 +16,7 @@ extern void metalog_commit_delete_dimension(RRDDIM *rd);
 
 extern RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid);
 extern RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
+extern RRDHOST *metalog_get_host_from_uuid(struct metalog_instance *ctx, uuid_t *uuid);
 extern void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
 
 /* must call once before using anything */

database/engine/metadata_log/metalogpluginsd.c

@@ -4,6 +4,63 @@
 #include "metadatalog.h"
 #include "metalogpluginsd.h"
 
+PARSER_RC metalog_pluginsd_host_action(
+    void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, char *timezone,
+    char *tags)
+{
+    struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
+
+    RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
+    if (host)
+        goto write_replay;
+
+    if (strcmp(machine_guid, registry_get_this_machine_guid()) == 0) {
+        struct metalog_record record;
+        struct metadata_logfile *metalogfile = state->metalogfile;
+
+        uuid_parse(machine_guid, record.uuid);
+        mlf_record_insert(metalogfile, &record);
+        return PARSER_RC_OK;
+    }
+
+    // Ignore HOST command for now
+    // TODO: Remove when the next task is completed ie. accept new children in the lcoalhost / multidb
+    return PARSER_RC_OK;
+
+    host = rrdhost_create(
+        hostname
+        , registry_hostname
+        , machine_guid
+        , os
+        , timezone
+        , tags
+        , NULL
+        , NULL
+        , update_every
+        , 3600
+        , RRD_MEMORY_MODE_DBENGINE
+        , 0   // health enabled
+        , 0   // Push enabled
+        , NULL
+        , NULL
+        , NULL
+        , callocz(1, sizeof(struct rrdhost_system_info))
+        , 0     // localhost
+        , 1     // archived
+    );
+
+write_replay:
+    if (host) { /* It's a valid object */
+        struct metalog_record record;
+        struct metadata_logfile *metalogfile = state->metalogfile;
+
+        uuid_copy(record.uuid, host->host_uuid);
+        mlf_record_insert(metalogfile, &record);
+    }
+
+    return PARSER_RC_OK;
+}
+
 PARSER_RC metalog_pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context,
                                         char *title, char *units, char *plugin, char *module, int priority,
                                         int update_every, RRDSET_TYPE chart_type, char *options)

database/engine/metadata_log/metalogpluginsd.h

@@ -25,5 +25,7 @@ extern PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char
 extern PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid);
 extern PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid);
 extern PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid);
+extern PARSER_RC metalog_pluginsd_host(char **words, void *user, PLUGINSD_ACTION  *plugins_action);
+extern PARSER_RC metalog_pluginsd_host_action(void *user, char *machine_guid, char *hostname, char *registry_hostname, int update_every, char *os, char *timezone, char *tags);
 
 #endif /* NETDATA_METALOGPLUGINSD_H */

database/engine/rrdengineapi.c

@@ -916,11 +916,16 @@ int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_p
     if (ctx->worker_config.error) {
         goto error_after_rrdeng_worker;
     }
-    error = metalog_init(ctx);
-    if(error) {
-        error("Failed to initialize metadata log file event loop.");
-        goto error_after_rrdeng_worker;
+    if ((strcmp(host->machine_guid, registry_get_this_machine_guid()) == 0) || (!rrdhost_flag_check(host, RRDHOST_FLAG_MULTIHOST))) {
+        info("Metadatalog init for host %s starting...", host->hostname);
+        error = metalog_init(ctx);
+        if (error) {
+            error("Failed to initialize metadata log file event loop.");
+            goto error_after_rrdeng_worker;
+        }
     }
+    else
+        info("No metadatalog init for host %s", host->hostname);
     return 0;
 
 error_after_rrdeng_worker:

database/rrd.h

@@ -560,6 +560,8 @@ typedef enum rrdhost_flags {
     RRDHOST_FLAG_DELETE_ORPHAN_HOST     = 1 << 2, // delete the entire host when orphan
     RRDHOST_FLAG_BACKEND_SEND           = 1 << 3, // send it to backends
     RRDHOST_FLAG_BACKEND_DONT_SEND      = 1 << 4, // don't send it to backends
+    RRDHOST_FLAG_ARCHIVED               = 1 << 5, // The host is archived, no collected charts yet
+    RRDHOST_FLAG_MULTIHOST              = 1 << 6, // Host belongs to localhost/megadb
 } RRDHOST_FLAGS;
 
 #ifdef HAVE_C___ATOMIC
@@ -1195,6 +1197,12 @@ extern void rrdset_delete_custom(RRDSET *st, int db_rotated);
 extern void rrdset_delete_obsolete_dimensions(RRDSET *st);
 
 extern void rrdhost_cleanup_obsolete_charts(RRDHOST *host);
+extern RRDHOST *rrdhost_create(
+    const char *hostname, const char *registry_hostname, const char *guid, const char *os, const char *timezone,
+    const char *tags, const char *program_name, const char *program_version, int update_every, long entries,
+    RRD_MEMORY_MODE memory_mode, unsigned int health_enabled, unsigned int rrdpush_enabled, char *rrdpush_destination,
+    char *rrdpush_api_key, char *rrdpush_send_charts_matching, struct rrdhost_system_info *system_info,
+    int is_localhost, int is_archived);
 
 #endif /* NETDATA_RRD_INTERNALS */