Labels issues (#7515)

Initial work on host labels from the dedicated branch. Includes work for issues #7096, #7400, #7411, #7369, #7410, #7458, #7459, #7412 and #7408 by @vlvkobal, @thiagoftsm, @cakrit and @amoss.

.gitignore

@@ -113,6 +113,7 @@ system/netdata-freebsd
 system/edit-config
 
 daemon/anonymous-statistics.sh
+daemon/get-kubernetes-labels.sh
 
 health/notifications/alarm-notify.sh
 collectors/cgroups.plugin/cgroup-name.sh

BUILD.md

@@ -1,3 +1,4 @@
+
 # The build system
 
 We are currently migrating from `autotools` to `CMake` as a build-system. This document 

daemon/Makefile.am

@@ -13,9 +13,11 @@ dist_noinst_DATA = \
     README.md \
     config/README.md \
     anonymous-statistics.sh.in \
+    get-kubernetes-labels.sh.in \
     $(NULL)
 
 dist_plugins_SCRIPTS = \
     anonymous-statistics.sh \
     system-info.sh \
+    get-kubernetes-labels.sh \
     $(NULL)

daemon/commands.c

@@ -40,6 +40,7 @@ static cmd_status_t cmd_save_database_execute(char *args, char **message);
 static cmd_status_t cmd_reopen_logs_execute(char *args, char **message);
 static cmd_status_t cmd_exit_execute(char *args, char **message);
 static cmd_status_t cmd_fatal_execute(char *args, char **message);
+static cmd_status_t cmd_reload_labels_execute(char *args, char **message);
 
 static command_info_t command_info_array[] = {
         {"help", cmd_help_execute, CMD_TYPE_HIGH_PRIORITY},                  // show help menu
@@ -48,6 +49,7 @@ static command_info_t command_info_array[] = {
         {"reopen-logs", cmd_reopen_logs_execute, CMD_TYPE_ORTHOGONAL},       // Close and reopen log files
         {"shutdown-agent", cmd_exit_execute, CMD_TYPE_EXCLUSIVE},            // exit cleanly
         {"fatal-agent", cmd_fatal_execute, CMD_TYPE_HIGH_PRIORITY},          // exit with fatal error
+        {"reload-labels", cmd_reload_labels_execute, CMD_TYPE_ORTHOGONAL},   // reload the labels
 };
 
 /* Mutexes for commands of type CMD_TYPE_ORTHOGONAL */
@@ -97,6 +99,8 @@ static cmd_status_t cmd_help_execute(char *args, char **message)
              "    Show this help menu.\n"
              "reload-health\n"
              "    Reload health configuration.\n"
+             "reload-labels\n"
+             "    Reload all labels.\n"
              "save-database\n"
              "    Save internal DB to disk for memory mode save.\n"
              "reopen-logs\n"
@@ -172,6 +176,28 @@ static cmd_status_t cmd_fatal_execute(char *args, char **message)
     return CMD_STATUS_SUCCESS;
 }
 
+static cmd_status_t cmd_reload_labels_execute(char *args, char **message)
+{
+    (void)args;
+    info("COMMAND: reloading host labels.");
+    reload_host_labels();
+
+    BUFFER *wb = buffer_create(10);
+
+    netdata_rwlock_rdlock(&localhost->labels_rwlock);
+    struct label *l=localhost->labels;
+    while (l != NULL) {
+        buffer_sprintf(wb,"Label [source id=%s]: \"%s\" -> \"%s\"\n", translate_label_source(l->label_source), l->key, l->value);
+        l = l->next;
+    }
+    netdata_rwlock_unlock(&localhost->labels_rwlock);
+
+    (*message)=strdupz(buffer_tostring(wb));
+    buffer_free(wb);
+
+    return CMD_STATUS_SUCCESS;
+}
+
 static void cmd_lock_exclusive(unsigned index)
 {
     (void)index;

daemon/commands.h

@@ -19,6 +19,7 @@ typedef enum cmd {
     CMD_REOPEN_LOGS,
     CMD_EXIT,
     CMD_FATAL,
+    CMD_RELOAD_LABELS,
     CMD_TOTAL_COMMANDS
 } cmd_t;
 

daemon/common.h

@@ -8,7 +8,7 @@
 // ----------------------------------------------------------------------------
 // shortcuts for the default netdata configuration
 
-#define config_load(filename, overwrite_used) appconfig_load(&netdata_config, filename, overwrite_used)
+#define config_load(filename, overwrite_used, section) appconfig_load(&netdata_config, filename, overwrite_used, section)
 #define config_get(section, name, default_value) appconfig_get(&netdata_config, section, name, default_value)
 #define config_get_number(section, name, value) appconfig_get_number(&netdata_config, section, name, value)
 #define config_get_float(section, name, value) appconfig_get_float(&netdata_config, section, name, value)

daemon/get-kubernetes-labels.sh.in

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Checks if netdata is running in a kubernetes pod and fetches that pod's labels
+
+if [ -n "${KUBERNETES_SERVICE_HOST}" ] && [ -n "${KUBERNETES_PORT_443_TCP_PORT}" ] && [ -n "${MY_POD_NAMESPACE}" ] && [ -n "${MY_POD_NAME}" ]; then
+	if command -v jq >/dev/null 2>&1; then
+			KUBE_TOKEN="$(</var/run/secrets/kubernetes.io/serviceaccount/token)"
+		URL="https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT/api/v1/namespaces/$MY_POD_NAMESPACE/pods/$MY_POD_NAME"
+		curl -sSk -H "Authorization: Bearer $KUBE_TOKEN" "$URL" |
+		jq -r '.metadata.labels' | grep ':' | tr -d '," '
+		exit 0
+	else
+		echo "jq command not available. Please install jq to get host labels for kubernetes pods."
+		exit 1
+	fi
+else
+	exit 0
+fi

daemon/main.c

@@ -703,20 +703,20 @@ static int load_netdata_conf(char *filename, char overwrite_used) {
     int ret = 0;
 
     if(filename && *filename) {
-        ret = config_load(filename, overwrite_used);
+        ret = config_load(filename, overwrite_used, NULL);
         if(!ret)
             error("CONFIG: cannot load config file '%s'.", filename);
     }
     else {
         filename = strdupz_path_subpath(netdata_configured_user_config_dir, "netdata.conf");
 
-        ret = config_load(filename, overwrite_used);
+        ret = config_load(filename, overwrite_used, NULL);
         if(!ret) {
             info("CONFIG: cannot load user config '%s'. Will try the stock version.", filename);
             freez(filename);
 
             filename = strdupz_path_subpath(netdata_configured_stock_config_dir, "netdata.conf");
-            ret = config_load(filename, overwrite_used);
+            ret = config_load(filename, overwrite_used, NULL);
             if(!ret)
                 info("CONFIG: cannot load stock config '%s'. Running with internal defaults.", filename);
         }
@@ -1271,6 +1271,9 @@ int main(int argc, char **argv) {
 
     error_log_limit_reset();
 
+    // Load host labels
+    reload_host_labels();
+
     // ------------------------------------------------------------------------
     // spawn the threads
 

daemon/system-info.sh

@@ -107,7 +107,7 @@ if [ "${CONTAINER}" = "unknown" ]; then
 	fi
 fi
 
-echo "NETDATA_SYSTEM_OS_NAME=\"${NAME}\""
+echo "NETDATA_SYSTEM_OS_NAME=${NAME}"
 echo "NETDATA_SYSTEM_OS_ID=${ID}"
 echo "NETDATA_SYSTEM_OS_ID_LIKE=${ID_LIKE}"
 echo "NETDATA_SYSTEM_OS_VERSION=${VERSION}"

database/rrd.h

@@ -148,6 +148,25 @@ typedef enum rrddim_flags {
 #define rrddim_flag_clear(rd, flag) (rd)->flags &= ~(flag)
 #endif
 
+typedef enum label_source {
+    LABEL_SOURCE_AUTO             = 0,
+    LABEL_SOURCE_NETDATA_CONF     = 1,
+    LABEL_SOURCE_DOCKER           = 2,
+    LABEL_SOURCE_ENVIRONMENT      = 3,
+    LABEL_SOURCE_KUBERNETES       = 4
+} LABEL_SOURCE;
+
+struct label {
+    char *key, *value;
+    uint32_t key_hash;
+    LABEL_SOURCE label_source;
+    struct label *next;
+};
+
+char *translate_label_source(LABEL_SOURCE l);
+struct label *create_label(char *key, char *value, LABEL_SOURCE label_source);
+struct label *add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source);
+void reload_host_labels();
 
 // ----------------------------------------------------------------------------
 // RRD DIMENSION - this is a metric
@@ -726,6 +745,11 @@ struct rrdhost {
 
     netdata_rwlock_t rrdhost_rwlock;                // lock for this RRDHOST (protects rrdset_root linked list)
 
+    // ------------------------------------------------------------------------
+    // Support for host-level labels
+    struct label *labels;
+    netdata_rwlock_t labels_rwlock;         // lock for the label list
+
     // ------------------------------------------------------------------------
     // indexes