Obvious memory reductions (#15204)

* remove rd->update_every

* reduce amount of memory for RRDDIM

* reorgnize rrddim->db entries

* optimize rrdset and statsd

* optimize dictionaries

* RW_SPINLOCK for dictionaries

* fix codeql warning

* rw_spinlock improvements

* remove obsolete assertion

* fix crash on health_alarm_log_process()

* use RW_SPINLOCK for AVL trees

* add RW_SPINLOCK read/write trylock

* pgc and mrg now use rw_spinlocks; cache line optimizations for mrg

* thread tag of dbegnine init

* append created datafile, lockless

* make DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE friendly for lockless use

* thread cancelability in spinlocks; optimize thread cancelability management

* introduce a JudyL to index datafiles and use it during queries to quickly find the relevant files

* use the last timestamp of each journal file for indexing

* when the previous cannot be found, start from the beginning

* add more stats to PDC to trace routing easier

* rename spinlock functions

* fix for spinlock renames

* revert statsd socket statistics to size_t

* turn fatal into internal_fatal()

* show candidates always

* show connected status and connection attempts

collectors/freebsd.plugin/freebsd_sysctl.c

@@ -459,9 +459,9 @@ int do_dev_cpu_temperature(int update_every, usec_t dt) {
     static RRDDIM **rd_pcpu_temperature;
 
     if (unlikely(number_of_cpus != old_number_of_cpus)) {
-        rd_pcpu_temperature = reallocz(rd_pcpu_temperature, sizeof(RRDDIM) * number_of_cpus);
+        rd_pcpu_temperature = reallocz(rd_pcpu_temperature, sizeof(RRDDIM *) * number_of_cpus);
         if (unlikely(number_of_cpus > old_number_of_cpus))
-            memset(&rd_pcpu_temperature[old_number_of_cpus], 0, sizeof(RRDDIM) * (number_of_cpus - old_number_of_cpus));
+            memset(&rd_pcpu_temperature[old_number_of_cpus], 0, sizeof(RRDDIM *) * (number_of_cpus - old_number_of_cpus));
     }
 
     if (unlikely(!st)) {

collectors/plugins.d/plugins_d.c

@@ -22,28 +22,28 @@ inline size_t pluginsd_initialize_plugin_directories()
 }
 
 static inline void plugin_set_disabled(struct plugind *cd) {
-    netdata_spinlock_lock(&cd->unsafe.spinlock);
+    spinlock_lock(&cd->unsafe.spinlock);
     cd->unsafe.enabled = false;
-    netdata_spinlock_unlock(&cd->unsafe.spinlock);
+    spinlock_unlock(&cd->unsafe.spinlock);
 }
 
 bool plugin_is_enabled(struct plugind *cd) {
-    netdata_spinlock_lock(&cd->unsafe.spinlock);
+    spinlock_lock(&cd->unsafe.spinlock);
     bool ret = cd->unsafe.enabled;
-    netdata_spinlock_unlock(&cd->unsafe.spinlock);
+    spinlock_unlock(&cd->unsafe.spinlock);
     return ret;
 }
 
 static inline void plugin_set_running(struct plugind *cd) {
-    netdata_spinlock_lock(&cd->unsafe.spinlock);
+    spinlock_lock(&cd->unsafe.spinlock);
     cd->unsafe.running = true;
-    netdata_spinlock_unlock(&cd->unsafe.spinlock);
+    spinlock_unlock(&cd->unsafe.spinlock);
 }
 
 static inline bool plugin_is_running(struct plugind *cd) {
-    netdata_spinlock_lock(&cd->unsafe.spinlock);
+    spinlock_lock(&cd->unsafe.spinlock);
     bool ret = cd->unsafe.running;
-    netdata_spinlock_unlock(&cd->unsafe.spinlock);
+    spinlock_unlock(&cd->unsafe.spinlock);
     return ret;
 }
 
@@ -53,7 +53,7 @@ static void pluginsd_worker_thread_cleanup(void *arg)
 
     worker_unregister();
 
-    netdata_spinlock_lock(&cd->unsafe.spinlock);
+    spinlock_lock(&cd->unsafe.spinlock);
 
     cd->unsafe.running = false;
     cd->unsafe.thread = 0;
@@ -61,7 +61,7 @@ static void pluginsd_worker_thread_cleanup(void *arg)
     pid_t pid = cd->unsafe.pid;
     cd->unsafe.pid = 0;
 
-    netdata_spinlock_unlock(&cd->unsafe.spinlock);
+    spinlock_unlock(&cd->unsafe.spinlock);
 
     if (pid) {
         siginfo_t info;
@@ -190,14 +190,14 @@ static void pluginsd_main_cleanup(void *data) {
 
     struct plugind *cd;
     for (cd = pluginsd_root; cd; cd = cd->next) {
-        netdata_spinlock_lock(&cd->unsafe.spinlock);
+        spinlock_lock(&cd->unsafe.spinlock);
         if (cd->unsafe.enabled && cd->unsafe.running && cd->unsafe.thread != 0) {
             info("PLUGINSD: 'host:%s', stopping plugin thread: %s",
                  rrdhost_hostname(cd->host), cd->id);
 
             netdata_thread_cancel(cd->unsafe.thread);
         }
-        netdata_spinlock_unlock(&cd->unsafe.spinlock);
+        spinlock_unlock(&cd->unsafe.spinlock);
     }
 
     info("PLUGINSD: cleanup completed.");

collectors/plugins.d/pluginsd_parser.c

@@ -11,7 +11,7 @@ static ssize_t send_to_plugin(const char *txt, void *data) {
         return 0;
 
     errno = 0;
-    netdata_spinlock_lock(&parser->writer.spinlock);
+    spinlock_lock(&parser->writer.spinlock);
     ssize_t bytes = -1;
 
 #ifdef ENABLE_HTTPS
@@ -24,7 +24,7 @@ static ssize_t send_to_plugin(const char *txt, void *data) {
         else
             error("PLUGINSD: cannot send command (SSL)");
 
-        netdata_spinlock_unlock(&parser->writer.spinlock);
+        spinlock_unlock(&parser->writer.spinlock);
         return bytes;
     }
 #endif
@@ -39,7 +39,7 @@ static ssize_t send_to_plugin(const char *txt, void *data) {
         else
             fflush(parser->fp_output);
 
-        netdata_spinlock_unlock(&parser->writer.spinlock);
+        spinlock_unlock(&parser->writer.spinlock);
         return bytes;
     }
 
@@ -52,18 +52,18 @@ static ssize_t send_to_plugin(const char *txt, void *data) {
             sent = write(parser->fd, &txt[bytes], total - bytes);
             if(sent <= 0) {
                 error("PLUGINSD: cannot send command (fd)");
-                netdata_spinlock_unlock(&parser->writer.spinlock);
+                spinlock_unlock(&parser->writer.spinlock);
                 return -3;
             }
             bytes += sent;
         }
         while(bytes < total);
 
-        netdata_spinlock_unlock(&parser->writer.spinlock);
+        spinlock_unlock(&parser->writer.spinlock);
         return (int)bytes;
     }
 
-    netdata_spinlock_unlock(&parser->writer.spinlock);
+    spinlock_unlock(&parser->writer.spinlock);
     error("PLUGINSD: cannot send command (no output socket/pipe/file given to plugins.d parser)");
     return -4;
 }
@@ -93,7 +93,7 @@ static inline RRDSET *pluginsd_get_chart_from_parent(void *user) {
 static inline void pluginsd_lock_rrdset_data_collection(void *user) {
     PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;
     if(u->st && !u->v2.locked_data_collection) {
-        netdata_spinlock_lock(&u->st->data_collection_lock);
+        spinlock_lock(&u->st->data_collection_lock);
         u->v2.locked_data_collection = true;
     }
 }
@@ -101,7 +101,7 @@ static inline void pluginsd_lock_rrdset_data_collection(void *user) {
 static inline bool pluginsd_unlock_rrdset_data_collection(void *user) {
     PARSER_USER_OBJECT *u = (PARSER_USER_OBJECT *) user;
     if(u->st && u->v2.locked_data_collection) {
-        netdata_spinlock_unlock(&u->st->data_collection_lock);
+        spinlock_unlock(&u->st->data_collection_lock);
         u->v2.locked_data_collection = false;
         return true;
     }
@@ -1233,9 +1233,9 @@ PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, void *user) {
             st->counter_done++;
 
             // these are only needed for db mode RAM, SAVE, MAP, ALLOC
-            st->current_entry++;
-            if(st->current_entry >= st->entries)
-                st->current_entry -= st->entries;
+            st->db.current_entry++;
+            if(st->db.current_entry >= st->db.entries)
+                st->db.current_entry -= st->db.entries;
 
             ((PARSER_USER_OBJECT *) user)->replay.start_time = start_time;
             ((PARSER_USER_OBJECT *) user)->replay.end_time = end_time;
@@ -1660,9 +1660,9 @@ PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user) {
     st->counter_done++;
 
     // these are only needed for db mode RAM, SAVE, MAP, ALLOC
-    st->current_entry++;
-    if(st->current_entry >= st->entries)
-        st->current_entry -= st->entries;
+    st->db.current_entry++;
+    if(st->db.current_entry >= st->db.entries)
+        st->db.current_entry -= st->db.entries;
 
     timing_step(TIMING_STEP_BEGIN2_STORE);
 
@@ -1774,7 +1774,7 @@ PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user) {
     rd->last_stored_value = value;
     rd->last_calculated_value = value;
     rd->collections_counter++;
-    rd->updated = true;
+    rrddim_set_updated(rd);
 
     timing_step(TIMING_STEP_SET2_STORE);
 
@@ -1831,7 +1831,7 @@ PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size_t num_words __maybe_
     rrddim_foreach_read(rd, st) {
                 rd->calculated_value = 0;
                 rd->collected_value = 0;
-                rd->updated = false;
+                rrddim_clear_updated(rd);
             }
     rrddim_foreach_done(rd);
 

collectors/statsd.plugin/statsd.c

@@ -34,7 +34,7 @@ typedef struct statsd_metric_gauge {
 } STATSD_METRIC_GAUGE;
 
 typedef struct statsd_metric_counter { // counter and meter
-    long long value;
+    collected_number value;
 } STATSD_METRIC_COUNTER;
 
 typedef struct statsd_histogram_extensions {
@@ -57,8 +57,8 @@ typedef struct statsd_histogram_extensions {
     RRDDIM *rd_stddev;
     //RRDDIM *rd_sum;
 
-    size_t size;
-    size_t used;
+    uint32_t size;
+    uint32_t used;
     NETDATA_DOUBLE *values;   // dynamic array of values collected
 } STATSD_METRIC_HISTOGRAM_EXTENSIONS;
 
@@ -68,24 +68,22 @@ typedef struct statsd_metric_histogram { // histogram and timer
 
 typedef struct statsd_metric_set {
     DICTIONARY *dict;
-    size_t unique;
 } STATSD_METRIC_SET;
 
 typedef struct statsd_metric_dictionary_item {
-    size_t count;
+    uint32_t count;
     RRDDIM *rd;
 } STATSD_METRIC_DICTIONARY_ITEM;
 
 typedef struct statsd_metric_dictionary {
     DICTIONARY *dict;
-    size_t unique;
 } STATSD_METRIC_DICTIONARY;
 
 
 // --------------------------------------------------------------------------------------------------------------------
 // this is a metric - for all types of metrics
 
-typedef enum statsd_metric_options {
+typedef enum __attribute__((packed)) statsd_metric_options {
     STATSD_METRIC_OPTION_NONE                         = 0x00000000, // no options set
     STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED = 0x00000001, // do not update the chart dimension, when this metric is not collected
     STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED        = 0x00000002, // render a private chart for this metric
@@ -99,7 +97,7 @@ typedef enum statsd_metric_options {
     STATSD_METRIC_OPTION_UPDATED_CHART_METADATA       = 0x00000200, // set when the private chart metadata have been updated via tags
 } STATS_METRIC_OPTIONS;
 
-typedef enum statsd_metric_type {
+typedef enum __attribute__((packed)) statsd_metric_type {
     STATSD_METRIC_TYPE_GAUGE,
     STATSD_METRIC_TYPE_COUNTER,
     STATSD_METRIC_TYPE_METER,
@@ -118,7 +116,7 @@ typedef struct statsd_metric {
 
     // metadata about data collection
     collected_number events;        // the number of times this metric has been collected (never resets)
-    size_t count;                   // the number of times this metric has been collected since the last flush
+    uint32_t count;                 // the number of times this metric has been collected since the last flush
 
     // the actual collected data
     union {
@@ -151,22 +149,21 @@ typedef struct statsd_metric {
 
 typedef struct statsd_index {
     char *name;                     // the name of the index of metrics
-    size_t events;                  // the number of events processed for this index
-    size_t metrics;                 // the number of metrics in this index
-    size_t useful;                  // the number of useful metrics in this index
+    uint32_t events;                // the number of events processed for this index
+    uint32_t metrics;               // the number of metrics in this index
+    uint32_t useful;                // the number of useful metrics in this index
 
+    STATS_METRIC_OPTIONS default_options;  // default options for all metrics in this index
     STATSD_METRIC_TYPE type;        // the type of index
     DICTIONARY *dict;
 
     STATSD_METRIC *first_useful;    // the linked list of useful metrics (new metrics are added in front)
-
-    STATS_METRIC_OPTIONS default_options;  // default options for all metrics in this index
 } STATSD_INDEX;
 
 // --------------------------------------------------------------------------------------------------------------------
 // synthetic charts
 
-typedef enum statsd_app_chart_dimension_value_type {
+typedef enum __attribute__((packed)) statsd_app_chart_dimension_value_type {
     STATSD_APP_CHART_DIM_VALUE_TYPE_EVENTS,
     STATSD_APP_CHART_DIM_VALUE_TYPE_LAST,
     STATSD_APP_CHART_DIM_VALUE_TYPE_AVERAGE,
@@ -183,18 +180,18 @@ typedef struct statsd_app_chart_dimension {
     const char *metric;             // the source metric name of this dimension
     uint32_t metric_hash;           // hash for fast string comparisons
 
-    SIMPLE_PATTERN *metric_pattern; // set when the 'metric' is a simple pattern
-
-    collected_number multiplier;    // the multiplier of the dimension
-    collected_number divisor;       // the divisor of the dimension
+    int32_t multiplier;             // the multiplier of the dimension
+    int32_t divisor;                // the divisor of the dimension
     RRDDIM_FLAGS flags;             // the RRDDIM flags for this dimension
     RRDDIM_OPTIONS options;         // the RRDDIM options for this dimension
+    RRD_ALGORITHM algorithm;        // the algorithm of this dimension
 
     STATSD_APP_CHART_DIM_VALUE_TYPE value_type; // which value to use of the source metric
 
+    SIMPLE_PATTERN *metric_pattern; // set when the 'metric' is a simple pattern
+
     RRDDIM *rd;                     // a pointer to the RRDDIM that has been created for this dimension
     collected_number *value_ptr;    // a pointer to the source metric value
-    RRD_ALGORITHM algorithm;        // the algorithm of this dimension
 
     struct statsd_app_chart_dimension *next; // the next dimension for this chart
 } STATSD_APP_CHART_DIM;
@@ -207,11 +204,11 @@ typedef struct statsd_app_chart {
     const char *context;
     const char *units;
     const char *module;
-    long priority;
+    int32_t priority;
     RRDSET_TYPE chart_type;
     STATSD_APP_CHART_DIM *dimensions;
-    size_t dimensions_count;
-    size_t dimensions_linked_count;
+    uint32_t dimensions_count;
+    uint32_t dimensions_linked_count;
 
     RRDSET *st;
     struct statsd_app_chart *next;
@@ -222,8 +219,8 @@ typedef struct statsd_app {
     SIMPLE_PATTERN *metrics;
     STATS_METRIC_OPTIONS default_options;
     RRD_MEMORY_MODE rrd_memory_mode;
+    int32_t rrd_history_entries;
     DICTIONARY *dict;
-    long rrd_history_entries;
 
     const char *source;
     STATSD_APP_CHART *charts;
@@ -236,7 +233,7 @@ typedef struct statsd_app {
 struct collection_thread_status {
     SPINLOCK spinlock;
     bool running;
-    size_t max_sockets;
+    uint32_t max_sockets;
 
     netdata_thread_t thread;
 };
@@ -262,23 +259,22 @@ static struct statsd {
     size_t udp_packets_received;
     size_t udp_bytes_read;
 
-    int enabled;
-    int update_every;
+    int32_t update_every;
+    bool enabled;
+    bool private_charts_hidden;
     SIMPLE_PATTERN *charts_for;
 
-    size_t tcp_idle_timeout;
+    uint32_t tcp_idle_timeout;
     collected_number decimal_detail;
-    size_t private_charts;
-    size_t max_private_charts_hard;
-    long private_charts_rrd_history_entries;
-    unsigned int private_charts_hidden:1;
+    uint32_t private_charts;
+    uint32_t max_private_charts_hard;
 
     STATSD_APP *apps;
-    size_t recvmmsg_size;
-    size_t histogram_increase_step;
+    uint32_t recvmmsg_size;
+    uint32_t histogram_increase_step;
+    uint32_t dictionary_max_unique;
     double histogram_percentile;
     char *histogram_percentile_str;
-    size_t dictionary_max_unique;
 
     int threads;
     struct collection_thread_status *collection_threads_status;
@@ -287,7 +283,7 @@ static struct statsd {
 } statsd = {
         .enabled = 1,
         .max_private_charts_hard = 1000,
-        .private_charts_hidden = 0,
+        .private_charts_hidden = false,
         .recvmmsg_size = 10,
         .decimal_detail = STATSD_DECIMAL_DETAIL,
 
@@ -571,13 +567,6 @@ static inline void statsd_process_histogram_or_timer(STATSD_METRIC *m, const cha
 #define statsd_process_timer(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "timer")
 #define statsd_process_histogram(m, value, sampling) statsd_process_histogram_or_timer(m, value, sampling, "histogram")
 
-static void dictionary_metric_set_value_insert_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
-    (void)item;
-    (void)value;
-    STATSD_METRIC *m = (STATSD_METRIC *)data;
-    m->set.unique++;
-}
-
 static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
     if(!is_metric_useful_for_collection(m)) return;
 
@@ -594,11 +583,8 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
         statsd_reset_metric(m);
     }
 
-    if (unlikely(!m->set.dict)) {
-        m->set.dict   = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
-        dictionary_register_insert_callback(m->set.dict, dictionary_metric_set_value_insert_callback, m);
-        m->set.unique = 0;
-    }
+    if (unlikely(!m->set.dict))
+        m->set.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
 
     if(unlikely(value_is_zinit(value))) {
         // magic loading of metric, without affecting anything
@@ -616,13 +602,6 @@ static inline void statsd_process_set(STATSD_METRIC *m, const char *value) {
     }
 }
 
-static void dictionary_metric_dict_value_insert_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
-    (void)item;
-    (void)value;
-    STATSD_METRIC *m = (STATSD_METRIC *)data;
-    m->dictionary.unique++;
-}
-
 static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value) {
     if(!is_metric_useful_for_collection(m)) return;
 
@@ -634,11 +613,8 @@ static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value
     if(unlikely(m->reset))
         statsd_reset_metric(m);
 
-    if (unlikely(!m->dictionary.dict)) {
-        m->dictionary.dict   = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
-        dictionary_register_insert_callback(m->dictionary.dict, dictionary_metric_dict_value_insert_callback, m);
-        m->dictionary.unique = 0;
-    }
+    if (unlikely(!m->dictionary.dict))
+        m->dictionary.dict = dictionary_create_advanced(STATSD_DICTIONARY_OPTIONS, &dictionary_stats_category_collectors, 0);
 
     if(unlikely(value_is_zinit(value))) {
         // magic loading of metric, without affecting anything
@@ -647,7 +623,7 @@ static inline void statsd_process_dictionary(STATSD_METRIC *m, const char *value
         STATSD_METRIC_DICTIONARY_ITEM *t = (STATSD_METRIC_DICTIONARY_ITEM *)dictionary_get(m->dictionary.dict, value);
 
         if (unlikely(!t)) {
-            if(!t && m->dictionary.unique >= statsd.dictionary_max_unique)
+            if(!t && dictionary_entries(m->dictionary.dict) >= statsd.dictionary_max_unique)
                 value = "other";
 
             t = (STATSD_METRIC_DICTIONARY_ITEM *)dictionary_set(m->dictionary.dict, value, NULL, sizeof(STATSD_METRIC_DICTIONARY_ITEM));
@@ -1096,9 +1072,9 @@ static int statsd_snd_callback(POLLINFO *pi, short int *events) {
 
 void statsd_collector_thread_cleanup(void *data) {
     struct statsd_udp *d = data;
-    netdata_spinlock_lock(&d->status->spinlock);
+    spinlock_lock(&d->status->spinlock);
     d->status->running = false;
-    netdata_spinlock_unlock(&d->status->spinlock);
+    spinlock_unlock(&d->status->spinlock);
 
     collector_info("cleaning up...");
 
@@ -1121,9 +1097,9 @@ static bool statsd_should_stop(void) {
 
 void *statsd_collector_thread(void *ptr) {
     struct collection_thread_status *status = ptr;
-    netdata_spinlock_lock(&status->spinlock);
+    spinlock_lock(&status->spinlock);
     status->running = true;
-    netdata_spinlock_unlock(&status->spinlock);
+    spinlock_unlock(&status->spinlock);
 
     worker_register("STATSD");
     worker_register_job_name(WORKER_JOB_TYPE_TCP_CONNECTED, "tcp connect");
@@ -1255,7 +1231,7 @@ static STATSD_APP_CHART_DIM *add_dimension_to_app_chart(
     }
     chart->dimensions_count++;
 
-    debug(D_STATSD, "Added dimension '%s' to chart '%s' of app '%s', for metric '%s', with type %u, multiplier " COLLECTED_NUMBER_FORMAT ", divisor " COLLECTED_NUMBER_FORMAT,
+    debug(D_STATSD, "Added dimension '%s' to chart '%s' of app '%s', for metric '%s', with type %u, multiplier %d, divisor %d",
             dim->name, chart->id, app->name, dim->metric, dim->value_type, dim->multiplier, dim->divisor);
 
     return dim;
@@ -1909,7 +1885,7 @@ static inline void statsd_flush_set(STATSD_METRIC *m) {
 
     int updated = 0;
     if(unlikely(!m->reset && m->count)) {
-        m->last = (collected_number)m->set.unique;
+        m->last = (collected_number)dictionary_entries(m->set.dict);
 
         m->reset = 1;
         updated = 1;
@@ -1927,7 +1903,7 @@ static inline void statsd_flush_dictionary(STATSD_METRIC *m) {
 
     int updated = 0;
     if(unlikely(!m->reset && m->count)) {
-        m->last = (collected_number)m->dictionary.unique;
+        m->last = (collected_number)dictionary_entries(m->dictionary.dict);
 
         m->reset = 1;
         updated = 1;
@@ -1939,13 +1915,13 @@ static inline void statsd_flush_dictionary(STATSD_METRIC *m) {
     if(unlikely(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED && (updated || !(m->options & STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED))))
         statsd_private_chart_dictionary(m);
 
-    if(m->dictionary.unique >= statsd.dictionary_max_unique) {
+    if(dictionary_entries(m->dictionary.dict) >= statsd.dictionary_max_unique) {
         if(!(m->options & STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED)) {
             m->options |= STATSD_METRIC_OPTION_COLLECTION_FULL_LOGGED;
             collector_info(
                 "STATSD dictionary '%s' reach max of %zu items - try increasing 'dictionaries max unique dimensions' in netdata.conf",
                 m->name,
-                m->dictionary.unique);
+                dictionary_entries(m->dictionary.dict));
         }
     }
 }
@@ -2314,8 +2290,13 @@ static inline void statsd_flush_index_metrics(STATSD_INDEX *index, void (*flush_
 
         if(unlikely(!(m->options & STATSD_METRIC_OPTION_PRIVATE_CHART_CHECKED))) {
             if(unlikely(statsd.private_charts >= statsd.max_private_charts_hard)) {
-                debug(D_STATSD, "STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts has been reached.", m->name);
-                collector_info("STATSD: metric '%s' will not be charted, because the hard limit of the maximum number of charts (%zu) has been reached. Increase the number of charts by editing netdata.conf, [statsd] section.", m->name, statsd.max_private_charts_hard);
+                debug(D_STATSD, "STATSD: metric '%s' will not be charted, because the hard limit of the maximum number "
+                                "of charts has been reached.", m->name);
+
+                collector_info("STATSD: metric '%s' will not be charted, because the hard limit of the maximum number "
+                               "of charts (%u) has been reached. Increase the number of charts by editing netdata.conf, "
+                               "[statsd] section.", m->name, statsd.max_private_charts_hard);
+
                 m->options &= ~STATSD_METRIC_OPTION_PRIVATE_CHART_ENABLED;
             }
             else {
@@ -2366,7 +2347,7 @@ static void statsd_main_cleanup(void *data) {
     if (statsd.collection_threads_status) {
         int i;
         for (i = 0; i < statsd.threads; i++) {
-            netdata_spinlock_lock(&statsd.collection_threads_status[i].spinlock);
+            spinlock_lock(&statsd.collection_threads_status[i].spinlock);
             if(statsd.collection_threads_status[i].running) {
                 collector_info("STATSD: stopping data collection thread %d...", i + 1);
                 netdata_thread_cancel(statsd.collection_threads_status[i].thread);
@@ -2374,7 +2355,7 @@ static void statsd_main_cleanup(void *data) {
             else {
                 collector_info("STATSD: data collection thread %d found stopped.", i + 1);
             }
-            netdata_spinlock_unlock(&statsd.collection_threads_status[i].spinlock);
+            spinlock_unlock(&statsd.collection_threads_status[i].spinlock);
         }
     }
 
@@ -2466,7 +2447,6 @@ void *statsd_main(void *ptr) {
             config_get(CONFIG_SECTION_STATSD, "create private charts for metrics matching", "*"), NULL,
             SIMPLE_PATTERN_EXACT, true);
     statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts_hard);
-    statsd.private_charts_rrd_history_entries = (int)config_get_number(CONFIG_SECTION_STATSD, "private charts history", default_rrd_history_entries);
     statsd.decimal_detail = (collected_number)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail);
     statsd.tcp_idle_timeout = (size_t) config_get_number(CONFIG_SECTION_STATSD, "disconnect idle tcp clients after seconds", (long long int)statsd.tcp_idle_timeout);
     statsd.private_charts_hidden = (unsigned int)config_get_boolean(CONFIG_SECTION_STATSD, "private charts hidden", statsd.private_charts_hidden);
@@ -2549,7 +2529,7 @@ void *statsd_main(void *ptr) {
         statsd.collection_threads_status[i].max_sockets = max_sockets / statsd.threads;
         char tag[NETDATA_THREAD_TAG_MAX + 1];
         snprintfz(tag, NETDATA_THREAD_TAG_MAX, "STATSD_IN[%d]", i + 1);
-        netdata_spinlock_init(&statsd.collection_threads_status[i].spinlock);
+        spinlock_init(&statsd.collection_threads_status[i].spinlock);
         netdata_thread_create(&statsd.collection_threads_status[i].thread, tag, NETDATA_THREAD_OPTION_DEFAULT, statsd_collector_thread, &statsd.collection_threads_status[i]);
     }
 

daemon/global_statistics.c

@@ -1718,7 +1718,7 @@ static void dbengine2_statistics_charts(void) {
     cache_efficiency_stats = rrdeng_get_cache_efficiency_stats();
 
     mrg_stats_old = mrg_stats;
-    mrg_stats = mrg_get_statistics(main_mrg);
+    mrg_get_statistics(main_mrg, &mrg_stats);
 
     struct rrdeng_buffer_sizes buffers = rrdeng_get_buffer_sizes();
     size_t buffers_total_size = buffers.handles + buffers.xt_buf + buffers.xt_io + buffers.pdc + buffers.descriptors +

daemon/main.c

@@ -60,7 +60,7 @@ SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t
     SERVICE_THREAD *sth = NULL;
     pid_t tid = gettid();
 
-    netdata_spinlock_lock(&service_globals.lock);
+    spinlock_lock(&service_globals.lock);
     Pvoid_t *PValue = JudyLIns(&service_globals.pid_judy, tid, PJE0);
     if(!*PValue) {
         sth = callocz(1, sizeof(SERVICE_THREAD));
@@ -87,7 +87,7 @@ SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t
     else {
         sth = *PValue;
     }
-    netdata_spinlock_unlock(&service_globals.lock);
+    spinlock_unlock(&service_globals.lock);
 
     return sth;
 }
@@ -95,13 +95,13 @@ SERVICE_THREAD *service_register(SERVICE_THREAD_TYPE thread_type, request_quit_t
 void service_exits(void) {
     pid_t tid = gettid();
 
-    netdata_spinlock_lock(&service_globals.lock);
+    spinlock_lock(&service_globals.lock);
     Pvoid_t *PValue = JudyLGet(service_globals.pid_judy, tid, PJE0);
     if(PValue) {
         freez(*PValue);
         JudyLDel(&service_globals.pid_judy, tid, PJE0);
     }
-    netdata_spinlock_unlock(&service_globals.lock);
+    spinlock_unlock(&service_globals.lock);
 }
 
 bool service_running(SERVICE_TYPE service) {
@@ -124,7 +124,7 @@ bool service_running(SERVICE_TYPE service) {
 void service_signal_exit(SERVICE_TYPE service) {
     __atomic_and_fetch(&service_globals.running, ~(service), __ATOMIC_RELAXED);
 
-    netdata_spinlock_lock(&service_globals.lock);
+    spinlock_lock(&service_globals.lock);
 
     Pvoid_t *PValue;
     Word_t tid = 0;
@@ -133,14 +133,14 @@ void service_signal_exit(SERVICE_TYPE service) {
         SERVICE_THREAD *sth = *PValue;
 
         if((sth->services & service) && sth->request_quit_callback) {
-            netdata_spinlock_unlock(&service_globals.lock);
+            spinlock_unlock(&service_globals.lock);
             sth->request_quit_callback(sth->data);
-            netdata_spinlock_lock(&service_globals.lock);
+            spinlock_lock(&service_globals.lock);
             continue;
         }
     }
 
-    netdata_spinlock_unlock(&service_globals.lock);
+    spinlock_unlock(&service_globals.lock);
 }
 
 static void service_to_buffer(BUFFER *wb, SERVICE_TYPE service) {
@@ -187,7 +187,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
     {
         buffer_flush(thread_list);
 
-        netdata_spinlock_lock(&service_globals.lock);
+        spinlock_lock(&service_globals.lock);
 
         Pvoid_t *PValue;
         Word_t tid = 0;
@@ -217,15 +217,15 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
                 running_services |= sth->services & service;
 
                 if(sth->force_quit_callback) {
-                    netdata_spinlock_unlock(&service_globals.lock);
+                    spinlock_unlock(&service_globals.lock);
                     sth->force_quit_callback(sth->data);
-                    netdata_spinlock_lock(&service_globals.lock);
+                    spinlock_lock(&service_globals.lock);
                     continue;
                 }
             }
         }
 
-        netdata_spinlock_unlock(&service_globals.lock);
+        spinlock_unlock(&service_globals.lock);
     }
 
     service_signal_exit(service);
@@ -244,7 +244,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
         running_services = 0;
         buffer_flush(thread_list);
 
-        netdata_spinlock_lock(&service_globals.lock);
+        spinlock_lock(&service_globals.lock);
 
         Pvoid_t *PValue;
         Word_t tid = 0;
@@ -262,7 +262,7 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
             }
         }
 
-        netdata_spinlock_unlock(&service_globals.lock);
+        spinlock_unlock(&service_globals.lock);
 
         if(running) {
             log_countdown_ut -= (log_countdown_ut >= sleep_ut) ? sleep_ut : log_countdown_ut;

daemon/unit_test.c

@@ -1334,13 +1334,14 @@ int run_test(struct test *test)
     int errors = 0;
 
     if(st->counter != test->result_entries) {
-        fprintf(stderr, "    %s stored %zu entries, but we were expecting %lu, ### E R R O R ###\n", test->name, st->counter, test->result_entries);
+        fprintf(stderr, "    %s stored %u entries, but we were expecting %lu, ### E R R O R ###\n",
+                test->name, st->counter, test->result_entries);
         errors++;
     }
 
     unsigned long max = (st->counter < test->result_entries)?st->counter:test->result_entries;
     for(c = 0 ; c < max ; c++) {
-        NETDATA_DOUBLE v = unpack_storage_number(rd->db[c]);
+        NETDATA_DOUBLE v = unpack_storage_number(rd->db.data[c]);
         NETDATA_DOUBLE n = unpack_storage_number(pack_storage_number(test->results[c], SN_DEFAULT_FLAGS));
         int same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;
         fprintf(stderr, "    %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT
@@ -1352,7 +1353,7 @@ int run_test(struct test *test)
         if(!same) errors++;
 
         if(rd2) {
-            v = unpack_storage_number(rd2->db[c]);
+            v = unpack_storage_number(rd2->db.data[c]);
             n = test->results2[c];
             same = (roundndd(v * 10000000.0) == roundndd(n * 10000000.0))?1:0;
             fprintf(stderr, "    %s/%s: checking position %lu (at %"PRId64" secs), expecting value " NETDATA_DOUBLE_FORMAT
@@ -1602,7 +1603,7 @@ int unit_test(long delay, long shift)
         fprintf(stderr, "\nPOSITION: c = %lu, EXPECTED VALUE %lu\n", c, (oincrement + c * increment + increment * (1000000 - shift) / 1000000 )* 10);
 
         rrddim_foreach_read(rd, st) {
-            sn = rd->db[c];
+            sn = rd->db.data[c];
             cn = unpack_storage_number(sn);
             fprintf(stderr, "\t %s " NETDATA_DOUBLE_FORMAT " (PACKED AS " STORAGE_NUMBER_FORMAT ")   ->   ", rrddim_id(rd), cn, sn);
 
@@ -1821,7 +1822,7 @@ static inline void rrddim_set_by_pointer_fake_time(RRDDIM *rd, collected_number
     rd->last_collected_time.tv_sec = now;
     rd->last_collected_time.tv_usec = 0;
     rd->collected_value = value;
-    rd->updated = 1;
+    rrddim_set_updated(rd);
 
     rd->collections_counter++;
 

database/contexts/api_v2.c

@@ -299,19 +299,17 @@ static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *ke
         return;
 
     buffer_json_member_add_object(wb, key);
+    {
+        buffer_json_member_add_uint64(wb, "id", s->stream.id);
+        buffer_json_member_add_uint64(wb, "hops", s->stream.hops);
+        buffer_json_member_add_string(wb, "status", rrdhost_streaming_status_to_string(s->stream.status));
+        buffer_json_member_add_time_t(wb, "since", s->stream.since);
+        buffer_json_member_add_time_t(wb, "age", s->now - s->stream.since);
 
-    buffer_json_member_add_uint64(wb, "id", s->stream.id);
-    buffer_json_member_add_uint64(wb, "hops", s->stream.hops);
-    buffer_json_member_add_string(wb, "status", rrdhost_streaming_status_to_string(s->stream.status));
-    buffer_json_member_add_time_t(wb, "since", s->stream.since);
-    buffer_json_member_add_time_t(wb, "age", s->now - s->stream.since);
-
-    if(s->stream.status == RRDHOST_STREAM_STATUS_OFFLINE)
-        buffer_json_member_add_string(wb, "reason", s->stream.reason);
-
-    if(s->stream.status == RRDHOST_STREAM_STATUS_REPLICATING || s->stream.status == RRDHOST_STREAM_STATUS_ONLINE) {
+        if (s->stream.status == RRDHOST_STREAM_STATUS_OFFLINE)
+            buffer_json_member_add_string(wb, "reason", s->stream.reason);
 
-        if(s->stream.status == RRDHOST_STREAM_STATUS_REPLICATING) {
+        if (s->stream.status == RRDHOST_STREAM_STATUS_REPLICATING) {
             buffer_json_member_add_object(wb, "replication");
             {
                 buffer_json_member_add_boolean(wb, "in_progress", s->stream.replication.in_progress);
@@ -346,6 +344,7 @@ static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *ke
             struct rrdpush_destinations *d;
             for (d = s->host->destinations; d; d = d->next) {
                 buffer_json_add_array_item_object(wb);
+                buffer_json_member_add_uint64(wb, "attempts", d->attempts);
                 {
 
                     if (d->ssl) {
@@ -357,13 +356,12 @@ static void rrdhost_sender_to_json(BUFFER *wb, RRDHOST_STATUS *s, const char *ke
 
                     buffer_json_member_add_time_t(wb, "last_check", d->last_attempt);
                     buffer_json_member_add_time_t(wb, "age", s->now - d->last_attempt);
-                    buffer_json_member_add_string(wb, "last_error", d->last_error);
-                    buffer_json_member_add_string(wb, "last_handshake",
-                                                  stream_handshake_error_to_string(d->last_handshake));
-                    buffer_json_member_add_time_t(wb, "next_check", d->postpone_reconnection_until);
-                    buffer_json_member_add_time_t(wb, "next_in",
-                                                  (d->postpone_reconnection_until > s->now) ?
-                                                  d->postpone_reconnection_until - s->now : 0);
+                    buffer_json_member_add_string_or_omit(wb, "last_error", d->last_error);
+                    buffer_json_member_add_string(wb, "last_handshake", stream_handshake_error_to_string(d->last_handshake));
+                    if(d->postpone_reconnection_until > s->now) {
+                        buffer_json_member_add_time_t(wb, "next_check", d->postpone_reconnection_until);
+                        buffer_json_member_add_time_t(wb, "next_in", d->postpone_reconnection_until - s->now);
+                    }
                 }
                 buffer_json_object_close(wb); // each candidate
             }

database/contexts/query_target.c

@@ -131,10 +131,10 @@ void query_target_release(QUERY_TARGET *qt) {
 
     qt->id[0] = '\0';
 
-    netdata_spinlock_lock(&query_target_base.used.spinlock);
+    spinlock_lock(&query_target_base.used.spinlock);
     DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(query_target_base.used.base, qt, internal.prev, internal.next);
     query_target_base.used.count--;
-    netdata_spinlock_unlock(&query_target_base.used.spinlock);
+    spinlock_unlock(&query_target_base.used.spinlock);
 
     qt->internal.used = false;
     thread_qt = NULL;
@@ -143,29 +143,29 @@ void query_target_release(QUERY_TARGET *qt) {
         query_target_destroy(qt);
     }
     else {
-        netdata_spinlock_lock(&query_target_base.available.spinlock);
+        spinlock_lock(&query_target_base.available.spinlock);
         DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(query_target_base.available.base, qt, internal.prev, internal.next);
         query_target_base.available.count++;
-        netdata_spinlock_unlock(&query_target_base.available.spinlock);
+        spinlock_unlock(&query_target_base.available.spinlock);
     }
 }
 
 static QUERY_TARGET *query_target_get(void) {
-    netdata_spinlock_lock(&query_target_base.available.spinlock);
+    spinlock_lock(&query_target_base.available.spinlock);
     QUERY_TARGET *qt = query_target_base.available.base;
     if (qt) {
         DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(query_target_base.available.base, qt, internal.prev, internal.next);
         query_target_base.available.count--;
     }
-    netdata_spinlock_unlock(&query_target_base.available.spinlock);
+    spinlock_unlock(&query_target_base.available.spinlock);
 
     if(unlikely(!qt))
         qt = callocz(1, sizeof(*qt));
 
-    netdata_spinlock_lock(&query_target_base.used.spinlock);
+    spinlock_lock(&query_target_base.used.spinlock);
     DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(query_target_base.used.base, qt, internal.prev, internal.next);
     query_target_base.used.count++;
-    netdata_spinlock_unlock(&query_target_base.used.spinlock);
+    spinlock_unlock(&query_target_base.used.spinlock);
 
     qt->internal.used = true;
     qt->internal.queries++;

database/contexts/worker.c

@@ -171,7 +171,7 @@ static void rrdhost_update_cached_retention(RRDHOST *host, time_t first_time_s,
     if(unlikely(!host))
         return;
 
-    netdata_spinlock_lock(&host->retention.spinlock);
+    spinlock_lock(&host->retention.spinlock);
 
     if(global) {
         host->retention.first_time_s = first_time_s;
@@ -185,7 +185,7 @@ static void rrdhost_update_cached_retention(RRDHOST *host, time_t first_time_s,
             host->retention.last_time_s = last_time_s;
     }
 
-    netdata_spinlock_unlock(&host->retention.spinlock);
+    spinlock_unlock(&host->retention.spinlock);
 }
 
 void rrdcontext_recalculate_context_retention(RRDCONTEXT *rc, RRD_FLAGS reason, bool worker_jobs) {