track memory footprint of Netdata (#14294)

* track memory footprint of Netdata

* track db modes alloc/ram/save/map

* track system info; track sender and receiver

* fixes

* more fixes

* track workers memory, onewayalloc memory; unify judyhs size estimation

* track replication structures and buffers

* Properly clear host RRDHOST_FLAG_METADATA_UPDATE flag

* flush the replication buffer every 1000 times the circular buffer is found empty

* dont take timestamp too frequently in sender loop

* sender buffers are not used by the same thread as the sender, so they were never recreated - fixed it

* free sender thread buffer on replication threads when replication is idle

* use the last sender flag as a timestamp of the last buffer recreation

* free cbuffer before reconnecting

* recreate cbuffer on every flush

* timings for journal v2 loading

* inlining of metric and cache functions

* aral likely/unlikely

* free left-over thread buffers

* fix NULL pointer dereference in replication

* free sender thread buffer on sender thread too

* mark ctx as used before flushing

* better logging on ctx datafiles closing

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

aclk/aclk.c

@@ -947,7 +947,7 @@ char *aclk_state(void)
 #ifndef ENABLE_ACLK
     return strdupz("ACLK Available: No");
 #else
-    BUFFER *wb = buffer_create(1024);
+    BUFFER *wb = buffer_create(1024, &netdata_buffers_statistics.buffers_aclk);
     struct tm *tmptr, tmbuf;
     char *ret;
 

aclk/aclk_otp.c

@@ -314,7 +314,7 @@ int aclk_get_otp_challenge(url_t *target, const char *agent_id, unsigned char **
     https_req_t req = HTTPS_REQ_T_INITIALIZER;
     https_req_response_t resp = HTTPS_REQ_RESPONSE_T_INITIALIZER;
 
-    BUFFER *url = buffer_create(strlen(OTP_URL_PREFIX) + UUID_STR_LEN + 20);
+    BUFFER *url = buffer_create(strlen(OTP_URL_PREFIX) + UUID_STR_LEN + 20, &netdata_buffers_statistics.buffers_aclk);
 
     req.host = target->host;
     req.port = target->port;
@@ -394,8 +394,8 @@ int aclk_send_otp_response(const char *agent_id, const unsigned char *response,
 
     base64_encode_helper(base64, &len, response, response_bytes);
 
-    BUFFER *url = buffer_create(strlen(OTP_URL_PREFIX) + UUID_STR_LEN + 20);
-    BUFFER *resp_json = buffer_create(strlen(OTP_URL_PREFIX) + UUID_STR_LEN + 20);
+    BUFFER *url = buffer_create(strlen(OTP_URL_PREFIX) + UUID_STR_LEN + 20, &netdata_buffers_statistics.buffers_aclk);
+    BUFFER *resp_json = buffer_create(strlen(OTP_URL_PREFIX) + UUID_STR_LEN + 20, &netdata_buffers_statistics.buffers_aclk);
 
     buffer_sprintf(url, "%s/node/%s/password", target->path, agent_id);
     buffer_sprintf(resp_json, "{\"response\":\"%s\"}", base64);
@@ -814,7 +814,7 @@ exit:
 }
 
 int aclk_get_env(aclk_env_t *env, const char* aclk_hostname, int aclk_port) {
-    BUFFER *buf = buffer_create(1024);
+    BUFFER *buf = buffer_create(1024, &netdata_buffers_statistics.buffers_aclk);
 
     https_req_t req = HTTPS_REQ_T_INITIALIZER;
     https_req_response_t resp = HTTPS_REQ_RESPONSE_T_INITIALIZER;

aclk/aclk_query.c

@@ -62,19 +62,19 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
     int retval = 0;
     usec_t t;
     BUFFER *local_buffer = NULL;
-    BUFFER *log_buffer = buffer_create(NETDATA_WEB_REQUEST_URL_SIZE);
+    BUFFER *log_buffer = buffer_create(NETDATA_WEB_REQUEST_URL_SIZE, &netdata_buffers_statistics.buffers_aclk);
     RRDHOST *query_host = localhost;
 
 #ifdef NETDATA_WITH_ZLIB
     int z_ret;
-    BUFFER *z_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
+    BUFFER *z_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk);
     char *start, *end;
 #endif
 
     struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client));
-    w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
-    w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE);
-    w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE);
+    w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk);
+    w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE, &netdata_buffers_statistics.buffers_aclk);
+    w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE, &netdata_buffers_statistics.buffers_aclk);
     strcpy(w->origin, "*"); // Simulate web_client_create_on_fd()
     w->cookie1[0] = 0;      // Simulate web_client_create_on_fd()
     w->cookie2[0] = 0;      // Simulate web_client_create_on_fd()
@@ -191,7 +191,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
 
     w->response.data->date = w->tv_ready.tv_sec;
     web_client_build_http_header(w);
-    local_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
+    local_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk);
     local_buffer->contenttype = CT_APPLICATION_JSON;
 
     buffer_strcat(local_buffer, w->response.header_output->buffer);

aclk/https_client.c

@@ -8,6 +8,8 @@
 
 #include "aclk_util.h"
 
+#include "daemon/global_statistics.h"
+
 enum http_parse_state {
     HTTP_PARSE_INITIAL = 0,
     HTTP_PARSE_HEADERS,
@@ -354,7 +356,7 @@ static int read_parse_response(https_req_ctx_t *ctx) {
 #define TX_BUFFER_SIZE 8192
 #define RX_BUFFER_SIZE (TX_BUFFER_SIZE*2)
 static int handle_http_request(https_req_ctx_t *ctx) {
-    BUFFER *hdr = buffer_create(TX_BUFFER_SIZE);
+    BUFFER *hdr = buffer_create(TX_BUFFER_SIZE, &netdata_buffers_statistics.buffers_aclk);
     int rc = 0;
 
     http_parse_ctx_clear(&ctx->parse_ctx);

collectors/apps.plugin/apps_plugin.c

@@ -4396,7 +4396,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
     unsigned int memory_divisor = 1024;
     unsigned int io_divisor = 1024 * RATES_DETAIL;
 
-    BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX);
+    BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
     buffer_sprintf(wb,
                    "{"
                    "\n   \"status\":%d"

collectors/diskspace.plugin/plugin_diskspace.c

@@ -319,7 +319,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
                 , SIMPLE_PATTERN_EXACT
         );
 
-        dict_mountpoints = dictionary_create(DICT_OPTION_NONE);
+        dict_mountpoints = dictionary_create_advanced(DICT_OPTION_NONE, &dictionary_stats_category_collectors);
     }
 
     struct mount_point_metadata *m = dictionary_get(dict_mountpoints, mi->mount_point);

collectors/plugins.d/pluginsd_parser.c

@@ -530,7 +530,7 @@ static void inflight_functions_delete_callback(const DICTIONARY_ITEM *item __may
 }
 
 void inflight_functions_init(PARSER *parser) {
-    parser->inflight.functions = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
+    parser->inflight.functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE, &dictionary_stats_category_functions);
     dictionary_register_insert_callback(parser->inflight.functions, inflight_functions_insert_callback, parser);
     dictionary_register_delete_callback(parser->inflight.functions, inflight_functions_delete_callback, parser);
     dictionary_register_conflict_callback(parser->inflight.functions, inflight_functions_conflict_callback, parser);

collectors/proc.plugin/proc_self_mountinfo.c

@@ -227,8 +227,9 @@ struct mountinfo *mountinfo_read(int do_statvfs) {
     struct mountinfo *root = NULL, *last = NULL, *mi = NULL;
 
     // create a dictionary to track uniqueness
-    DICTIONARY *dict = dictionary_create(
-        DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_NAME_LINK_DONT_CLONE);
+    DICTIONARY *dict = dictionary_create_advanced(
+            DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_NAME_LINK_DONT_CLONE,
+            &dictionary_stats_category_collectors);
 
     unsigned long l, lines = procfile_lines(ff);
     for(l = 0; l < lines ;l++) {

collectors/proc.plugin/proc_spl_kstat_zfs.c

@@ -322,7 +322,7 @@ int do_proc_spl_kstat_zfs_pool_state(int update_every, usec_t dt)
         snprintfz(filename, FILENAME_MAX, "%s%s", netdata_configured_host_prefix, "/proc/spl/kstat/zfs");
         dirname = config_get("plugin:proc:" ZFS_PROC_POOLS, "directory to monitor", filename);
 
-        zfs_pools = dictionary_create(DICT_OPTION_SINGLE_THREADED);
+        zfs_pools = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors);
 
         do_zfs_pool_state = 1;
     }

collectors/proc.plugin/sys_block_zram.c

@@ -267,7 +267,7 @@ int do_sys_block_zram(int update_every, usec_t dt) {
         }
         procfile_close(ff);
 
-        devices = dictionary_create(DICT_OPTION_SINGLE_THREADED);
+        devices = dictionary_create_advanced(DICT_OPTION_SINGLE_THREADED, &dictionary_stats_category_collectors);
         device_count = init_devices(devices, (unsigned int)zram_id, update_every);
     }