DBENGINE v2 - improvements part 5 (#14289)

* cleanup journal v2 mounts periodically

* fix for last commit

* re-enable loading page from disk when the arrangement of pages requires it

* Remove unused statistics

* Estimate diskspace when the current datafile is full and queue a rotate command (Currently it will not attempt to estimate end size for journals)
Queue a command to check quota on startup per tier

* apps.plugin now exposes RSS chart

* shorter thread names to make debugging easier, since thread names can only be 15 characters

* more thread names fixes

* allow an apps_groups.conf target to be pid 0 or 1

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

aclk/aclk.c

@@ -731,8 +731,8 @@ void *aclk_main(void *ptr)
         stats_thread->client = mqttwss_client;
         aclk_stats_thread_prepare(query_threads.count, proto_hdl_cnt);
         netdata_thread_create(
-            stats_thread->thread, ACLK_STATS_THREAD_NAME, NETDATA_THREAD_OPTION_JOINABLE, aclk_stats_main_thread,
-            stats_thread);
+                stats_thread->thread, "ACLK_STATS", NETDATA_THREAD_OPTION_JOINABLE, aclk_stats_main_thread,
+                stats_thread);
     }
 
     // Keep reconnecting and talking until our time has come

aclk/aclk_query.c

@@ -4,8 +4,6 @@
 #include "aclk_stats.h"
 #include "aclk_tx_msgs.h"
 
-#define ACLK_QUERY_THREAD_NAME "ACLK_Query"
-
 #define WEB_HDR_ACCEPT_ENC "Accept-Encoding:"
 
 pthread_cond_t query_cond_wait = PTHREAD_COND_INITIALIZER;
@@ -368,7 +366,7 @@ void aclk_query_threads_start(struct aclk_query_threads *query_threads, mqtt_wss
         query_threads->thread_list[i].idx = i; //thread needs to know its index for statistics
         query_threads->thread_list[i].client = client;
 
-        if(unlikely(snprintfz(thread_name, TASK_LEN_MAX, "%s_%d", ACLK_QUERY_THREAD_NAME, i) < 0))
+        if(unlikely(snprintfz(thread_name, TASK_LEN_MAX, "ACLK_QRY[%d]", i) < 0))
             error("snprintf encoding error");
         netdata_thread_create(
             &query_threads->thread_list[i].thread, thread_name, NETDATA_THREAD_OPTION_JOINABLE, aclk_query_main_thread,

aclk/aclk_stats.h

@@ -8,8 +8,6 @@
 #include "aclk_query_queue.h"
 #include "mqtt_wss_client.h"
 
-#define ACLK_STATS_THREAD_NAME "ACLK_Stats"
-
 extern netdata_mutex_t aclk_stats_mutex;
 
 #define ACLK_STATS_LOCK netdata_mutex_lock(&aclk_stats_mutex)

collectors/apps.plugin/apps_plugin.c

@@ -420,6 +420,7 @@ struct pid_stat {
     int sortlist;                   // higher numbers = top on the process tree
                                     // each process gets a unique number
 
+    bool matched_by_config;
     struct target *target;          // app_groups.conf targets
     struct target *user_target;     // uid based targets
     struct target *group_target;    // gid based targets
@@ -1103,6 +1104,7 @@ static inline void assign_target_to_pid(struct pid_stat *p) {
                       || (proc_pid_cmdline_is_needed && w->starts_with && w->ends_with && p->cmdline && strstr(p->cmdline, w->compare))
                     ))) {
 
+            p->matched_by_config = true;
             if(w->target) p->target = w->target;
             else p->target = w;
 
@@ -2832,11 +2834,11 @@ static void apply_apps_groups_targets_inheritance(void) {
     }
 
     // init goes always to default target
-    if(all_pids[INIT_PID])
+    if(all_pids[INIT_PID] && !all_pids[INIT_PID]->matched_by_config)
         all_pids[INIT_PID]->target = apps_groups_default_target;
 
     // pid 0 goes always to default target
-    if(all_pids[0])
+    if(all_pids[0] && !all_pids[INIT_PID]->matched_by_config)
         all_pids[0]->target = apps_groups_default_target;
 
     // give a default target on all top level processes
@@ -3589,6 +3591,13 @@ static void send_collected_data_to_netdata(struct target *root, const char *type
     }
     send_END();
 
+    send_BEGIN(type, "rss", dt);
+    for (w = root; w ; w = w->next) {
+        if(unlikely(w->exposed && w->processes))
+            send_SET(w->name, w->status_vmrss);
+    }
+    send_END();
+
     send_BEGIN(type, "vmem", dt);
     for (w = root; w ; w = w->next) {
         if(unlikely(w->exposed && w->processes))
@@ -3728,6 +3737,12 @@ static void send_charts_updates_to_netdata(struct target *root, const char *type
     }
     APPS_PLUGIN_FUNCTIONS();
 
+    fprintf(stdout, "CHART %s.rss '' '%s Resident Set Size (w/shared)' 'MiB' mem %s.rss stacked 20004 %d\n", type, title, type, update_every);
+    for (w = root; w ; w = w->next) {
+        if(unlikely(w->exposed))
+            fprintf(stdout, "DIMENSION %s '' absolute %ld %ld\n", w->name, 1L, 1024L);
+    }
+    APPS_PLUGIN_FUNCTIONS();
 
     fprintf(stdout, "CHART %s.vmem '' '%s Virtual Memory Size' 'MiB' mem %s.vmem stacked 20005 %d\n", type, title, type, update_every);
     for (w = root; w ; w = w->next) {

collectors/diskspace.plugin/plugin_diskspace.c

@@ -3,7 +3,6 @@
 #include "../proc.plugin/plugin_proc.h"
 
 #define PLUGIN_DISKSPACE_NAME "diskspace.plugin"
-#define THREAD_DISKSPACE_SLOW_NAME "PLUGIN[diskspace slow]"
 
 #define DEFAULT_EXCLUDED_PATHS "/proc/* /sys/* /var/run/user/* /run/user/* /snap/* /var/lib/docker/*"
 #define DEFAULT_EXCLUDED_FILESYSTEMS "*gvfs *gluster* *s3fs *ipfs *davfs2 *httpfs *sshfs *gdfs *moosefs fusectl autofs"
@@ -632,7 +631,7 @@ void *diskspace_main(void *ptr) {
 
     netdata_thread_create(
         diskspace_slow_thread,
-        THREAD_DISKSPACE_SLOW_NAME,
+        "P[diskspace slow]",
         NETDATA_THREAD_OPTION_JOINABLE,
         diskspace_slow_worker,
         &slow_worker_data);

collectors/plugins.d/plugins_d.c

@@ -271,7 +271,7 @@ void *pluginsd_main(void *ptr)
 
                     if (cd->enabled) {
                         char tag[NETDATA_THREAD_TAG_MAX + 1];
-                        snprintfz(tag, NETDATA_THREAD_TAG_MAX, "PLUGINSD[%s]", pluginname);
+                        snprintfz(tag, NETDATA_THREAD_TAG_MAX, "PD[%s]", pluginname);
                         // spawn a new thread for it
                         netdata_thread_create(
                             &cd->thread, tag, NETDATA_THREAD_OPTION_DEFAULT, pluginsd_worker_thread, cd);

collectors/proc.plugin/plugin_proc.h

@@ -8,7 +8,7 @@
 #define PLUGIN_PROC_CONFIG_NAME "proc"
 #define PLUGIN_PROC_NAME PLUGIN_PROC_CONFIG_NAME ".plugin"
 
-#define THREAD_NETDEV_NAME "PLUGIN[proc netdev]"
+#define THREAD_NETDEV_NAME "P[proc netdev]"
 void *netdev_main(void *ptr);
 
 int do_proc_net_wireless(int update_every, usec_t dt);

collectors/statsd.plugin/statsd.c

@@ -2548,7 +2548,7 @@ void *statsd_main(void *ptr) {
     for(i = 0; i < statsd.threads ;i++) {
         statsd.collection_threads_status[i].max_sockets = max_sockets / statsd.threads;
         char tag[NETDATA_THREAD_TAG_MAX + 1];
-        snprintfz(tag, NETDATA_THREAD_TAG_MAX, "STATSD_COLLECTOR[%d]", i + 1);
+        snprintfz(tag, NETDATA_THREAD_TAG_MAX, "STATSD_IN[%d]", i + 1);
         netdata_spinlock_init(&statsd.collection_threads_status[i].spinlock);
         netdata_thread_create(&statsd.collection_threads_status[i].thread, tag, NETDATA_THREAD_OPTION_DEFAULT, statsd_collector_thread, &statsd.collection_threads_status[i]);
     }

daemon/static_threads.c

@@ -19,7 +19,7 @@ extern bool global_statistics_enabled;
 
 const struct netdata_static_thread static_threads_common[] = {
     {
-        .name = "PLUGIN[timex]",
+        .name = "P[timex]",
         .config_section = CONFIG_SECTION_PLUGINS,
         .config_name = "timex",
         .enabled = 1,
@@ -28,7 +28,7 @@ const struct netdata_static_thread static_threads_common[] = {
         .start_routine = timex_main
     },
     {
-        .name = "PLUGIN[idlejitter]",
+        .name = "P[idlejitter]",
         .config_section = CONFIG_SECTION_PLUGINS,
         .config_name = "idlejitter",
         .enabled = 1,
@@ -55,7 +55,7 @@ const struct netdata_static_thread static_threads_common[] = {
         .start_routine = analytics_main
     },
     {
-        .name = "GLOBAL_STATS",
+        .name = "STATS_GLOBAL",
         .config_section = CONFIG_SECTION_PLUGINS,
         .config_name = "netdata monitoring",
         .env_name = "NETDATA_INTERNALS_MONITORING",
@@ -66,7 +66,7 @@ const struct netdata_static_thread static_threads_common[] = {
         .start_routine = global_statistics_main
     },
     {
-        .name = "WORKERS_STATS",
+        .name = "STATS_WORKERS",
         .config_section = CONFIG_SECTION_PLUGINS,
         .config_name = "netdata monitoring",
         .env_name = "NETDATA_INTERNALS_MONITORING",
@@ -77,7 +77,7 @@ const struct netdata_static_thread static_threads_common[] = {
         .start_routine = global_statistics_workers_main
     },
     {
-        .name = "SQLITE3_STATS",
+        .name = "STATS_SQLITE3",
         .config_section = CONFIG_SECTION_PLUGINS,
         .config_name = "netdata monitoring",
         .env_name = "NETDATA_INTERNALS_MONITORING",
@@ -106,7 +106,7 @@ const struct netdata_static_thread static_threads_common[] = {
         .start_routine = service_main
     },
     {
-        .name = "STATSD",
+        .name = "STATSD_FLUSH",
         .config_section = NULL,
         .config_name = NULL,
         .enabled = 1,
@@ -124,7 +124,7 @@ const struct netdata_static_thread static_threads_common[] = {
         .start_routine = exporting_main
     },
     {
-        .name = "STREAM",
+        .name = "SNDR[localhost]",
         .config_section = NULL,
         .config_name = NULL,
         .enabled = 0,
@@ -133,7 +133,7 @@ const struct netdata_static_thread static_threads_common[] = {
         .start_routine = rrdpush_sender_thread
     },
     {
-        .name = "WEB_SERVER[static1]",
+        .name = "WEB[1]",
         .config_section = NULL,
         .config_name = NULL,
         .enabled = 0,
@@ -144,7 +144,7 @@ const struct netdata_static_thread static_threads_common[] = {
 
 #ifdef ENABLE_ACLK
     {
-        .name = "ACLK_Main",
+        .name = "ACLK_MAIN",
         .config_section = NULL,
         .config_name = NULL,
         .enabled = 1,
@@ -165,13 +165,13 @@ const struct netdata_static_thread static_threads_common[] = {
     },
 
     {
-            .name = "REPLICATION",
-            .config_section = NULL,
-            .config_name = NULL,
-            .enabled = 1,
-            .thread = NULL,
-            .init_routine = NULL,
-            .start_routine = replication_thread_main
+        .name = "REPLAY[1]",
+        .config_section = NULL,
+        .config_name = NULL,
+        .enabled = 1,
+        .thread = NULL,
+        .init_routine = NULL,
+        .start_routine = replication_thread_main
     },
 
     // terminator

daemon/static_threads_freebsd.c

@@ -6,7 +6,7 @@ extern void *freebsd_main(void *ptr);
 
 const struct netdata_static_thread static_threads_freebsd[] = {
     {
-        .name = "PLUGIN[freebsd]",
+        .name = "P[freebsd]",
         .config_section = CONFIG_SECTION_PLUGINS,
         .config_name = "freebsd",
         .enabled = 1,