Improve eBPF exit (#14012)

collectors/ebpf.plugin/ebpf.c

@@ -27,8 +27,10 @@ struct config collector_config = { .first_section = NULL,
 int running_on_kernel = 0;
 int ebpf_nprocs;
 int isrh = 0;
+int main_thread_id = 0;
 
 pthread_mutex_t lock;
+pthread_mutex_t ebpf_exit_cleanup;
 pthread_mutex_t collect_data_mutex;
 pthread_cond_t collect_data_cond_var;
 
@@ -41,7 +43,7 @@ ebpf_module_t ebpf_modules[] = {
       .config_file = NETDATA_PROCESS_CONFIG_FILE,
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_10 |
                   NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "socket", .config_name = "socket", .enabled = 0, .start_routine = ebpf_socket_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
@@ -49,7 +51,7 @@ ebpf_module_t ebpf_modules[] = {
       .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &socket_config,
       .config_file = NETDATA_NETWORK_CONFIG_FILE,
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = socket_targets, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = socket_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "cachestat", .config_name = "cachestat", .enabled = 0, .start_routine = ebpf_cachestat_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
@@ -58,7 +60,7 @@ ebpf_module_t ebpf_modules[] = {
       .config_file = NETDATA_CACHESTAT_CONFIG_FILE,
       .kernels = NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18|
                  NETDATA_V5_4 | NETDATA_V5_14 | NETDATA_V5_15 | NETDATA_V5_16,
-      .load = EBPF_LOAD_LEGACY, .targets = cachestat_targets, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = cachestat_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "sync", .config_name = "sync", .enabled = 0, .start_routine = ebpf_sync_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
@@ -66,7 +68,7 @@ ebpf_module_t ebpf_modules[] = {
       .config_file = NETDATA_SYNC_CONFIG_FILE,
       // All syscalls have the same kernels
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = sync_targets, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = sync_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "dc", .config_name = "dc", .enabled = 0, .start_routine = ebpf_dcstat_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
@@ -74,7 +76,7 @@ ebpf_module_t ebpf_modules[] = {
       .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &dcstat_config,
       .config_file = NETDATA_DIRECTORY_DCSTAT_CONFIG_FILE,
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = dc_targets, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = dc_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "swap", .config_name = "swap", .enabled = 0, .start_routine = ebpf_swap_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
@@ -82,7 +84,7 @@ ebpf_module_t ebpf_modules[] = {
       .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &swap_config,
       .config_file = NETDATA_DIRECTORY_SWAP_CONFIG_FILE,
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = swap_targets, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = swap_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "vfs", .config_name = "vfs", .enabled = 0, .start_routine = ebpf_vfs_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
@@ -90,28 +92,28 @@ ebpf_module_t ebpf_modules[] = {
       .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &vfs_config,
       .config_file = NETDATA_DIRECTORY_VFS_CONFIG_FILE,
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = vfs_targets, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = vfs_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "filesystem", .config_name = "filesystem", .enabled = 0, .start_routine = ebpf_filesystem_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
       .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &fs_config,
       .config_file = NETDATA_FILESYSTEM_CONFIG_FILE,
       //We are setting kernels as zero, because we load eBPF programs according the kernel running.
-      .kernels = 0, .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL },
+      .kernels = 0, .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL },
     { .thread_name = "disk", .config_name = "disk", .enabled = 0, .start_routine = ebpf_disk_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
       .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &disk_config,
       .config_file = NETDATA_DISK_CONFIG_FILE,
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "mount", .config_name = "mount", .enabled = 0, .start_routine = ebpf_mount_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
       .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &mount_config,
       .config_file = NETDATA_MOUNT_CONFIG_FILE,
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = mount_targets, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = mount_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "fd", .config_name = "fd", .enabled = 0, .start_routine = ebpf_fd_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
@@ -120,21 +122,21 @@ ebpf_module_t ebpf_modules[] = {
       .config_file = NETDATA_FD_CONFIG_FILE,
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_11 |
                   NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = fd_targets, .probe_links = NULL, .objects = NULL},
+     .load = EBPF_LOAD_LEGACY, .targets = fd_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "hardirq", .config_name = "hardirq", .enabled = 0, .start_routine = ebpf_hardirq_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
       .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &hardirq_config,
       .config_file = NETDATA_HARDIRQ_CONFIG_FILE,
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "softirq", .config_name = "softirq", .enabled = 0, .start_routine = ebpf_softirq_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
       .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &softirq_config,
       .config_file = NETDATA_SOFTIRQ_CONFIG_FILE,
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "oomkill", .config_name = "oomkill", .enabled = 0, .start_routine = ebpf_oomkill_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
@@ -142,7 +144,7 @@ ebpf_module_t ebpf_modules[] = {
       .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &oomkill_config,
       .config_file = NETDATA_OOMKILL_CONFIG_FILE,
       .kernels =  NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "shm", .config_name = "shm", .enabled = 0, .start_routine = ebpf_shm_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_LEVEL_REAL_PARENT, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
@@ -150,19 +152,19 @@ ebpf_module_t ebpf_modules[] = {
       .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &shm_config,
       .config_file = NETDATA_DIRECTORY_SHM_CONFIG_FILE,
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = shm_targets, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = shm_targets, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = "mdflush", .config_name = "mdflush", .enabled = 0, .start_routine = ebpf_mdflush_thread,
       .update_every = EBPF_DEFAULT_UPDATE_EVERY, .global_charts = 1, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO,
       .apps_level = NETDATA_APPS_NOT_SET, .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0,
       .apps_routine = NULL, .maps = NULL, .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &mdflush_config,
       .config_file = NETDATA_DIRECTORY_MDFLUSH_CONFIG_FILE,
       .kernels =  NETDATA_V3_10 | NETDATA_V4_14 | NETDATA_V4_16 | NETDATA_V4_18 | NETDATA_V5_4 | NETDATA_V5_14,
-      .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL},
+      .load = EBPF_LOAD_LEGACY, .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
     { .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_every = EBPF_DEFAULT_UPDATE_EVERY,
       .global_charts = 0, .apps_charts = NETDATA_EBPF_APPS_FLAG_NO, .apps_level = NETDATA_APPS_NOT_SET,
       .cgroup_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY, .optional = 0, .apps_routine = NULL, .maps = NULL,
       .pid_map_size = 0, .names = NULL, .cfg = NULL, .config_name = NULL, .kernels = 0, .load = EBPF_LOAD_LEGACY,
-      .targets = NULL, .probe_links = NULL, .objects = NULL},
+      .targets = NULL, .probe_links = NULL, .objects = NULL, .thread = NULL},
 };
 
 struct netdata_static_thread ebpf_threads[] = {
@@ -432,6 +434,7 @@ ebpf_sync_syscalls_t local_syscalls[] = {
     }
 };
 
+
 // Link with apps.plugin
 ebpf_process_stat_t *global_process_stat = NULL;
 
@@ -463,16 +466,16 @@ char *btf_path = NULL;
 
 /**
  * Close the collector gracefully
- *
- * @param sig is the signal number used to close the collector
  */
-static void ebpf_exit(int sig)
+static void ebpf_exit()
 {
 #ifdef LIBBPF_MAJOR_VERSION
+    pthread_mutex_lock(&ebpf_exit_cleanup);
     if (default_btf) {
         btf__free(default_btf);
         default_btf = NULL;
     }
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
 #endif
 
     char filename[FILENAME_MAX + 1];
@@ -480,7 +483,7 @@ static void ebpf_exit(int sig)
     if (unlink(filename))
         error("Cannot remove PID file %s", filename);
 
-    exit(sig);
+    exit(0);
 }
 
 /**
@@ -513,44 +516,66 @@ int ebpf_exit_plugin = 0;
  */
 static void ebpf_stop_threads(int sig)
 {
-    ebpf_exit_plugin = 1;
+    UNUSED(sig);
+    static int only_one = 0;
+
     int i;
-    for (i = 0; ebpf_threads[i].name != NULL; i++);
+    // Child thread should be closed by itself.
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    if (main_thread_id != gettid() || only_one) {
+        pthread_mutex_unlock(&ebpf_exit_cleanup);
+        return;
+    }
+    only_one = 1;
+    for (i = 0; ebpf_threads[i].name != NULL; i++) {
+        if (ebpf_threads[i].enabled != NETDATA_THREAD_EBPF_STOPPED)
+            netdata_thread_cancel(*ebpf_threads[i].thread);
+    }
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
 
-    usec_t max = 2 * USEC_PER_SEC, step = 100000;
+    ebpf_exit_plugin = 1;
+    usec_t max = 3 * USEC_PER_SEC, step = 100000;
     while (i && max) {
         max -= step;
         sleep_usec(step);
         i = 0;
         int j;
+        pthread_mutex_lock(&ebpf_exit_cleanup);
         for (j = 0; ebpf_threads[j].name != NULL; j++) {
-            if (ebpf_threads[j].enabled != NETDATA_MAIN_THREAD_EXITED)
+            if (ebpf_threads[j].enabled != NETDATA_THREAD_EBPF_STOPPED)
                 i++;
         }
+        pthread_mutex_unlock(&ebpf_exit_cleanup);
     }
 
     //Unload threads(except sync and filesystem)
+    pthread_mutex_lock(&ebpf_exit_cleanup);
     for (i = 0; ebpf_threads[i].name != NULL; i++) {
-        if (ebpf_threads[i].enabled == NETDATA_MAIN_THREAD_EXITED && i != EBPF_MODULE_FILESYSTEM_IDX &&
+        if (ebpf_threads[i].enabled == NETDATA_THREAD_EBPF_STOPPED && i != EBPF_MODULE_FILESYSTEM_IDX &&
             i != EBPF_MODULE_SYNC_IDX)
             ebpf_unload_legacy_code(ebpf_modules[i].objects, ebpf_modules[i].probe_links);
     }
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
 
     //Unload filesystem
-    if (ebpf_threads[EBPF_MODULE_FILESYSTEM_IDX].enabled  == NETDATA_MAIN_THREAD_EXITED) {
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    if (ebpf_threads[EBPF_MODULE_FILESYSTEM_IDX].enabled  == NETDATA_THREAD_EBPF_STOPPED) {
         for (i = 0; localfs[i].filesystem != NULL; i++) {
             ebpf_unload_legacy_code(localfs[i].objects, localfs[i].probe_links);
         }
     }
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
 
     //Unload Sync
-    if (ebpf_threads[EBPF_MODULE_SYNC_IDX].enabled  == NETDATA_MAIN_THREAD_EXITED) {
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    if (ebpf_threads[EBPF_MODULE_SYNC_IDX].enabled  == NETDATA_THREAD_EBPF_STOPPED) {
         for (i = 0; local_syscalls[i].syscall != NULL; i++) {
             ebpf_unload_legacy_code(local_syscalls[i].objects, local_syscalls[i].probe_links);
         }
     }
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
 
-    ebpf_exit(sig);
+    ebpf_exit();
 }
 
 /*****************************************************************
@@ -1310,6 +1335,7 @@ static void read_local_addresses()
 int ebpf_start_pthread_variables()
 {
     pthread_mutex_init(&lock, NULL);
+    pthread_mutex_init(&ebpf_exit_cleanup, NULL);
     pthread_mutex_init(&collect_data_mutex, NULL);
 
     if (pthread_cond_init(&collect_data_cond_var, NULL)) {
@@ -1954,7 +1980,7 @@ static void ebpf_parse_args(int argc, char **argv)
                 &apps_groups_default_target, &apps_groups_root_target, ebpf_stock_config_dir, "groups")) {
             error("Cannot read process groups '%s/apps_groups.conf'. There are no internal defaults. Failing.",
                   ebpf_stock_config_dir);
-            ebpf_exit(1);
+            ebpf_exit();
         }
     } else
         info("Loaded config file '%s/apps_groups.conf'", ebpf_user_config_dir);
@@ -2133,6 +2159,7 @@ static void ebpf_manage_pid(pid_t pid)
 int main(int argc, char **argv)
 {
     clocks_init();
+    main_thread_id = gettid();
 
     set_global_variables();
     ebpf_parse_args(argc, argv);
@@ -2173,7 +2200,7 @@ int main(int argc, char **argv)
 
     if (ebpf_start_pthread_variables()) {
         error("Cannot start mutex to control overall charts.");
-        ebpf_exit(5);
+        ebpf_exit();
     }
 
     netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX");
@@ -2196,18 +2223,25 @@ int main(int argc, char **argv)
     int i;
     for (i = 0; ebpf_threads[i].name != NULL; i++) {
         struct netdata_static_thread *st = &ebpf_threads[i];
-        st->thread = mallocz(sizeof(netdata_thread_t));
 
         ebpf_module_t *em = &ebpf_modules[i];
-        em->thread_id = i;
-        netdata_thread_create(st->thread, st->name, NETDATA_THREAD_OPTION_DEFAULT, st->start_routine, em);
+        em->thread = st;
+        // We always initialize process, because it is responsible to take care of apps integration
+        if (em->enabled || !i) {
+            st->thread = mallocz(sizeof(netdata_thread_t));
+            em->thread_id = i;
+            st->enabled = NETDATA_THREAD_EBPF_RUNNING;
+            netdata_thread_create(st->thread, st->name, NETDATA_THREAD_OPTION_DEFAULT, st->start_routine, em);
+        } else {
+            st->enabled = NETDATA_THREAD_EBPF_STOPPED;
+        }
     }
 
-    usec_t step = 60 * USEC_PER_SEC;
+    usec_t step = EBPF_DEFAULT_UPDATE_EVERY * USEC_PER_SEC;
     heartbeat_t hb;
     heartbeat_init(&hb);
     //Plugin will be killed when it receives a signal
-    for (;;) {
+    while (!ebpf_exit_plugin) {
         (void)heartbeat_next(&hb, step);
     }
 

collectors/ebpf.plugin/ebpf.h

@@ -168,6 +168,7 @@ void *ebpf_socket_thread(void *ptr);
 
 // Common variables
 extern pthread_mutex_t lock;
+extern pthread_mutex_t ebpf_exit_cleanup;
 extern int ebpf_nprocs;
 extern int running_on_kernel;
 extern int isrh;

collectors/ebpf.plugin/ebpf_cachestat.c

@@ -44,7 +44,6 @@ struct config cachestat_config = { .first_section = NULL,
     .mutex = NETDATA_MUTEX_INITIALIZER,
     .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
         .rwlock = AVL_LOCK_INITIALIZER } };
-static enum ebpf_threads_status ebpf_cachestat_exited = NETDATA_THREAD_EBPF_RUNNING;
 
 netdata_ebpf_targets_t cachestat_targets[] = { {.name = "add_to_page_cache_lru", .mode = EBPF_LOAD_TRAMPOLINE},
                                                {.name = "mark_page_accessed", .mode = EBPF_LOAD_TRAMPOLINE},
@@ -322,6 +321,38 @@ static inline int ebpf_cachestat_load_and_attach(struct cachestat_bpf *obj, ebpf
  *
  *****************************************************************/
 
+/**
+ * Cachestat Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_cachestat_free(ebpf_module_t *em)
+{
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+        pthread_mutex_unlock(&ebpf_exit_cleanup);
+        return;
+    }
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+    ebpf_cleanup_publish_syscall(cachestat_counter_publish_aggregated);
+
+    freez(cachestat_vector);
+    freez(cachestat_values);
+    freez(cachestat_threads.thread);
+
+#ifdef LIBBPF_MAJOR_VERSION
+    if (bpf_obj)
+        cachestat_bpf__destroy(bpf_obj);
+#endif
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
 /**
  * Cachestat exit.
  *
@@ -332,12 +363,8 @@ static inline int ebpf_cachestat_load_and_attach(struct cachestat_bpf *obj, ebpf
 static void ebpf_cachestat_exit(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (!em->enabled) {
-        em->enabled = NETDATA_MAIN_THREAD_EXITED;
-        return;
-    }
-
-    ebpf_cachestat_exited = NETDATA_THREAD_EBPF_STOPPING;
+    netdata_thread_cancel(*cachestat_threads.thread);
+    ebpf_cachestat_free(em);
 }
 
 /**
@@ -350,21 +377,7 @@ static void ebpf_cachestat_exit(void *ptr)
 static void ebpf_cachestat_cleanup(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (ebpf_cachestat_exited != NETDATA_THREAD_EBPF_STOPPED)
-        return;
-
-    ebpf_cleanup_publish_syscall(cachestat_counter_publish_aggregated);
-
-    freez(cachestat_vector);
-    freez(cachestat_values);
-    freez(cachestat_threads.thread);
-
-#ifdef LIBBPF_MAJOR_VERSION
-    if (bpf_obj)
-        cachestat_bpf__destroy(bpf_obj);
-#endif
-    cachestat_threads.enabled = NETDATA_MAIN_THREAD_EXITED;
-    em->enabled = NETDATA_MAIN_THREAD_EXITED;
+    ebpf_cachestat_free(em);
 }
 
 /*****************************************************************
@@ -682,17 +695,12 @@ void *ebpf_cachestat_read_hash(void *ptr)
     ebpf_module_t *em = (ebpf_module_t *)ptr;
 
     usec_t step = NETDATA_LATENCY_CACHESTAT_SLEEP_MS * em->update_every;
-    while (ebpf_cachestat_exited == NETDATA_THREAD_EBPF_RUNNING) {
-        usec_t dt = heartbeat_next(&hb, step);
-        (void)dt;
-        if (ebpf_cachestat_exited == NETDATA_THREAD_EBPF_STOPPING)
-            break;
+    while (!ebpf_exit_plugin) {
+        (void)heartbeat_next(&hb, step);
 
         read_global_table();
     }
 
-    ebpf_cachestat_exited = NETDATA_THREAD_EBPF_STOPPED;
-
     netdata_thread_cleanup_pop(1);
     return NULL;
 }
@@ -1292,16 +1300,13 @@ void *ebpf_cachestat_thread(void *ptr)
 
     ebpf_update_pid_table(&cachestat_maps[NETDATA_CACHESTAT_PID_STATS], em);
 
-    if (!em->enabled)
-        goto endcachestat;
-
     ebpf_cachestat_set_internal_value();
 
 #ifdef LIBBPF_MAJOR_VERSION
     ebpf_adjust_thread_load(em, default_btf);
 #endif
     if (ebpf_cachestat_load_bpf(em)) {
-        em->enabled = CONFIG_BOOLEAN_NO;
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
         goto endcachestat;
     }
 
@@ -1323,8 +1328,7 @@ void *ebpf_cachestat_thread(void *ptr)
     cachestat_collector(em);
 
 endcachestat:
-    if (!em->enabled)
-        ebpf_update_disabled_plugin_stats(em);
+    ebpf_update_disabled_plugin_stats(em);
 
     netdata_thread_cleanup_pop(1);
     return NULL;

collectors/ebpf.plugin/ebpf_dcstat.c

@@ -27,7 +27,6 @@ struct netdata_static_thread dcstat_threads = {"DCSTAT KERNEL",
                                                .thread = NULL,
                                                .init_routine = NULL,
                                                .start_routine = NULL};
-static enum ebpf_threads_status ebpf_dcstat_exited = NETDATA_THREAD_EBPF_RUNNING;
 
 ebpf_local_maps_t dcstat_maps[] = {{.name = "dcstat_global", .internal_input = NETDATA_DIRECTORY_CACHE_END,
                                            .user_input = 0, .type = NETDATA_EBPF_MAP_STATIC,
@@ -295,33 +294,21 @@ void ebpf_dcstat_clean_names()
 }
 
 /**
- * DCstat exit
+ * DCstat Free
  *
- * Cancel child and exit.
+ * Cleanup variables after child threads to stop
  *
  * @param ptr thread data.
  */
-static void ebpf_dcstat_exit(void *ptr)
+static void ebpf_dcstat_free(ebpf_module_t *em )
 {
-    ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (!em->enabled) {
-        em->enabled = NETDATA_MAIN_THREAD_EXITED;
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+        pthread_mutex_unlock(&ebpf_exit_cleanup);
         return;
     }
-
-    ebpf_dcstat_exited = NETDATA_THREAD_EBPF_STOPPING;
-}
-
-/**
- * Clean up the main thread.
- *
- * @param ptr thread data.
- */
-static void ebpf_dcstat_cleanup(void *ptr)
-{
-    ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (ebpf_dcstat_exited != NETDATA_THREAD_EBPF_STOPPED)
-        return;
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
 
     freez(dcstat_vector);
     freez(dcstat_values);
@@ -336,8 +323,34 @@ static void ebpf_dcstat_cleanup(void *ptr)
         dc_bpf__destroy(bpf_obj);
 #endif
 
-    dcstat_threads.enabled = NETDATA_MAIN_THREAD_EXITED;
-    em->enabled = NETDATA_MAIN_THREAD_EXITED;
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
+/**
+ * DCstat exit
+ *
+ * Cancel child and exit.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_dcstat_exit(void *ptr)
+{
+    ebpf_module_t *em = (ebpf_module_t *)ptr;
+    netdata_thread_cancel(*dcstat_threads.thread);
+    ebpf_dcstat_free(em);
+}
+
+/**
+ * Clean up the main thread.
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_dcstat_cleanup(void *ptr)
+{
+    ebpf_module_t *em = (ebpf_module_t *)ptr;
+    ebpf_dcstat_free(em);
 }
 
 /*****************************************************************
@@ -564,17 +577,12 @@ void *ebpf_dcstat_read_hash(void *ptr)
     ebpf_module_t *em = (ebpf_module_t *)ptr;
 
     usec_t step = NETDATA_LATENCY_DCSTAT_SLEEP_MS * em->update_every;
-    while (ebpf_dcstat_exited == NETDATA_THREAD_EBPF_RUNNING) {
-        usec_t dt = heartbeat_next(&hb, step);
-        (void)dt;
-        if (ebpf_dcstat_exited == NETDATA_THREAD_EBPF_STOPPING)
-            break;
+    while (!ebpf_exit_plugin) {
+        (void)heartbeat_next(&hb, step);
 
         read_global_table();
     }
 
-    ebpf_dcstat_exited = NETDATA_THREAD_EBPF_STOPPED;
-
     netdata_thread_cleanup_pop(1);
     return NULL;
 }
@@ -1183,14 +1191,11 @@ void *ebpf_dcstat_thread(void *ptr)
 
     ebpf_update_names(dc_optional_name, em);
 
-    if (!em->enabled)
-        goto enddcstat;
-
 #ifdef LIBBPF_MAJOR_VERSION
     ebpf_adjust_thread_load(em, default_btf);
 #endif
     if (ebpf_dcstat_load_bpf(em)) {
-        em->enabled = CONFIG_BOOLEAN_NO;
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
         goto enddcstat;
     }
 
@@ -1213,8 +1218,7 @@ void *ebpf_dcstat_thread(void *ptr)
     dcstat_collector(em);
 
 enddcstat:
-    if (!em->enabled)
-        ebpf_update_disabled_plugin_stats(em);
+    ebpf_update_disabled_plugin_stats(em);
 
     netdata_thread_cleanup_pop(1);
     return NULL;

collectors/ebpf.plugin/ebpf_disk.c

@@ -43,7 +43,6 @@ static struct netdata_static_thread disk_threads = {
                                         .init_routine = NULL,
                                         .start_routine = NULL
 };
-static enum ebpf_threads_status ebpf_disk_exited = NETDATA_THREAD_EBPF_RUNNING;
 
 ebpf_publish_disk_t *plot_disks = NULL;
 pthread_mutex_t plot_mutex;
@@ -430,6 +429,40 @@ static void ebpf_cleanup_disk_list()
     }
 }
 
+/**
+ * DISK Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_disk_free(ebpf_module_t *em)
+{
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+        pthread_mutex_unlock(&ebpf_exit_cleanup);
+        return;
+    }
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+    ebpf_disk_disable_tracepoints();
+
+    if (dimensions)
+        ebpf_histogram_dimension_cleanup(dimensions, NETDATA_EBPF_HIST_MAX_BINS);
+
+    freez(disk_hash_values);
+    freez(disk_threads.thread);
+    pthread_mutex_destroy(&plot_mutex);
+
+    ebpf_cleanup_plot_disks();
+    ebpf_cleanup_disk_list();
+
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
 /**
  * Disk exit.
  *
@@ -440,12 +473,8 @@ static void ebpf_cleanup_disk_list()
 static void ebpf_disk_exit(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (!em->enabled) {
-        em->enabled = NETDATA_MAIN_THREAD_EXITED;
-        return;
-    }
-
-    ebpf_disk_exited = NETDATA_THREAD_EBPF_STOPPING;
+    netdata_thread_cancel(*disk_threads.thread);
+    ebpf_disk_free(em);
 }
 
 /**
@@ -458,23 +487,7 @@ static void ebpf_disk_exit(void *ptr)
 static void ebpf_disk_cleanup(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (ebpf_disk_exited != NETDATA_THREAD_EBPF_STOPPED)
-        return;
-
-    ebpf_disk_disable_tracepoints();
-
-    if (dimensions)
-        ebpf_histogram_dimension_cleanup(dimensions, NETDATA_EBPF_HIST_MAX_BINS);
-
-    freez(disk_hash_values);
-    freez(disk_threads.thread);
-    pthread_mutex_destroy(&plot_mutex);
-
-    ebpf_cleanup_plot_disks();
-    ebpf_cleanup_disk_list();
-
-    disk_threads.enabled = NETDATA_MAIN_THREAD_EXITED;
-    em->enabled = NETDATA_MAIN_THREAD_EXITED;
+    ebpf_disk_free(em);
 }
 
 /*****************************************************************
@@ -597,17 +610,12 @@ void *ebpf_disk_read_hash(void *ptr)
     ebpf_module_t *em = (ebpf_module_t *)ptr;
 
     usec_t step = NETDATA_LATENCY_DISK_SLEEP_MS * em->update_every;
-    while (ebpf_disk_exited == NETDATA_THREAD_EBPF_RUNNING) {
-        usec_t dt = heartbeat_next(&hb, step);
-        (void)dt;
-        if (ebpf_disk_exited == NETDATA_THREAD_EBPF_STOPPING)
-            break;
+    while (!ebpf_exit_plugin) {
+        (void)heartbeat_next(&hb, step);
 
         read_hard_disk_tables(disk_maps[NETDATA_DISK_READ].map_fd);
     }
 
-    ebpf_disk_exited = NETDATA_THREAD_EBPF_STOPPED;
-
     netdata_thread_cleanup_pop(1);
     return NULL;
 }
@@ -812,29 +820,26 @@ void *ebpf_disk_thread(void *ptr)
     ebpf_module_t *em = (ebpf_module_t *)ptr;
     em->maps = disk_maps;
 
-    if (!em->enabled)
-        goto enddisk;
-
     if (ebpf_disk_enable_tracepoints()) {
-        em->enabled = CONFIG_BOOLEAN_NO;
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
         goto enddisk;
     }
 
     avl_init_lock(&disk_tree, ebpf_compare_disks);
     if (read_local_disks()) {
-        em->enabled = CONFIG_BOOLEAN_NO;
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
         goto enddisk;
     }
 
     if (pthread_mutex_init(&plot_mutex, NULL)) {
-        em->enabled = 0;
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
         error("Cannot initialize local mutex");
         goto enddisk;
     }
 
     em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
     if (!em->probe_links) {
-        em->enabled = 0;
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
         goto enddisk;
     }
 
@@ -852,8 +857,7 @@ void *ebpf_disk_thread(void *ptr)
     disk_collector(em);
 
 enddisk:
-    if (!em->enabled)
-        ebpf_update_disabled_plugin_stats(em);
+    ebpf_update_disabled_plugin_stats(em);
 
     netdata_thread_cleanup_pop(1);
 

collectors/ebpf.plugin/ebpf_fd.c

@@ -38,7 +38,6 @@ struct netdata_static_thread fd_thread = {"FD KERNEL",
                                           .init_routine = NULL,
                                           .start_routine = NULL};
 
-static enum ebpf_threads_status ebpf_fd_exited = NETDATA_THREAD_EBPF_RUNNING;
 static netdata_idx_t fd_hash_values[NETDATA_FD_COUNTER];
 static netdata_idx_t *fd_values = NULL;
 
@@ -331,6 +330,38 @@ static inline int ebpf_fd_load_and_attach(struct fd_bpf *obj, ebpf_module_t *em)
  *
  *****************************************************************/
 
+/**
+ * FD Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_fd_free(ebpf_module_t *em)
+{
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+        pthread_mutex_unlock(&ebpf_exit_cleanup);
+        return;
+    }
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+    ebpf_cleanup_publish_syscall(fd_publish_aggregated);
+    freez(fd_thread.thread);
+    freez(fd_values);
+    freez(fd_vector);
+
+#ifdef LIBBPF_MAJOR_VERSION
+    if (bpf_obj)
+        fd_bpf__destroy(bpf_obj);
+#endif
+
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
 /**
  * FD Exit
  *
@@ -341,12 +372,8 @@ static inline int ebpf_fd_load_and_attach(struct fd_bpf *obj, ebpf_module_t *em)
 static void ebpf_fd_exit(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (!em->enabled) {
-        em->enabled = NETDATA_MAIN_THREAD_EXITED;
-        return;
-    }
-
-    ebpf_fd_exited = NETDATA_THREAD_EBPF_STOPPING;
+    netdata_thread_cancel(*fd_thread.thread);
+    ebpf_fd_free(em);
 }
 
 /**
@@ -357,21 +384,7 @@ static void ebpf_fd_exit(void *ptr)
 static void ebpf_fd_cleanup(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (ebpf_fd_exited != NETDATA_THREAD_EBPF_STOPPED)
-        return;
-
-    ebpf_cleanup_publish_syscall(fd_publish_aggregated);
-    freez(fd_thread.thread);
-    freez(fd_values);
-    freez(fd_vector);
-
-#ifdef LIBBPF_MAJOR_VERSION
-    if (bpf_obj)
-        fd_bpf__destroy(bpf_obj);
-#endif
-
-    fd_thread.enabled = NETDATA_MAIN_THREAD_EXITED;
-    em->enabled = NETDATA_MAIN_THREAD_EXITED;
+    ebpf_fd_free(em);
 }
 
 /*****************************************************************
@@ -445,17 +458,12 @@ void *ebpf_fd_read_hash(void *ptr)
 
     ebpf_module_t *em = (ebpf_module_t *)ptr;
     usec_t step = NETDATA_FD_SLEEP_MS * em->update_every;
-    while (ebpf_fd_exited == NETDATA_THREAD_EBPF_RUNNING) {
-        usec_t dt = heartbeat_next(&hb, step);
-        (void)dt;
-        if (ebpf_fd_exited == NETDATA_THREAD_EBPF_STOPPING)
-            break;
+    while (!ebpf_exit_plugin) {
+        (void)heartbeat_next(&hb, step);
 
         read_global_table();
     }
 
-    ebpf_fd_exited = NETDATA_THREAD_EBPF_STOPPED;
-
     netdata_thread_cleanup_pop(1);
     return NULL;
 }
@@ -1143,14 +1151,13 @@ void *ebpf_fd_thread(void *ptr)
     ebpf_module_t *em = (ebpf_module_t *)ptr;
     em->maps = fd_maps;
 
-    if (!em->enabled)
-        goto endfd;
-
 #ifdef LIBBPF_MAJOR_VERSION
     ebpf_adjust_thread_load(em, default_btf);
 #endif
-    if (ebpf_fd_load_bpf(em))
+    if (ebpf_fd_load_bpf(em))  {
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
         goto endfd;
+    }
 
     ebpf_fd_allocate_global_vectors(em->apps_charts);
 
@@ -1169,8 +1176,7 @@ void *ebpf_fd_thread(void *ptr)
     fd_collector(em);
 
 endfd:
-    if (!em->enabled)
-        ebpf_update_disabled_plugin_stats(em);
+    ebpf_update_disabled_plugin_stats(em);
 
     netdata_thread_cleanup_pop(1);
     return NULL;

collectors/ebpf.plugin/ebpf_filesystem.c

@@ -41,8 +41,6 @@ struct netdata_static_thread filesystem_threads = {
                                  .start_routine = NULL
 };
 
-static enum ebpf_threads_status ebpf_fs_exited = NETDATA_THREAD_EBPF_RUNNING;
-
 static netdata_syscall_stat_t filesystem_aggregated_data[NETDATA_EBPF_HIST_MAX_BINS];
 static netdata_publish_syscall_t filesystem_publish_aggregated[NETDATA_EBPF_HIST_MAX_BINS];
 
@@ -325,21 +323,40 @@ void ebpf_filesystem_cleanup_ebpf_data()
 
             freez(efp->hadditional.name);
             freez(efp->hadditional.title);
-
-            struct bpf_link **probe_links = efp->probe_links;
-            size_t j = 0 ;
-            struct bpf_program *prog;
-            bpf_object__for_each_program(prog, efp->objects) {
-                bpf_link__destroy(probe_links[j]);
-                j++;
-            }
-            freez(probe_links);
-            if (efp->objects)
-                bpf_object__close(efp->objects);
         }
     }
 }
 
+/**
+ * Filesystem Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_filesystem_free(ebpf_module_t *em)
+{
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+        pthread_mutex_unlock(&ebpf_exit_cleanup);
+        return;
+    }
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+    freez(filesystem_threads.thread);
+    ebpf_cleanup_publish_syscall(filesystem_publish_aggregated);
+
+    ebpf_filesystem_cleanup_ebpf_data();
+    if (dimensions)
+        ebpf_histogram_dimension_cleanup(dimensions, NETDATA_EBPF_HIST_MAX_BINS);
+    freez(filesystem_hash_values);
+
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
 /**
  * Filesystem exit
  *
@@ -350,12 +367,8 @@ void ebpf_filesystem_cleanup_ebpf_data()
 static void ebpf_filesystem_exit(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (!em->enabled) {
-        em->enabled = NETDATA_MAIN_THREAD_EXITED;
-        return;
-    }
-
-    ebpf_fs_exited = NETDATA_THREAD_EBPF_STOPPING;
+    netdata_thread_cancel(*filesystem_threads.thread);
+    ebpf_filesystem_free(em);
 }
 
 /**
@@ -368,19 +381,7 @@ static void ebpf_filesystem_exit(void *ptr)
 static void ebpf_filesystem_cleanup(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (ebpf_fs_exited != NETDATA_THREAD_EBPF_STOPPED)
-        return;
-
-    freez(filesystem_threads.thread);
-    ebpf_cleanup_publish_syscall(filesystem_publish_aggregated);
-
-    ebpf_filesystem_cleanup_ebpf_data();
-    if (dimensions)
-        ebpf_histogram_dimension_cleanup(dimensions, NETDATA_EBPF_HIST_MAX_BINS);
-    freez(filesystem_hash_values);
-
-    filesystem_threads.enabled = NETDATA_MAIN_THREAD_EXITED;
-    em->enabled = NETDATA_MAIN_THREAD_EXITED;
+    ebpf_filesystem_free(em);
 }
 
 /*****************************************************************
@@ -491,11 +492,8 @@ void *ebpf_filesystem_read_hash(void *ptr)
     heartbeat_init(&hb);
     usec_t step = NETDATA_FILESYSTEM_READ_SLEEP_MS * em->update_every;
     int update_every = em->update_every;
-    while (ebpf_fs_exited == NETDATA_THREAD_EBPF_RUNNING) {
-        usec_t dt = heartbeat_next(&hb, step);
-        (void)dt;
-        if (ebpf_fs_exited == NETDATA_THREAD_EBPF_STOPPING)
-            break;
+    while (!ebpf_exit_plugin) {
+        (void)heartbeat_next(&hb, step);
 
         (void) ebpf_update_partitions(em);
         ebpf_obsolete_fs_charts(update_every);
@@ -507,8 +505,6 @@ void *ebpf_filesystem_read_hash(void *ptr)
         read_filesystem_tables();
     }
 
-    ebpf_fs_exited = NETDATA_THREAD_EBPF_STOPPED;
-
     netdata_thread_cleanup_pop(1);
     return NULL;
 }
@@ -611,9 +607,6 @@ void *ebpf_filesystem_thread(void *ptr)
     em->maps = fs_maps;
     ebpf_update_filesystem();
 
-    if (!em->enabled)
-        goto endfilesystem;
-
     // Initialize optional as zero, to identify when there are not partitions to monitor
     em->optional = 0;
 
@@ -621,7 +614,7 @@ void *ebpf_filesystem_thread(void *ptr)
         if (em->optional)
             info("Netdata cannot monitor the filesystems used on this host.");
 
-        em->enabled = CONFIG_BOOLEAN_NO;
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
         goto endfilesystem;
     }
 
@@ -638,8 +631,7 @@ void *ebpf_filesystem_thread(void *ptr)
     filesystem_collector(em);
 
 endfilesystem:
-    if (!em->enabled)
-        ebpf_update_disabled_plugin_stats(em);
+    ebpf_update_disabled_plugin_stats(em);
 
     netdata_thread_cleanup_pop(1);
     return NULL;

collectors/ebpf.plugin/ebpf_hardirq.c

@@ -145,7 +145,33 @@ static struct netdata_static_thread hardirq_threads = {
                                         .init_routine = NULL,
                                         .start_routine = NULL
 };
-static enum ebpf_threads_status ebpf_hardirq_exited = NETDATA_THREAD_EBPF_RUNNING;
+
+/**
+ * Hardirq Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_hardirq_free(ebpf_module_t *em)
+{
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+        pthread_mutex_unlock(&ebpf_exit_cleanup);
+        return;
+    }
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+    freez(hardirq_threads.thread);
+    for (int i = 0; hardirq_tracepoints[i].class != NULL; i++) {
+        ebpf_disable_tracepoint(&hardirq_tracepoints[i]);
+    }
+    freez(hardirq_ebpf_vals);
+    freez(hardirq_ebpf_static_vals);
+
+    em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+}
 
 /**
  * Hardirq Exit
@@ -157,12 +183,8 @@ static enum ebpf_threads_status ebpf_hardirq_exited = NETDATA_THREAD_EBPF_RUNNIN
 static void hardirq_exit(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (!em->enabled) {
-        em->enabled = NETDATA_MAIN_THREAD_EXITED;
-        return;
-    }
-
-    ebpf_hardirq_exited = NETDATA_THREAD_EBPF_STOPPING;
+    netdata_thread_cancel(*hardirq_threads.thread);
+    ebpf_hardirq_free(em);
 }
 
 /**
@@ -175,19 +197,7 @@ static void hardirq_exit(void *ptr)
 static void hardirq_cleanup(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-
-    if (ebpf_hardirq_exited != NETDATA_THREAD_EBPF_STOPPED)
-        return;
-
-    freez(hardirq_threads.thread);
-    for (int i = 0; hardirq_tracepoints[i].class != NULL; i++) {
-        ebpf_disable_tracepoint(&hardirq_tracepoints[i]);
-    }
-    freez(hardirq_ebpf_vals);
-    freez(hardirq_ebpf_static_vals);
-
-    hardirq_threads.enabled = NETDATA_MAIN_THREAD_EXITED;
-    em->enabled = NETDATA_MAIN_THREAD_EXITED;
+    ebpf_hardirq_free(em);
 }
 
 /*****************************************************************
@@ -330,16 +340,12 @@ static void *hardirq_reader(void *ptr)
     ebpf_module_t *em = (ebpf_module_t *)ptr;
 
     usec_t step = NETDATA_HARDIRQ_SLEEP_MS * em->update_every;
-    while (ebpf_hardirq_exited == NETDATA_THREAD_EBPF_RUNNING) {
-        usec_t dt = heartbeat_next(&hb, step);
-        UNUSED(dt);
-        if (ebpf_hardirq_exited == NETDATA_THREAD_EBPF_STOPPING)
-            break;
+    while (!ebpf_exit_plugin) {
+        (void)heartbeat_next(&hb, step);
 
         hardirq_read_latency_map(hardirq_maps[HARDIRQ_MAP_LATENCY].map_fd);
         hardirq_read_latency_static_map(hardirq_maps[HARDIRQ_MAP_LATENCY_STATIC].map_fd);
     }
-    ebpf_hardirq_exited = NETDATA_THREAD_EBPF_STOPPED;
 
     netdata_thread_cleanup_pop(1);
     return NULL;
@@ -479,26 +485,21 @@ void *ebpf_hardirq_thread(void *ptr)
     ebpf_module_t *em = (ebpf_module_t *)ptr;
     em->maps = hardirq_maps;
 
-    if (!em->enabled) {
-        goto endhardirq;
-    }
-
     if (ebpf_enable_tracepoints(hardirq_tracepoints) == 0) {
-        em->enabled = CONFIG_BOOLEAN_NO;
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
         goto endhardirq;
     }
 
     em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
     if (!em->probe_links) {
-        em->enabled = CONFIG_BOOLEAN_NO;
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
         goto endhardirq;
     }
 
     hardirq_collector(em);
 
 endhardirq:
-    if (!em->enabled)
-        ebpf_update_disabled_plugin_stats(em);
+    ebpf_update_disabled_plugin_stats(em);
 
     netdata_thread_cleanup_pop(1);
 

collectors/ebpf.plugin/ebpf_mdflush.c

@@ -45,7 +45,29 @@ static struct netdata_static_thread mdflush_threads = {
                                         .init_routine = NULL,
                                         .start_routine = NULL
 };
-static enum ebpf_threads_status ebpf_mdflush_exited = NETDATA_THREAD_EBPF_RUNNING;
+
+/**
+ * MDflush Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_mdflush_free(ebpf_module_t *em)
+{
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+        pthread_mutex_unlock(&ebpf_exit_cleanup);
+        return;
+    }
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+    freez(mdflush_ebpf_vals);
+    freez(mdflush_threads.thread);
+
+    em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+}
 
 /**
  * MDflush exit
@@ -57,12 +79,7 @@ static enum ebpf_threads_status ebpf_mdflush_exited = NETDATA_THREAD_EBPF_RUNNIN
 static void mdflush_exit(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (!em->enabled) {
-        em->enabled = NETDATA_MAIN_THREAD_EXITED;
-        return;
-    }
-
-    ebpf_mdflush_exited = NETDATA_THREAD_EBPF_STOPPING;
+    ebpf_mdflush_free(em);
 }
 
 /**
@@ -75,14 +92,8 @@ static void mdflush_exit(void *ptr)
 static void mdflush_cleanup(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (ebpf_mdflush_exited != NETDATA_THREAD_EBPF_STOPPED)
-        return;
-
-    freez(mdflush_ebpf_vals);
-    freez(mdflush_threads.thread);
-
-    mdflush_threads.enabled = NETDATA_MAIN_THREAD_EXITED;
-    em->enabled = NETDATA_MAIN_THREAD_EXITED;
+    netdata_thread_cancel(*mdflush_threads.thread);
+    ebpf_mdflush_free(em);
 }
 
 /**
@@ -189,17 +200,12 @@ static void *mdflush_reader(void *ptr)
     ebpf_module_t *em = (ebpf_module_t *)ptr;
 
     usec_t step = NETDATA_MDFLUSH_SLEEP_MS * em->update_every;
-    while (ebpf_mdflush_exited == NETDATA_THREAD_EBPF_RUNNING) {
-        usec_t dt = heartbeat_next(&hb, step);
-        UNUSED(dt);
-        if (ebpf_mdflush_exited == NETDATA_THREAD_EBPF_STOPPING)
-            break;
+    while (!ebpf_exit_plugin) {
+        (void)heartbeat_next(&hb, step);
 
         mdflush_read_count_map();
     }
 
-    ebpf_mdflush_exited = NETDATA_THREAD_EBPF_STOPPED;
-
     netdata_thread_cleanup_pop(1);
     return NULL;
 }
@@ -302,26 +308,25 @@ void *ebpf_mdflush_thread(void *ptr)
 
     char *md_flush_request = ebpf_find_symbol("md_flush_request");
     if (!md_flush_request) {
-        em->enabled = CONFIG_BOOLEAN_NO;
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
         error("Cannot monitor MD devices, because md is not loaded.");
     }
     freez(md_flush_request);
 
-    if (!em->enabled) {
+    if (em->thread->enabled == NETDATA_THREAD_EBPF_STOPPED) {
         goto endmdflush;
     }
 
     em->probe_links = ebpf_load_program(ebpf_plugin_dir, em, running_on_kernel, isrh, &em->objects);
     if (!em->probe_links) {
-        em->enabled = CONFIG_BOOLEAN_NO;
+        em->enabled = NETDATA_THREAD_EBPF_STOPPED;
         goto endmdflush;
     }
 
     mdflush_collector(em);
 
 endmdflush:
-    if (!em->enabled)
-        ebpf_update_disabled_plugin_stats(em);
+    ebpf_update_disabled_plugin_stats(em);
 
     netdata_thread_cleanup_pop(1);
 

collectors/ebpf.plugin/ebpf_mount.c

@@ -36,7 +36,6 @@ struct netdata_static_thread mount_thread = {
 netdata_ebpf_targets_t mount_targets[] = { {.name = "mount", .mode = EBPF_LOAD_TRAMPOLINE},
                                            {.name = "umount", .mode = EBPF_LOAD_TRAMPOLINE},
                                            {.name = NULL, .mode = EBPF_LOAD_TRAMPOLINE}};
-static enum ebpf_threads_status ebpf_mount_exited = NETDATA_THREAD_EBPF_RUNNING;
 
 #ifdef LIBBPF_MAJOR_VERSION
 #include "includes/mount.skel.h" // BTF code
@@ -230,6 +229,36 @@ static inline int ebpf_mount_load_and_attach(struct mount_bpf *obj, ebpf_module_
  *
  *****************************************************************/
 
+/**
+ * Mount Free
+ *
+ * Cleanup variables after child threads to stop
+ *
+ * @param ptr thread data.
+ */
+static void ebpf_mount_free(ebpf_module_t *em)
+{
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    if (em->thread->enabled == NETDATA_THREAD_EBPF_RUNNING) {
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPING;
+        pthread_mutex_unlock(&ebpf_exit_cleanup);
+        return;
+    }
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+
+    freez(mount_thread.thread);
+    freez(mount_values);
+
+#ifdef LIBBPF_MAJOR_VERSION
+    if (bpf_obj)
+        mount_bpf__destroy(bpf_obj);
+#endif
+
+    pthread_mutex_lock(&ebpf_exit_cleanup);
+    em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
+    pthread_mutex_unlock(&ebpf_exit_cleanup);
+}
+
 /**
  * Mount Exit
  *
@@ -240,12 +269,8 @@ static inline int ebpf_mount_load_and_attach(struct mount_bpf *obj, ebpf_module_
 static void ebpf_mount_exit(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (!em->enabled) {
-        em->enabled = NETDATA_MAIN_THREAD_EXITED;
-        return;
-    }
-
-    ebpf_mount_exited = NETDATA_THREAD_EBPF_STOPPING;
+    netdata_thread_cancel(*mount_thread.thread);
+    ebpf_mount_free(em);
 }
 
 /**
@@ -258,19 +283,7 @@ static void ebpf_mount_exit(void *ptr)
 static void ebpf_mount_cleanup(void *ptr)
 {
     ebpf_module_t *em = (ebpf_module_t *)ptr;
-    if (ebpf_mount_exited != NETDATA_THREAD_EBPF_STOPPED)
-        return;
-
-    freez(mount_thread.thread);
-    freez(mount_values);
-
-#ifdef LIBBPF_MAJOR_VERSION
-    if (bpf_obj)
-        mount_bpf__destroy(bpf_obj);
-#endif
-
-    mount_thread.enabled = NETDATA_MAIN_THREAD_EXITED;
-    em->enabled = NETDATA_MAIN_THREAD_EXITED;
+    ebpf_mount_free(em);
 }
 
 /*****************************************************************
@@ -324,17 +337,12 @@ void *ebpf_mount_read_hash(void *ptr)
 
     usec_t step = NETDATA_LATENCY_MOUNT_SLEEP_MS * em->update_every;
     //This will be cancelled by its parent
-    while (ebpf_mount_exited == NETDATA_THREAD_EBPF_RUNNING) {
-        usec_t dt = heartbeat_next(&hb, step);
-        (void)dt;
-        if (ebpf_mount_exited == NETDATA_THREAD_EBPF_STOPPING)
-            break;
+    while (!ebpf_exit_plugin) {
+        (void)heartbeat_next(&hb, step);
 
         read_global_table();
     }
 
-    ebpf_mount_exited = NETDATA_THREAD_EBPF_STOPPED;
-
     netdata_thread_cleanup_pop(1);
     return NULL;
 }
@@ -481,14 +489,11 @@ void *ebpf_mount_thread(void *ptr)
     ebpf_module_t *em = (ebpf_module_t *)ptr;
     em->maps = mount_maps;
 
-    if (!em->enabled)
-        goto endmount;
-
 #ifdef LIBBPF_MAJOR_VERSION
     ebpf_adjust_thread_load(em, default_btf);
 #endif
     if (ebpf_mount_load_bpf(em)) {
-        em->enabled = CONFIG_BOOLEAN_NO;
+        em->thread->enabled = NETDATA_THREAD_EBPF_STOPPED;
         goto endmount;
     }
 
@@ -505,8 +510,7 @@ void *ebpf_mount_thread(void *ptr)
     mount_collector(em);
 
 endmount:
-    if (!em->enabled)
-        ebpf_update_disabled_plugin_stats(em);
+    ebpf_update_disabled_plugin_stats(em);
 
     netdata_thread_cleanup_pop(1);
     return NULL;