Improve agent shutdown (#16959)

* Clear detection thread on stop
Handle multi db close

* cancel_threads should be called once

src/daemon/main.c

@@ -691,6 +691,9 @@ static void set_nofile_limit(struct rlimit *rl) {
 void cancel_main_threads() {
     nd_log_limits_unlimited();
 
+    if (!static_threads)
+        return;
+
     int i, found = 0;
     usec_t max = 5 * USEC_PER_SEC, step = 100000;
     for (i = 0; static_threads[i].name != NULL ; i++) {
@@ -732,6 +735,7 @@ void cancel_main_threads() {
         freez(static_threads[i].thread);
 
     freez(static_threads);
+    static_threads = NULL;
 }
 
 static const struct option_def {

src/database/sqlite/sqlite_context.c

@@ -86,6 +86,7 @@ void sql_close_context_database(void)
     rc = sqlite3_close_v2(db_context_meta);
     if (unlikely(rc != SQLITE_OK))
         error_report("Error %d while closing the context SQLite database, %s", rc, sqlite3_errstr(rc));
+    db_context_meta = NULL;
 }
 
 //

src/database/sqlite/sqlite_functions.c

@@ -518,6 +518,7 @@ void sql_close_database(void)
     rc = sqlite3_close_v2(db_meta);
     if (unlikely(rc != SQLITE_OK))
         error_report("Error %d while closing the SQLite database, %s", rc, sqlite3_errstr(rc));
+    db_meta = NULL;
 }
 
 int exec_statement_with_uuid(const char *sql, uuid_t *uuid)

src/database/sqlite/sqlite_metadata.c

@@ -1826,9 +1826,12 @@ void metadata_sync_shutdown(void)
 
 void metadata_sync_shutdown_prepare(void)
 {
-    if (unlikely(!metasync_worker.loop))
+    static bool running = false;
+    if (unlikely(!metasync_worker.loop || running))
         return;
 
+    running = true;
+
     struct metadata_cmd cmd;
     memset(&cmd, 0, sizeof(cmd));
 

src/ml/ml.cc

@@ -1805,12 +1805,13 @@ void ml_init()
 }
 
 void ml_fini() {
-    if (!Cfg.enable_anomaly_detection)
+    if (!Cfg.enable_anomaly_detection || !db)
         return;
 
     int rc = sqlite3_close_v2(db);
     if (unlikely(rc != SQLITE_OK))
         error_report("Error %d while closing the SQLite database, %s", rc, sqlite3_errstr(rc));
+    db = NULL;
 }
 
 void ml_start_threads() {
@@ -1845,6 +1846,7 @@ void ml_stop_threads()
         return;
 
     netdata_thread_join(Cfg.detection_thread, NULL);
+    Cfg.detection_thread = 0;
 
     // signal the training queue of each thread
     for (size_t idx = 0; idx != Cfg.num_training_threads; idx++) {