Allow for an easy way to do metadata migrations (#13196)

Allow for an easy way to migrate metadata to a new database schema (versioning)

CMakeLists.txt

@@ -658,6 +658,8 @@ set(RRD_PLUGIN_FILES
         database/ram/rrddim_mem.h
         database/sqlite/sqlite_functions.c
         database/sqlite/sqlite_functions.h
+        database/sqlite/sqlite_db_migration.c
+        database/sqlite/sqlite_db_migration.h
         database/sqlite/sqlite_aclk.c
         database/sqlite/sqlite_aclk.h
         database/sqlite/sqlite_health.c

Makefile.am

@@ -454,6 +454,8 @@ RRD_PLUGIN_FILES = \
     database/ram/rrddim_mem.h \
     database/sqlite/sqlite_functions.c \
     database/sqlite/sqlite_functions.h \
+    database/sqlite/sqlite_db_migration.c \
+    database/sqlite/sqlite_db_migration.h \
     database/sqlite/sqlite_aclk.c \
     database/sqlite/sqlite_aclk.h \
     database/sqlite/sqlite_health.c \

database/sqlite/sqlite_db_migration.c

@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "sqlite_db_migration.h"
+
+static int do_migration_noop(sqlite3 *database, const char *name)
+{
+    UNUSED(database);
+    UNUSED(name);
+    info("Running database migration %s", name);
+    return 0;
+}
+
+static struct database_func_migration_list {
+    char *name;
+    int (*func)(sqlite3 *database, const char *name);
+} migration_action[] = {
+    {.name = "v0 to v1",  .func = do_migration_noop},
+    // the terminator of this array
+    {.name = NULL, .func = NULL}
+};
+
+
+static int perform_database_migration_cb(void *data, int argc, char **argv, char **column)
+{
+    int *status = data;
+    UNUSED(argc);
+    UNUSED(column);
+    *status = str2uint32_t(argv[0]);
+    return 0;
+}
+
+int perform_database_migration(sqlite3 *database, int target_version)
+{
+    int user_version = 0;
+    char *err_msg = NULL;
+
+    int rc = sqlite3_exec(database, "PRAGMA user_version;", perform_database_migration_cb, (void *) &user_version, &err_msg);
+    if (rc != SQLITE_OK) {
+        info("Error checking the database version; %s", err_msg);
+        sqlite3_free(err_msg);
+    }
+
+    if (likely(user_version == target_version)) {
+        info("Metadata database version is %d", target_version);
+        return target_version;
+    }
+
+    info("Database version is %d, current version is %d. Running migration ...", user_version, target_version);
+    for (int i = user_version; migration_action[i].func && i < target_version; i++) {
+        rc = (migration_action[i].func)(database, migration_action[i].name);
+        if (unlikely(rc)) {
+            error_report("Database migration from version %d to version %d failed", i, i + 1);
+            return i;
+        }
+    }
+    return target_version;
+}

database/sqlite/sqlite_db_migration.h

@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+#ifndef NETDATA_SQLITE_DB_MIGRATION_H
+#define NETDATA_SQLITE_DB_MIGRATION_H
+
+#include "daemon/common.h"
+#include "sqlite3.h"
+
+
+int perform_database_migration(sqlite3 *database, int target_version);
+
+#endif //NETDATA_SQLITE_DB_MIGRATION_H

database/sqlite/sqlite_functions.c

@@ -1,8 +1,9 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "sqlite_functions.h"
+#include "sqlite_db_migration.h"
 
-#define DB_METADATA_VERSION "1"
+#define DB_METADATA_VERSION 1
 
 const char *database_config[] = {
     "CREATE TABLE IF NOT EXISTS host(host_id blob PRIMARY KEY, hostname text, "
@@ -52,7 +53,6 @@ const char *database_config[] = {
     "INSERT INTO chart_hash_map (chart_id, hash_id) values (new.chart_id, new.hash_id) "
     "on conflict (chart_id, hash_id) do nothing; END; ",
 
-    "PRAGMA user_version="DB_METADATA_VERSION";",
     NULL
 };
 
@@ -404,6 +404,8 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
     char buf[1024 + 1] = "";
     const char *list[2] = { buf, NULL };
 
+    int target_version = perform_database_migration(db_meta, DB_METADATA_VERSION);
+
     // https://www.sqlite.org/pragma.html#pragma_auto_vacuum
     // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
     snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
@@ -435,6 +437,9 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
     snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
     if(init_database_batch(rebuild, 0, list)) return 1;
 
+    snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
+    if(init_database_batch(rebuild, 0, list)) return 1;
+
     if (init_database_batch(rebuild, 0, &database_config[0]))
         return 1;