dbengine free from RRDSET and RRDDIM (#13772)

* dbengine free from RRDSET and RRDDIM

* fix for excess parameters to query ops

* add comment about ML

* update_every from int to uint32_t

* rrddim_mem storage engine working

* fixes for update_every_s

* working dbengine

* a lot of changes in dbengine regarding timestamps

* better logging of not sequential points

* rrdset_done() now gives aligned timestamps for higher tiers

* dont change the end_time of descriptors, because they cant be loaded back

* fixes for cmake

* fixes for db mode ram

* Global counters for dbengine loading errors.
Ensure dbengine store metrics always has aligned metrics or breaks the page when storing new data.

* update lgtm config

* fixes for 32-bit systems

* update unittests

* Don't try to find and create a host on the fly if not already in memory

* Remove unused functions

* print backtrace in case of fatal

* always set ctx to page_index

* detect ctx and metric uuid discrepancies

* use legacy uuid if multihost is not available

* fix for last commit

* prevent repeating log

* Do not try to access archived charts when executing a data query

* Remove unused function

* log inconsistent collections once every 10 mins

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

.lgtm.yml

@@ -13,10 +13,12 @@ path_classifiers:
     - collectors/python.d.plugin/python_modules/pyyaml2/
     - collectors/python.d.plugin/python_modules/pyyaml3/
     - ml/kmeans/dlib/
+    - ml/dlib/dlib/
     - ml/json/
     - web/gui/lib/
     - web/gui/src/
     - web/gui/css/
+    - web/gui/dashboard/lib/
     - libnetdata/libjudy/
   test:
     - tests/

CMakeLists.txt

@@ -9,31 +9,25 @@ project(netdata C CXX)
 find_package(Threads REQUIRED)
 find_package(PkgConfig REQUIRED)
 
+include(CheckFunctionExists)
+include(CheckLibraryExists)
+
 # default is "Debug"
 #set(CMAKE_BUILD_TYPE "Release")
 
 # set this to see the compilation commands
-# set(CMAKE_VERBOSE_MAKEFILE 1)
-
+#set(CMAKE_VERBOSE_MAKEFILE 1)
 
 # -----------------------------------------------------------------------------
 # Set compilation options according to build type
 
+set(CMAKE_C_STANDARD 11)
+
 IF("${CMAKE_BUILD_TYPE}" MATCHES "Debug")
+    set(CXX_DEFAULT_CFLAGS "-O0 -g -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_DEV_MODE=1 -fstack-protector-all -fno-omit-frame-pointer")
     message(STATUS "building for: debugging")
-
-    ## unfortunately these produce errors
-    #include(CheckCXXCompilerFlag)
-    #CHECK_CXX_COMPILER_FLAG("-Wformat-signedness" CXX_FORMAT_SIGNEDNESS)
-    #CHECK_CXX_COMPILER_FLAG("-Werror=format-security" CXX_FORMAT_SECURITY)
-    #CHECK_CXX_COMPILER_FLAG("-fstack-protector-all" CXX_STACK_PROTECTOR)
-    set(CXX_FORMAT_SIGNEDNESS "-Wformat-signedness")
-    set(CXX_FORMAT_SECURITY "-Werror=format-security")
-    set(CXX_STACK_PROTECTOR "-fstack-protector-all")
-    set(CXX_FLAGS_DEBUG "-O0")
-    set(CMAKE_C_STANDARD 99)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O1 -ggdb -Wall -Wextra -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1 ${CXX_FORMAT_SIGNEDNESS} ${CXX_FORMAT_SECURITY} ${CXX_STACK_PROTECTOR} ${CXX_FLAGS_DEBUG}")
 ELSE()
+    set(CXX_DEFAULT_CFLAGS "-O2")
     message(STATUS "building for: release")
     cmake_policy(SET CMP0069 "NEW")
     include(CheckIPOSupported)
@@ -46,6 +40,7 @@ ELSE()
     ENDIF()
 ENDIF()
 
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O1 -ggdb -Wall -Wextra -Wformat-signedness -Werror=format-security -Wunused-result ${CXX_DEFAULT_CFLAGS}")
 
 # -----------------------------------------------------------------------------
 # O/S Detection
@@ -70,6 +65,12 @@ message(STATUS "system name: ${CMAKE_SYSTEM_NAME}")
 set(GENERATED_CONFIG_H_DIR ${CMAKE_BINARY_DIR})
 set(GENERATED_CONFIG_H ${GENERATED_CONFIG_H_DIR}/config.h)
 
+# -----------------------------------------------------------------------------
+# Math
+
+set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} m)
+set(NETDATA_REQUIRED_DEFINES "-DSTORAGE_WITH_MATH=1 ${NETDATA_REQUIRED_DEFINES}")
+
 # -----------------------------------------------------------------------------
 # Detect libuuid
 
@@ -85,6 +86,7 @@ pkg_check_modules(ZLIB REQUIRED zlib)
 set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${ZLIB_CFLAGS_OTHER})
 set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${ZLIB_LIBRARIES})
 set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIRS})
+# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DNETDATA_WITH_ZLIB=1")
 
 # -----------------------------------------------------------------------------
 # libuv multi-platform support library with a focus on asynchronous I/O
@@ -101,6 +103,7 @@ pkg_check_modules(LIBLZ4 REQUIRED liblz4)
 set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${LIBLZ4_CFLAGS_OTHER})
 set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${LIBLZ4_LIBRARIES})
 set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${LIBLZ4_INCLUDE_DIRS})
+# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DENABLE_COMPRESSION=1")
 
 # -----------------------------------------------------------------------------
 # Judy General purpose dynamic array
@@ -114,6 +117,7 @@ pkg_check_modules(OPENSSL REQUIRED openssl)
 set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${OPENSSL_CFLAGS_OTHER})
 set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${OPENSSL_LIBRARIES})
 set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIRS})
+# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DENABLE_HTTPS=1")
 
 # -----------------------------------------------------------------------------
 # JSON-C used to health
@@ -1023,7 +1027,11 @@ set(ML_FILES
     ml/ml-dummy.c
 )
 
+# -----------------------------------------------------------------------------
+# ML
+
 IF(ENABLE_ML)
+    message(STATUS "ML: enabled")
     list(APPEND ML_FILES
             ml/Config.h
             ml/Config.cc
@@ -1041,6 +1049,8 @@ IF(ENABLE_ML)
             ml/ml.cc
             ml/ml-private.h
     )
+ELSE()
+    message(STATUS "ML: disabled")
 ENDIF()
 
 set(NETDATA_FILES
@@ -1731,7 +1741,6 @@ if (HAVE_SYS_SYSMACROS_H)
 endif()
 
 if (CRYPTO_FOUND)
-    include(CheckLibraryExists)
     set(HAVE_CRYPTO True)
     FIND_LIBRARY(CRYPTO_LIBRARY_LOCATION NAMES crypto)
     check_library_exists(crypto X509_VERIFY_PARAM_set1_host ${CRYPTO_LIBRARY_LOCATION} HAVE_X509_VERIFY_PARAM_set1_host)
@@ -1763,4 +1772,23 @@ ELSE()
     SET(MAYBE_UNUSED_MACRO "")
 ENDIF()
 
+# -----------------------------------------------------------------------------
+
+check_library_exists(c clock_gettime "" HAVE_CLOCK_GETTIME)
+
+IF(NOT HAVE_CLOCK_GETTIME)
+    CHECK_LIBRARY_EXISTS(rt clock_gettime "" HAVE_CLOCK_GETTIME)
+ENDIF()
+
+IF(HAVE_CLOCK_GETTIME)
+    message("-- clock_gettime(): found")
+    set(NETDATA_REQUIRED_DEFINES "-DHAVE_CLOCK_GETTIME=1 ${NETDATA_REQUIRED_DEFINES}")
+ELSE()
+    message("-- clock_gettime(): not found")
+ENDIF()
+
+# -----------------------------------------------------------------------------
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${NETDATA_REQUIRED_DEFINES}")
+message("CFLAGS=\"${CMAKE_C_FLAGS}\"")
+
 configure_file(config.cmake.h.in config.h)

aclk/aclk_query.c

@@ -99,7 +99,6 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
         }
     }
 
-    RRDHOST *temp_host = NULL;
     if (!strncmp(query->data.http_api_v2.query, NODE_ID_QUERY, strlen(NODE_ID_QUERY))) {
         char *node_uuid = query->data.http_api_v2.query + strlen(NODE_ID_QUERY);
         char nodeid[UUID_STR_LEN];
@@ -114,14 +113,11 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
 
         query_host = node_id_2_rrdhost(nodeid);
         if (!query_host) {
-            temp_host = sql_create_host_by_uuid(nodeid);
-            if (!temp_host) {
-                error_report("Host with node_id \"%s\" not found! Returning 404 to Cloud!", nodeid);
-                retval = 1;
-                w->response.code = 404;
-                aclk_http_msg_v2_err(query_thr->client, query->callback_topic, query->msg_id, w->response.code, CLOUD_EC_NODE_NOT_FOUND, CLOUD_EMSG_NODE_NOT_FOUND, NULL, 0);
-                goto cleanup;
-            }
+            error_report("Host with node_id \"%s\" not found! Returning 404 to Cloud!", nodeid);
+            retval = 1;
+            w->response.code = 404;
+            aclk_http_msg_v2_err(query_thr->client, query->callback_topic, query->msg_id, w->response.code, CLOUD_EC_NODE_NOT_FOUND, CLOUD_EMSG_NODE_NOT_FOUND, NULL, 0);
+            goto cleanup;
         }
     }
 
@@ -142,8 +138,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
     }
 
     // execute the query
-    t = aclk_web_api_v1_request(query_host ? query_host : temp_host, w, mysep ? mysep + 1 : "noop");
-    free_temporary_host(temp_host);
+    t = aclk_web_api_v1_request(query_host, w, mysep ? mysep + 1 : "noop");
     size = (w->mode == WEB_CLIENT_MODE_FILECOPY) ? w->response.rlen : w->response.data->len;
     sent = size;
 

collectors/statsd.plugin/statsd.c

@@ -1421,7 +1421,10 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
                     app->default_options |= STATSD_METRIC_OPTION_SHOW_GAPS_WHEN_NOT_COLLECTED;
             }
             else if (!strcmp(name, "memory mode")) {
-                app->rrd_memory_mode = rrd_memory_mode_id(value);
+                // this is not supported anymore
+                // with the implementation of storage engines, all charts have the same storage engine always
+                // app->rrd_memory_mode = rrd_memory_mode_id(value);
+                ;
             }
             else if (!strcmp(name, "history")) {
                 app->rrd_history_entries = atol(value);

configure.ac

@@ -317,6 +317,10 @@ AM_CONDITIONAL([MACOS], [test "${build_target}" = "macos"])
 AM_CONDITIONAL([LINUX], [test "${build_target}" = "linux"])
 AC_MSG_RESULT([Host OS: ${build_target}])
 
+# -----------------------------------------------------------------------------
+# backtrace
+
+AC_SEARCH_LIBS([backtrace], [execinfo], [AC_DEFINE([HAVE_BACKTRACE], [1], [backtrace availability])])
 
 # -----------------------------------------------------------------------------
 # pthreads

daemon/unit_test.c

@@ -1816,6 +1816,8 @@ static time_t test_dbengine_create_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS
     // feed it with the test data
     for (i = 0 ; i < CHARTS ; ++i) {
         for (j = 0 ; j < DIMS ; ++j) {
+            rd[i][j]->tiers[0]->collect_ops.change_collection_frequency(rd[i][j]->tiers[0]->db_collection_handle, update_every);
+
             rd[i][j]->last_collected_time.tv_sec =
             st[i]->last_collected_time.tv_sec = st[i]->last_updated.tv_sec = time_now;
             rd[i][j]->last_collected_time.tv_usec =
@@ -1859,7 +1861,7 @@ static int test_dbengine_check_metrics(RRDSET *st[CHARTS], RRDDIM *rd[CHARTS][DI
         time_now = time_start + (c + 1) * update_every;
         for (i = 0 ; i < CHARTS ; ++i) {
             for (j = 0; j < DIMS; ++j) {
-                rd[i][j]->tiers[0]->query_ops.init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every, TIER_QUERY_FETCH_SUM);
+                rd[i][j]->tiers[0]->query_ops.init(rd[i][j]->tiers[0]->db_metric_handle, &handle, time_now, time_now + QUERY_BATCH * update_every);
                 for (k = 0; k < QUERY_BATCH; ++k) {
                     last = ((collected_number)i * DIMS) * REGION_POINTS[current_region] +
                            j * REGION_POINTS[current_region] + c + k;
@@ -2316,7 +2318,7 @@ static void query_dbengine_chart(void *arg)
             time_before = MIN(time_after + duration, time_max); /* up to 1 hour queries */
         }
 
-        rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before, TIER_QUERY_FETCH_SUM);
+        rd->tiers[0]->query_ops.init(rd->tiers[0]->db_metric_handle, &handle, time_after, time_before);
         ++thread_info->queries_nr;
         for (time_now = time_after ; time_now <= time_before ; time_now += update_every) {
             generatedv = generate_dbengine_chart_value(i, j, time_now);

database/engine/journalfile.c

@@ -311,20 +311,46 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
             }
             continue;
         }
-        uint64_t start_time = jf_metric_data->descr[i].start_time;
-        uint64_t end_time = jf_metric_data->descr[i].end_time;
+        uint64_t start_time_ut = jf_metric_data->descr[i].start_time_ut;
+        uint64_t end_time_ut = jf_metric_data->descr[i].end_time_ut;
+        size_t entries = jf_metric_data->descr[i].page_length / page_type_size[page_type];
+        time_t update_every_s = (entries > 1) ? ((end_time_ut - start_time_ut) / USEC_PER_SEC / (entries - 1)) : 0;
+
+        if (unlikely(start_time_ut > end_time_ut)) {
+            ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].counter++;
+            if(ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].latest_end_time_ut < end_time_ut)
+                ctx->load_errors[LOAD_ERRORS_PAGE_FLIPPED_TIME].latest_end_time_ut = end_time_ut;
+            continue;
+        }
 
-        if (unlikely(start_time > end_time)) {
-            error("Invalid page encountered, start time %"PRIu64" > end time %"PRIu64"", start_time , end_time);
+        if (unlikely(start_time_ut == end_time_ut && entries != 1)) {
+            ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].counter++;
+            if(ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].latest_end_time_ut < end_time_ut)
+                ctx->load_errors[LOAD_ERRORS_PAGE_EQUAL_TIME].latest_end_time_ut = end_time_ut;
             continue;
         }
 
-        if (unlikely(start_time == end_time)) {
-            size_t entries = jf_metric_data->descr[i].page_length / page_type_size[page_type];
-            if (unlikely(entries > 1)) {
-                error("Invalid page encountered, start time %"PRIu64" = end time but %zu entries were found", start_time, entries);
-                continue;
-            }
+        if (unlikely(!entries)) {
+            ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].counter++;
+            if(ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].latest_end_time_ut < end_time_ut)
+                ctx->load_errors[LOAD_ERRORS_PAGE_ZERO_ENTRIES].latest_end_time_ut = end_time_ut;
+            continue;
+        }
+
+        if(entries > 1 && update_every_s == 0) {
+            ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].counter++;
+            if(ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].latest_end_time_ut < end_time_ut)
+                ctx->load_errors[LOAD_ERRORS_PAGE_UPDATE_ZERO].latest_end_time_ut = end_time_ut;
+            continue;
+        }
+
+        if(start_time_ut + update_every_s * USEC_PER_SEC * (entries - 1) != end_time_ut) {
+            ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].counter++;
+            if(ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].latest_end_time_ut < end_time_ut)
+                ctx->load_errors[LOAD_ERRORS_PAGE_FLEXY_TIME].latest_end_time_ut = end_time_ut;
+
+            // let this be
+            // end_time_ut = start_time_ut + update_every_s * USEC_PER_SEC * (entries - 1);
         }
 
         temp_id = (uuid_t *)jf_metric_data->descr[i].uuid;
@@ -340,7 +366,7 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
             uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
             PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, temp_id, sizeof(uuid_t), PJE0);
             fatal_assert(NULL == *PValue); /* TODO: figure out concurrency model */
-            *PValue = page_index = create_page_index(temp_id);
+            *PValue = page_index = create_page_index(temp_id, ctx);
             page_index->prev = pg_cache->metrics_index.last_page_index;
             pg_cache->metrics_index.last_page_index = page_index;
             uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
@@ -348,21 +374,32 @@ static void restore_extent_metadata(struct rrdengine_instance *ctx, struct rrden
 
         descr = pg_cache_create_descr();
         descr->page_length = jf_metric_data->descr[i].page_length;
-        descr->start_time = start_time;
-        descr->end_time = end_time;
+        descr->start_time_ut = start_time_ut;
+        descr->end_time_ut = end_time_ut;
+        descr->update_every_s = (update_every_s > 0) ? (uint32_t)update_every_s : (page_index->latest_update_every_s);
         descr->id = &page_index->id;
         descr->extent = extent;
         descr->type = page_type;
         extent->pages[valid_pages++] = descr;
         pg_cache_insert(ctx, page_index, descr);
+
+        if(page_index->latest_time_ut == descr->end_time_ut)
+            page_index->latest_update_every_s = descr->update_every_s;
+
+        if(descr->update_every_s == 0)
+            fatal(
+                "DBENGINE: page descriptor update every is zero, end_time_ut = %llu, start_time_ut = %llu, entries = %zu",
+                (unsigned long long)end_time_ut, (unsigned long long)start_time_ut, entries);
     }
 
     extent->number_of_pages = valid_pages;
 
     if (likely(valid_pages))
         df_extent_insert(extent);
-    else
+    else {
         freez(extent);
+        ctx->load_errors[LOAD_ERRORS_DROPPED_EXTENT].counter++;
+    }
 }
 
 /*

database/engine/pagecache.c

@@ -127,12 +127,13 @@ struct rrdeng_page_descr *pg_cache_create_descr(void)
 
     descr = rrdeng_page_descr_mallocz();
     descr->page_length = 0;
-    descr->start_time = INVALID_TIME;
-    descr->end_time = INVALID_TIME;
+    descr->start_time_ut = INVALID_TIME;
+    descr->end_time_ut = INVALID_TIME;
     descr->id = NULL;
     descr->extent = NULL;
     descr->pg_cache_descr_state = 0;
     descr->pg_cache_descr = NULL;
+    descr->update_every_s = 0;
 
     return descr;
 }
@@ -476,7 +477,7 @@ uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_d
     uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
 
     uv_rwlock_wrlock(&page_index->lock);
-    ret = JudyLDel(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
+    ret = JudyLDel(&page_index->JudyL_array, (Word_t)(descr->start_time_ut / USEC_PER_SEC), PJE0);
     if (unlikely(0 == ret)) {
         uv_rwlock_wrunlock(&page_index->lock);
         if (unlikely(debug_flags & D_RRDENGINE)) {
@@ -506,7 +507,7 @@ uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_d
         while (!pg_cache_try_get_unsafe(descr, 1)) {
             debug(D_RRDENGINE, "%s: Waiting for locked page:", __func__);
             if (unlikely(debug_flags & D_RRDENGINE))
-                print_page_cache_descr(descr);
+                print_page_cache_descr(descr, "", true);
             pg_cache_wait_event_unsafe(descr);
         }
     }
@@ -517,7 +518,7 @@ uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_d
         while (unlikely(pg_cache_descr->flags & RRD_PAGE_DIRTY)) {
             debug(D_RRDENGINE, "%s: Found dirty page, waiting for it to be flushed:", __func__);
             if (unlikely(debug_flags & D_RRDENGINE))
-                print_page_cache_descr(descr);
+                print_page_cache_descr(descr, "", true);
             pg_cache_wait_event_unsafe(descr);
         }
     }
@@ -548,8 +549,8 @@ static inline int is_page_in_time_range(struct rrdeng_page_descr *descr, usec_t
 {
     usec_t pg_start, pg_end;
 
-    pg_start = descr->start_time;
-    pg_end = descr->end_time;
+    pg_start = descr->start_time_ut;
+    pg_end = descr->end_time_ut;
 
     return (pg_start < start_time && pg_end >= start_time) ||
            (pg_start >= start_time && pg_start <= end_time);
@@ -557,7 +558,7 @@ static inline int is_page_in_time_range(struct rrdeng_page_descr *descr, usec_t
 
 static inline int is_point_in_time_in_page(struct rrdeng_page_descr *descr, usec_t point_in_time)
 {
-    return (point_in_time >= descr->start_time && point_in_time <= descr->end_time);
+    return (point_in_time >= descr->start_time_ut && point_in_time <= descr->end_time_ut);
 }
 
 /* The caller must hold the page index lock */
@@ -592,14 +593,14 @@ static inline struct rrdeng_page_descr *
 /* Update metric oldest and latest timestamps efficiently when adding new values */
 void pg_cache_add_new_metric_time(struct pg_cache_page_index *page_index, struct rrdeng_page_descr *descr)
 {
-    usec_t oldest_time = page_index->oldest_time;
-    usec_t latest_time = page_index->latest_time;
+    usec_t oldest_time = page_index->oldest_time_ut;
+    usec_t latest_time = page_index->latest_time_ut;
 
-    if (unlikely(oldest_time == INVALID_TIME || descr->start_time < oldest_time)) {
-        page_index->oldest_time = descr->start_time;
+    if (unlikely(oldest_time == INVALID_TIME || descr->start_time_ut < oldest_time)) {
+        page_index->oldest_time_ut = descr->start_time_ut;
     }
-    if (likely(descr->end_time > latest_time || latest_time == INVALID_TIME)) {
-        page_index->latest_time = descr->end_time;
+    if (likely(descr->end_time_ut > latest_time || latest_time == INVALID_TIME)) {
+        page_index->latest_time_ut = descr->end_time_ut;
     }
 }
 
@@ -618,23 +619,23 @@ void pg_cache_update_metric_times(struct pg_cache_page_index *page_index)
     firstPValue = JudyLFirst(page_index->JudyL_array, &firstIndex, PJE0);
     if (likely(NULL != firstPValue)) {
         descr = *firstPValue;
-        oldest_time = descr->start_time;
+        oldest_time = descr->start_time_ut;
     }
     lastIndex = (Word_t)-1;
     lastPValue = JudyLLast(page_index->JudyL_array, &lastIndex, PJE0);
     if (likely(NULL != lastPValue)) {
         descr = *lastPValue;
-        latest_time = descr->end_time;
+        latest_time = descr->end_time_ut;
     }
     uv_rwlock_rdunlock(&page_index->lock);
 
     if (unlikely(NULL == firstPValue)) {
         fatal_assert(NULL == lastPValue);
-        page_index->oldest_time = page_index->latest_time = INVALID_TIME;
+        page_index->oldest_time_ut = page_index->latest_time_ut = INVALID_TIME;
         return;
     }
-    page_index->oldest_time = oldest_time;
-    page_index->latest_time = latest_time;
+    page_index->oldest_time_ut = oldest_time;
+    page_index->latest_time_ut = latest_time;
 }
 
 /* If index is NULL lookup by UUID (descr->id) */
@@ -669,7 +670,7 @@ void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index
     }
 
     uv_rwlock_wrlock(&page_index->lock);
-    PValue = JudyLIns(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
+    PValue = JudyLIns(&page_index->JudyL_array, (Word_t)(descr->start_time_ut / USEC_PER_SEC), PJE0);
     *PValue = descr;
     ++page_index->page_count;
     pg_cache_add_new_metric_time(page_index, descr);
@@ -681,7 +682,7 @@ void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index
     uv_rwlock_wrunlock(&pg_cache->pg_cache_rwlock);
 }
 
-usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time, usec_t end_time)
+usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time_ut, usec_t end_time_ut)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
     struct rrdeng_page_descr *descr = NULL;
@@ -699,25 +700,25 @@ usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id,
     }
 
     uv_rwlock_rdlock(&page_index->lock);
-    descr = find_first_page_in_time_range(page_index, start_time, end_time);
+    descr = find_first_page_in_time_range(page_index, start_time_ut, end_time_ut);
     if (NULL == descr) {
         uv_rwlock_rdunlock(&page_index->lock);
         return INVALID_TIME;
     }
     uv_rwlock_rdunlock(&page_index->lock);
-    return descr->start_time;
+    return descr->start_time_ut;
 }
 
 /**
  * Return page information for the first page before point_in_time that satisfies the filter.
  * @param ctx DB context
  * @param page_index page index of a metric
- * @param point_in_time the pages that are searched must be older than this timestamp
+ * @param point_in_time_ut the pages that are searched must be older than this timestamp
  * @param filter decides if the page satisfies the caller's criteria
  * @param page_info the result of the search is set in this pointer
  */
 void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_cache_page_index *page_index,
-                                     usec_t point_in_time, pg_cache_page_info_filter_t *filter,
+                                     usec_t point_in_time_ut, pg_cache_page_info_filter_t *filter,
                                      struct rrdeng_page_info *page_info)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
@@ -728,7 +729,7 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c
     (void)pg_cache;
     fatal_assert(NULL != page_index);
 
-    Index = (Word_t)(point_in_time / USEC_PER_SEC);
+    Index = (Word_t)(point_in_time_ut / USEC_PER_SEC);
     uv_rwlock_rdlock(&page_index->lock);
     do {
         PValue = JudyLPrev(page_index->JudyL_array, &Index, PJE0);
@@ -736,12 +737,12 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c
     } while (descr != NULL && !filter(descr));
     if (unlikely(NULL == descr)) {
         page_info->page_length = 0;
-        page_info->start_time = INVALID_TIME;
-        page_info->end_time = INVALID_TIME;
+        page_info->start_time_ut = INVALID_TIME;
+        page_info->end_time_ut = INVALID_TIME;
     } else {
         page_info->page_length = descr->page_length;
-        page_info->start_time = descr->start_time;
-        page_info->end_time = descr->end_time;
+        page_info->start_time_ut = descr->start_time_ut;
+        page_info->end_time_ut = descr->end_time_ut;
     }
     uv_rwlock_rdunlock(&page_index->lock);
 }
@@ -750,7 +751,7 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c
  * Searches for an unallocated page without triggering disk I/O. Attempts to reserve the page and get a reference.
  * @param ctx DB context
  * @param id lookup by UUID
- * @param start_time exact starting time in usec
+ * @param start_time_ut exact starting time in usec
  * @param ret_page_indexp Sets the page index pointer (*ret_page_indexp) for the given UUID.
  * @return the page descriptor or NULL on failure. It can fail if:
  *         1. The page is already allocated to the page cache.
@@ -758,7 +759,7 @@ void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_c
  *         3. It did not succeed to reserve a spot in the page cache.
  */
 struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_instance *ctx, uuid_t *id,
-                                                               usec_t start_time)
+                                                               usec_t start_time_ut)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
     struct rrdeng_page_descr *descr = NULL;
@@ -781,7 +782,7 @@ struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_
     }
 
     uv_rwlock_rdlock(&page_index->lock);
-    Index = (Word_t)(start_time / USEC_PER_SEC);
+    Index = (Word_t)(start_time_ut / USEC_PER_SEC);
     PValue = JudyLGet(page_index->JudyL_array, Index, PJE0);
     if (likely(NULL != PValue)) {
         descr = *PValue;
@@ -818,15 +819,15 @@ struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_
  * Does not get a reference.
  * @param ctx DB context
  * @param id UUID
- * @param start_time inclusive starting time in usec
- * @param end_time inclusive ending time in usec
+ * @param start_time_ut inclusive starting time in usec
+ * @param end_time_ut inclusive ending time in usec
  * @param page_info_arrayp It allocates (*page_arrayp) and populates it with information of pages that overlap
  *        with the time range [start_time,end_time]. The caller must free (*page_info_arrayp) with freez().
  *        If page_info_arrayp is set to NULL nothing was allocated.
  * @param ret_page_indexp Sets the page index pointer (*ret_page_indexp) for the given UUID.
  * @return the number of pages that overlap with the time range [start_time,end_time].
  */
-unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time, usec_t end_time,
+unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time_ut, usec_t end_time_ut,
                           struct rrdeng_page_info **page_info_arrayp, struct pg_cache_page_index **ret_page_indexp)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
@@ -854,14 +855,14 @@ unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t sta
     }
 
     uv_rwlock_rdlock(&page_index->lock);
-    descr = find_first_page_in_time_range(page_index, start_time, end_time);
+    descr = find_first_page_in_time_range(page_index, start_time_ut, end_time_ut);
     if (NULL == descr) {
         uv_rwlock_rdunlock(&page_index->lock);
         debug(D_RRDENGINE, "%s: No page was found to attempt preload.", __func__);
         *ret_page_indexp = NULL;
         return 0;
     } else {
-        Index = (Word_t)(descr->start_time / USEC_PER_SEC);
+        Index = (Word_t)(descr->start_time_ut / USEC_PER_SEC);
     }
     if (page_info_arrayp) {
         page_info_array_max_size = PAGE_CACHE_MAX_PRELOAD_PAGES * sizeof(struct rrdeng_page_info);
@@ -869,7 +870,7 @@ unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t sta
     }
 
     for (count = 0, preload_count = 0 ;
-         descr != NULL && is_page_in_time_range(descr, start_time, end_time) ;
+         descr != NULL && is_page_in_time_range(descr, start_time_ut, end_time_ut) ;
          PValue = JudyLNext(page_index->JudyL_array, &Index, PJE0),
          descr = unlikely(NULL == PValue) ? NULL : *PValue) {
         /* Iterate all pages in range */
@@ -881,8 +882,8 @@ unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t sta
                 page_info_array_max_size += PAGE_CACHE_MAX_PRELOAD_PAGES * sizeof(struct rrdeng_page_info);
                 *page_info_arrayp = reallocz(*page_info_arrayp, page_info_array_max_size);
             }
-            (*page_info_arrayp)[count].start_time = descr->start_time;
-            (*page_info_arrayp)[count].end_time = descr->end_time;
+            (*page_info_arrayp)[count].start_time_ut = descr->start_time_ut;
+            (*page_info_arrayp)[count].end_time_ut = descr->end_time_ut;
             (*page_info_arrayp)[count].page_length = descr->page_length;
         }
         ++count;
@@ -974,7 +975,7 @@ unsigned pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t sta
  */
 struct rrdeng_page_descr *
         pg_cache_lookup(struct rrdengine_instance *ctx, struct pg_cache_page_index *index, uuid_t *id,
-                        usec_t point_in_time)
+                        usec_t point_in_time_ut)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
     struct rrdeng_page_descr *descr = NULL;
@@ -1003,15 +1004,15 @@ struct rrdeng_page_descr *
     page_not_in_cache = 0;
     uv_rwlock_rdlock(&page_index->lock);
     while (1) {
-        Index = (Word_t)(point_in_time / USEC_PER_SEC);
+        Index = (Word_t)(point_in_time_ut / USEC_PER_SEC);
         PValue = JudyLLast(page_index->JudyL_array, &Index, PJE0);
         if (likely(NULL != PValue)) {
             descr = *PValue;
         }
         if (NULL == PValue ||
             0 == descr->page_length ||
-            (INVALID_TIME != point_in_time &&
-             !is_point_in_time_in_page(descr, point_in_time))) {
+            (INVALID_TIME != point_in_time_ut &&
+             !is_point_in_time_in_page(descr, point_in_time_ut))) {
             /* non-empty page not found */
             uv_rwlock_rdunlock(&page_index->lock);
 
@@ -1038,7 +1039,7 @@ struct rrdeng_page_descr *
 
             debug(D_RRDENGINE, "%s: Waiting for page to be asynchronously read from disk:", __func__);
             if(unlikely(debug_flags & D_RRDENGINE))
-                print_page_cache_descr(descr);
+                print_page_cache_descr(descr, "", true);
             while (!(pg_cache_descr->flags & RRD_PAGE_POPULATED)) {
                 pg_cache_wait_event_unsafe(descr);
             }
@@ -1053,7 +1054,7 @@ struct rrdeng_page_descr *
         uv_rwlock_rdunlock(&page_index->lock);
         debug(D_RRDENGINE, "%s: Waiting for page to be unlocked:", __func__);
         if(unlikely(debug_flags & D_RRDENGINE))
-            print_page_cache_descr(descr);
+            print_page_cache_descr(descr, "", true);
         if (!(flags & RRD_PAGE_POPULATED))
             page_not_in_cache = 1;
         pg_cache_wait_event_unsafe(descr);
@@ -1081,7 +1082,7 @@ struct rrdeng_page_descr *
  */
 struct rrdeng_page_descr *
 pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index *index, uuid_t *id,
-                     usec_t start_time, usec_t end_time)
+                     usec_t start_time_ut, usec_t end_time_ut)
 {
     struct page_cache *pg_cache = &ctx->pg_cache;
     struct rrdeng_page_descr *descr = NULL;
@@ -1110,7 +1111,7 @@ pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index
     uv_rwlock_rdlock(&page_index->lock);
     int retry_count = 0;
     while (1) {
-        descr = find_first_page_in_time_range(page_index, start_time, end_time);
+        descr = find_first_page_in_time_range(page_index, start_time_ut, end_time_ut);
         if (NULL == descr || 0 == descr->page_length || retry_count == default_rrdeng_page_fetch_retries) {
             /* non-empty page not found */
             if (retry_count == default_rrdeng_page_fetch_retries)
@@ -1140,7 +1141,7 @@ pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index
 
             debug(D_RRDENGINE, "%s: Waiting for page to be asynchronously read from disk:", __func__);
             if(unlikely(debug_flags & D_RRDENGINE))
-                print_page_cache_descr(descr);
+                print_page_cache_descr(descr, "", true);
             while (!(pg_cache_descr->flags & RRD_PAGE_POPULATED)) {
                 pg_cache_wait_event_unsafe(descr);
             }
@@ -1155,7 +1156,7 @@ pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index
         uv_rwlock_rdunlock(&page_index->lock);
         debug(D_RRDENGINE, "%s: Waiting for page to be unlocked:", __func__);
         if(unlikely(debug_flags & D_RRDENGINE))
-            print_page_cache_descr(descr);
+            print_page_cache_descr(descr, "", true);
         if (!(flags & RRD_PAGE_POPULATED))
             page_not_in_cache = 1;
 
@@ -1180,7 +1181,7 @@ pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index
     return descr;
 }
 
-struct pg_cache_page_index *create_page_index(uuid_t *id)
+struct pg_cache_page_index *create_page_index(uuid_t *id, struct rrdengine_instance *ctx)
 {
     struct pg_cache_page_index *page_index;
 
@@ -1188,11 +1189,15 @@ struct pg_cache_page_index *create_page_index(uuid_t *id)
     page_index->JudyL_array = (Pvoid_t) NULL;
     uuid_copy(page_index->id, *id);
     fatal_assert(0 == uv_rwlock_init(&page_index->lock));
-    page_index->oldest_time = INVALID_TIME;
-    page_index->latest_time = INVALID_TIME;
+    page_index->oldest_time_ut = INVALID_TIME;
+    page_index->latest_time_ut = INVALID_TIME;
     page_index->prev = NULL;
     page_index->page_count = 0;
+    page_index->refcount = 0;
     page_index->writers = 0;
+    page_index->ctx = ctx;
+    page_index->alignment = NULL;
+    page_index->latest_update_every_s = default_rrd_update_every;
 
     return page_index;
 }

database/engine/pagecache.h

@@ -60,18 +60,19 @@ struct rrdeng_page_descr {
     volatile unsigned long pg_cache_descr_state;
 
     /* page information */
-    usec_t start_time;
-    usec_t end_time;
-    uint32_t page_length;
+    usec_t start_time_ut;
+    usec_t end_time_ut;
+    uint32_t update_every_s:24;
     uint8_t type;
+    uint32_t page_length;
 };
 
 #define PAGE_INFO_SCRATCH_SZ (8)
 struct rrdeng_page_info {
     uint8_t scratch[PAGE_INFO_SCRATCH_SZ]; /* scratch area to be used by page-cache users */
 
-    usec_t start_time;
-    usec_t end_time;
+    usec_t start_time_ut;
+    usec_t end_time_ut;
     uint32_t page_length;
 };
 
@@ -80,6 +81,11 @@ typedef int pg_cache_page_info_filter_t(struct rrdeng_page_descr *);
 
 #define PAGE_CACHE_MAX_PRELOAD_PAGES    (256)
 
+struct pg_alignment {
+    uint32_t page_length;
+    uint32_t refcount;
+};
+
 /* maps time ranges to pages */
 struct pg_cache_page_index {
     uuid_t id;
@@ -89,6 +95,7 @@ struct pg_cache_page_index {
      */
     Pvoid_t JudyL_array;
     Word_t page_count;
+    unsigned short refcount;
     unsigned short writers;
     uv_rwlock_t lock;
 
@@ -96,13 +103,17 @@ struct pg_cache_page_index {
      * Only one effective writer, data deletion workqueue.
      * It's also written during the DB loading phase.
      */
-    usec_t oldest_time;
+    usec_t oldest_time_ut;
 
     /*
      * Only one effective writer, data collection thread.
      * It's also written by the data deletion workqueue when data collection is disabled for this metric.
      */
-    usec_t latest_time;
+    usec_t latest_time_ut;
+
+    struct rrdengine_instance *ctx;
+    struct pg_alignment *alignment;
+    uint32_t latest_update_every_s;
 
     struct pg_cache_page_index *prev;
 };
@@ -171,22 +182,22 @@ void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index
 uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr,
                                    uint8_t remove_dirty, uint8_t is_exclusive_holder, uuid_t *metric_id);
 usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id,
-                                            usec_t start_time, usec_t end_time);
+                                            usec_t start_time_ut, usec_t end_time_ut);
 void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_cache_page_index *page_index,
-                                            usec_t point_in_time, pg_cache_page_info_filter_t *filter,
+                                            usec_t point_in_time_ut, pg_cache_page_info_filter_t *filter,
                                             struct rrdeng_page_info *page_info);
 struct rrdeng_page_descr *pg_cache_lookup_unpopulated_and_lock(struct rrdengine_instance *ctx, uuid_t *id,
-                                                                      usec_t start_time);
-extern unsigned
-        pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time, usec_t end_time,
+                                                                      usec_t start_time_ut);
+unsigned
+        pg_cache_preload(struct rrdengine_instance *ctx, uuid_t *id, usec_t start_time_ut, usec_t end_time_ut,
                          struct rrdeng_page_info **page_info_arrayp, struct pg_cache_page_index **ret_page_indexp);
-extern struct rrdeng_page_descr *
+struct rrdeng_page_descr *
         pg_cache_lookup(struct rrdengine_instance *ctx, struct pg_cache_page_index *index, uuid_t *id,
-                        usec_t point_in_time);
-extern struct rrdeng_page_descr *
+                        usec_t point_in_time_ut);
+struct rrdeng_page_descr *
         pg_cache_lookup_next(struct rrdengine_instance *ctx, struct pg_cache_page_index *index, uuid_t *id,
-                     usec_t start_time, usec_t end_time);
-struct pg_cache_page_index *create_page_index(uuid_t *id);
+                     usec_t start_time_ut, usec_t end_time_ut);
+struct pg_cache_page_index *create_page_index(uuid_t *id, struct rrdengine_instance *ctx);
 void init_page_cache(struct rrdengine_instance *ctx);
 void free_page_cache(struct rrdengine_instance *ctx);
 void pg_cache_add_new_metric_time(struct pg_cache_page_index *page_index, struct rrdeng_page_descr *descr);
@@ -204,41 +215,41 @@ struct rrdeng_page_descr *rrdeng_page_descr_mallocz(void);
 void rrdeng_page_descr_freez(struct rrdeng_page_descr *descr);
 
 static inline void
-    pg_cache_atomic_get_pg_info(struct rrdeng_page_descr *descr, usec_t *end_timep, uint32_t *page_lengthp)
+    pg_cache_atomic_get_pg_info(struct rrdeng_page_descr *descr, usec_t *end_time_ut_p, uint32_t *page_lengthp)
 {
-    usec_t end_time, old_end_time;
+    usec_t end_time_ut, old_end_time_ut;
     uint32_t page_length;
 
     if (NULL == descr->extent) {
         /* this page is currently being modified, get consistent info locklessly */
         do {
-            end_time = descr->end_time;
+            end_time_ut = descr->end_time_ut;
             __sync_synchronize();
-            old_end_time = end_time;
+            old_end_time_ut = end_time_ut;
             page_length = descr->page_length;
             __sync_synchronize();
-            end_time = descr->end_time;
+            end_time_ut = descr->end_time_ut;
             __sync_synchronize();
-        } while ((end_time != old_end_time || (end_time & 1) != 0));
+        } while ((end_time_ut != old_end_time_ut || (end_time_ut & 1) != 0));
 
-        *end_timep = end_time;
+        *end_time_ut_p = end_time_ut;
         *page_lengthp = page_length;
     } else {
-        *end_timep = descr->end_time;
+        *end_time_ut_p = descr->end_time_ut;
         *page_lengthp = descr->page_length;
     }
 }
 
 /* The caller must hold a reference to the page and must have already set the new data */
-static inline void pg_cache_atomic_set_pg_info(struct rrdeng_page_descr *descr, usec_t end_time, uint32_t page_length)
+static inline void pg_cache_atomic_set_pg_info(struct rrdeng_page_descr *descr, usec_t end_time_ut, uint32_t page_length)
 {
-    fatal_assert(!(end_time & 1));
+    fatal_assert(!(end_time_ut & 1));
     __sync_synchronize();
-    descr->end_time |= 1; /* mark start of uncertainty period by adding 1 microsecond */
+    descr->end_time_ut |= 1; /* mark start of uncertainty period by adding 1 microsecond */
     __sync_synchronize();
     descr->page_length = page_length;
     __sync_synchronize();
-    descr->end_time = end_time; /* mark end of uncertainty period */
+    descr->end_time_ut = end_time_ut; /* mark end of uncertainty period */
 }
 
 #endif /* NETDATA_PAGECACHE_H */

database/engine/rrddiskprotocol.h

@@ -46,8 +46,8 @@ struct rrdeng_extent_page_descr {
 
     uint8_t uuid[UUID_SZ];
     uint32_t page_length;
-    uint64_t start_time;
-    uint64_t end_time;
+    uint64_t start_time_ut;
+    uint64_t end_time_ut;
 } __attribute__ ((packed));
 
 /*