Improved the data query when using the context parameter (#9978)

database/engine/rrdengineapi.c

@@ -376,7 +376,7 @@ static inline uint32_t *pginfo_to_points(struct rrdeng_page_info *page_info)
  * @return number of regions with different data collection intervals.
  */
 unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
-                                         struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, RRDDIM *temp_rd)
+                                         struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list)
 {
     struct pg_cache_page_index *page_index;
     struct rrdengine_instance *ctx;
@@ -396,6 +396,7 @@ unsigned rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t e
     *region_info_arrayp = NULL;
     page_info_array = NULL;
 
+    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
     rrdset_rdlock(st);
     for(rd_iter = temp_rd?temp_rd:st->dimensions, rd = NULL, min_time = (usec_t)-1 ; rd_iter ; rd_iter = rd_iter->next) {
         /*

database/engine/rrdengineapi.h

@@ -43,7 +43,7 @@ extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_n
 extern int rrdeng_store_metric_finalize(RRDDIM *rd);
 extern unsigned
     rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
-                                    struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, RRDDIM *temp_rd);
+                                    struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp, struct context_param *context_param_list);
 extern void rrdeng_load_metric_init(RRDDIM *rd, struct rrddim_query_handle *rrdimm_handle,
                                     time_t start_time, time_t end_time);
 extern storage_number rrdeng_load_metric_next(struct rrddim_query_handle *rrdimm_handle, time_t *current_time);

database/rrd.h

@@ -13,6 +13,7 @@ typedef struct rrddimvar RRDDIMVAR;
 typedef struct rrdcalc RRDCALC;
 typedef struct rrdcalctemplate RRDCALCTEMPLATE;
 typedef struct alarm_entry ALARM_ENTRY;
+typedef struct context_param CONTEXT_PARAM;
 
 // forward declarations
 struct rrddim_volatile;

web/api/formatters/rrd2json.c

@@ -22,6 +22,57 @@ static inline void free_temp_rrddim(RRDDIM *temp_rd)
     }
 }
 
+void free_context_param_list(struct context_param **param_list)
+{
+    if (unlikely(!param_list || !*param_list))
+        return;
+
+    free_temp_rrddim(((*param_list)->rd));
+    freez((*param_list));
+    *param_list = NULL;
+}
+
+void build_context_param_list(struct context_param **param_list, RRDSET *st)
+{
+    if (unlikely(!param_list || !st))
+        return;
+
+    if (unlikely(!(*param_list))) {
+        *param_list = mallocz(sizeof(struct context_param));
+        (*param_list)->first_entry_t = LONG_MAX;
+        (*param_list)->last_entry_t = 0;
+        (*param_list)->rd = NULL;
+    }
+
+    RRDDIM *rd1;
+    rrdset_rdlock(st);
+
+    st->last_accessed_time = now_realtime_sec();
+    (*param_list)->first_entry_t = MIN((*param_list)->first_entry_t, rrdset_first_entry_t(st));
+    (*param_list)->last_entry_t  = MAX((*param_list)->last_entry_t, rrdset_last_entry_t(st));
+
+    rrddim_foreach_read(rd1, st) {
+        RRDDIM *rd = mallocz(rd1->memsize);
+        memcpy(rd, rd1, rd1->memsize);
+        rd->id = strdupz(rd1->id);
+        rd->name = strdupz(rd1->name);
+        rd->state = mallocz(sizeof(*rd->state));
+        memcpy(rd->state, rd1->state, sizeof(*rd->state));
+        memcpy(&rd->state->collect_ops, &rd1->state->collect_ops, sizeof(struct rrddim_collect_ops));
+        memcpy(&rd->state->query_ops, &rd1->state->query_ops, sizeof(struct rrddim_query_ops));
+#ifdef ENABLE_DBENGINE
+        if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
+            rd->state->metric_uuid = mallocz(sizeof(uuid_t));
+            uuid_copy(*rd->state->metric_uuid, *rd1->state->metric_uuid);
+        }
+#endif
+        rd->next = (*param_list)->rd;
+        (*param_list)->rd = rd;
+    }
+
+    rrdset_unlock(st);
+}
+
 void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb) {
     rrdset2json(st, wb, NULL, NULL, 0);
 }
@@ -89,9 +140,8 @@ int rrdset2value_api_v1(
         , time_t *db_before
         , int *value_is_null
 ) {
-    RRDDIM *temp_rd = NULL;
 
-    RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, temp_rd);
+    RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions, NULL);
 
     if(!r) {
         if(value_is_null) *value_is_null = 1;
@@ -121,8 +171,6 @@ int rrdset2value_api_v1(
     long i = (!(options & RRDR_OPTION_REVERSED))?rrdr_rows(r) - 1:0;
     *n = rrdr2value(r, i, options, value_is_null);
 
-    free_temp_rrddim(temp_rd);
-
     rrdr_free(r);
     return HTTP_RESP_OK;
 }
@@ -139,47 +187,14 @@ int rrdset2anything_api_v1(
         , long group_time
         , uint32_t options
         , time_t *latest_timestamp
-        , char *context
+        , struct context_param *context_param_list
 ) {
     time_t last_accessed_time = now_realtime_sec();
     st->last_accessed_time = last_accessed_time;
 
-    RRDDIM *temp_rd = NULL;
-
-    if (context) {
-        rrdhost_rdlock(st->rrdhost);
-        RRDSET *st1;
-        rrdset_foreach_read(st1, st->rrdhost) {
-            if (strcmp(st1->context, context) == 0) {
-                // Loop the dimensions of the chart
-                RRDDIM  *rd1;
-                rrdset_rdlock(st1);
-                st1->last_accessed_time = last_accessed_time;
-                rrddim_foreach_read(rd1, st1) {
-                    RRDDIM *rd = mallocz(rd1->memsize);
-                    memcpy(rd, rd1, rd1->memsize);
-                    rd->id = strdupz(rd1->id);
-                    rd->name = strdupz(rd1->name);
-                    rd->state = mallocz(sizeof(*rd->state));
-                    memcpy(rd->state, rd1->state, sizeof(*rd->state));
-                    memcpy(&rd->state->collect_ops, &rd1->state->collect_ops, sizeof(struct rrddim_collect_ops));
-                    memcpy(&rd->state->query_ops, &rd1->state->query_ops, sizeof(struct rrddim_query_ops));
-#ifdef ENABLE_DBENGINE
-                    if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
-                        rd->state->metric_uuid = mallocz(sizeof(uuid_t));
-                        uuid_copy(*rd->state->metric_uuid, *rd1->state->metric_uuid);
-                    }
-#endif
-                    rd->next = temp_rd;
-                    temp_rd = rd;
-                }
-                rrdset_unlock(st1);
-            }
-        }
-        rrdhost_unlock(st->rrdhost);
-    }
+    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
 
-    RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, temp_rd);
+    RRDR *r = rrd2rrdr(st, points, after, before, group_method, group_time, options, dimensions?buffer_tostring(dimensions):NULL, context_param_list);
     if(!r) {
         buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
         return HTTP_RESP_INTERNAL_SERVER_ERROR;
@@ -355,8 +370,6 @@ int rrdset2anything_api_v1(
         break;
     }
 
-    free_temp_rrddim(temp_rd);
-
     rrdr_free(r);
     return HTTP_RESP_OK;
 }

web/api/formatters/rrd2json.h

@@ -53,6 +53,12 @@
 extern void rrd_stats_api_v1_chart(RRDSET *st, BUFFER *wb);
 extern void rrdr_buffer_print_format(BUFFER *wb, uint32_t format);
 
+typedef struct context_param {
+    RRDDIM *rd;
+    time_t first_entry_t;
+    time_t last_entry_t;
+} CONTEXT_PARAM;
+
 extern int rrdset2anything_api_v1(
           RRDSET *st
         , BUFFER *wb
@@ -65,7 +71,7 @@ extern int rrdset2anything_api_v1(
         , long group_time
         , uint32_t options
         , time_t *latest_timestamp
-        , char *context
+        , struct context_param *context_param_list
 );
 
 extern int rrdset2value_api_v1(
@@ -84,4 +90,7 @@ extern int rrdset2value_api_v1(
         , int *value_is_null
 );
 
+extern void build_context_param_list(struct context_param **param_list, RRDSET *st);
+extern void free_context_param_list(struct context_param **param_list);
+
 #endif /* NETDATA_RRD2JSON_H */

web/api/queries/query.c

@@ -815,8 +815,8 @@ static RRDR *rrd2rrdr_fixedstep(
         , int update_every
         , time_t first_entry_t
         , time_t last_entry_t
-        , int absolute_period_requested,
-        RRDDIM *temp_rd
+        , int absolute_period_requested
+        , struct context_param *context_param_list
 ) {
     int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
 
@@ -824,8 +824,10 @@ static RRDR *rrd2rrdr_fixedstep(
     time_t duration = before_requested - after_requested;
     long available_points = duration / update_every;
 
+    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
+
     if(duration <= 0 || available_points <= 0)
-        return rrdr_create(st, 1, temp_rd);
+        return rrdr_create(st, 1, context_param_list);
 
     // check the number of wanted points in the result
     if(unlikely(points_requested < 0)) points_requested = -points_requested;
@@ -983,7 +985,7 @@ static RRDR *rrd2rrdr_fixedstep(
     // initialize our result set
     // this also locks the chart for us
 
-    RRDR *r = rrdr_create(st, points_wanted, temp_rd);
+    RRDR *r = rrdr_create(st, points_wanted, context_param_list);
     if(unlikely(!r)) {
         #ifdef NETDATA_INTERNAL_CHECKS
         error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
@@ -1185,7 +1187,7 @@ static RRDR *rrd2rrdr_variablestep(
         , time_t last_entry_t
         , int absolute_period_requested
         , struct rrdeng_region_info *region_info_array
-        , RRDDIM *temp_rd
+        , struct context_param *context_param_list
 ) {
     int aligned = !(options & RRDR_OPTION_NOT_ALIGNED);
 
@@ -1193,9 +1195,11 @@ static RRDR *rrd2rrdr_variablestep(
     time_t duration = before_requested - after_requested;
     long available_points = duration / update_every;
 
+    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
+
     if(duration <= 0 || available_points <= 0) {
         freez(region_info_array);
-        return rrdr_create(st, 1, temp_rd);
+        return rrdr_create(st, 1, context_param_list);
     }
 
     // check the number of wanted points in the result
@@ -1354,7 +1358,7 @@ static RRDR *rrd2rrdr_variablestep(
     // initialize our result set
     // this also locks the chart for us
 
-    RRDR *r = rrdr_create(st, points_wanted, temp_rd);
+    RRDR *r = rrdr_create(st, points_wanted, context_param_list);
     if(unlikely(!r)) {
         #ifdef NETDATA_INTERNAL_CHECKS
         error("INTERNAL CHECK: Cannot create RRDR for %s, after=%u, before=%u, duration=%u, points=%ld", st->id, (uint32_t)after_wanted, (uint32_t)before_wanted, (uint32_t)duration, points_wanted);
@@ -1556,17 +1560,19 @@ RRDR *rrd2rrdr(
         , long resampling_time_requested
         , RRDR_OPTIONS options
         , const char *dimensions
-        , RRDDIM *temp_rd
+        , struct context_param *context_param_list
 )
 {
     int rrd_update_every;
     int absolute_period_requested;
 
+//    RRDDIM *temp_rd = context_param_list ? context_param_list->rd : NULL;
+
     time_t first_entry_t;
     time_t last_entry_t;
-    if (temp_rd) {
-        first_entry_t = rrddim_first_entry_t(temp_rd);
-        last_entry_t = rrddim_last_entry_t(temp_rd);
+    if (context_param_list) {
+        first_entry_t = context_param_list->first_entry_t;
+        last_entry_t = context_param_list->last_entry_t;
     } else {
         first_entry_t = rrdset_first_entry_t(st);
         last_entry_t = rrdset_last_entry_t(st);
@@ -1583,7 +1589,7 @@ RRDR *rrd2rrdr(
 
         /* This call takes the chart read-lock */
         regions = rrdeng_variable_step_boundaries(st, after_requested, before_requested,
-                                                  &region_info_array, &max_interval, temp_rd);
+                                                  &region_info_array, &max_interval, context_param_list);
         if (1 == regions) {
             if (region_info_array) {
                 if (rrd_update_every != region_info_array[0].update_every) {
@@ -1597,7 +1603,7 @@ RRDR *rrd2rrdr(
             }
             return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method,
                                       resampling_time_requested, options, dimensions, rrd_update_every,
-                                      first_entry_t, last_entry_t, absolute_period_requested, temp_rd);
+                                      first_entry_t, last_entry_t, absolute_period_requested, context_param_list);
         } else {
             if (rrd_update_every != (uint16_t)max_interval) {
                 rrd_update_every = (uint16_t) max_interval;
@@ -1608,11 +1614,11 @@ RRDR *rrd2rrdr(
             }
             return rrd2rrdr_variablestep(st, points_requested, after_requested, before_requested, group_method,
                                          resampling_time_requested, options, dimensions, rrd_update_every,
-                                         first_entry_t, last_entry_t, absolute_period_requested, region_info_array, temp_rd);
+                                         first_entry_t, last_entry_t, absolute_period_requested, region_info_array, context_param_list);
         }
     }
 #endif
     return rrd2rrdr_fixedstep(st, points_requested, after_requested, before_requested, group_method,
                               resampling_time_requested, options, dimensions,
-                              rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, temp_rd);
+                              rrd_update_every, first_entry_t, last_entry_t, absolute_period_requested, context_param_list);
 }

web/api/queries/rrdr.c

@@ -98,7 +98,7 @@ inline void rrdr_free(RRDR *r)
     freez(r);
 }
 
-RRDR *rrdr_create(struct rrdset *st, long n, struct rrddim *temp_rd)
+RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param_list)
 {
     if(unlikely(!st)) {
         error("NULL value given!");
@@ -110,6 +110,7 @@ RRDR *rrdr_create(struct rrdset *st, long n, struct rrddim *temp_rd)
 
     rrdr_lock_rrdset(r);
 
+    RRDDIM *temp_rd =  context_param_list ? context_param_list->rd : NULL;
     RRDDIM *rd;
     if (temp_rd) {
         RRDDIM *t = temp_rd;

web/api/queries/rrdr.h

@@ -99,12 +99,15 @@ typedef struct rrdresult {
 
 #include "../../../database/rrd.h"
 extern void rrdr_free(RRDR *r);
-extern RRDR *rrdr_create(struct rrdset *st, long n, struct rrddim *id);
+extern RRDR *rrdr_create(struct rrdset *st, long n, struct context_param *context_param_list);
 
 #include "../web_api_v1.h"
 #include "web/api/queries/query.h"
 
-extern RRDR *rrd2rrdr(RRDSET *st, long points_requested, long long after_requested, long long before_requested, RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions, RRDDIM *temp_rd);
+extern RRDR *rrd2rrdr(
+    RRDSET *st, long points_requested, long long after_requested, long long before_requested,
+    RRDR_GROUPING group_method, long resampling_time_requested, RRDR_OPTIONS options, const char *dimensions,
+    struct context_param *context_param_list);
 
 #include "query.h"
 

web/api/web_api_v1.c

@@ -491,36 +491,39 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
         goto cleanup;
     }
 
+    struct context_param  *context_param_list = NULL;
     if (context) {
-        st = NULL;
-        // TODO: Scan all charts of host
-        rrdhost_rdlock(localhost);
         RRDSET *st1;
+        uint32_t context_hash = simple_hash(context);
+        rrdhost_rdlock(localhost);
         rrdset_foreach_read(st1, localhost) {
-            if (strcmp(st1->context, context) == 0) {
-                st = st1;
-                break;
-            }
+            if (st1->hash_context == context_hash && !strcmp(st1->context, context))
+                build_context_param_list(&context_param_list, st1);
         }
         rrdhost_unlock(localhost);
+        if (likely(context_param_list && context_param_list->rd))  // Just set the first one
+            st = context_param_list->rd->rrdset;
     }
-
-    if (!st) {
-        if (!chart || !*chart) {
-            buffer_sprintf(w->response.data, "No chart id is given at the request.");
-            goto cleanup;
-        }
+    else {
         st = rrdset_find(host, chart);
         if (!st)
             st = rrdset_find_byname(host, chart);
-        if (!st) {
+        if (likely(st))
+            st->last_accessed_time = now_realtime_sec();
+    }
+
+    if (!st && !context_param_list) {
+        if (context) {
+            buffer_strcat(w->response.data, "Context is not found: ");
+            buffer_strcat_htmlescape(w->response.data, context);
+        }
+        else {
             buffer_strcat(w->response.data, "Chart is not found: ");
             buffer_strcat_htmlescape(w->response.data, chart);
-            ret = HTTP_RESP_NOT_FOUND;
-            goto cleanup;
         }
+        ret = HTTP_RESP_NOT_FOUND;
+        goto cleanup;
     }
-    st->last_accessed_time = now_realtime_sec();
 
     long long before = (before_str && *before_str)?str2l(before_str):0;
     long long after  = (after_str  && *after_str) ?str2l(after_str):-600;
@@ -565,7 +568,9 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
     }
 
     ret = rrdset2anything_api_v1(st, w->response.data, dimensions, format, points, after, before, group, group_time
-                                 , options, &last_timestamp_in_data, context);
+                                 , options, &last_timestamp_in_data, context_param_list);
+
+    free_context_param_list(&context_param_list);
 
     if(format == DATASOURCE_DATATABLE_JSONP) {
         if(google_timestamp < last_timestamp_in_data)