/api/v2/X part 7 (#14797)

* /api/v2/weights, points key renamed to result

* /api/v2/weights, add node ids in response

* /api/v2/data remove NONZERO flag when all dimensions are zero and fix MIN/MAX grouping and statistics

* /api/v2/data expose view.dimensions.sts{}

* /api/v2 endpoints expose agents and additional info per node, that is needed to unify cloud responses

* /api/v2 nodes output now includes the duration of time spent per node

* jsonwrap view object renames and cleanup

* rework of the statistics returned by the query engine

* swagger work

* swagger work

* more swagger work

* updated swagger json

* added the remaining of the /api/v2 endpoints to swagger

* point.ar has been renamed point.arp

* updated weights endpoint

* fix compilation warnings

database/contexts/api_v2.c

@@ -97,6 +97,7 @@ struct rrdcontext_to_json_v2_data {
     struct {
         SIMPLE_PATTERN *scope_pattern;
         SIMPLE_PATTERN *pattern;
+        size_t ni;
     } nodes;
 
     struct {
@@ -222,6 +223,21 @@ static ssize_t rrdcontext_to_json_v2_add_context(void *data, RRDCONTEXT_ACQUIRED
     return 1;
 }
 
+void buffer_json_node_add_v2(BUFFER *wb, RRDHOST *host, size_t ni, usec_t duration_ut) {
+    buffer_json_member_add_string(wb, "mg", host->machine_guid);
+    if(host->node_id)
+        buffer_json_member_add_uuid(wb, "nd", host->node_id);
+    buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));
+    buffer_json_member_add_uint64(wb, "ni", ni);
+    buffer_json_member_add_object(wb, "st");
+    buffer_json_member_add_uint64(wb, "ai", 0);
+    buffer_json_member_add_uint64(wb, "code", 200);
+    buffer_json_member_add_string(wb, "msg", "");
+    if(duration_ut)
+        buffer_json_member_add_double(wb, "ms", (NETDATA_DOUBLE)duration_ut / 1000.0);
+    buffer_json_object_close(wb);
+}
+
 static ssize_t rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool queryable_host) {
     if(!queryable_host || !host->rrdctx.contexts)
         // the host matches the 'scope_host' but does not match the 'host' patterns
@@ -279,9 +295,7 @@ static ssize_t rrdcontext_to_json_v2_add_host(void *data, RRDHOST *host, bool qu
 
     if(host_matched && (ctl->options & (CONTEXTS_V2_NODES | CONTEXTS_V2_NODES_DETAILED | CONTEXTS_V2_DEBUG))) {
         buffer_json_add_array_item_object(wb);
-        buffer_json_member_add_string(wb, "mg", host->machine_guid);
-        buffer_json_member_add_uuid(wb, "nd", host->node_id);
-        buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));
+        buffer_json_node_add_v2(wb, host, ctl->nodes.ni++, 0);
 
         if(ctl->options & CONTEXTS_V2_NODES_DETAILED) {
             buffer_json_member_add_string(wb, "version", rrdhost_program_version(host));
@@ -372,6 +386,21 @@ static void buffer_json_contexts_v2_options_to_array(BUFFER *wb, CONTEXTS_V2_OPT
         buffer_json_add_array_item_string(wb, "search");
 }
 
+void buffer_json_agents_array_v2(BUFFER *wb, time_t now_s) {
+    if(!now_s)
+        now_s = now_realtime_sec();
+
+    buffer_json_member_add_array(wb, "agents");
+    buffer_json_add_array_item_object(wb);
+    buffer_json_member_add_string(wb, "mg", localhost->machine_guid);
+    buffer_json_member_add_uuid(wb, "nd", localhost->node_id);
+    buffer_json_member_add_string(wb, "nm", rrdhost_hostname(localhost));
+    buffer_json_member_add_time_t(wb, "now", now_s);
+    buffer_json_member_add_uint64(wb, "ai", 0);
+    buffer_json_object_close(wb);
+    buffer_json_array_close(wb);
+}
+
 int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_OPTIONS options) {
     int resp = HTTP_RESP_OK;
 
@@ -398,17 +427,10 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE
 
     time_t now_s = now_realtime_sec();
     buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+    buffer_json_member_add_uint64(wb, "api", 2);
+    buffer_json_agents_array_v2(wb, now_s);
 
     if(options & CONTEXTS_V2_DEBUG) {
-        buffer_json_member_add_object(wb, "agent");
-        buffer_json_member_add_string(wb, "mg", localhost->machine_guid);
-        buffer_json_member_add_uuid(wb, "nd", localhost->node_id);
-        buffer_json_member_add_string(wb, "nm", rrdhost_hostname(localhost));
-        if (req->q)
-            buffer_json_member_add_string(wb, "q", req->q);
-        buffer_json_member_add_time_t(wb, "now", now_s);
-        buffer_json_object_close(wb);
-
         buffer_json_member_add_object(wb, "request");
 
         buffer_json_member_add_object(wb, "scope");

database/contexts/query_target.c

@@ -977,7 +977,7 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
         qtr->scope_contexts = qtr->contexts;
 
     memset(&qt->db, 0, sizeof(qt->db));
-    memset(&qt->query_stats, 0, sizeof(qt->query_stats));
+    qt->query_points = STORAGE_POINT_UNSET;
 
     // copy the request into query_thread_target
     qt->request = *qtr;
@@ -985,6 +985,7 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
     query_target_generate_name(qt);
     qt->window.after = qt->request.after;
     qt->window.before = qt->request.before;
+    qt->window.options = qt->request.options;
     rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now);
 
     // prepare our local variables - we need these across all these functions

database/contexts/rrdcontext.h

@@ -144,55 +144,47 @@ typedef struct query_plan_entry {
 
 #define QUERY_PLANS_MAX (RRD_STORAGE_TIERS)
 
-struct query_metrics_counts {
-    size_t selected;
-    size_t excluded;
-    size_t queried;
-    size_t failed;
-};
-
-struct query_instances_counts {
-    size_t selected;
-    size_t excluded;
-    size_t queried;
-    size_t failed;
-};
-
-struct query_alerts_counts {
-    size_t clear;
-    size_t warning;
-    size_t critical;
-    size_t other;
-};
-
-struct query_data_statistics {      // time-aggregated (group points) statistics
-    size_t group_points;            // the number of group points the query generated
-    NETDATA_DOUBLE min;             // the min value of the group points
-    NETDATA_DOUBLE max;             // the max value of the group points
-    NETDATA_DOUBLE sum;             // the sum of the group points
-    NETDATA_DOUBLE volume;          // the volume of the group points
-    NETDATA_DOUBLE anomaly_sum;     // the anomaly sum of the group points
-};
+typedef struct query_metrics_counts {   // counts the number of metrics related to an object
+    size_t selected;                    // selected to be queried
+    size_t excluded;                    // not selected to be queried
+    size_t queried;                     // successfully queried
+    size_t failed;                      // failed to be queried
+} QUERY_METRICS_COUNTS;
+
+typedef struct query_instances_counts { // counts the number of instances related to an object
+    size_t selected;                    // selected to be queried
+    size_t excluded;                    // not selected to be queried
+    size_t queried;                     // successfully queried
+    size_t failed;                      // failed to be queried
+} QUERY_INSTANCES_COUNTS;
+
+typedef struct query_alerts_counts {    // counts the number of alerts related to an object
+    size_t clear;                       // number of alerts in clear state
+    size_t warning;                     // number of alerts in warning state
+    size_t critical;                    // number of alerts in critical state
+    size_t other;                       // number of alerts in any other state
+} QUERY_ALERTS_COUNTS;
 
 typedef struct query_node {
     uint32_t slot;
     RRDHOST *rrdhost;
     char node_id[UUID_STR_LEN];
+    usec_t duration_ut;
 
-    struct query_data_statistics query_stats;
-    struct query_instances_counts instances;
-    struct query_metrics_counts metrics;
-    struct query_alerts_counts alerts;
+    STORAGE_POINT query_points;
+    QUERY_INSTANCES_COUNTS instances;
+    QUERY_METRICS_COUNTS metrics;
+    QUERY_ALERTS_COUNTS alerts;
 } QUERY_NODE;
 
 typedef struct query_context {
     uint32_t slot;
     RRDCONTEXT_ACQUIRED *rca;
 
-    struct query_data_statistics query_stats;
-    struct query_instances_counts instances;
-    struct query_metrics_counts metrics;
-    struct query_alerts_counts alerts;
+    STORAGE_POINT query_points;
+    QUERY_INSTANCES_COUNTS instances;
+    QUERY_METRICS_COUNTS metrics;
+    QUERY_ALERTS_COUNTS alerts;
 } QUERY_CONTEXT;
 
 typedef struct query_instance {
@@ -202,9 +194,9 @@ typedef struct query_instance {
     STRING *id_fqdn;        // never access this directly - it is created on demand via query_instance_id_fqdn()
     STRING *name_fqdn;      // never access this directly - it is created on demand via query_instance_name_fqdn()
 
-    struct query_data_statistics query_stats;
-    struct query_metrics_counts metrics;
-    struct query_alerts_counts alerts;
+    STORAGE_POINT query_points;
+    QUERY_METRICS_COUNTS metrics;
+    QUERY_ALERTS_COUNTS alerts;
 } QUERY_INSTANCE;
 
 typedef struct query_dimension {
@@ -236,7 +228,7 @@ typedef struct query_metric {
         uint32_t query_dimension_id;
     } link;
 
-    struct query_data_statistics query_stats;
+    STORAGE_POINT query_points;
 
     struct {
         size_t slot;
@@ -245,6 +237,7 @@ typedef struct query_metric {
         STRING *units;
     } grouped_as;
 
+    usec_t duration_ut;
 } QUERY_METRIC;
 
 #define MAX_QUERY_TARGET_ID_LENGTH 255
@@ -320,6 +313,8 @@ struct query_versions {
     uint64_t alerts_soft_hash;
 };
 
+#define query_view_update_every(qt) ((qt)->window.group * (qt)->window.query_granularity)
+
 typedef struct query_target {
     char id[MAX_QUERY_TARGET_ID_LENGTH + 1]; // query identifier (for logging)
     QUERY_TARGET_REQUEST request;
@@ -334,10 +329,10 @@ typedef struct query_target {
         time_t after;                       // the absolute timestamp this query is about
         time_t before;                      // the absolute timestamp this query is about
         time_t query_granularity;
-        size_t points;                        // the number of points the query will return (maybe different from the request)
+        size_t points;                      // the number of points the query will return (maybe different from the request)
         size_t group;
-        RRDR_TIME_GROUPING group_method;
-        const char *group_options;
+        RRDR_TIME_GROUPING time_group_method;
+        const char *time_group_options;
         size_t resampling_group;
         NETDATA_DOUBLE resampling_divisor;
         RRDR_OPTIONS options;
@@ -396,7 +391,7 @@ typedef struct query_target {
         char *label_keys[GROUP_BY_MAX_LABEL_KEYS];
     } group_by;
 
-    struct query_data_statistics query_stats;
+    STORAGE_POINT query_points;
 
     struct query_versions versions;
 
@@ -404,7 +399,6 @@ typedef struct query_target {
         usec_t received_ut;
         usec_t preprocessed_ut;
         usec_t executed_ut;
-        usec_t group_by_ut;
         usec_t finished_ut;
     } timings;
 } QUERY_TARGET;
@@ -485,6 +479,8 @@ typedef enum __attribute__ ((__packed__)) {
 int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTEXTS_V2_OPTIONS options);
 
 RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o);
+void buffer_json_agents_array_v2(BUFFER *wb, time_t now_s);
+void buffer_json_node_add_v2(BUFFER *wb, RRDHOST *host, size_t ni, usec_t duration_ut);
 
 // ----------------------------------------------------------------------------
 // scope

libnetdata/libnetdata.h

@@ -377,8 +377,8 @@ typedef struct storage_point {
     time_t start_time_s;    // the time the point starts
     time_t end_time_s;      // the time the point ends
 
-    size_t count;           // the number of original points aggregated
-    size_t anomaly_count;   // the number of original points found anomalous
+    uint32_t count;         // the number of original points aggregated
+    uint32_t anomaly_count; // the number of original points found anomalous
 
     SN_FLAGS flags;         // flags stored with the point
 } STORAGE_POINT;
@@ -482,6 +482,9 @@ typedef struct storage_point {
 #define storage_point_anomaly_rate(sp) \
     (NETDATA_DOUBLE)(storage_point_is_unset(sp) ? 0.0 : (NETDATA_DOUBLE)((sp).anomaly_count) * 100.0 / (NETDATA_DOUBLE)((sp).count))
 
+#define storage_point_average_value(sp) \
+    ((sp).count ? (sp).sum / (NETDATA_DOUBLE)((sp).count) : 0.0)
+
 // ---------------------------------------------------------------------------------------------
 
 void netdata_fix_chart_id(char *s);

web/api/formatters/json/json.c

@@ -247,7 +247,7 @@ void rrdr2json(RRDR *r, BUFFER *wb, RRDR_OPTIONS options, int datatable) {
 
 void rrdr2json_v2(RRDR *r, BUFFER *wb) {
     QUERY_TARGET *qt = r->internal.qt;
-    RRDR_OPTIONS options = qt->request.options;
+    RRDR_OPTIONS options = qt->window.options;
 
     bool expose_gbc = query_target_aggregatable(qt);
 
@@ -268,7 +268,7 @@ void rrdr2json_v2(RRDR *r, BUFFER *wb) {
 
     buffer_json_member_add_object(wb, "point");
     buffer_json_member_add_uint64(wb, "value", 0);
-    buffer_json_member_add_uint64(wb, "ar", 1);
+    buffer_json_member_add_uint64(wb, "arp", 1);
     buffer_json_member_add_uint64(wb, "pa", 2);
     if(expose_gbc)
         buffer_json_member_add_uint64(wb, "count", 3);

web/api/formatters/json_wrapper.c

@@ -101,7 +101,7 @@ struct summary_total_counts {
     size_t failed;
 };
 
-static inline void aggregate_into_summary_totals(struct summary_total_counts *totals, struct query_metrics_counts *metrics) {
+static inline void aggregate_into_summary_totals(struct summary_total_counts *totals, QUERY_METRICS_COUNTS *metrics) {
     if(unlikely(!totals || !metrics))
         return;
 
@@ -139,7 +139,7 @@ static inline void query_target_total_counts(BUFFER *wb, const char *key, struct
     buffer_json_object_close(wb);
 }
 
-static inline void query_target_metric_counts(BUFFER *wb, struct query_metrics_counts *metrics) {
+static inline void query_target_metric_counts(BUFFER *wb, QUERY_METRICS_COUNTS *metrics) {
     if(!metrics->selected && !metrics->queried && !metrics->failed && !metrics->excluded)
         return;
 
@@ -160,7 +160,7 @@ static inline void query_target_metric_counts(BUFFER *wb, struct query_metrics_c
     buffer_json_object_close(wb);
 }
 
-static inline void query_target_instance_counts(BUFFER *wb, struct query_instances_counts *instances) {
+static inline void query_target_instance_counts(BUFFER *wb, QUERY_INSTANCES_COUNTS *instances) {
     if(!instances->selected && !instances->queried && !instances->failed && !instances->excluded)
         return;
 
@@ -181,7 +181,7 @@ static inline void query_target_instance_counts(BUFFER *wb, struct query_instanc
     buffer_json_object_close(wb);
 }
 
-static inline void query_target_alerts_counts(BUFFER *wb, struct query_alerts_counts *alerts, const char *name, bool array) {
+static inline void query_target_alerts_counts(BUFFER *wb, QUERY_ALERTS_COUNTS *alerts, const char *name, bool array) {
     if(!alerts->clear && !alerts->other && !alerts->critical && !alerts->warning)
         return;
 
@@ -208,40 +208,36 @@ static inline void query_target_alerts_counts(BUFFER *wb, struct query_alerts_co
     buffer_json_object_close(wb);
 }
 
-static inline void query_target_data_statistics(BUFFER *wb, QUERY_TARGET *qt, struct query_data_statistics *d) {
-    if(!d->group_points)
+static inline void query_target_points_statistics(BUFFER *wb, QUERY_TARGET *qt, STORAGE_POINT *sp) {
+    if(!sp->count)
         return;
 
     buffer_json_member_add_object(wb, "sts");
 
-    buffer_json_member_add_double(wb, "min", d->min);
-    buffer_json_member_add_double(wb, "max", d->max);
+    buffer_json_member_add_double(wb, "min", sp->min);
+    buffer_json_member_add_double(wb, "max", sp->max);
 
     if(query_target_aggregatable(qt)) {
-        buffer_json_member_add_uint64(wb, "cnt", d->group_points);
+        buffer_json_member_add_uint64(wb, "cnt", sp->count);
 
-        if(d->sum != 0.0)
-            buffer_json_member_add_double(wb, "sum", d->sum);
-
-        if(d->volume != 0.0)
-            buffer_json_member_add_double(wb, "vol", d->volume);
+        if(sp->sum != 0.0) {
+            buffer_json_member_add_double(wb, "sum", sp->sum);
+            buffer_json_member_add_double(wb, "vol", sp->sum * (NETDATA_DOUBLE) query_view_update_every(qt));
+        }
 
-        if(d->anomaly_sum != 0.0)
-            buffer_json_member_add_double(wb, "ars", d->anomaly_sum);
+        if(sp->anomaly_count != 0)
+            buffer_json_member_add_double(wb, "ars", storage_point_anomaly_rate(*sp));
     }
     else {
-//        buffer_json_member_add_double(wb, "min", d->min);
-//        buffer_json_member_add_double(wb, "max", d->max);
-
-        NETDATA_DOUBLE avg = (d->group_points) ? d->sum / (NETDATA_DOUBLE)d->group_points : 0.0;
+        NETDATA_DOUBLE avg = (sp->count) ? sp->sum / (NETDATA_DOUBLE)sp->count : 0.0;
         if(avg != 0.0)
             buffer_json_member_add_double(wb, "avg", avg);
 
-        NETDATA_DOUBLE arp = (d->group_points) ? d->anomaly_sum / (NETDATA_DOUBLE)d->group_points : 0.0;
+        NETDATA_DOUBLE arp = storage_point_anomaly_rate(*sp);
         if(arp != 0.0)
             buffer_json_member_add_double(wb, "arp", arp);
 
-        NETDATA_DOUBLE con = (qt->query_stats.volume > 0) ? d->volume * 100.0 / qt->query_stats.volume : 0.0;
+        NETDATA_DOUBLE con = (qt->query_points.sum > 0.0) ? sp->sum * 100.0 / qt->query_points.sum : 0.0;
         if(con != 0.0)
             buffer_json_member_add_double(wb, "con", con);
     }
@@ -254,15 +250,11 @@ static void query_target_summary_nodes_v2(BUFFER *wb, QUERY_TARGET *qt, const ch
         QUERY_NODE *qn = query_node(qt, c);
         RRDHOST *host = qn->rrdhost;
         buffer_json_add_array_item_object(wb);
-        buffer_json_member_add_uint64(wb, "ni", qn->slot);
-        buffer_json_member_add_string(wb, "mg", host->machine_guid);
-        if(qn->node_id[0])
-            buffer_json_member_add_string(wb, "nd", qn->node_id);
-        buffer_json_member_add_string(wb, "nm", rrdhost_hostname(host));
+        buffer_json_node_add_v2(wb, host, qn->slot, qn->duration_ut);
         query_target_instance_counts(wb, &qn->instances);
         query_target_metric_counts(wb, &qn->metrics);
         query_target_alerts_counts(wb, &qn->alerts, NULL, false);
-        query_target_data_statistics(wb, qt, &qn->query_stats);
+        query_target_points_statistics(wb, qt, &qn->query_points);
         buffer_json_object_close(wb);
 
         aggregate_into_summary_totals(totals, &qn->metrics);
@@ -275,16 +267,16 @@ static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, con
     DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
 
     struct {
-        struct query_data_statistics query_stats;
-        struct query_instances_counts instances;
-        struct query_metrics_counts metrics;
-        struct query_alerts_counts alerts;
-    } x = { 0 }, *z;
+        STORAGE_POINT query_points;
+        QUERY_INSTANCES_COUNTS instances;
+        QUERY_METRICS_COUNTS metrics;
+        QUERY_ALERTS_COUNTS alerts;
+    } *z;
 
     for (long c = 0; c < (long) qt->contexts.used; c++) {
         QUERY_CONTEXT *qc = query_context(qt, c);
 
-        z = dictionary_set(dict, rrdcontext_acquired_id(qc->rca), &x, sizeof(x));
+        z = dictionary_set(dict, rrdcontext_acquired_id(qc->rca), NULL, sizeof(*z));
 
         z->instances.selected += qc->instances.selected;
         z->instances.excluded += qc->instances.selected;
@@ -300,7 +292,7 @@ static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, con
         z->alerts.warning += qc->alerts.warning;
         z->alerts.critical += qc->alerts.critical;
 
-        query_target_merge_data_statistics(&z->query_stats, &qc->query_stats);
+        storage_point_merge_to(z->query_points, qc->query_points);
     }
 
     size_t unique_contexts = dictionary_entries(dict);
@@ -310,7 +302,7 @@ static size_t query_target_summary_contexts_v2(BUFFER *wb, QUERY_TARGET *qt, con
         query_target_instance_counts(wb, &z->instances);
         query_target_metric_counts(wb, &z->metrics);
         query_target_alerts_counts(wb, &z->alerts, NULL, false);
-        query_target_data_statistics(wb, qt, &z->query_stats);
+                query_target_points_statistics(wb, qt, &z->query_points);
         buffer_json_object_close(wb);
 
         aggregate_into_summary_totals(totals, &z->metrics);
@@ -334,8 +326,7 @@ static void query_target_summary_instances_v1(BUFFER *wb, QUERY_TARGET *qt, cons
                   rrdinstance_acquired_id(qi->ria),
                   rrdinstance_acquired_name(qi->ria));
 
-        bool existing = 0;
-        bool *set = dictionary_set(dict, name, &existing, sizeof(bool));
+        bool *set = dictionary_set(dict, name, NULL, sizeof(*set));
         if (!*set) {
             *set = true;
             buffer_json_add_array_item_array(wb);
@@ -369,7 +360,7 @@ static void query_target_summary_instances_v2(BUFFER *wb, QUERY_TARGET *qt, cons
 //            buffer_json_member_add_string(wb, "nd", qh->node_id);
         query_target_metric_counts(wb, &qi->metrics);
         query_target_alerts_counts(wb, &qi->alerts, NULL, false);
-        query_target_data_statistics(wb, qt, &qi->query_stats);
+        query_target_points_statistics(wb, qt, &qi->query_points);
         buffer_json_object_close(wb);
 
         aggregate_into_summary_totals(totals, &qi->metrics);
@@ -385,8 +376,8 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co
     struct {
         const char *id;
         const char *name;
-        struct query_data_statistics query_stats;
-        struct query_metrics_counts metrics;
+        STORAGE_POINT query_points;
+        QUERY_METRICS_COUNTS metrics;
     } *z;
     size_t q = 0;
     for (long c = 0; c < (long) qt->dimensions.used; c++) {
@@ -426,7 +417,7 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co
 
             if(qm->status & RRDR_DIMENSION_QUERIED) {
                 z->metrics.queried++;
-                query_target_merge_data_statistics(&z->query_stats, &qm->query_stats);
+                storage_point_merge_to(z->query_points, qm->query_points);
             }
         }
         else
@@ -440,7 +431,7 @@ static void query_target_summary_dimensions_v12(BUFFER *wb, QUERY_TARGET *qt, co
                         buffer_json_member_add_string(wb, "nm", z->name);
 
                     query_target_metric_counts(wb, &z->metrics);
-                    query_target_data_statistics(wb, qt, &z->query_stats);
+                    query_target_points_statistics(wb, qt, &z->query_points);
                     buffer_json_object_close(wb);
 
                     aggregate_into_summary_totals(totals, &z->metrics);
@@ -466,38 +457,34 @@ struct rrdlabels_formatting_v2 {
 struct rrdlabels_keys_dict_entry {
     const char *name;
     DICTIONARY *values;
-    struct query_data_statistics query_stats;
-    struct query_metrics_counts metrics;
+    STORAGE_POINT query_points;
+    QUERY_METRICS_COUNTS metrics;
 };
 
 struct rrdlabels_key_value_dict_entry {
     const char *key;
     const char *value;
-    struct query_data_statistics query_stats;
-    struct query_metrics_counts metrics;
+    STORAGE_POINT query_points;
+    QUERY_METRICS_COUNTS metrics;
 };
 
 static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
     struct rrdlabels_formatting_v2 *t = data;
 
-    struct rrdlabels_keys_dict_entry k = {
-            .name = name,
-            .values = NULL,
-            .metrics = (struct query_metrics_counts){ 0 },
-    }, *d = dictionary_set(t->keys, name, &k, sizeof(k));
-
-    if(!d->values)
+    struct rrdlabels_keys_dict_entry *d = dictionary_set(t->keys, name, NULL, sizeof(*d));
+    if(!d->values) {
+        d->name = name;
         d->values = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
+    }
 
     char n[RRD_ID_LENGTH_MAX * 2 + 2];
     snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, value);
 
-    struct rrdlabels_key_value_dict_entry x = {
-            .key = name,
-            .value = value,
-            .query_stats = (struct query_data_statistics) { 0 },
-            .metrics = (struct query_metrics_counts){ 0 },
-    }, *z = dictionary_set(d->values, n, &x, sizeof(x));
+    struct rrdlabels_key_value_dict_entry *z = dictionary_set(d->values, n, NULL, sizeof(*z));
+    if(!z->key) {
+        z->key = name;
+        z->value = value;
+    }
 
     if(t->v2) {
         QUERY_INSTANCE *qi = t->qi;
@@ -512,8 +499,8 @@ static int rrdlabels_formatting_v2(const char *name, const char *value, RRDLABEL
         d->metrics.queried += qi->metrics.queried;
         d->metrics.failed += qi->metrics.failed;
 
-        query_target_merge_data_statistics(&z->query_stats, &qi->query_stats);
-        query_target_merge_data_statistics(&d->query_stats, &qi->query_stats);
+        storage_point_merge_to(z->query_points, qi->query_points);
+        storage_point_merge_to(d->query_points, qi->query_points);
     }
 
     return 1;
@@ -537,7 +524,7 @@ static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const
                     buffer_json_add_array_item_object(wb);
                     buffer_json_member_add_string(wb, "id", d_dfe.name);
                     query_target_metric_counts(wb, &d->metrics);
-                    query_target_data_statistics(wb, qt, &d->query_stats);
+                    query_target_points_statistics(wb, qt, &d->query_points);
                     aggregate_into_summary_totals(key_totals, &d->metrics);
                     buffer_json_member_add_array(wb, "vl");
                 }
@@ -547,7 +534,7 @@ static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const
                                 buffer_json_add_array_item_object(wb);
                                 buffer_json_member_add_string(wb, "id", z->value);
                                 query_target_metric_counts(wb, &z->metrics);
-                                query_target_data_statistics(wb, qt, &z->query_stats);
+                                query_target_points_statistics(wb, qt, &z->query_points);
                                 buffer_json_object_close(wb);
                                 aggregate_into_summary_totals(value_totals, &z->metrics);
                             } else {
@@ -571,7 +558,7 @@ static void query_target_summary_labels_v12(BUFFER *wb, QUERY_TARGET *qt, const
 
 static void query_target_summary_alerts_v2(BUFFER *wb, QUERY_TARGET *qt, const char *key) {
     buffer_json_member_add_array(wb, key);
-    struct query_alerts_counts x = { 0 }, *z;
+    QUERY_ALERTS_COUNTS *z;
 
     DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
     for (long c = 0; c < (long) qt->instances.used; c++) {
@@ -581,7 +568,7 @@ static void query_target_summary_alerts_v2(BUFFER *wb, QUERY_TARGET *qt, const c
             netdata_rwlock_rdlock(&st->alerts.rwlock);
             if (st->alerts.base) {
                 for (RRDCALC *rc = st->alerts.base; rc; rc = rc->next) {
-                    z = dictionary_set(dict, string2str(rc->name), &x, sizeof(x));
+                    z = dictionary_set(dict, string2str(rc->name), NULL, sizeof(*z));
 
                     switch(rc->status) {
                         case RRDCALC_STATUS_CLEAR:
@@ -720,52 +707,102 @@ static inline size_t rrdr_dimension_view_latest_values(BUFFER *wb, const char *k
     return i;
 }
 
-static inline void rrdr_dimension_view_average_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
-    if(!r->dv)
+static inline void rrdr_dimension_query_points_statistics(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options, bool dview) {
+    STORAGE_POINT *sp = (dview) ? r->dview : r->dqp;
+    NETDATA_DOUBLE anomaly_rate_multiplier = (dview) ? RRDR_DVIEW_ANOMALY_COUNT_MULTIPLIER : 1.0;
+
+    if(unlikely(!sp))
         return;
 
-    buffer_json_member_add_array(wb, key);
+    if(key)
+        buffer_json_member_add_object(wb, key);
 
+    buffer_json_member_add_array(wb, "min");
     for(size_t c = 0; c < r->d ; c++) {
-        if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+        if (!rrdr_dimension_should_be_exposed(r->od[c], options))
             continue;
 
-        buffer_json_add_array_item_double(wb, r->dv[c]);
+        buffer_json_add_array_item_double(wb, sp[c].min);
     }
+    buffer_json_array_close(wb);
+
+    buffer_json_member_add_array(wb, "max");
+    for(size_t c = 0; c < r->d ; c++) {
+        if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+            continue;
 
+        buffer_json_add_array_item_double(wb, sp[c].max);
+    }
     buffer_json_array_close(wb);
-}
 
-static inline void rrdr_dimension_view_minimum_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
-    if(!r->dmin)
-        return;
+    if(options & RRDR_OPTION_RETURN_RAW) {
+        buffer_json_member_add_array(wb, "sum");
+        for(size_t c = 0; c < r->d ; c++) {
+            if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
 
-    buffer_json_member_add_array(wb, key);
+            buffer_json_add_array_item_double(wb, sp[c].sum);
+        }
+        buffer_json_array_close(wb);
 
-    for(size_t c = 0; c < r->d ; c++) {
-        if(!rrdr_dimension_should_be_exposed(r->od[c], options))
-            continue;
+        buffer_json_member_add_array(wb, "cnt");
+        for(size_t c = 0; c < r->d ; c++) {
+            if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
+
+            buffer_json_add_array_item_uint64(wb, sp[c].count);
+        }
+        buffer_json_array_close(wb);
 
-        buffer_json_add_array_item_double(wb, r->dmin[c]);
+        buffer_json_member_add_array(wb, "ars");
+        for(size_t c = 0; c < r->d ; c++) {
+            if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
+
+            buffer_json_add_array_item_uint64(wb, sp[c].anomaly_count * 100 / anomaly_rate_multiplier);
+        }
+        buffer_json_array_close(wb);
     }
+    else {
+        NETDATA_DOUBLE sum = 0.0;
+        for(size_t c = 0; c < r->d ; c++) {
+            if(!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
 
-    buffer_json_array_close(wb);
-}
+            sum += ABS(sp[c].sum);
+        }
 
-static inline void rrdr_dimension_view_maximum_values(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTIONS options) {
-    if(!r->dmin)
-        return;
+        buffer_json_member_add_array(wb, "avg");
+        for(size_t c = 0; c < r->d ; c++) {
+            if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
 
-    buffer_json_member_add_array(wb, key);
+            buffer_json_add_array_item_double(wb, storage_point_average_value(sp[c]));
+        }
+        buffer_json_array_close(wb);
 
-    for(size_t c = 0; c < r->d ; c++) {
-        if(!rrdr_dimension_should_be_exposed(r->od[c], options))
-            continue;
+        buffer_json_member_add_array(wb, "arp");
+        for(size_t c = 0; c < r->d ; c++) {
+            if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
 
-        buffer_json_add_array_item_double(wb, r->dmax[c]);
+            buffer_json_add_array_item_double(wb, storage_point_anomaly_rate(sp[c]) / anomaly_rate_multiplier);
+        }
+        buffer_json_array_close(wb);
+
+        buffer_json_member_add_array(wb, "con");
+        for(size_t c = 0; c < r->d ; c++) {
+            if (!rrdr_dimension_should_be_exposed(r->od[c], options))
+                continue;
+
+            NETDATA_DOUBLE con = (sum > 0.0) ? ABS(sp[c].sum) * 100.0 / sum : 0.0;
+            buffer_json_add_array_item_double(wb, con);
+        }
+        buffer_json_array_close(wb);
     }
 
-    buffer_json_array_close(wb);
+    if(key)
+        buffer_json_object_close(wb);
 }
 
 static void rrdr_timings_v12(BUFFER *wb, const char *key, RRDR *r) {
@@ -775,8 +812,7 @@ static void rrdr_timings_v12(BUFFER *wb, const char *key, RRDR *r) {
     buffer_json_member_add_object(wb, key);
     buffer_json_member_add_double(wb, "prep_ms", (NETDATA_DOUBLE)(qt->timings.preprocessed_ut - qt->timings.received_ut) / USEC_PER_MS);
     buffer_json_member_add_double(wb, "query_ms", (NETDATA_DOUBLE)(qt->timings.executed_ut - qt->timings.preprocessed_ut) / USEC_PER_MS);
-    buffer_json_member_add_double(wb, "group_by_ms", (NETDATA_DOUBLE)(qt->timings.group_by_ut - qt->timings.executed_ut) / USEC_PER_MS);
-    buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(qt->timings.finished_ut - qt->timings.group_by_ut) / USEC_PER_MS);
+    buffer_json_member_add_double(wb, "output_ms", (NETDATA_DOUBLE)(qt->timings.finished_ut - qt->timings.executed_ut) / USEC_PER_MS);
     buffer_json_member_add_double(wb, "total_ms", (NETDATA_DOUBLE)(qt->timings.finished_ut - qt->timings.received_ut) / USEC_PER_MS);
     buffer_json_object_close(wb);
 }
@@ -784,7 +820,7 @@ static void rrdr_timings_v12(BUFFER *wb, const char *key, RRDR *r) {
 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
     QUERY_TARGET *qt = r->internal.qt;
     DATASOURCE_FORMAT format = qt->request.format;
-    RRDR_OPTIONS options = qt->request.options;
+    RRDR_OPTIONS options = qt->window.options;
 
     long rows = rrdr_rows(r);
 
@@ -812,7 +848,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
     buffer_json_member_add_time_t(wb, "after", r->view.after);
     buffer_json_member_add_time_t(wb, "before", r->view.before);
     buffer_json_member_add_string(wb, "group", time_grouping_tostring(qt->request.time_group_method));
-    web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", r->view.options);
+    web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", options);
 
     if(!rrdr_dimension_names(wb, "dimension_names", r, options))
         rows = 0;
@@ -820,7 +856,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb) {
     if(!rrdr_dimension_ids(wb, "dimension_ids", r, options))
         rows = 0;
 
-    if (r->view.options & RRDR_OPTION_ALL_DIMENSIONS) {
+    if (options & RRDR_OPTION_ALL_DIMENSIONS) {
         query_target_summary_instances_v1(wb, qt, "full_chart_list");
         query_target_summary_dimensions_v12(wb, qt, "full_dimension_list", false, NULL);
         query_target_summary_labels_v12(wb, qt, "full_chart_labels", false, NULL, NULL);
@@ -1006,8 +1042,7 @@ static void query_target_title(BUFFER *wb, QUERY_TARGET *qt, size_t contexts) {
 
         size_t added = 0;
         for(size_t c = 0; c < qt->contexts.used ;c++) {
-            bool old = false;
-            bool *set = dictionary_set(dict, rrdcontext_acquired_id(qt->contexts.array[c].rca), &old, sizeof(old));
+            bool *set = dictionary_set(dict, rrdcontext_acquired_id(qt->contexts.array[c].rca), NULL, sizeof(*set));
             if(!*set) {
                 *set = true;
                 if(added)
@@ -1148,7 +1183,7 @@ static void query_target_detailed_objects_tree(BUFFER *wb, RRDR *r, RRDR_OPTIONS
                                 buffer_json_member_add_string(wb, "as", string2str(qm->grouped_as.name));
                             }
 
-                            query_target_data_statistics(wb, qt, &qm->query_stats);
+                            query_target_points_statistics(wb, qt, &qm->query_points);
 
                             if(options & RRDR_OPTION_DEBUG)
                                 jsonwrap_query_metric_plan(wb, qm);
@@ -1190,7 +1225,7 @@ void version_hashes_api_v2(BUFFER *wb, struct query_versions *versions) {
 
 void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
     QUERY_TARGET *qt = r->internal.qt;
-    RRDR_OPTIONS options = qt->request.options;
+    RRDR_OPTIONS options = qt->window.options;
 
     char kq[2] = "\"",                    // key quote
          sq[2] = "\"";                    // string quote
@@ -1201,8 +1236,8 @@ void rrdr_json_wrapper_begin2(RRDR *r, BUFFER *wb) {
     }
 
     buffer_json_initialize(wb, kq, sq, 0, true, options & RRDR_OPTION_MINIFY);
-
     buffer_json_member_add_uint64(wb, "api", 2);
+    buffer_json_agents_array_v2(wb, 0);
 
     if(options & RRDR_OPTION_DEBUG) {
         buffer_json_member_add_string(wb, "id", qt->id);
@@ -1416,25 +1451,32 @@ void rrdr_json_wrapper_end(RRDR *r, BUFFER *wb) {
 void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {
     QUERY_TARGET *qt = r->internal.qt;
     DATASOURCE_FORMAT format = qt->request.format;
-    RRDR_OPTIONS options = qt->request.options;
+    RRDR_OPTIONS options = qt->window.options;
 
     buffer_json_member_add_object(wb, "view");
     {
         query_target_title(wb, qt, r->internal.contexts);
-        buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format));
-        web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", r->view.options);
-        buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method));
         buffer_json_member_add_time_t(wb, "update_every", r->view.update_every);
         buffer_json_member_add_time_t(wb, "after", r->view.after);
         buffer_json_member_add_time_t(wb, "before", r->view.before);
 
-        buffer_json_member_add_object(wb, "partial_data_trimming");
-        buffer_json_member_add_time_t(wb, "max_update_every", r->partial_data_trimming.max_update_every);
-        buffer_json_member_add_time_t(wb, "expected_after", r->partial_data_trimming.expected_after);
-        buffer_json_member_add_time_t(wb, "trimmed_after", r->partial_data_trimming.trimmed_after);
-        buffer_json_object_close(wb);
+        if(options & RRDR_OPTION_DEBUG) {
+            buffer_json_member_add_string(wb, "format", rrdr_format_to_string(format));
+            web_client_api_request_v1_data_options_to_buffer_json_array(wb, "options", options);
+            buffer_json_member_add_string(wb, "time_group", time_grouping_tostring(qt->request.time_group_method));
+        }
+
+        if(options & RRDR_OPTION_DEBUG) {
+            buffer_json_member_add_object(wb, "partial_data_trimming");
+            buffer_json_member_add_time_t(wb, "max_update_every", r->partial_data_trimming.max_update_every);
+            buffer_json_member_add_time_t(wb, "expected_after", r->partial_data_trimming.expected_after);
+            buffer_json_member_add_time_t(wb, "trimmed_after", r->partial_data_trimming.trimmed_after);
+            buffer_json_object_close(wb);
+        }
+
+        if(options & RRDR_OPTION_RETURN_RAW)
+            buffer_json_member_add_uint64(wb, "points", rrdr_rows(r));
 
-        buffer_json_member_add_uint64(wb, "points", rrdr_rows(r));
         query_target_combined_units_v2(wb, qt, r->internal.contexts);
         query_target_combined_chart_type(wb, qt, r->internal.contexts);
         buffer_json_member_add_object(wb, "dimensions");
@@ -1445,11 +1487,8 @@ void rrdr_json_wrapper_end2(RRDR *r, BUFFER *wb) {
             rrdr_dimension_units_array_v2(wb, "units", r, options);
             rrdr_dimension_priority_array_v2(wb, "priorities", r, options);
             rrdr_dimension_aggregated_array_v2(wb, "aggregated", r, options);
-            rrdr_dimension_view_minimum_values(wb, "view_minimum_values", r, options);
-            rrdr_dimension_view_maximum_values(wb, "view_maximum_values", r, options);
-            rrdr_dimension_view_average_values(wb, "view_average_values", r, options);
-            size_t dims = rrdr_dimension_view_latest_values(wb, "view_latest_values", r, options);
-            buffer_json_member_add_uint64(wb, "count", dims);
+            rrdr_dimension_query_points_statistics(wb, NULL, r, options, true);
+            rrdr_dimension_query_points_statistics(wb, "sts", r, options, false);
             rrdr_json_group_by_labels(wb, "labels", r, options);
         }
         buffer_json_object_close(wb); // dimensions

web/api/formatters/rrd2json.c

@@ -4,8 +4,8 @@
 #include "database/storage_engine.h"
 
 inline bool query_target_has_percentage_units(struct query_target *qt) {
-    if(qt->request.options & RRDR_OPTION_PERCENTAGE ||
-       qt->request.time_group_method == RRDR_GROUPING_CV)
+    if(qt->window.options & RRDR_OPTION_PERCENTAGE ||
+       qt->window.time_group_method == RRDR_GROUPING_CV)
         return true;
 
     return false;
@@ -170,7 +170,6 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
     }
 
     RRDR *r = rrd2rrdr(owa, qt);
-    qt->timings.executed_ut = now_monotonic_usec();
 
     if(!r) {
         buffer_strcat(wb, "Cannot generate output with these parameters on this chart.");
@@ -191,9 +190,7 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
         *latest_timestamp = r->view.before;
 
     DATASOURCE_FORMAT format = qt->request.format;
-    RRDR_OPTIONS options = qt->request.options;
-
-    qt->timings.group_by_ut = now_monotonic_usec();
+    RRDR_OPTIONS options = qt->window.options;
 
     switch(format) {
     case DATASOURCE_SSV:

web/api/formatters/rrd2json.h

@@ -63,7 +63,7 @@ void rrdr_json_group_by_labels(BUFFER *wb, const char *key, RRDR *r, RRDR_OPTION
 
 struct query_target;
 bool query_target_has_percentage_units(struct query_target *qt);
-#define query_target_aggregatable(qt) (qt->request.options & RRDR_OPTION_RETURN_RAW)
+#define query_target_aggregatable(qt) ((qt)->window.options & RRDR_OPTION_RETURN_RAW)
 
 int rrdset2value_api_v1(
         RRDSET *st

web/api/formatters/value/value.c

@@ -107,7 +107,8 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,
                         .max = NAN,
                         .sum = NAN,
                         .anomaly_count = 0,
-                }
+                },
+                .duration_ut = (r) ? r->internal.qt->timings.executed_ut - r->internal.qt->timings.received_ut : 0,
         };
     }
     else {
@@ -118,11 +119,16 @@ QUERY_VALUE rrdmetric2value(RRDHOST *host,
                 .result_points = r->stats.result_points_generated,
                 .sp = {
                         .count = 0,
-                }
+                },
+                .duration_ut = r->internal.qt->timings.executed_ut - r->internal.qt->timings.received_ut,
         };
 
-        for(size_t d = 0; d < r->d ;d++)
-            storage_point_merge_to(qv.sp, r->drs[d]);
+        for(size_t d = 0; d < r->internal.qt->query.used ;d++) {
+            if(!rrdr_dimension_should_be_exposed(r->internal.qt->query.array[d].status, options))
+                continue;
+
+            storage_point_merge_to(qv.sp, r->internal.qt->query.array[d].query_points);
+        }
 
         for(size_t t = 0; t < storage_tiers ;t++)
             qv.storage_points_per_tier[t] = r->internal.qt->db.tiers[t].points;

web/api/formatters/value/value.h

@@ -14,6 +14,7 @@ typedef struct storage_value {
     size_t storage_points_per_tier[RRD_STORAGE_TIERS];
     size_t result_points;
     STORAGE_POINT sp;
+    usec_t duration_ut;
 } QUERY_VALUE;
 
 struct rrdmetric_acquired;