/api/v2/contexts (#14592)

* preparation for /api/v2/contexts

* working /api/v2/contexts

* add anomaly rate information in all statistics; when sum-count is requested, return sums and counts instead of averages

* minor fix

* query targegt now accurately counts hosts, contexts, instances, dimensions, metrics

* cleanup /api/v2/contexts

* full text search with /api/v2/contexts

* simple patterns now support the option to search ignoring case

* full text search API with /api/v2/q

* simple pattern execution optimization

* do not show q when not given

* full text search accounting

* separated /api/v2/nodes from /api/v2/contexts

* fix ssv queries for group_by

* count query instances queried and failed per context and host

* split rrdcontext.c to multiple files

* add query totals

* fix anomaly rate calculation; provide "ni" for indexing hosts

* do not generate zero valued members

* faster calculation of anomaly rate; by just summing integers for each db points and doing math once for every generated point

* fix typo when printing dimensions totals

* added option minify to remove spaces and newlines fron JSON output

* send instance ids and names when they differ

* do not add in query target dimensions, instances, contexts and hosts for which there is no retention in the current timeframe

* fix for the previous + renames and code cleanup

* when a dimension is filtered, include in the response all the other dimensions that are selectable

* do not add nodes that do not have retention in the current window

* move selection of dimensions to query_dimension_add(), instead of query_metric_add()

* increase the pre-processing capacity of queries

* generate instance fqdn ids and names only when they are needed

* provide detailed statistics about tiers retention, queries, points, update_every

* late allocation of query dimensions

* cleanup

* more cleanup

* support for annotations per displayed point, RESET and PARTIAL

* new type annotations

* if a chart is not linked to contexts and it is collected, link it when it is collected

* make ML run reentrant

* make ML rrdr query synchronous

* optimize replication memory allocation of replication_sort_entry

* change units to percentage, when requesting a coefficinet of variation, or a percentage query

* initialize replication before starting main threads

* properly decrement no room requests counter

* propagate the non-zero flag to group-by

* the same by avoiding the extra loop

* respect non-zero in all dimension arrays

* remove dictionary garbage collection from dictionary_entries() and dictionary_version()

* be more verbose when jv2 indexing is postponed

* prevent infinite loop

* use hidden dimensions even when dimensions pattern is unset

* traverse hosts using dictionaries

* fix dictionary unittests

Makefile.am

@@ -410,12 +410,21 @@ PLUGINSD_PLUGIN_FILES = \
     $(NULL)
 
 RRD_PLUGIN_FILES = \
+    database/contexts/rrdcontext.c \
+    database/contexts/rrdcontext.h \
+    database/contexts/metric.c \
+    database/contexts/instance.c \
+    database/contexts/context.c \
+    database/contexts/worker.c \
+    database/contexts/query_target.c \
+    database/contexts/query_scope.c \
+    database/contexts/api_v1.c \
+    database/contexts/api_v2.c \
+    database/contexts/internal.h \
     database/rrdcalc.c \
     database/rrdcalc.h \
     database/rrdcalctemplate.c \
     database/rrdcalctemplate.h \
-    database/rrdcontext.c \
-    database/rrdcontext.h \
     database/rrddim.c \
     database/rrddimvar.c \
     database/rrddimvar.h \

collectors/apps.plugin/apps_plugin.c

@@ -4400,7 +4400,7 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
     unsigned int io_divisor = 1024 * RATES_DETAIL;
 
     BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
-    buffer_json_initialize(wb, "\"", "\"", 0, true);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, false);
     buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
     buffer_json_member_add_string(wb, "type", "table");
     buffer_json_member_add_time_t(wb, "update_every", update_every);

collectors/cgroups.plugin/sys_fs_cgroup.c

@@ -449,70 +449,70 @@ void read_cgroup_plugin_configuration() {
             config_get("plugin:cgroups", "enable by default cgroups matching",
             // ----------------------------------------------------------------
 
-                    " !*/init.scope "                      // ignore init.scope
-                    " !/system.slice/run-*.scope "         // ignore system.slice/run-XXXX.scope
-                    " *.scope "                            // we need all other *.scope for sure
-
-            // ----------------------------------------------------------------
-
-                    " /machine.slice/*.service "           // #3367 systemd-nspawn
-
-            // ----------------------------------------------------------------
-
-                    " */kubepods/pod*/* "                   // k8s containers
-                    " */kubepods/*/pod*/* "                 // k8s containers
-                    " */*-kubepods-pod*/* "                 // k8s containers
-                    " */*-kubepods-*-pod*/* "               // k8s containers
-                    " !*kubepods* !*kubelet* "              // all other k8s cgroups
-
-            // ----------------------------------------------------------------
-
-                    " !*/vcpu* "                           // libvirtd adds these sub-cgroups
-                    " !*/emulator "                        // libvirtd adds these sub-cgroups
-                    " !*.mount "
-                    " !*.partition "
-                    " !*.service "
-                    " !*.socket "
-                    " !*.slice "
-                    " !*.swap "
-                    " !*.user "
-                    " !/ "
-                    " !/docker "
-                    " !*/libvirt "
-                    " !/lxc "
-                    " !/lxc/*/* "                          //  #1397 #2649
-                    " !/lxc.monitor* "
-                    " !/lxc.pivot "
-                    " !/lxc.payload "
-                    " !/machine "
-                    " !/qemu "
-                    " !/system "
-                    " !/systemd "
-                    " !/user "
-                    " * "                                  // enable anything else
-            ), NULL, SIMPLE_PATTERN_EXACT);
+                       " !*/init.scope "                      // ignore init.scope
+                       " !/system.slice/run-*.scope "         // ignore system.slice/run-XXXX.scope
+                       " *.scope "                            // we need all other *.scope for sure
+
+                       // ----------------------------------------------------------------
+
+                       " /machine.slice/*.service "           // #3367 systemd-nspawn
+
+                       // ----------------------------------------------------------------
+
+                       " */kubepods/pod*/* "                   // k8s containers
+                       " */kubepods/*/pod*/* "                 // k8s containers
+                       " */*-kubepods-pod*/* "                 // k8s containers
+                       " */*-kubepods-*-pod*/* "               // k8s containers
+                       " !*kubepods* !*kubelet* "              // all other k8s cgroups
+
+                       // ----------------------------------------------------------------
+
+                       " !*/vcpu* "                           // libvirtd adds these sub-cgroups
+                       " !*/emulator "                        // libvirtd adds these sub-cgroups
+                       " !*.mount "
+                       " !*.partition "
+                       " !*.service "
+                       " !*.socket "
+                       " !*.slice "
+                       " !*.swap "
+                       " !*.user "
+                       " !/ "
+                       " !/docker "
+                       " !*/libvirt "
+                       " !/lxc "
+                       " !/lxc/*/* "                          //  #1397 #2649
+                       " !/lxc.monitor* "
+                       " !/lxc.pivot "
+                       " !/lxc.payload "
+                       " !/machine "
+                       " !/qemu "
+                       " !/system "
+                       " !/systemd "
+                       " !/user "
+                       " * "                                  // enable anything else
+            ), NULL, SIMPLE_PATTERN_EXACT, true);
 
     enabled_cgroup_names = simple_pattern_create(
             config_get("plugin:cgroups", "enable by default cgroups names matching",
-                    " * "
-            ), NULL, SIMPLE_PATTERN_EXACT);
+                       " * "
+            ), NULL, SIMPLE_PATTERN_EXACT, true);
 
     search_cgroup_paths = simple_pattern_create(
             config_get("plugin:cgroups", "search for cgroups in subpaths matching",
-                    " !*/init.scope "                      // ignore init.scope
-                    " !*-qemu "                            //  #345
-                    " !*.libvirt-qemu "                    //  #3010
-                    " !/init.scope "
-                    " !/system "
-                    " !/systemd "
-                    " !/user "
-                    " !/user.slice "
-                    " !/lxc/*/* "                          //  #2161 #2649
-                    " !/lxc.monitor "
-                    " !/lxc.payload/*/* "
-                    " !/lxc.payload.* "
-                    " * "
-            ), NULL, SIMPLE_PATTERN_EXACT);
+                       " !*/init.scope "                      // ignore init.scope
+                       " !*-qemu "                            //  #345
+                       " !*.libvirt-qemu "                    //  #3010
+                       " !/init.scope "
+                       " !/system "
+                       " !/systemd "
+                       " !/user "
+                       " !/user.slice "
+                       " !/lxc/*/* "                          //  #2161 #2649
+                       " !/lxc.monitor "
+                       " !/lxc.payload/*/* "
+                       " !/lxc.payload.* "
+                       " * "
+            ), NULL, SIMPLE_PATTERN_EXACT, true);
 
     snprintfz(filename, FILENAME_MAX, "%s/cgroup-name.sh", netdata_configured_primary_plugins_dir);
     cgroups_rename_script = config_get("plugin:cgroups", "script to get cgroup names", filename);
@@ -522,37 +522,37 @@ void read_cgroup_plugin_configuration() {
 
     enabled_cgroup_renames = simple_pattern_create(
             config_get("plugin:cgroups", "run script to rename cgroups matching",
-                    " !/ "
-                    " !*.mount "
-                    " !*.socket "
-                    " !*.partition "
-                    " /machine.slice/*.service "          // #3367 systemd-nspawn
-                    " !*.service "
-                    " !*.slice "
-                    " !*.swap "
-                    " !*.user "
-                    " !init.scope "
-                    " !*.scope/vcpu* "                    // libvirtd adds these sub-cgroups
-                    " !*.scope/emulator "                 // libvirtd adds these sub-cgroups
-                    " *.scope "
-                    " *docker* "
-                    " *lxc* "
-                    " *qemu* "
-                    " */kubepods/pod*/* "                   // k8s containers
-                    " */kubepods/*/pod*/* "                 // k8s containers
-                    " */*-kubepods-pod*/* "                 // k8s containers
-                    " */*-kubepods-*-pod*/* "               // k8s containers
-                    " !*kubepods* !*kubelet* "              // all other k8s cgroups
-                    " *.libvirt-qemu "                    // #3010
-                    " * "
-            ), NULL, SIMPLE_PATTERN_EXACT);
+                       " !/ "
+                       " !*.mount "
+                       " !*.socket "
+                       " !*.partition "
+                       " /machine.slice/*.service "          // #3367 systemd-nspawn
+                       " !*.service "
+                       " !*.slice "
+                       " !*.swap "
+                       " !*.user "
+                       " !init.scope "
+                       " !*.scope/vcpu* "                    // libvirtd adds these sub-cgroups
+                       " !*.scope/emulator "                 // libvirtd adds these sub-cgroups
+                       " *.scope "
+                       " *docker* "
+                       " *lxc* "
+                       " *qemu* "
+                       " */kubepods/pod*/* "                   // k8s containers
+                       " */kubepods/*/pod*/* "                 // k8s containers
+                       " */*-kubepods-pod*/* "                 // k8s containers
+                       " */*-kubepods-*-pod*/* "               // k8s containers
+                       " !*kubepods* !*kubelet* "              // all other k8s cgroups
+                       " *.libvirt-qemu "                    // #3010
+                       " * "
+            ), NULL, SIMPLE_PATTERN_EXACT, true);
 
     if(cgroup_enable_systemd_services) {
         systemd_services_cgroups = simple_pattern_create(
                 config_get("plugin:cgroups", "cgroups to match as systemd services",
-                        " !/system.slice/*/*.service "
-                        " /system.slice/*.service "
-                ), NULL, SIMPLE_PATTERN_EXACT);
+                           " !/system.slice/*/*.service "
+                           " /system.slice/*.service "
+                ), NULL, SIMPLE_PATTERN_EXACT, true);
     }
 
     mountinfo_free_all(root);
@@ -2784,10 +2784,10 @@ void cgroup_discovery_worker(void *ptr)
     worker_register_job_name(WORKER_DISCOVERY_LOCK,               "lock");
 
     entrypoint_parent_process_comm = simple_pattern_create(
-        " runc:[* " // http://terenceli.github.io/%E6%8A%80%E6%9C%AF/2021/12/28/runc-internals-3)
-        " exe ", // https://github.com/falcosecurity/falco/blob/9d41b0a151b83693929d3a9c84f7c5c85d070d3a/rules/falco_rules.yaml#L1961
-        NULL,
-        SIMPLE_PATTERN_EXACT);
+            " runc:[* " // http://terenceli.github.io/%E6%8A%80%E6%9C%AF/2021/12/28/runc-internals-3)
+            " exe ", // https://github.com/falcosecurity/falco/blob/9d41b0a151b83693929d3a9c84f7c5c85d070d3a/rules/falco_rules.yaml#L1961
+            NULL,
+            SIMPLE_PATTERN_EXACT, true);
 
     while (service_running(SERVICE_COLLECTORS)) {
         worker_is_idle();

collectors/diskspace.plugin/plugin_diskspace.c

@@ -308,16 +308,12 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
         }
 
         excluded_mountpoints = simple_pattern_create(
-                config_get(CONFIG_SECTION_DISKSPACE, "exclude space metrics on paths", DEFAULT_EXCLUDED_PATHS)
-                , NULL
-                , mode
-        );
+                config_get(CONFIG_SECTION_DISKSPACE, "exclude space metrics on paths", DEFAULT_EXCLUDED_PATHS), NULL,
+                mode, true);
 
         excluded_filesystems = simple_pattern_create(
-                config_get(CONFIG_SECTION_DISKSPACE, "exclude space metrics on filesystems", DEFAULT_EXCLUDED_FILESYSTEMS)
-                , NULL
-                , SIMPLE_PATTERN_EXACT
-        );
+                config_get(CONFIG_SECTION_DISKSPACE, "exclude space metrics on filesystems",
+                           DEFAULT_EXCLUDED_FILESYSTEMS), NULL, SIMPLE_PATTERN_EXACT, true);
 
         dict_mountpoints = dictionary_create_advanced(DICT_OPTION_NONE, &dictionary_stats_category_collectors, 0);
     }

collectors/ebpf.plugin/ebpf_socket.c

@@ -3767,7 +3767,7 @@ static void link_hostnames(char *parse)
         ebpf_network_viewer_hostname_list_t *hostname = callocz(1 , sizeof(ebpf_network_viewer_hostname_list_t));
         hostname->value = strdupz(parse);
         hostname->hash = simple_hash(parse);
-        hostname->value_pattern = simple_pattern_create(parse, NULL, SIMPLE_PATTERN_EXACT);
+        hostname->value_pattern = simple_pattern_create(parse, NULL, SIMPLE_PATTERN_EXACT, true);
 
         link_hostname((!neg)?&network_viewer_opt.included_hostnames:&network_viewer_opt.excluded_hostnames,
                       hostname);

collectors/proc.plugin/proc_diskstats.c

@@ -934,16 +934,12 @@ int do_proc_diskstats(int update_every, usec_t dt) {
         name_disks_by_id = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "name disks by id", name_disks_by_id);
 
         preferred_ids = simple_pattern_create(
-                config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "preferred disk ids", DEFAULT_PREFERRED_IDS)
-                , NULL
-                , SIMPLE_PATTERN_EXACT
-        );
+                config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "preferred disk ids", DEFAULT_PREFERRED_IDS), NULL,
+                SIMPLE_PATTERN_EXACT, true);
 
         excluded_disks = simple_pattern_create(
-                config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "exclude disks", DEFAULT_EXCLUDED_DISKS)
-                , NULL
-                , SIMPLE_PATTERN_EXACT
-        );
+                config_get(CONFIG_SECTION_PLUGIN_PROC_DISKSTATS, "exclude disks", DEFAULT_EXCLUDED_DISKS), NULL,
+                SIMPLE_PATTERN_EXACT, true);
     }
 
     // --------------------------------------------------------------------------

collectors/proc.plugin/proc_net_dev.c

@@ -725,7 +725,9 @@ int do_proc_net_dev(int update_every, usec_t dt) {
         do_carrier      = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "carrier for all interfaces", CONFIG_BOOLEAN_AUTO);
         do_mtu          = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "mtu for all interfaces", CONFIG_BOOLEAN_AUTO);
 
-        disabled_list = simple_pattern_create(config_get(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "disable by default interfaces matching", "lo fireqos* *-ifb fwpr* fwbr* fwln*"), NULL, SIMPLE_PATTERN_EXACT);
+        disabled_list = simple_pattern_create(
+                config_get(CONFIG_SECTION_PLUGIN_PROC_NETDEV, "disable by default interfaces matching",
+                           "lo fireqos* *-ifb fwpr* fwbr* fwln*"), NULL, SIMPLE_PATTERN_EXACT, true);
     }
 
     if(unlikely(!ff)) {

collectors/proc.plugin/proc_pagetypeinfo.c

@@ -120,10 +120,8 @@ int do_proc_pagetypeinfo(int update_every, usec_t dt) {
         do_global = config_get_boolean(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "enable system summary", CONFIG_BOOLEAN_YES);
         do_detail = config_get_boolean_ondemand(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "enable detail per-type", CONFIG_BOOLEAN_AUTO);
         filter_types = simple_pattern_create(
-                config_get(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "hide charts id matching", "")
-                , NULL
-                , SIMPLE_PATTERN_SUFFIX
-        );
+                config_get(CONFIG_SECTION_PLUGIN_PROC_PAGETYPEINFO, "hide charts id matching", ""), NULL,
+                SIMPLE_PATTERN_SUFFIX, true);
 
         pagelines_cnt = 0;
 

collectors/proc.plugin/sys_class_infiniband.c

@@ -327,8 +327,9 @@ int do_sys_class_infiniband(int update_every, usec_t dt)
         enable_only_active = config_get_boolean_ondemand(
             CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "monitor only active ports", CONFIG_BOOLEAN_AUTO);
         disabled_list = simple_pattern_create(
-            config_get(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "disable by default interfaces matching", ""), NULL,
-            SIMPLE_PATTERN_EXACT);
+                config_get(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "disable by default interfaces matching", ""),
+                NULL,
+                SIMPLE_PATTERN_EXACT, true);
 
         dt_to_refresh_ports =
             config_get_number(CONFIG_SECTION_PLUGIN_SYS_CLASS_INFINIBAND, "refresh ports state every seconds", 30) *

collectors/statsd.plugin/statsd.c

@@ -1418,7 +1418,7 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
             }
             else if (!strcmp(name, "metrics")) {
                 simple_pattern_free(app->metrics);
-                app->metrics = simple_pattern_create(value, NULL, SIMPLE_PATTERN_EXACT);
+                app->metrics = simple_pattern_create(value, NULL, SIMPLE_PATTERN_EXACT, true);
             }
             else if (!strcmp(name, "private charts")) {
                 if (!strcmp(value, "yes") || !strcmp(value, "on"))
@@ -1533,7 +1533,7 @@ static int statsd_readfile(const char *filename, STATSD_APP *app, STATSD_APP_CHA
                 );
 
                 if(pattern)
-                    dim->metric_pattern = simple_pattern_create(dim->metric, NULL, SIMPLE_PATTERN_EXACT);
+                    dim->metric_pattern = simple_pattern_create(dim->metric, NULL, SIMPLE_PATTERN_EXACT, true);
             }
             else {
                 error("STATSD: ignoring line %zu ('%s') of file '%s'. Unknown keyword for the [%s] section.", line, name, filename, chart->id);
@@ -2462,7 +2462,9 @@ void *statsd_main(void *ptr) {
     statsd.recvmmsg_size = (size_t)config_get_number(CONFIG_SECTION_STATSD, "udp messages to process at once", (long long)statsd.recvmmsg_size);
 #endif
 
-    statsd.charts_for = simple_pattern_create(config_get(CONFIG_SECTION_STATSD, "create private charts for metrics matching", "*"), NULL, SIMPLE_PATTERN_EXACT);
+    statsd.charts_for = simple_pattern_create(
+            config_get(CONFIG_SECTION_STATSD, "create private charts for metrics matching", "*"), NULL,
+            SIMPLE_PATTERN_EXACT, true);
     statsd.max_private_charts_hard = (size_t)config_get_number(CONFIG_SECTION_STATSD, "max private charts hard limit", (long long)statsd.max_private_charts_hard);
     statsd.private_charts_rrd_history_entries = (int)config_get_number(CONFIG_SECTION_STATSD, "private charts history", default_rrd_history_entries);
     statsd.decimal_detail = (collected_number)config_get_number(CONFIG_SECTION_STATSD, "decimal detail", (long long int)statsd.decimal_detail);