JSON internal API, IEEE754 base64/hex streaming, weights endpoint optimization (#14493)

* first work on standardizing json formatting

* renamed old grouping to time_grouping and added group_by

* add dummy functions to enable compilation

* buffer json api work

* jsonwrap opening with buffer_json_X() functions

* cleanup

* storage for quotes

* optimize buffer printing for both numbers and strings

* removed ; from define

* contexts json generation using the new json functions

* fix buffer overflow at unit test

* weights endpoint using new json api

* fixes to weights endpoint

* check buffer overflow on all buffer functions

* do synchronous queries for weights

* buffer_flush() now resets json state too

* content type typedef

* print double values that are above the max 64-bit value

* str2ndd() can now parse values above UINT64_MAX

* faster number parsing by avoiding double calculations as much as possible

* faster number parsing

* faster hex parsing

* accurate printing and parsing of double values, even for very large numbers that cannot fit in 64bit integers

* full printing and parsing without using library functions - and related unit tests

* added IEEE754 streaming capability to enable streaming of double values in hex

* streaming and replication to transfer all values in hex

* use our own str2ndd for set2

* remove subnormal check from ieee

* base64 encoding for numbers, instead of hex

* when increasing double precision, also make sure the fractional number printed is aligned to the wanted precision

* str2ndd_encoded() parses all encoding formats, including integers

* prevent uninitialized use

* /api/v1/info using the new json API

* Fix error when compiling with --disable-ml

* Remove redundant 'buffer_unittest' declaration

* Fix formatting

* Fix formatting

* Fix formatting

* fix buffer unit test

* apps.plugin using the new JSON API

* make sure the metrics registry does not accept negative timestamps

* do not allow pages with negative timestamps to be loaded from db files; do not accept pages with negative timestamps in the cache

* Fix more formatting

---------

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

aclk/aclk_query.c

@@ -192,7 +192,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
     w->response.data->date = w->tv_ready.tv_sec;
     web_client_build_http_header(w);
     local_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE, &netdata_buffers_statistics.buffers_aclk);
-    local_buffer->contenttype = CT_APPLICATION_JSON;
+    local_buffer->content_type = CT_APPLICATION_JSON;
 
     buffer_strcat(local_buffer, w->response.header_output->buffer);
 

collectors/apps.plugin/apps_plugin.c

@@ -638,9 +638,9 @@ int read_user_or_group_ids(struct user_or_group_ids *ids, struct timespec *last_
         struct user_or_group_id *user_or_group_id = callocz(1, sizeof(struct user_or_group_id));
 
         if(ids->type == USER_ID)
-            user_or_group_id->id.uid = (uid_t)str2ull(id_string);
+            user_or_group_id->id.uid = (uid_t) str2ull(id_string, NULL);
         else
-            user_or_group_id->id.gid = (uid_t)str2ull(id_string);
+            user_or_group_id->id.gid = (uid_t) str2ull(id_string, NULL);
 
         user_or_group_id->name = strdupz(name);
         user_or_group_id->updated = 1;
@@ -1452,7 +1452,7 @@ static inline int read_proc_pid_stat(struct pid_stat *p, void *ptr) {
     pid_incremental_rate(stat, p->cstime,  str2kernel_uint_t(procfile_lineword(ff, 0, 16)));
     // p->priority      = str2kernel_uint_t(procfile_lineword(ff, 0, 17));
     // p->nice          = str2kernel_uint_t(procfile_lineword(ff, 0, 18));
-    p->num_threads      = (int32_t)str2uint32_t(procfile_lineword(ff, 0, 19));
+    p->num_threads      = (int32_t) str2uint32_t(procfile_lineword(ff, 0, 19), NULL);
     // p->itrealvalue   = str2kernel_uint_t(procfile_lineword(ff, 0, 20));
     p->collected_starttime        = str2kernel_uint_t(procfile_lineword(ff, 0, 21)) / system_hz;
     p->uptime           = (global_uptime > p->collected_starttime)?(global_uptime - p->collected_starttime):0;
@@ -4234,7 +4234,7 @@ static void get_MemTotal(void) {
     for(line = 0; line < lines ;line++) {
         size_t words = procfile_linewords(ff, line);
         if(words == 3 && strcmp(procfile_lineword(ff, line, 0), "MemTotal") == 0 && strcmp(procfile_lineword(ff, line, 2), "kB") == 0) {
-            kernel_uint_t n = str2ull(procfile_lineword(ff, line, 1));
+            kernel_uint_t n = str2ull(procfile_lineword(ff, line, 1), NULL);
             if(n) MemTotal = n;
             break;
         }
@@ -4289,47 +4289,41 @@ static void apps_plugin_function_processes_help(const char *transaction) {
 }
 
 #define add_table_field(wb, key, name, visible, type, visualization, transform, decimal_points, units, max, sort, sortable, sticky, unique_key, pointer_to, summary, range) do { \
-    if(fields_added) buffer_strcat(wb, ",");                                                                    \
-    buffer_sprintf(wb, "\n      \"%s\": {", key);                                                               \
-    buffer_sprintf(wb, "\n         \"index\":%d,", fields_added);                                               \
-    buffer_sprintf(wb, "\n         \"unique_key\":%s,", (unique_key)?"true":"false");                           \
-    buffer_sprintf(wb, "\n         \"name\":\"%s\",", name);                                                    \
-    buffer_sprintf(wb, "\n         \"visible\":%s,", (visible)?"true":"false");                                 \
-    buffer_sprintf(wb, "\n         \"type\":\"%s\",", type);                                                    \
-    if(units)                                                                                                   \
-       buffer_sprintf(wb, "\n         \"units\":\"%s\",", (char*)(units));                                      \
-    buffer_sprintf(wb, "\n         \"visualization\":\"%s\",", visualization);                                  \
-    buffer_sprintf(wb, "\n         \"value_options\":{");                                                       \
-    if(units)                                                                                                   \
-       buffer_sprintf(wb, "\n           \"units\":\"%s\",", (char*)(units));                                    \
-    buffer_sprintf(wb, "\n           \"transform\":\"%s\",", transform);                                        \
-    buffer_sprintf(wb, "\n           \"decimal_points\":%d", decimal_points);                                   \
-    buffer_sprintf(wb, "\n         },");                                                                        \
+    buffer_json_member_add_object(wb, key);                                                                     \
+    buffer_json_member_add_uint64(wb, "index", fields_added);                                                   \
+    buffer_json_member_add_boolean(wb, "unique_key", unique_key);                                               \
+    buffer_json_member_add_string(wb, "name", name);                                                            \
+    buffer_json_member_add_boolean(wb, "visible", visible);                                                     \
+    buffer_json_member_add_string(wb, "type", type);                                                            \
+    buffer_json_member_add_string_or_omit(wb, "units", (char*)(units));                                         \
+    buffer_json_member_add_string(wb, "visualization", visualization);                                          \
+    buffer_json_member_add_object(wb, "value_options");                                                         \
+    buffer_json_member_add_string_or_omit(wb, "units", (char*)(units));                                         \
+    buffer_json_member_add_string(wb, "transform", transform);                                                  \
+    buffer_json_member_add_uint64(wb, "decimal_points", decimal_points);                                        \
+    buffer_json_object_close(wb);                                                                               \
     if(!isnan((NETDATA_DOUBLE)(max)))                                                                           \
-       buffer_sprintf(wb, "\n         \"max\":%f,", (NETDATA_DOUBLE)(max));                                     \
-    if(pointer_to)                                                                                              \
-        buffer_sprintf(wb, "\n         \"pointer_to\":\"%s\",", (char *)(pointer_to));                          \
-    buffer_sprintf(wb, "\n         \"sort\":\"%s\",", sort);                                                    \
-    buffer_sprintf(wb, "\n         \"sortable\":%s,", (sortable)?"true":"false");                               \
-    buffer_sprintf(wb, "\n         \"sticky\":%s,", (sticky)?"true":"false");                                   \
-    buffer_sprintf(wb, "\n         \"summary\":\"%s\",", summary);                                              \
-    buffer_sprintf(wb, "\n         \"filter\":\"%s\"", (range)?"range":"multiselect");                          \
-    buffer_sprintf(wb, "\n      }");                                                                            \
+       buffer_json_member_add_double(wb, "max", (NETDATA_DOUBLE)(max));                                         \
+    buffer_json_member_add_string_or_omit(wb, "pointer_to", (char *)(pointer_to));                              \
+    buffer_json_member_add_string(wb, "sort", sort);                                                            \
+    buffer_json_member_add_boolean(wb, "sortable", sortable);                                                   \
+    buffer_json_member_add_boolean(wb, "sticky", sticky);                                                       \
+    buffer_json_member_add_string(wb, "summary", summary);                                                      \
+    buffer_json_member_add_string(wb, "filter", (range)?"range":"multiselect");                                 \
+    buffer_json_object_close(wb);                                                                               \
     fields_added++;                                                                                             \
 } while(0)
 
 #define add_value_field_llu_with_max(wb, key, value) do {                                                       \
     unsigned long long _tmp = (value);                                                                          \
     key ## _max = (rows == 0) ? (_tmp) : MAX(key ## _max, _tmp);                                                \
-    buffer_fast_strcat(wb, ",", 1);                                                                             \
-    buffer_print_llu(wb, _tmp);                                                                                 \
+    buffer_json_add_array_item_uint64(wb, _tmp);                                                                \
 } while(0)
 
 #define add_value_field_ndd_with_max(wb, key, value) do {                                                       \
     NETDATA_DOUBLE _tmp = (value);                                                                              \
     key ## _max = (rows == 0) ? (_tmp) : MAX(key ## _max, _tmp);                                                \
-    buffer_fast_strcat(wb, ",", 1);                                                                             \
-    buffer_rrd_value(wb, _tmp);                                                                                 \
+    buffer_json_add_array_item_double(wb, _tmp);                                                                \
 } while(0)
 
 static void apps_plugin_function_processes(const char *transaction, char *function __maybe_unused, char *line_buffer __maybe_unused, int line_max __maybe_unused, int timeout __maybe_unused) {
@@ -4406,18 +4400,12 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
     unsigned int io_divisor = 1024 * RATES_DETAIL;
 
     BUFFER *wb = buffer_create(PLUGINSD_LINE_MAX, NULL);
-    buffer_sprintf(wb,
-                   "{"
-                   "\n   \"status\":%d"
-                   ",\n   \"type\":\"table\""
-                   ",\n   \"update_every\":%d"
-                   ",\n   \"help\":\"%s\""
-                   ",\n   \"data\":["
-                   "\n"
-            , HTTP_RESP_OK
-            , update_every
-            , APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION
-    );
+    buffer_json_initialize(wb, "\"", "\"", 0, true);
+    buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
+    buffer_json_member_add_string(wb, "type", "table");
+    buffer_json_member_add_time_t(wb, "update_every", update_every);
+    buffer_json_member_add_string(wb, "help", APPS_PLUGIN_PROCESSES_FUNCTION_DESCRIPTION);
+    buffer_json_member_add_array(wb, "data");
 
     NETDATA_DOUBLE
               UserCPU_max = 0.0
@@ -4493,52 +4481,41 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
         if(filter_gid && p->gid != gid)
             continue;
 
-        if(rows) buffer_fast_strcat(wb, ",\n", 2);
         rows++;
 
-        buffer_strcat(wb, "      [");
+        buffer_json_add_array_item_array(wb);
 
         // IMPORTANT!
         // THE ORDER SHOULD BE THE SAME WITH THE FIELDS!
 
         // pid
-        buffer_print_llu(wb, p->pid);
+        buffer_json_add_array_item_uint64(wb, p->pid);
 
         // cmd
-        buffer_fast_strcat(wb, ",\"", 2);
-        buffer_strcat_jsonescape(wb, p->comm);
-        buffer_fast_strcat(wb, "\"", 1);
+        buffer_json_add_array_item_string(wb, p->comm);
 
 #ifdef NETDATA_DEV_MODE
         // cmdline
-        buffer_fast_strcat(wb, ",\"", 2);
-        buffer_strcat_jsonescape(wb, (p->cmdline && *p->cmdline) ? p->cmdline : p->comm);
-        buffer_fast_strcat(wb, "\"", 1);
+        buffer_json_add_array_item_string(wb, (p->cmdline && *p->cmdline) ? p->cmdline : p->comm);
 #endif
 
         // ppid
-        buffer_fast_strcat(wb, ",", 1); buffer_print_llu(wb, p->ppid);
+        buffer_json_add_array_item_uint64(wb, p->ppid);
 
         // category
-        buffer_fast_strcat(wb, ",\"", 2);
-        buffer_strcat_jsonescape(wb, p->target ? p->target->name : "-");
-        buffer_fast_strcat(wb, "\"", 1);
+        buffer_json_add_array_item_string(wb, p->target ? p->target->name : "-");
 
         // user
-        buffer_fast_strcat(wb, ",\"", 2);
-        buffer_strcat_jsonescape(wb, p->user_target ? p->user_target->name : "-");
-        buffer_fast_strcat(wb, "\"", 1);
+        buffer_json_add_array_item_string(wb, p->user_target ? p->user_target->name : "-");
 
         // uid
-        buffer_fast_strcat(wb, ",", 1); buffer_print_llu(wb, p->uid);
+        buffer_json_add_array_item_uint64(wb, p->uid);
 
         // group
-        buffer_fast_strcat(wb, ",\"", 2);
-        buffer_strcat_jsonescape(wb, p->group_target ? p->group_target->name : "-");
-        buffer_fast_strcat(wb, "\"", 1);
+        buffer_json_add_array_item_string(wb, p->group_target ? p->group_target->name : "-");
 
         // gid
-        buffer_fast_strcat(wb, ",", 1); buffer_print_llu(wb, p->gid);
+        buffer_json_add_array_item_uint64(wb, p->gid);
 
         // CPU utilization %
         add_value_field_ndd_with_max(wb, CPU, (NETDATA_DOUBLE)(p->utime + p->stime + p->gtime + p->cutime + p->cstime + p->cgtime) / cpu_divisor);
@@ -4599,18 +4576,15 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
         add_value_field_llu_with_max(wb, Threads, p->num_threads);
         add_value_field_llu_with_max(wb, Uptime, p->uptime);
 
-        buffer_fast_strcat(wb, "]", 1);
-
-        fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);
-        buffer_flush(wb);
+        buffer_json_array_close(wb);
     }
 
+    buffer_json_array_close(wb);
+    buffer_json_member_add_object(wb, "columns");
+
     {
         int fields_added = 0;
 
-        buffer_flush(wb);
-        buffer_sprintf(wb, "\n   ],\n   \"columns\": {");
-
         // IMPORTANT!
         // THE ORDER SHOULD BE THE SAME WITH THE VALUES!
         add_table_field(wb, "PID", "Process ID", true, "integer", "value", "number", 0, NULL, NAN, "ascending", true, true, true, NULL, "count_unique", false);
@@ -4684,118 +4658,271 @@ static void apps_plugin_function_processes(const char *transaction, char *functi
         add_table_field(wb, "Processes", "Processes", true, "bar-with-integer", "bar", "number", 0, "processes", Processes_max, "descending", true, false, false, NULL, "sum", true);
         add_table_field(wb, "Threads", "Threads", true, "bar-with-integer", "bar", "number", 0, "threads", Threads_max, "descending", true, false, false, NULL, "sum", true);
         add_table_field(wb, "Uptime", "Uptime in seconds", true, "duration", "bar", "duration", 2, "seconds", Uptime_max, "descending", true, false, false, NULL, "max", true);
+    }
+    buffer_json_object_close(wb);
 
-        buffer_strcat(
-                wb,
-                ""
-                "\n   },"
-                "\n   \"default_sort_column\": \"CPU\","
-                "\n   \"charts\": {"
-                "\n      \"CPU\": {"
-                "\n         \"name\":\"CPU Utilization\","
-                "\n         \"type\":\"stacked-bar\","
-                "\n         \"columns\": [ \"UserCPU\", \"SysCPU\", \"GuestCPU\", \"CUserCPU\", \"CSysCPU\", \"CGuestCPU\" ]"
-                "\n      },"
-                "\n      \"Memory\": {"
-                "\n         \"name\":\"Memory\","
-                "\n         \"type\":\"stacked-bar\","
-                "\n         \"columns\": [ \"Virtual\", \"Resident\", \"Shared\", \"Swap\" ]"
-                "\n      },"
-        );
+    buffer_json_member_add_string(wb, "default_sort_column", "CPU");
 
-        if(MemTotal)
-            buffer_strcat(
-                    wb,
-                    ""
-                    "\n      \"MemoryPercent\": {"
-                    "\n         \"name\":\"Memory Percentage\","
-                    "\n         \"type\":\"stacked-bar\","
-                    "\n         \"columns\": [ \"Memory\" ]"
-                    "\n      },"
-            );
+    buffer_json_member_add_object(wb, "charts");
+    {
+        // CPU chart
+        buffer_json_member_add_object(wb, "CPU");
+        {
+            buffer_json_member_add_string(wb, "name", "CPU Utilization");
+            buffer_json_member_add_string(wb, "type", "stacked-bar");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "UserCPU");
+                buffer_json_add_array_item_string(wb, "SysCPU");
+                buffer_json_add_array_item_string(wb, "GuestCPU");
+                buffer_json_add_array_item_string(wb, "CUserCPU");
+                buffer_json_add_array_item_string(wb, "CSysCPU");
+                buffer_json_add_array_item_string(wb, "CGuestCPU");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        // Memory chart
+        buffer_json_member_add_object(wb, "Memory");
+        {
+            buffer_json_member_add_string(wb, "name", "Memory");
+            buffer_json_member_add_string(wb, "type", "stacked-bar");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "Virtual");
+                buffer_json_add_array_item_string(wb, "Resident");
+                buffer_json_add_array_item_string(wb, "Shared");
+                buffer_json_add_array_item_string(wb, "Swap");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        if(MemTotal) {
+            // Memory chart
+            buffer_json_member_add_object(wb, "MemoryPercent");
+            {
+                buffer_json_member_add_string(wb, "name", "Memory Percentage");
+                buffer_json_member_add_string(wb, "type", "stacked-bar");
+                buffer_json_member_add_array(wb, "columns");
+                {
+                    buffer_json_add_array_item_string(wb, "Memory");
+                }
+                buffer_json_array_close(wb);
+            }
+            buffer_json_object_close(wb);
+        }
 
-        buffer_strcat(
-                wb, ""
-                    #ifndef __FreeBSD__
-                    "\n      \"Reads\": {"
-                                       "\n         \"name\":\"I/O Reads\","
-                                       "\n         \"type\":\"stacked-bar\","
-                                       "\n         \"columns\": [ \"LReads\", \"PReads\" ]"
-                                       "\n      },"
-                                       "\n      \"Writes\": {"
-                                       "\n         \"name\":\"I/O Writes\","
-                                       "\n         \"type\":\"stacked-bar\","
-                                       "\n         \"columns\": [ \"LWrites\", \"PWrites\" ]"
-                                       "\n      },"
-                                       "\n      \"LogicalIO\": {"
-                                       "\n         \"name\":\"Logical I/O\","
-                                       "\n         \"type\":\"stacked-bar\","
-                                       "\n         \"columns\": [ \"LReads\", \"LWrites\" ]"
-                                       "\n      },"
-                    #endif
-                    "\n      \"PhysicalIO\": {"
-                                       "\n         \"name\":\"Physical I/O\","
-                                       "\n         \"type\":\"stacked-bar\","
-                                       "\n         \"columns\": [ \"PReads\", \"PWrites\" ]"
-                                       "\n      },"
-                                       "\n      \"IOCalls\": {"
-                                       "\n         \"name\":\"I/O Calls\","
-                                       "\n         \"type\":\"stacked-bar\","
-                                       "\n         \"columns\": [ \"RCalls\", \"WCalls\" ]"
-                                       "\n      },"
-                                       "\n      \"MinFlt\": {"
-                                       "\n         \"name\":\"Minor Page Faults\","
-                                       "\n         \"type\":\"stacked-bar\","
-                                       "\n         \"columns\": [ \"MinFlt\", \"CMinFlt\" ]"
-                                       "\n      },"
-                                       "\n      \"MajFlt\": {"
-                                       "\n         \"name\":\"Major Page Faults\","
-                                       "\n         \"type\":\"stacked-bar\","
-                                       "\n         \"columns\": [ \"MajFlt\", \"CMajFlt\" ]"
-                                       "\n      },"
-                                       "\n      \"Threads\": {"
-                                       "\n         \"name\":\"Threads\","
-                                       "\n         \"type\":\"stacked-bar\","
-                                       "\n         \"columns\": [ \"Threads\" ]"
-                                       "\n      },"
-                                       "\n      \"Processes\": {"
-                                       "\n         \"name\":\"Processes\","
-                                       "\n         \"type\":\"stacked-bar\","
-                                       "\n         \"columns\": [ \"Processes\" ]"
-                                       "\n      },"
-                                       "\n      \"FDs\": {"
-                                       "\n         \"name\":\"File Descriptors\","
-                                       "\n         \"type\":\"stacked-bar\","
-                                       "\n         \"columns\": [ \"Files\", \"Pipes\", \"Sockets\", \"iNotiFDs\", \"EventFDs\", \"TimerFDs\", \"SigFDs\", \"EvPollFDs\", \"OtherFDs\" ]"
-                                       "\n      }"
-                                       "\n   },"
-                                       "\n   \"group_by\": {"
-                                       "\n     \"pid\": {"
-                                       "\n         \"name\":\"Process Tree by PID\","
-                                       "\n         \"columns\":[ \"PPID\" ]"
-                                       "\n     },"
-                                       "\n     \"category\": {"
-                                       "\n         \"name\":\"Process Tree by Category\","
-                                       "\n         \"columns\":[ \"Category\", \"PPID\" ]"
-                                       "\n     },"
-                                       "\n     \"user\": {"
-                                       "\n         \"name\":\"Process Tree by User\","
-                                       "\n         \"columns\":[ \"User\", \"PPID\" ]"
-                                       "\n     },"
-                                       "\n     \"group\": {"
-                                       "\n         \"name\":\"Process Tree by Group\","
-                                       "\n         \"columns\":[ \"Group\", \"PPID\" ]"
-                                       "\n     }"
-                                       "\n   }"
-        );
+#ifndef __FreeBSD__
+        // I/O Reads chart
+        buffer_json_member_add_object(wb, "Reads");
+        {
+            buffer_json_member_add_string(wb, "name", "I/O Reads");
+            buffer_json_member_add_string(wb, "type", "stacked-bar");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "LReads");
+                buffer_json_add_array_item_string(wb, "PReads");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        // I/O Writes chart
+        buffer_json_member_add_object(wb, "Writes");
+        {
+            buffer_json_member_add_string(wb, "name", "I/O Writes");
+            buffer_json_member_add_string(wb, "type", "stacked-bar");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "LWrites");
+                buffer_json_add_array_item_string(wb, "PWrites");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        // Logical I/O chart
+        buffer_json_member_add_object(wb, "LogicalIO");
+        {
+            buffer_json_member_add_string(wb, "name", "Logical I/O");
+            buffer_json_member_add_string(wb, "type", "stacked-bar");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "LReads");
+                buffer_json_add_array_item_string(wb, "LWrites");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+#endif
+
+        // Physical I/O chart
+        buffer_json_member_add_object(wb, "PhysicalIO");
+        {
+            buffer_json_member_add_string(wb, "name", "Physical I/O");
+            buffer_json_member_add_string(wb, "type", "stacked-bar");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "PReads");
+                buffer_json_add_array_item_string(wb, "PWrites");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        // I/O Calls chart
+        buffer_json_member_add_object(wb, "IOCalls");
+        {
+            buffer_json_member_add_string(wb, "name", "I/O Calls");
+            buffer_json_member_add_string(wb, "type", "stacked-bar");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "RCalls");
+                buffer_json_add_array_item_string(wb, "WCalls");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        // Minor Page Faults chart
+        buffer_json_member_add_object(wb, "MinFlt");
+        {
+            buffer_json_member_add_string(wb, "name", "Minor Page Faults");
+            buffer_json_member_add_string(wb, "type", "stacked-bar");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "MinFlt");
+                buffer_json_add_array_item_string(wb, "CMinFlt");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        // Major Page Faults chart
+        buffer_json_member_add_object(wb, "MajFlt");
+        {
+            buffer_json_member_add_string(wb, "name", "Major Page Faults");
+            buffer_json_member_add_string(wb, "type", "stacked-bar");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "MajFlt");
+                buffer_json_add_array_item_string(wb, "CMajFlt");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        // Threads chart
+        buffer_json_member_add_object(wb, "Threads");
+        {
+            buffer_json_member_add_string(wb, "name", "Threads");
+            buffer_json_member_add_string(wb, "type", "stacked-bar");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "Threads");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        // Processes chart
+        buffer_json_member_add_object(wb, "Processes");
+        {
+            buffer_json_member_add_string(wb, "name", "Processes");
+            buffer_json_member_add_string(wb, "type", "stacked-bar");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "Processes");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        // FDs chart
+        buffer_json_member_add_object(wb, "FDs");
+        {
+            buffer_json_member_add_string(wb, "name", "File Descriptors");
+            buffer_json_member_add_string(wb, "type", "stacked-bar");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "Files");
+                buffer_json_add_array_item_string(wb, "Pipes");
+                buffer_json_add_array_item_string(wb, "Sockets");
+                buffer_json_add_array_item_string(wb, "iNotiFDs");
+                buffer_json_add_array_item_string(wb, "EventFDs");
+                buffer_json_add_array_item_string(wb, "TimerFDs");
+                buffer_json_add_array_item_string(wb, "SigFDs");
+                buffer_json_add_array_item_string(wb, "EvPollFDs");
+                buffer_json_add_array_item_string(wb, "OtherFDs");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+    }
+    buffer_json_object_close(wb); // charts
 
-        fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);
+    buffer_json_member_add_object(wb, "group_by");
+    {
+        // group by PID
+        buffer_json_member_add_object(wb, "PID");
+        {
+            buffer_json_member_add_string(wb, "name", "Process Tree by PID");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "PPID");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        // group by Category
+        buffer_json_member_add_object(wb, "Category");
+        {
+            buffer_json_member_add_string(wb, "name", "Process Tree by Category");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "Category");
+                buffer_json_add_array_item_string(wb, "PPID");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        // group by User
+        buffer_json_member_add_object(wb, "User");
+        {
+            buffer_json_member_add_string(wb, "name", "Process Tree by User");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "User");
+                buffer_json_add_array_item_string(wb, "PPID");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
+
+        // group by Group
+        buffer_json_member_add_object(wb, "Group");
+        {
+            buffer_json_member_add_string(wb, "name", "Process Tree by Group");
+            buffer_json_member_add_array(wb, "columns");
+            {
+                buffer_json_add_array_item_string(wb, "Group");
+                buffer_json_add_array_item_string(wb, "PPID");
+            }
+            buffer_json_array_close(wb);
+        }
+        buffer_json_object_close(wb);
     }
+    buffer_json_object_close(wb); // group_by
 
-    buffer_free(wb);
+    buffer_json_member_add_time_t(wb, "expires", expires);
+    buffer_json_finalize(wb);
 
-    fprintf(stdout, ",\n   \"expires\":%lld", (long long)expires);
-    fprintf(stdout, "\n}");
+    fwrite(buffer_tostring(wb), buffer_strlen(wb), 1, stdout);
+    buffer_free(wb);
 
     pluginsd_function_result_end_to_stdout();
 }

collectors/cgroups.plugin/sys_fs_cgroup.c

@@ -1089,10 +1089,10 @@ static inline void cgroup_read_cpuacct_stat(struct cpuacct_stat *cp) {
             uint32_t hash = simple_hash(s);
 
             if(unlikely(hash == user_hash && !strcmp(s, "user")))
-                cp->user = str2ull(procfile_lineword(ff, i, 1));
+                cp->user = str2ull(procfile_lineword(ff, i, 1), NULL);
 
             else if(unlikely(hash == system_hash && !strcmp(s, "system")))
-                cp->system = str2ull(procfile_lineword(ff, i, 1));
+                cp->system = str2ull(procfile_lineword(ff, i, 1), NULL);
         }
 
         cp->updated = 1;
@@ -1138,11 +1138,11 @@ static inline void cgroup_read_cpuacct_cpu_stat(struct cpuacct_cpu_throttling *c
         uint32_t hash = simple_hash(s);
 
         if (unlikely(hash == nr_periods_hash && !strcmp(s, "nr_periods"))) {
-            cp->nr_periods = str2ull(procfile_lineword(ff, i, 1));
+            cp->nr_periods = str2ull(procfile_lineword(ff, i, 1), NULL);
         } else if (unlikely(hash == nr_throttled_hash && !strcmp(s, "nr_throttled"))) {
-            cp->nr_throttled = str2ull(procfile_lineword(ff, i, 1));
+            cp->nr_throttled = str2ull(procfile_lineword(ff, i, 1), NULL);
         } else if (unlikely(hash == throttled_time_hash && !strcmp(s, "throttled_time"))) {
-            cp->throttled_time = str2ull(procfile_lineword(ff, i, 1));
+            cp->throttled_time = str2ull(procfile_lineword(ff, i, 1), NULL);
         }
     }
     cp->nr_throttled_perc =
@@ -1195,15 +1195,15 @@ static inline void cgroup2_read_cpuacct_cpu_stat(struct cpuacct_stat *cp, struct
         uint32_t hash = simple_hash(s);
 
         if (unlikely(hash == user_usec_hash && !strcmp(s, "user_usec"))) {
-            cp->user = str2ull(procfile_lineword(ff, i, 1));
+            cp->user = str2ull(procfile_lineword(ff, i, 1), NULL);
         } else if (unlikely(hash == system_usec_hash && !strcmp(s, "system_usec"))) {
-            cp->system = str2ull(procfile_lineword(ff, i, 1));
+            cp->system = str2ull(procfile_lineword(ff, i, 1), NULL);
         } else if (unlikely(hash == nr_periods_hash && !strcmp(s, "nr_periods"))) {
-            cpt->nr_periods = str2ull(procfile_lineword(ff, i, 1));
+            cpt->nr_periods = str2ull(procfile_lineword(ff, i, 1), NULL);
         } else if (unlikely(hash == nr_throttled_hash && !strcmp(s, "nr_throttled"))) {
-            cpt->nr_throttled = str2ull(procfile_lineword(ff, i, 1));
+            cpt->nr_throttled = str2ull(procfile_lineword(ff, i, 1), NULL);
         } else if (unlikely(hash == throttled_usec_hash && !strcmp(s, "throttled_usec"))) {
-            cpt->throttled_time = str2ull(procfile_lineword(ff, i, 1)) * 1000; // usec -> ns
+            cpt->throttled_time = str2ull(procfile_lineword(ff, i, 1), NULL) * 1000; // usec -> ns
         }
     }
     cpt->nr_throttled_perc =
@@ -1289,7 +1289,7 @@ static inline void cgroup_read_cpuacct_usage(struct cpuacct_usage *ca) {
 
         unsigned long long total = 0;
         for(i = 0; i < ca->cpus ;i++) {
-            unsigned long long n = str2ull(procfile_lineword(ff, 0, i));
+            unsigned long long n = str2ull(procfile_lineword(ff, 0, i), NULL);
             ca->cpu_percpu[i] = n;
             total += n;
         }
@@ -1346,10 +1346,10 @@ static inline void cgroup_read_blkio(struct blkio *io) {
             uint32_t hash = simple_hash(s);
 
             if(unlikely(hash == Read_hash && !strcmp(s, "Read")))
-                io->Read += str2ull(procfile_lineword(ff, i, 2));
+                io->Read += str2ull(procfile_lineword(ff, i, 2), NULL);
 
             else if(unlikely(hash == Write_hash && !strcmp(s, "Write")))
-                io->Write += str2ull(procfile_lineword(ff, i, 2));
+                io->Write += str2ull(procfile_lineword(ff, i, 2), NULL);
 
 /*
             else if(unlikely(hash == Sync_hash && !strcmp(s, "Sync")))
@@ -1409,8 +1409,8 @@ static inline void cgroup2_read_blkio(struct blkio *io, unsigned int word_offset
             io->Write = 0;
 
             for (i = 0; i < lines; i++) {
-                io->Read += str2ull(procfile_lineword(ff, i, 2 + word_offset));
-                io->Write += str2ull(procfile_lineword(ff, i, 4 + word_offset));
+                io->Read += str2ull(procfile_lineword(ff, i, 2 + word_offset), NULL);
+                io->Write += str2ull(procfile_lineword(ff, i, 4 + word_offset), NULL);
             }
 
             io->updated = 1;
@@ -1452,13 +1452,13 @@ static inline void cgroup2_read_pressure(struct pressure *res) {
         res->some.share_time.value10 = strtod(procfile_lineword(ff, 0, 2), NULL);
         res->some.share_time.value60 = strtod(procfile_lineword(ff, 0, 4), NULL);
         res->some.share_time.value300 = strtod(procfile_lineword(ff, 0, 6), NULL);
-        res->some.total_time.value_total = str2ull(procfile_lineword(ff, 0, 8)) / 1000; // us->ms
+        res->some.total_time.value_total = str2ull(procfile_lineword(ff, 0, 8), NULL) / 1000; // us->ms
 
         if (lines > 2) {
             res->full.share_time.value10 = strtod(procfile_lineword(ff, 1, 2), NULL);
             res->full.share_time.value60 = strtod(procfile_lineword(ff, 1, 4), NULL);
             res->full.share_time.value300 = strtod(procfile_lineword(ff, 1, 6), NULL);
-            res->full.total_time.value_total = str2ull(procfile_lineword(ff, 1, 8)) / 1000; // us->ms
+            res->full.total_time.value_total = str2ull(procfile_lineword(ff, 1, 8), NULL) / 1000; // us->ms
         }
 
         res->updated = 1;
@@ -3566,14 +3566,14 @@ static inline void update_cpu_limits2(struct cgroup *cg) {
             return;
         }
 
-        cg->cpu_cfs_period = str2ull(procfile_lineword(ff, 0, 1));
+        cg->cpu_cfs_period = str2ull(procfile_lineword(ff, 0, 1), NULL);
         cg->cpuset_cpus = get_system_cpus();
 
         char *s = "max\n\0";
         if(strcmp(s, procfile_lineword(ff, 0, 0)) == 0){
             cg->cpu_cfs_quota = cg->cpu_cfs_period * cg->cpuset_cpus;
         } else {
-            cg->cpu_cfs_quota = str2ull(procfile_lineword(ff, 0, 0));
+            cg->cpu_cfs_quota = str2ull(procfile_lineword(ff, 0, 0), NULL);
         }
         debug(D_CGROUP, "CPU limits values: %llu %llu %llu", cg->cpu_cfs_period, cg->cpuset_cpus, cg->cpu_cfs_quota);
         return;
@@ -3623,7 +3623,7 @@ static inline int update_memory_limits(char **filename, const RRDSETVAR_ACQUIRED
                     rrdsetvar_custom_chart_variable_set(cg->st_mem_usage, *chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024)));
                     return 1;
                 }
-                *value = str2ull(buffer);
+                *value = str2ull(buffer, NULL);
                 rrdsetvar_custom_chart_variable_set(cg->st_mem_usage, *chart_var, (NETDATA_DOUBLE)(*value / (1024 * 1024)));
                 return 1;
             }
@@ -4111,7 +4111,7 @@ void update_cgroup_charts(int update_every) {
                     if(likely(ff))
                         ff = procfile_readall(ff);
                     if(likely(ff && procfile_lines(ff) && !strncmp(procfile_word(ff, 0), "MemTotal", 8)))
-                        ram_total = str2ull(procfile_word(ff, 1)) * 1024;
+                        ram_total = str2ull(procfile_word(ff, 1), NULL) * 1024;
                     else {
                         collector_error("Cannot read file %s. Will not update cgroup %s RAM limit anymore.", filename, cg->id);
                         freez(cg->filename_memory_limit);

collectors/ebpf.plugin/ebpf.c

@@ -2088,7 +2088,7 @@ static pid_t ebpf_read_previous_pid(char *filename)
             length = 63;
 
         buffer[length] = '\0';
-        old_pid = (pid_t)str2uint32_t(buffer);
+        old_pid = (pid_t) str2uint32_t(buffer, NULL);
     }
     fclose(fp);
 

collectors/plugins.d/pluginsd_parser.c

@@ -221,7 +221,7 @@ PARSER_RC pluginsd_set(char **words, size_t num_words, void *user)
               rrdhost_hostname(host), rrdset_id(st), dimension, value && *value ? value : "UNSET");
 
     if (value && *value)
-        rrddim_set_by_pointer(st, rd, str2ll_hex_or_dec(value));
+        rrddim_set_by_pointer(st, rd, str2ll_encoded(value));
 
     return PARSER_RC_OK;
 }
@@ -616,14 +616,14 @@ PARSER_RC pluginsd_dimension(char **words, size_t num_words, void *user)
 
     long multiplier = 1;
     if (multiplier_s && *multiplier_s) {
-        multiplier = str2ll_hex_or_dec(multiplier_s);
+        multiplier = str2ll_encoded(multiplier_s);
         if (unlikely(!multiplier))
             multiplier = 1;
     }
 
     long divisor = 1;
     if (likely(divisor_s && *divisor_s)) {
-        divisor = str2ll_hex_or_dec(divisor_s);
+        divisor = str2ll_encoded(divisor_s);
         if (unlikely(!divisor))
             divisor = 1;
     }
@@ -901,7 +901,7 @@ PARSER_RC pluginsd_function_result_begin(char **words, size_t num_words, void *u
     }
     else {
         if(format && *format)
-            pf->destination_wb->contenttype = functions_format_to_content_type(format);
+            pf->destination_wb->content_type = functions_format_to_content_type(format);
 
         pf->code = code;
 
@@ -967,7 +967,7 @@ PARSER_RC pluginsd_variable(char **words, size_t num_words, void *user)
         return PLUGINSD_DISABLE_PLUGIN(user, PLUGINSD_KEYWORD_VARIABLE, "no chart is defined and no GLOBAL is given");
 
     char *endptr = NULL;
-    v = (NETDATA_DOUBLE)str2ndd(value, &endptr);
+    v = (NETDATA_DOUBLE) str2ndd_encoded(value, &endptr);
     if (unlikely(endptr && *endptr)) {
         if (endptr == value)
             error("PLUGINSD: 'host:%s/chart:%s' the value '%s' of VARIABLE '%s' cannot be parsed as a number",
@@ -1162,13 +1162,13 @@ PARSER_RC pluginsd_replay_begin(char **words, size_t num_words, void *user) {
     pluginsd_set_chart_from_parent(user, st, PLUGINSD_KEYWORD_REPLAY_BEGIN);
 
     if(start_time_str && end_time_str) {
-        time_t start_time = (time_t)str2ul(start_time_str);
-        time_t end_time = (time_t)str2ul(end_time_str);
+        time_t start_time = (time_t) str2ull_encoded(start_time_str);
+        time_t end_time = (time_t) str2ull_encoded(end_time_str);
 
         time_t wall_clock_time = 0, tolerance;
         bool wall_clock_comes_from_child; (void)wall_clock_comes_from_child;
         if(child_now_str) {
-            wall_clock_time = (time_t)str2ul(child_now_str);
+            wall_clock_time = (time_t) str2ull_encoded(child_now_str);
             tolerance = st->update_every + 1;
             wall_clock_comes_from_child = true;
         }
@@ -1314,7 +1314,7 @@ PARSER_RC pluginsd_replay_set(char **words, size_t num_words, void *user)
         RRDDIM_FLAGS rd_flags = rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE | RRDDIM_FLAG_ARCHIVED);
 
         if(!(rd_flags & RRDDIM_FLAG_ARCHIVED)) {
-            NETDATA_DOUBLE value = strtondd(value_str, NULL);
+            NETDATA_DOUBLE value = str2ndd_encoded(value_str, NULL);
             SN_FLAGS flags = pluginsd_parse_storage_number_flags(flags_str);
 
             if (!netdata_double_isnumber(value) || (flags == SN_EMPTY_SLOT)) {
@@ -1358,15 +1358,15 @@ PARSER_RC pluginsd_replay_rrddim_collection_state(char **words, size_t num_words
     if(!rd) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
 
     usec_t dim_last_collected_ut = (usec_t)rd->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)rd->last_collected_time.tv_usec;
-    usec_t last_collected_ut = last_collected_ut_str ? str2ull(last_collected_ut_str) : 0;
+    usec_t last_collected_ut = last_collected_ut_str ? str2ull_encoded(last_collected_ut_str) : 0;
     if(last_collected_ut > dim_last_collected_ut) {
         rd->last_collected_time.tv_sec = (time_t)(last_collected_ut / USEC_PER_SEC);
         rd->last_collected_time.tv_usec = (last_collected_ut % USEC_PER_SEC);
     }
 
-    rd->last_collected_value = last_collected_value_str ? str2ll(last_collected_value_str, NULL) : 0;
-    rd->last_calculated_value = last_calculated_value_str ? str2ndd(last_calculated_value_str, NULL) : 0;
-    rd->last_stored_value = last_stored_value_str ? str2ndd(last_stored_value_str, NULL) : 0.0;
+    rd->last_collected_value = last_collected_value_str ? str2ll_encoded(last_collected_value_str) : 0;
+    rd->last_calculated_value = last_calculated_value_str ? str2ndd_encoded(last_calculated_value_str, NULL) : 0;
+    rd->last_stored_value = last_stored_value_str ? str2ndd_encoded(last_stored_value_str, NULL) : 0.0;
 
     return PARSER_RC_OK;
 }
@@ -1386,14 +1386,14 @@ PARSER_RC pluginsd_replay_rrdset_collection_state(char **words, size_t num_words
     if(!st) return PLUGINSD_DISABLE_PLUGIN(user, NULL, NULL);
 
     usec_t chart_last_collected_ut = (usec_t)st->last_collected_time.tv_sec * USEC_PER_SEC + (usec_t)st->last_collected_time.tv_usec;
-    usec_t last_collected_ut = last_collected_ut_str ? str2ull(last_collected_ut_str) : 0;
+    usec_t last_collected_ut = last_collected_ut_str ? str2ull_encoded(last_collected_ut_str) : 0;
     if(last_collected_ut > chart_last_collected_ut) {
         st->last_collected_time.tv_sec = (time_t)(last_collected_ut / USEC_PER_SEC);
         st->last_collected_time.tv_usec = (last_collected_ut % USEC_PER_SEC);
     }
 
     usec_t chart_last_updated_ut = (usec_t)st->last_updated.tv_sec * USEC_PER_SEC + (usec_t)st->last_updated.tv_usec;
-    usec_t last_updated_ut = last_updated_ut_str ? str2ull(last_updated_ut_str) : 0;
+    usec_t last_updated_ut = last_updated_ut_str ? str2ull_encoded(last_updated_ut_str) : 0;
     if(last_updated_ut > chart_last_updated_ut) {
         st->last_updated.tv_sec = (time_t)(last_updated_ut / USEC_PER_SEC);
         st->last_updated.tv_usec = (last_updated_ut % USEC_PER_SEC);
@@ -1420,16 +1420,17 @@ PARSER_RC pluginsd_replay_end(char **words, size_t num_words, void *user)
     const char *last_entry_requested_txt = get_word(words, num_words, 6);
     const char *child_world_time_txt = get_word(words, num_words, 7); // optional
 
-    time_t update_every_child = (time_t)str2ul(update_every_child_txt);
-    time_t first_entry_child = (time_t)str2ul(first_entry_child_txt);
-    time_t last_entry_child = (time_t)str2ul(last_entry_child_txt);
+    time_t update_every_child = (time_t) str2ull_encoded(update_every_child_txt);
+    time_t first_entry_child = (time_t) str2ull_encoded(first_entry_child_txt);
+    time_t last_entry_child = (time_t) str2ull_encoded(last_entry_child_txt);
 
     bool start_streaming = (strcmp(start_streaming_txt, "true") == 0);
-    time_t first_entry_requested = (time_t)str2ul(first_entry_requested_txt);
-    time_t last_entry_requested = (time_t)str2ul(last_entry_requested_txt);
+    time_t first_entry_requested = (time_t) str2ull_encoded(first_entry_requested_txt);
+    time_t last_entry_requested = (time_t) str2ull_encoded(last_entry_requested_txt);
 
     // the optional child world time
-    time_t child_world_time = (child_world_time_txt && *child_world_time_txt) ? (time_t)str2ul(child_world_time_txt) : now_realtime_sec();
+    time_t child_world_time = (child_world_time_txt && *child_world_time_txt) ? (time_t) str2ull_encoded(
+            child_world_time_txt) : now_realtime_sec();
 
     PARSER_USER_OBJECT *user_object = user;
 
@@ -1543,14 +1544,14 @@ PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user) {
     // ------------------------------------------------------------------------
     // parse the parameters
 
-    time_t update_every = (time_t)str2ull_hex_or_dec(update_every_str);
-    time_t end_time = (time_t)str2ull_hex_or_dec(end_time_str);
+    time_t update_every = (time_t) str2ull_encoded(update_every_str);
+    time_t end_time = (time_t) str2ull_encoded(end_time_str);
 
     time_t wall_clock_time;
     if(likely(*wall_clock_time_str == '#'))
         wall_clock_time = end_time;
     else
-        wall_clock_time = (time_t)str2ull_hex_or_dec(wall_clock_time_str);
+        wall_clock_time = (time_t) str2ull_encoded(wall_clock_time_str);
 
     if (unlikely(update_every != st->update_every))
         rrdset_set_update_every_s(st, update_every);
@@ -1577,6 +1578,10 @@ PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user) {
         u->v2.stream_buffer = rrdset_push_metric_initialize(u->st, wall_clock_time);
 
     if(u->v2.stream_buffer.v2 && u->v2.stream_buffer.wb) {
+        // check if receiver and sender have the same number parsing capabilities
+        bool can_copy = stream_has_capability(u, STREAM_CAP_IEEE754) == stream_has_capability(&u->v2.stream_buffer, STREAM_CAP_IEEE754);
+        NUMBER_ENCODING encoding = stream_has_capability(&u->v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
+
         BUFFER *wb = u->v2.stream_buffer.wb;
 
         buffer_need_bytes(wb, 1024);
@@ -1587,11 +1592,26 @@ PARSER_RC pluginsd_begin_v2(char **words, size_t num_words, void *user) {
         buffer_fast_strcat(wb, PLUGINSD_KEYWORD_BEGIN_V2 " '", sizeof(PLUGINSD_KEYWORD_BEGIN_V2) - 1 + 2);
         buffer_fast_strcat(wb, rrdset_id(st), string_strlen(st->id));
         buffer_fast_strcat(wb, "' ", 2);
-        buffer_strcat(wb, update_every_str);
+
+        if(can_copy)
+            buffer_strcat(wb, update_every_str);
+        else
+            buffer_print_uint64_encoded(wb, encoding, update_every);
+
         buffer_fast_strcat(wb, " ", 1);
-        buffer_strcat(wb, end_time_str);
+
+        if(can_copy)
+            buffer_strcat(wb, end_time_str);
+        else
+            buffer_print_uint64_encoded(wb, encoding, end_time);
+
         buffer_fast_strcat(wb, " ", 1);
-        buffer_strcat(wb, wall_clock_time_str);
+
+        if(can_copy)
+            buffer_strcat(wb, wall_clock_time_str);
+        else
+            buffer_print_uint64_encoded(wb, encoding, wall_clock_time);
+
         buffer_fast_strcat(wb, "\n", 1);
 
         u->v2.stream_buffer.last_point_end_time_s = end_time;
@@ -1652,13 +1672,13 @@ PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user) {
     // ------------------------------------------------------------------------
     // parse the parameters
 
-    collected_number collected_value = (collected_number)str2ll_hex_or_dec(collected_str);
+    collected_number collected_value = (collected_number) str2ll_encoded(collected_str);
 
     NETDATA_DOUBLE value;
     if(*value_str == '#')
         value = (NETDATA_DOUBLE)collected_value;
     else
-        value = strtondd(value_str, NULL);
+        value = str2ndd_encoded(value_str, NULL);
 
     SN_FLAGS flags = pluginsd_parse_storage_number_flags(flags_str);
 
@@ -1689,14 +1709,25 @@ PARSER_RC pluginsd_set_v2(char **words, size_t num_words, void *user) {
     // propagate it forward in v2
 
     if(u->v2.stream_buffer.v2 && u->v2.stream_buffer.begin_v2_added && u->v2.stream_buffer.wb) {
+        // check if receiver and sender have the same number parsing capabilities
+        bool can_copy = stream_has_capability(u, STREAM_CAP_IEEE754) == stream_has_capability(&u->v2.stream_buffer, STREAM_CAP_IEEE754);
+        NUMBER_ENCODING integer_encoding = stream_has_capability(&u->v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
+        NUMBER_ENCODING doubles_encoding = stream_has_capability(&u->v2.stream_buffer, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
+
         BUFFER *wb = u->v2.stream_buffer.wb;
         buffer_need_bytes(wb, 1024);
         buffer_fast_strcat(wb, PLUGINSD_KEYWORD_SET_V2 " '", sizeof(PLUGINSD_KEYWORD_SET_V2) - 1 + 2);
         buffer_fast_strcat(wb, rrddim_id(rd), string_strlen(rd->id));
         buffer_fast_strcat(wb, "' ", 2);
-        buffer_strcat(wb, collected_str);
+        if(can_copy)
+            buffer_strcat(wb, collected_str);
+        else
+            buffer_print_int64_encoded(wb, integer_encoding, collected_value); // original v2 had hex
         buffer_fast_strcat(wb, " ", 1);
-        buffer_strcat(wb, value_str);
+        if(can_copy)
+            buffer_strcat(wb, value_str);
+        else
+            buffer_print_netdata_double_encoded(wb, doubles_encoding, value); // original v2 had decimal
         buffer_fast_strcat(wb, " ", 1);
         buffer_print_sn_flags(wb, flags, true);
         buffer_fast_strcat(wb, "\n", 1);

collectors/plugins.d/pluginsd_parser.h

@@ -22,6 +22,8 @@ typedef struct parser_user_object {
     size_t data_collections_count;
     int enabled;
 
+    STREAM_CAPABILITIES capabilities; // receiver capabilities
+
     struct {
         bool parsing_host;
         uuid_t machine_guid;
@@ -44,7 +46,7 @@ typedef struct parser_user_object {
 
     struct parser_user_object_v2 {
         bool locked_data_collection;
-        RRDSET_STREAM_BUFFER stream_buffer;
+        RRDSET_STREAM_BUFFER stream_buffer; // sender capabilities in this
         time_t update_every;
         time_t end_time;
         time_t wall_clock_time;

collectors/proc.plugin/ipc.c

@@ -212,7 +212,7 @@ int ipc_msq_get_info(char *msg_filename, struct message_queue **message_queue_ro
         // find the id in the linked list or create a new structure
         int found = 0;
 
-        unsigned long long id = str2ull(procfile_lineword(ff, l, 1));
+        unsigned long long id = str2ull(procfile_lineword(ff, l, 1), NULL);
         for(msq = *message_queue_root; msq ; msq = msq->next) {
             if(unlikely(id == msq->id)) {
                 found = 1;
@@ -227,8 +227,8 @@ int ipc_msq_get_info(char *msg_filename, struct message_queue **message_queue_ro
             msq->id = id;
         }
 
-        msq->messages = str2ull(procfile_lineword(ff, l, 4));
-        msq->bytes    = str2ull(procfile_lineword(ff, l, 3));
+        msq->messages = str2ull(procfile_lineword(ff, l, 4), NULL);
+        msq->bytes    = str2ull(procfile_lineword(ff, l, 3), NULL);
         msq->found    = 1;
     }
 
@@ -268,7 +268,7 @@ int ipc_shm_get_info(char *shm_filename, struct shm_stats *shm) {
         }
 
         shm->segments++;
-        shm->bytes += str2ull(procfile_lineword(ff, l, 3));
+        shm->bytes += str2ull(procfile_lineword(ff, l, 3), NULL);
     }
 
     return 0;

collectors/proc.plugin/proc_diskstats.c

@@ -993,35 +993,35 @@ int do_proc_diskstats(int update_every, usec_t dt) {
 
         // # of reads completed # of writes completed
         // This is the total number of reads or writes completed successfully.
-        reads           = str2ull(procfile_lineword(ff, l, 3));  // rd_ios
-        writes          = str2ull(procfile_lineword(ff, l, 7));  // wr_ios
+        reads           = str2ull(procfile_lineword(ff, l, 3), NULL);  // rd_ios
+        writes          = str2ull(procfile_lineword(ff, l, 7), NULL);  // wr_ios
 
         // # of reads merged # of writes merged
         // Reads and writes which are adjacent to each other may be merged for
         // efficiency.  Thus two 4K reads may become one 8K read before it is
         // ultimately handed to the disk, and so it will be counted (and queued)
-        mreads          = str2ull(procfile_lineword(ff, l, 4));  // rd_merges_or_rd_sec
-        mwrites         = str2ull(procfile_lineword(ff, l, 8));  // wr_merges
+        mreads          = str2ull(procfile_lineword(ff, l, 4), NULL);  // rd_merges_or_rd_sec
+        mwrites         = str2ull(procfile_lineword(ff, l, 8), NULL);  // wr_merges
 
         // # of sectors read # of sectors written
         // This is the total number of sectors read or written successfully.
-        readsectors     = str2ull(procfile_lineword(ff, l, 5));  // rd_sec_or_wr_ios
-        writesectors    = str2ull(procfile_lineword(ff, l, 9));  // wr_sec
+        readsectors     = str2ull(procfile_lineword(ff, l, 5), NULL);  // rd_sec_or_wr_ios
+        writesectors    = str2ull(procfile_lineword(ff, l, 9), NULL);  // wr_sec
 
         // # of milliseconds spent reading # of milliseconds spent writing
         // This is the total number of milliseconds spent by all reads or writes (as
         // measured from __make_request() to end_that_request_last()).
-        readms          = str2ull(procfile_lineword(ff, l, 6));  // rd_ticks_or_wr_sec
-        writems         = str2ull(procfile_lineword(ff, l, 10)); // wr_ticks
+        readms          = str2ull(procfile_lineword(ff, l, 6), NULL);  // rd_ticks_or_wr_sec
+        writems         = str2ull(procfile_lineword(ff, l, 10), NULL); // wr_ticks
 
         // # of I/Os currently in progress
         // The only field that should go to zero. Incremented as requests are
         // given to appropriate struct request_queue and decremented as they finish.
-        queued_ios      = str2ull(procfile_lineword(ff, l, 11)); // ios_pgr
+        queued_ios      = str2ull(procfile_lineword(ff, l, 11), NULL); // ios_pgr
 
         // # of milliseconds spent doing I/Os
         // This field increases so long as field queued_ios is nonzero.
-        busy_ms         = str2ull(procfile_lineword(ff, l, 12)); // tot_ticks
+        busy_ms         = str2ull(procfile_lineword(ff, l, 12), NULL); // tot_ticks
 
         // weighted # of milliseconds spent doing I/Os
         // This field is incremented at each I/O start, I/O completion, I/O
@@ -1029,27 +1029,27 @@ int do_proc_diskstats(int update_every, usec_t dt) {
         // (field queued_ios) times the number of milliseconds spent doing I/O since the
         // last update of this field.  This can provide an easy measure of both
         // I/O completion time and the backlog that may be accumulating.
-        backlog_ms      = str2ull(procfile_lineword(ff, l, 13)); // rq_ticks
+        backlog_ms      = str2ull(procfile_lineword(ff, l, 13), NULL); // rq_ticks
 
         if (unlikely(words > 13)) {
             do_dc_stats = 1;
 
             // # of discards completed
             // This is the total number of discards completed successfully.
-            discards       = str2ull(procfile_lineword(ff, l, 14)); // dc_ios
+            discards       = str2ull(procfile_lineword(ff, l, 14), NULL); // dc_ios
 
             // # of discards merged
             // See the description of mreads/mwrites
-            mdiscards      = str2ull(procfile_lineword(ff, l, 15)); // dc_merges
+            mdiscards      = str2ull(procfile_lineword(ff, l, 15), NULL); // dc_merges
 
             // # of sectors discarded
             // This is the total number of sectors discarded successfully.
-            discardsectors = str2ull(procfile_lineword(ff, l, 16)); // dc_sec
+            discardsectors = str2ull(procfile_lineword(ff, l, 16), NULL); // dc_sec
 
             // # of milliseconds spent discarding
             // This is the total number of milliseconds spent by all discards (as
             // measured from __make_request() to end_that_request_last()).
-            discardms      = str2ull(procfile_lineword(ff, l, 17)); // dc_ticks
+            discardms      = str2ull(procfile_lineword(ff, l, 17), NULL); // dc_ticks
         }
 
         if (unlikely(words > 17)) {
@@ -1059,10 +1059,10 @@ int do_proc_diskstats(int update_every, usec_t dt) {
             // These values increment when an flush I/O request completes.
             // Block layer combines flush requests and executes at most one at a time.
             // This counts flush requests executed by disk. Not tracked for partitions.
-            flushes        = str2ull(procfile_lineword(ff, l, 18)); // fl_ios
+            flushes        = str2ull(procfile_lineword(ff, l, 18), NULL); // fl_ios
 
             // total wait time for flush requests
-            flushms        = str2ull(procfile_lineword(ff, l, 19)); // fl_ticks
+            flushms        = str2ull(procfile_lineword(ff, l, 19), NULL); // fl_ticks
         }
 
         // --------------------------------------------------------------------------

collectors/proc.plugin/proc_interrupts.c

@@ -120,7 +120,7 @@ int do_proc_interrupts(int update_every, usec_t dt) {
         int c;
         for(c = 0; c < cpus ;c++) {
             if(likely((c + 1) < (int)words))
-                irr->cpu[c].value = str2ull(procfile_lineword(ff, l, (uint32_t)(c + 1)));
+                irr->cpu[c].value = str2ull(procfile_lineword(ff, l, (uint32_t) (c + 1)), NULL);
             else
                 irr->cpu[c].value = 0;
 

collectors/proc.plugin/proc_loadavg.c

@@ -45,7 +45,7 @@ int do_proc_loadavg(int update_every, usec_t dt) {
     double load15 = strtod(procfile_lineword(ff, 0, 2), NULL);
 
     //unsigned long long running_processes  = str2ull(procfile_lineword(ff, 0, 3));
-    unsigned long long active_processes     = str2ull(procfile_lineword(ff, 0, 4));
+    unsigned long long active_processes     = str2ull(procfile_lineword(ff, 0, 4), NULL);
     
     //get system pid_max
     unsigned long long max_processes        = get_system_pid_max();