Просмотр исходного кода

added rrd memory mode "none" to allow netdata act as a proxy to forward metrics to another netdata

Costa Tsaousis (ktsaou) 8 лет назад
Родитель
Сommit
1958a34558
10 измененных файлов с 268 добавлено и 9 удалено
  1. 1 1
      CMakeLists.txt
  2. 2 0
      src/Makefile.am
  3. 2 0
      src/common.h
  4. 1 1
      src/main.c
  5. 5 0
      src/rrd.c
  6. 5 3
      src/rrd.h
  7. 3 2
      src/rrddim.c
  8. 5 2
      src/rrdset.c
  9. 231 0
      src/web_api_old.c
  10. 13 0
      src/web_api_old.h

+ 1 - 1
CMakeLists.txt

@@ -124,7 +124,7 @@ set(NETDATA_SOURCE_FILES
         src/web_client.h
         src/web_server.c
         src/web_server.h
-        src/rrdhost.c src/rrdfamily.c src/rrdset.c src/rrddim.c src/health_log.c src/health_config.c src/health_json.c src/rrdcalc.c src/rrdcalctemplate.c src/rrdvar.c src/rrddimvar.c src/rrdsetvar.c src/rrdpush.c src/rrdpush.h)
+        src/rrdhost.c src/rrdfamily.c src/rrdset.c src/rrddim.c src/health_log.c src/health_config.c src/health_json.c src/rrdcalc.c src/rrdcalctemplate.c src/rrdvar.c src/rrddimvar.c src/rrdsetvar.c src/rrdpush.c src/rrdpush.h src/web_api_old.c src/web_api_old.h src/web_api_v1.c src/web_api_v1.h)
 
 set(APPS_PLUGIN_SOURCE_FILES
         src/appconfig.c

+ 2 - 0
src/Makefile.am

@@ -79,6 +79,8 @@ netdata_SOURCES = \
 	storage_number.c storage_number.h \
 	unit_test.c unit_test.h \
 	url.c url.h \
+	web_api_old.c web_api_old.h \
+	web_api_v1.c web_api_v1.h \
 	web_buffer.c web_buffer.h \
 	web_buffer_svg.c web_buffer_svg.h \
 	web_client.c web_client.h \

+ 2 - 0
src/common.h

@@ -217,6 +217,8 @@
 #include "inlined.h"
 #include "adaptive_resortable_list.h"
 #include "rrdpush.h"
+#include "web_api_v1.h"
+#include "web_api_old.h"
 
 extern char *netdata_configured_config_dir;
 extern char *netdata_configured_log_dir;

+ 1 - 1
src/main.c

@@ -703,7 +703,7 @@ int main(int argc, char **argv) {
         // get default memory mode for the database
 
         if(rrdpush_exclusive) {
-            default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;
+            default_rrd_memory_mode = RRD_MEMORY_MODE_NONE;
             config_set("global", "memory mode", rrd_memory_mode_name(default_rrd_memory_mode));
         }
         else

+ 5 - 0
src/rrd.c

@@ -27,6 +27,9 @@ inline const char *rrd_memory_mode_name(RRD_MEMORY_MODE id) {
         case RRD_MEMORY_MODE_MAP:
             return RRD_MEMORY_MODE_MAP_NAME;
 
+        case RRD_MEMORY_MODE_NONE:
+            return RRD_MEMORY_MODE_NONE_NAME;
+
         case RRD_MEMORY_MODE_SAVE:
         default:
             return RRD_MEMORY_MODE_SAVE_NAME;
@@ -38,6 +41,8 @@ RRD_MEMORY_MODE rrd_memory_mode_id(const char *name) {
         return RRD_MEMORY_MODE_RAM;
     else if(unlikely(!strcmp(name, RRD_MEMORY_MODE_MAP_NAME)))
         return RRD_MEMORY_MODE_MAP;
+    else if(unlikely(!strcmp(name, RRD_MEMORY_MODE_NONE_NAME)))
+        return RRD_MEMORY_MODE_NONE;
 
     return RRD_MEMORY_MODE_SAVE;
 }

+ 5 - 3
src/rrd.h

@@ -39,11 +39,13 @@ const char *rrdset_type_name(RRDSET_TYPE chart_type);
 // memory mode
 
 typedef enum rrd_memory_mode {
-    RRD_MEMORY_MODE_RAM  = 0,
-    RRD_MEMORY_MODE_MAP  = 1,
-    RRD_MEMORY_MODE_SAVE = 2
+    RRD_MEMORY_MODE_NONE = 0,
+    RRD_MEMORY_MODE_RAM  = 1,
+    RRD_MEMORY_MODE_MAP  = 2,
+    RRD_MEMORY_MODE_SAVE = 3
 } RRD_MEMORY_MODE;
 
+#define RRD_MEMORY_MODE_NONE_NAME "none"
 #define RRD_MEMORY_MODE_RAM_NAME "ram"
 #define RRD_MEMORY_MODE_MAP_NAME "map"
 #define RRD_MEMORY_MODE_SAVE_NAME "save"

+ 3 - 2
src/rrddim.c

@@ -71,7 +71,7 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe
     rrdset_strncpyz_name(filename, id, FILENAME_MAX);
     snprintfz(fullfilename, FILENAME_MAX, "%s/%s.db", st->cache_dir, filename);
 
-    if(st->rrd_memory_mode != RRD_MEMORY_MODE_RAM) {
+    if(st->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
         rd = (RRDDIM *)mymmap(fullfilename, size, ((st->rrd_memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 1);
         if(likely(rd)) {
             // we have a file mapped for rd
@@ -132,7 +132,7 @@ RRDDIM *rrddim_add(RRDSET *st, const char *id, const char *name, collected_numbe
     if(unlikely(!rd)) {
         // if we didn't manage to get a mmap'd dimension, just create one
         rd = callocz(1, size);
-        rd->rrd_memory_mode = RRD_MEMORY_MODE_RAM;
+        rd->rrd_memory_mode = (st->rrd_memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_RAM;
     }
 
     rd->memsize = size;
@@ -243,6 +243,7 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
             munmap(rd, rd->memsize);
             break;
 
+        case RRD_MEMORY_MODE_NONE:
         case RRD_MEMORY_MODE_RAM:
             debug(D_RRD_CALLS, "Removing dimension '%s'.", rd->name);
             freez((void *)rd->id);

+ 5 - 2
src/rrdset.c

@@ -325,6 +325,9 @@ RRDSET *rrdset_create(RRDHOST *host, const char *type, const char *id, const cha
     long entries = align_entries_to_pagesize(rentries);
     if(entries != rentries) entries = config_set_number(config_section, "history", entries);
 
+    if(host->rrd_memory_mode == RRD_MEMORY_MODE_NONE && entries != rentries)
+        entries = config_set_number(config_section, "history", 10);
+
     int enabled = config_get_boolean(config_section, "enabled", 1);
     if(!enabled) entries = 5;
 
@@ -339,7 +342,7 @@ RRDSET *rrdset_create(RRDHOST *host, const char *type, const char *id, const cha
     debug(D_RRD_CALLS, "Creating RRD_STATS for '%s.%s'.", type, id);
 
     snprintfz(fullfilename, FILENAME_MAX, "%s/main.db", cache_dir);
-    if(host->rrd_memory_mode != RRD_MEMORY_MODE_RAM) {
+    if(host->rrd_memory_mode == RRD_MEMORY_MODE_SAVE || host->rrd_memory_mode == RRD_MEMORY_MODE_MAP) {
         st = (RRDSET *) mymmap(fullfilename, size, ((host->rrd_memory_mode == RRD_MEMORY_MODE_MAP) ? MAP_SHARED : MAP_PRIVATE), 0);
         if(st) {
             memset(&st->avl, 0, sizeof(avl));
@@ -405,7 +408,7 @@ RRDSET *rrdset_create(RRDHOST *host, const char *type, const char *id, const cha
 
     if(unlikely(!st)) {
         st = callocz(1, size);
-        st->rrd_memory_mode = RRD_MEMORY_MODE_RAM;
+        st->rrd_memory_mode = (host->rrd_memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_RAM;
     }
 
     st->config_section = strdup(config_section);

+ 231 - 0
src/web_api_old.c

@@ -0,0 +1,231 @@
+#include "common.h"
+
+int web_client_api_old_data_request(RRDHOST *host, struct web_client *w, char *url, int datasource_type) {
+    if(!url || !*url) {
+        buffer_flush(w->response.data);
+        buffer_sprintf(w->response.data, "Incomplete request.");
+        return 400;
+    }
+
+    RRDSET *st = NULL;
+
+    char *args = strchr(url, '?');
+    if(args) {
+        *args='\0';
+        args = &args[1];
+    }
+
+    // get the name of the data to show
+    char *tok = mystrsep(&url, "/");
+    if(!tok) tok = "";
+
+    // do we have such a data set?
+    if(*tok) {
+        debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
+        st = rrdset_find_byname(host, tok);
+        if(!st) st = rrdset_find(host, tok);
+    }
+
+    if(!st) {
+        // we don't have it
+        // try to send a file with that name
+        buffer_flush(w->response.data);
+        return(mysendfile(w, tok));
+    }
+
+    // we have it
+    debug(D_WEB_CLIENT, "%llu: Found RRD data with name '%s'.", w->id, tok);
+
+    // how many entries does the client want?
+    int lines = (int)st->entries;
+    int group_count = 1;
+    time_t after = 0, before = 0;
+    int group_method = GROUP_AVERAGE;
+    int nonzero = 0;
+
+    if(url) {
+        // parse the lines required
+        tok = mystrsep(&url, "/");
+        if(tok) lines = str2i(tok);
+        if(lines < 1) lines = 1;
+    }
+    if(url) {
+        // parse the group count required
+        tok = mystrsep(&url, "/");
+        if(tok && *tok) group_count = str2i(tok);
+        if(group_count < 1) group_count = 1;
+        //if(group_count > save_history / 20) group_count = save_history / 20;
+    }
+    if(url) {
+        // parse the grouping method required
+        tok = mystrsep(&url, "/");
+        if(tok && *tok) {
+            if(strcmp(tok, "max") == 0) group_method = GROUP_MAX;
+            else if(strcmp(tok, "average") == 0) group_method = GROUP_AVERAGE;
+            else if(strcmp(tok, "sum") == 0) group_method = GROUP_SUM;
+            else debug(D_WEB_CLIENT, "%llu: Unknown group method '%s'", w->id, tok);
+        }
+    }
+    if(url) {
+        // parse after time
+        tok = mystrsep(&url, "/");
+        if(tok && *tok) after = str2ul(tok);
+        if(after < 0) after = 0;
+    }
+    if(url) {
+        // parse before time
+        tok = mystrsep(&url, "/");
+        if(tok && *tok) before = str2ul(tok);
+        if(before < 0) before = 0;
+    }
+    if(url) {
+        // parse nonzero
+        tok = mystrsep(&url, "/");
+        if(tok && *tok && strcmp(tok, "nonzero") == 0) nonzero = 1;
+    }
+
+    w->response.data->contenttype = CT_APPLICATION_JSON;
+    buffer_flush(w->response.data);
+
+    char *google_version = "0.6";
+    char *google_reqId = "0";
+    char *google_sig = "0";
+    char *google_out = "json";
+    char *google_responseHandler = "google.visualization.Query.setResponse";
+    char *google_outFileName = NULL;
+    time_t last_timestamp_in_data = 0;
+    if(datasource_type == DATASOURCE_DATATABLE_JSON || datasource_type == DATASOURCE_DATATABLE_JSONP) {
+
+        w->response.data->contenttype = CT_APPLICATION_X_JAVASCRIPT;
+
+        while(args) {
+            tok = mystrsep(&args, "&");
+            if(tok && *tok) {
+                char *name = mystrsep(&tok, "=");
+                if(name && *name && strcmp(name, "tqx") == 0) {
+                    char *key = mystrsep(&tok, ":");
+                    char *value = mystrsep(&tok, ";");
+                    if(key && value && *key && *value) {
+                        if(strcmp(key, "version") == 0)
+                            google_version = value;
+
+                        else if(strcmp(key, "reqId") == 0)
+                            google_reqId = value;
+
+                        else if(strcmp(key, "sig") == 0)
+                            google_sig = value;
+
+                        else if(strcmp(key, "out") == 0)
+                            google_out = value;
+
+                        else if(strcmp(key, "responseHandler") == 0)
+                            google_responseHandler = value;
+
+                        else if(strcmp(key, "outFileName") == 0)
+                            google_outFileName = value;
+                    }
+                }
+            }
+        }
+
+        debug(D_WEB_CLIENT_ACCESS, "%llu: GOOGLE JSONP: version = '%s', reqId = '%s', sig = '%s', out = '%s', responseHandler = '%s', outFileName = '%s'",
+                w->id, google_version, google_reqId, google_sig, google_out, google_responseHandler, google_outFileName
+        );
+
+        if(datasource_type == DATASOURCE_DATATABLE_JSONP) {
+            last_timestamp_in_data = strtoul(google_sig, NULL, 0);
+
+            // check the client wants json
+            if(strcmp(google_out, "json") != 0) {
+                buffer_sprintf(w->response.data,
+                        "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'invalid_query',message:'output format is not supported',detailed_message:'the format %s requested is not supported by netdata.'}]});",
+                        google_responseHandler, google_version, google_reqId, google_out);
+                return 200;
+            }
+        }
+    }
+
+    if(datasource_type == DATASOURCE_DATATABLE_JSONP) {
+        buffer_sprintf(w->response.data,
+                "%s({version:'%s',reqId:'%s',status:'ok',sig:'%ld',table:",
+                google_responseHandler, google_version, google_reqId, st->last_updated.tv_sec);
+    }
+
+    debug(D_WEB_CLIENT_ACCESS, "%llu: Sending RRD data '%s' (id %s, %d lines, %d group, %d group_method, %ld after, %ld before).",
+            w->id, st->name, st->id, lines, group_count, group_method, after, before);
+
+    time_t timestamp_in_data = rrd_stats_json(datasource_type, st, w->response.data, lines, group_count, group_method, (unsigned long)after, (unsigned long)before, nonzero);
+
+    if(datasource_type == DATASOURCE_DATATABLE_JSONP) {
+        if(timestamp_in_data > last_timestamp_in_data)
+            buffer_strcat(w->response.data, "});");
+
+        else {
+            // the client already has the latest data
+            buffer_flush(w->response.data);
+            buffer_sprintf(w->response.data,
+                    "%s({version:'%s',reqId:'%s',status:'error',errors:[{reason:'not_modified',message:'Data not modified'}]});",
+                    google_responseHandler, google_version, google_reqId);
+        }
+    }
+
+    return 200;
+}
+
+inline int web_client_api_old_data_request_json(RRDHOST *host, struct web_client *w, char *url) {
+    return web_client_api_old_data_request(host, w, url, DATASOURCE_JSON);
+}
+
+inline int web_client_api_old_data_request_jsonp(RRDHOST *host, struct web_client *w, char *url) {
+    return web_client_api_old_data_request(host, w, url, DATASOURCE_DATATABLE_JSONP);
+}
+
+inline int web_client_api_old_graph_request(RRDHOST *host, struct web_client *w, char *url) {
+    // get the name of the data to show
+    char *tok = mystrsep(&url, "/?&");
+    if(tok && *tok) {
+        debug(D_WEB_CLIENT, "%llu: Searching for RRD data with name '%s'.", w->id, tok);
+
+        // do we have such a data set?
+        RRDSET *st = rrdset_find_byname(host, tok);
+        if(!st) st = rrdset_find(host, tok);
+        if(!st) {
+            // we don't have it
+            // try to send a file with that name
+            buffer_flush(w->response.data);
+            return mysendfile(w, tok);
+        }
+
+        debug(D_WEB_CLIENT_ACCESS, "%llu: Sending %s.json of RRD_STATS...", w->id, st->name);
+        w->response.data->contenttype = CT_APPLICATION_JSON;
+        buffer_flush(w->response.data);
+        rrd_stats_graph_json(st, url, w->response.data);
+        return 200;
+    }
+
+    buffer_flush(w->response.data);
+    buffer_strcat(w->response.data, "Graph name?\r\n");
+    return 400;
+}
+
+inline int web_client_api_old_list_request(RRDHOST *host, struct web_client *w, char *url) {
+    (void)url;
+
+    buffer_flush(w->response.data);
+    RRDSET *st;
+
+    rrdhost_rdlock(host);
+    rrdset_foreach_read(st, host) buffer_sprintf(w->response.data, "%s\n", st->name);
+    rrdhost_unlock(host);
+
+    return 200;
+}
+
+inline int web_client_api_old_all_json(RRDHOST *host, struct web_client *w, char *url) {
+    (void)url;
+
+    w->response.data->contenttype = CT_APPLICATION_JSON;
+    buffer_flush(w->response.data);
+    rrd_stats_all_json(host, w->response.data);
+    return 200;
+}

+ 13 - 0
src/web_api_old.h

@@ -0,0 +1,13 @@
+#ifndef NETDATA_WEB_API_OLD_H
+#define NETDATA_WEB_API_OLD_H
+
+#include "common.h"
+
+extern int web_client_api_old_data_request(RRDHOST *host, struct web_client *w, char *url, int datasource_type);
+extern int web_client_api_old_data_request_json(RRDHOST *host, struct web_client *w, char *url);
+extern int web_client_api_old_data_request_jsonp(RRDHOST *host, struct web_client *w, char *url);
+extern int web_client_api_old_graph_request(RRDHOST *host, struct web_client *w, char *url);
+extern int web_client_api_old_list_request(RRDHOST *host, struct web_client *w, char *url);
+extern int web_client_api_old_all_json(RRDHOST *host, struct web_client *w, char *url);
+
+#endif //NETDATA_WEB_API_OLD_H

Некоторые файлы не были показаны из-за большого количества измененных файлов