Add HTTP and HTTPS support to the simple exporting connector (#9911)

.gitignore

@@ -146,6 +146,8 @@ cmake-build-release/
 CMakeCache.txt
 CMakeFiles/
 cmake_install.cmake
+.cmake
+compile_commands.json
 
 # jetbrains IDE
 .jetbrains*

CMakeLists.txt

@@ -1270,6 +1270,7 @@ endif()
         -Wl,--wrap=connect_to_one_of
         -Wl,--wrap=create_main_rusage_chart
         -Wl,--wrap=send_main_rusage
+        -Wl,--wrap=simple_connector_end_batch
         ${PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS}
         ${KINESIS_LINK_OPTIONS}
         ${PUBSUB_LINK_OPTIONS}

Makefile.am

@@ -955,6 +955,7 @@ if ENABLE_UNITTESTS
         -Wl,--wrap=connect_to_one_of \
         -Wl,--wrap=create_main_rusage_chart \
         -Wl,--wrap=send_main_rusage \
+        -Wl,--wrap=simple_connector_end_batch \
         $(TEST_LDFLAGS) \
         $(NULL)
     exporting_tests_exporting_engine_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)

backends/backends.c

@@ -551,7 +551,7 @@ void *backends_main(void *ptr) {
         case BACKEND_TYPE_OPENTSDB_USING_HTTP: {
 #ifdef ENABLE_HTTPS
             if (!strcmp(type, "opentsdb:https")) {
-                security_start_ssl(NETDATA_SSL_CONTEXT_OPENTSDB);
+                security_start_ssl(NETDATA_SSL_CONTEXT_EXPORTING);
             }
 #endif
             backend_set_opentsdb_http_variables(&default_port,&backend_response_checker,&backend_request_formatter);
@@ -1001,9 +1001,9 @@ void *backends_main(void *ptr) {
                 sock = connect_to_one_of(destination, default_port, &timeout, &reconnects, NULL, 0);
 #ifdef ENABLE_HTTPS
                 if(sock != -1) {
-                    if(netdata_opentsdb_ctx) {
+                    if(netdata_exporting_ctx) {
                         if(!opentsdb_ssl.conn) {
-                            opentsdb_ssl.conn = SSL_new(netdata_opentsdb_ctx);
+                            opentsdb_ssl.conn = SSL_new(netdata_exporting_ctx);
                             if(!opentsdb_ssl.conn) {
                                 error("Failed to allocate SSL structure %d.", sock);
                                 opentsdb_ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
@@ -1229,7 +1229,7 @@ cleanup:
     buffer_free(response);
 
 #ifdef ENABLE_HTTPS
-    if(netdata_opentsdb_ctx) {
+    if(netdata_exporting_ctx) {
         if(opentsdb_ssl.conn) {
             SSL_free(opentsdb_ssl.conn);
         }

exporting/README.md

@@ -44,7 +44,7 @@ X seconds (though, it can send them per second if you need it to).
         also be configured). Learn more in our guide to [export and visualize Netdata metrics in
         Graphite](/docs/guides/export/export-netdata-metrics-graphite.md).
     -   [**JSON** document databases](/exporting/json/README.md)
-    -   [**OpenTSDB**](/exporting/opentsdb/README.md): Use a plaintext, HTTP, or HTTPS interfaces. Metrics are sent to
+    -   [**OpenTSDB**](/exporting/opentsdb/README.md): Use a plaintext or HTTP interfaces. Metrics are sent to
         OpenTSDB as `prefix.chart.dimension` with tag `host=hostname`.
     -   [**MongoDB**](/exporting/mongodb/README.md): Metrics are sent to the database in `JSON` format.
     -   [**Prometheus**](/exporting/prometheus/README.md): Use an existing Prometheus installation to scrape metrics
@@ -173,8 +173,10 @@ You can configure each connector individually using the available [options](#opt
 -   `[prometheus:exporter]` defines settings for Prometheus exporter API queries (e.g.:
     `http://NODE:19999/api/v1/allmetrics?format=prometheus&help=yes&source=as-collected`).
 -   `[<type>:<name>]` keeps settings for a particular exporting connector instance, where:
-  -   `type` selects the exporting connector type: graphite | opentsdb:telnet | opentsdb:http | opentsdb:https |
-      prometheus_remote_write | json | kinesis | pubsub | mongodb
+  -   `type` selects the exporting connector type: graphite | opentsdb:telnet | opentsdb:http |
+      prometheus_remote_write | json | kinesis | pubsub | mongodb. For graphite, opentsdb,
+      json, and prometheus_remote_write connectors you can also use `:http` or `:https` modifiers
+      (e.g.: `opentsdb:https`).
   -   `name` can be arbitrary instance name you chose.
 
 ### Options
@@ -270,6 +272,13 @@ Configure individual connectors and override any global settings with the follow
 > You can check how the host tags were parsed using the /api/v1/info API call. But, keep in mind that backends subsystem
 > is deprecated and will be deleted soon. Please move your existing tags to the `[host labels]` section.
 
+## HTTPS
+
+Netdata can send metrics to external databases using the TLS/SSL protocol. Unfortunately, some of
+them does not support encrypted connections, so you will have to configure a reverse proxy to enable
+HTTPS communication between Netdata and an external database. You can set up a reverse proxy with
+[Nginx](/docs/Running-behind-nginx.md).
+
 ## Exporting engine monitoring
 
 Netdata creates five charts in the dashboard, under the **Netdata Monitoring** section, to help you monitor the health

exporting/aws_kinesis/aws_kinesis.c

@@ -48,7 +48,7 @@ int init_aws_kinesis_instance(struct instance *instance)
     instance->end_host_formatting = flush_host_labels;
     instance->end_batch_formatting = NULL;
 
-    instance->send_header = NULL;
+    instance->prepare_header = NULL;
     instance->check_response = NULL;
 
     instance->buffer = (void *)buffer_create(0);

exporting/clean_connectors.c

@@ -35,3 +35,46 @@ void clean_instance(struct instance *instance)
     uv_cond_destroy(&instance->cond_var);
     // uv_mutex_destroy(&instance->mutex);
 }
+
+/**
+ * Clean up a simple connector instance on Netdata exit
+ *
+ * @param instance an instance data structure.
+ */
+void simple_connector_cleanup(struct instance *instance)
+{
+    info("EXPORTING: cleaning up instance %s ...", instance->config.name);
+
+    struct simple_connector_data *simple_connector_data =
+        (struct simple_connector_data *)instance->connector_specific_data;
+
+    buffer_free(instance->buffer);
+    buffer_free(simple_connector_data->buffer);
+    buffer_free(simple_connector_data->header);
+
+    struct simple_connector_buffer *next_buffer = simple_connector_data->first_buffer;
+    for (int i = 0; i < instance->config.buffer_on_failures; i++) {
+        struct simple_connector_buffer *current_buffer = next_buffer;
+        next_buffer = next_buffer->next;
+
+        buffer_free(current_buffer->header);
+        buffer_free(current_buffer->buffer);
+        freez(current_buffer);
+    }
+
+#ifdef ENABLE_HTTPS
+    if (simple_connector_data->conn)
+        SSL_free(simple_connector_data->conn);
+#endif
+
+    freez(simple_connector_data);
+
+    struct simple_connector_config *simple_connector_config =
+        (struct simple_connector_config *)instance->config.connector_specific_config;
+    freez(simple_connector_config);
+
+    info("EXPORTING: instance %s exited", instance->config.name);
+    instance->exited = 1;
+
+    return;
+}

exporting/exporting_engine.h

@@ -46,10 +46,12 @@ typedef enum exporting_options {
 typedef enum exporting_connector_types {
     EXPORTING_CONNECTOR_TYPE_UNKNOWN,                 // Invalid type
     EXPORTING_CONNECTOR_TYPE_GRAPHITE,                // Send plain text to Graphite
-    EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_TELNET,   // Send data to OpenTSDB using telnet API
-    EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP,     // Send data to OpenTSDB using HTTP API
-    EXPORTING_CONNECTOR_TYPE_JSON,                    // Stores the data using JSON.
-    EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE, // The user selected to use Prometheus backend
+    EXPORTING_CONNECTOR_TYPE_GRAPHITE_HTTP,           // Send data to Graphite using HTTP API
+    EXPORTING_CONNECTOR_TYPE_JSON,                    // Send data in JSON format
+    EXPORTING_CONNECTOR_TYPE_JSON_HTTP,               // Send data in JSON format using HTTP API
+    EXPORTING_CONNECTOR_TYPE_OPENTSDB,                // Send data to OpenTSDB using telnet API
+    EXPORTING_CONNECTOR_TYPE_OPENTSDB_HTTP,           // Send data to OpenTSDB using HTTP API
+    EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE, // User selected to use Prometheus backend
     EXPORTING_CONNECTOR_TYPE_KINESIS,                 // Send message to AWS Kinesis
     EXPORTING_CONNECTOR_TYPE_PUBSUB,                  // Send message to Google Cloud Pub/Sub
     EXPORTING_CONNECTOR_TYPE_MONGODB,                 // Send data to MongoDB collection
@@ -81,6 +83,38 @@ struct simple_connector_config {
     int default_port;
 };
 
+struct simple_connector_buffer {
+    BUFFER *header;
+    BUFFER *buffer;
+
+    size_t buffered_metrics;
+    size_t buffered_bytes;
+
+    int used;
+
+    struct simple_connector_buffer *next;
+};
+
+struct simple_connector_data {
+    void *connector_specific_data;
+
+    size_t total_buffered_metrics;
+
+    BUFFER *header;
+    BUFFER *buffer;
+    size_t buffered_metrics;
+    size_t buffered_bytes;
+
+    struct simple_connector_buffer *previous_buffer;
+    struct simple_connector_buffer *first_buffer;
+    struct simple_connector_buffer *last_buffer;
+
+#ifdef ENABLE_HTTPS
+    SSL *conn; //SSL connection
+    int flags; //The flags for SSL connection
+#endif
+};
+
 struct prometheus_remote_write_specific_config {
     char *remote_write_path;
 };
@@ -175,7 +209,7 @@ struct instance {
     int (*end_host_formatting)(struct instance *instance, RRDHOST *host);
     int (*end_batch_formatting)(struct instance *instance);
 
-    int (*send_header)(int *sock, struct instance *instance);
+    void (*prepare_header)(struct instance *instance);
     int (*check_response)(BUFFER *buffer, struct instance *instance);
 
     void *connector_specific_data;
@@ -210,6 +244,7 @@ struct engine *read_exporting_config();
 EXPORTING_CONNECTOR_TYPE exporting_select_type(const char *type);
 
 int init_connectors(struct engine *engine);
+void simple_connector_init(struct instance *instance);
 
 int mark_scheduled_instances(struct engine *engine);
 void prepare_buffers(struct engine *engine);
@@ -232,11 +267,12 @@ void end_chart_formatting(struct engine *engine, RRDSET *st);
 void end_host_formatting(struct engine *engine, RRDHOST *host);
 void end_batch_formatting(struct engine *engine);
 int flush_host_labels(struct instance *instance, RRDHOST *host);
-int simple_connector_update_buffered_bytes(struct instance *instance);
+int simple_connector_end_batch(struct instance *instance);
 
 int exporting_discard_response(BUFFER *buffer, struct instance *instance);
 void simple_connector_receive_response(int *sock, struct instance *instance);
-void simple_connector_send_buffer(int *sock, int *failures, struct instance *instance);
+void simple_connector_send_buffer(
+    int *sock, int *failures, struct instance *instance, BUFFER *header, BUFFER *buffer, size_t buffered_metrics);
 void simple_connector_worker(void *instance_p);
 
 void create_main_rusage_chart(RRDSET **st_rusage, RRDDIM **rd_user, RRDDIM **rd_system);
@@ -244,6 +280,7 @@ void send_main_rusage(RRDSET *st_rusage, RRDDIM *rd_user, RRDDIM *rd_system);
 void send_internal_metrics(struct instance *instance);
 
 extern void clean_instance(struct instance *ptr);
+void simple_connector_cleanup(struct instance *instance);
 
 static inline void disable_instance(struct instance *instance)
 {

exporting/graphite/README.md

@@ -3,6 +3,7 @@ title: "Export metrics to Graphite providers"
 sidebar_label: Graphite
 description: "Archive your Agent's metrics to a any Graphite database provider for long-term storage, further analysis, or correlation with data from other sources."
 custom_edit_url: https://github.com/netdata/netdata/edit/master/exporting/graphite/README.md
+sidebar_label: Graphite
 -->
 
 # Export metrics to Graphite providers
@@ -21,6 +22,9 @@ directory and set the following options:
     destination = localhost:2003
 ```
 
+Add `:http` or `:https` modifiers to the connector type if you need to use other than a plaintext protocol. For example: `graphite:http:my_graphite_instance`,
+`graphite:https:my_graphite_instance`.
+
 The Graphite connector is further configurable using additional settings. See the [exporting reference
 doc](/exporting/README.md#options) for details.
 

exporting/graphite/graphite.c

@@ -16,6 +16,17 @@ int init_graphite_instance(struct instance *instance)
     instance->config.connector_specific_config = (void *)connector_specific_config;
     connector_specific_config->default_port = 2003;
 
+    struct simple_connector_data *connector_specific_data = callocz(1, sizeof(struct simple_connector_data));
+    instance->connector_specific_data = connector_specific_data;
+
+#ifdef ENABLE_HTTPS
+    connector_specific_data->flags = NETDATA_SSL_START;
+    connector_specific_data->conn = NULL;
+    if (instance->config.options & EXPORTING_OPTION_USE_TLS) {
+        security_start_ssl(NETDATA_SSL_CONTEXT_EXPORTING);
+    }
+#endif
+
     instance->start_batch_formatting = NULL;
     instance->start_host_formatting = format_host_labels_graphite_plaintext;
     instance->start_chart_formatting = NULL;
@@ -27,9 +38,13 @@ int init_graphite_instance(struct instance *instance)
 
     instance->end_chart_formatting = NULL;
     instance->end_host_formatting = flush_host_labels;
-    instance->end_batch_formatting = simple_connector_update_buffered_bytes;
+    instance->end_batch_formatting = simple_connector_end_batch;
+
+    if (instance->config.type == EXPORTING_CONNECTOR_TYPE_GRAPHITE_HTTP)
+        instance->prepare_header = graphite_http_prepare_header;
+    else
+        instance->prepare_header = NULL;
 
-    instance->send_header = NULL;
     instance->check_response = exporting_discard_response;
 
     instance->buffer = (void *)buffer_create(0);
@@ -37,6 +52,9 @@ int init_graphite_instance(struct instance *instance)
         error("EXPORTING: cannot create buffer for graphite exporting connector instance %s", instance->config.name);
         return 1;
     }
+
+    simple_connector_init(instance);
+
     if (uv_mutex_init(&instance->mutex))
         return 1;
     if (uv_cond_init(&instance->cond_var))
@@ -187,3 +205,26 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
 
     return 0;
 }
+
+/**
+ * Ppepare HTTP header
+ *
+ * @param instance an instance data structure.
+ * @return Returns 0 on success, 1 on failure.
+ */
+void graphite_http_prepare_header(struct instance *instance)
+{
+    struct simple_connector_data *simple_connector_data = instance->connector_specific_data;
+
+    buffer_sprintf(
+        simple_connector_data->last_buffer->header,
+        "POST /api/put HTTP/1.1\r\n"
+        "Host: %s\r\n"
+        "Content-Type: application/graphite\r\n"
+        "Content-Length: %lu\r\n"
+        "\r\n",
+        instance->config.destination,
+        buffer_strlen(simple_connector_data->last_buffer->buffer));
+
+    return;
+}