Streaming re-organization (#18941)

split streaming into multiple files

CMakeLists.txt

@@ -973,7 +973,7 @@ set(LIBH2O_FILES
 set(DAEMON_FILES
         src/daemon/buildinfo.c
         src/daemon/buildinfo.h
-        src/daemon/common.c
+        src/daemon/h2o-common.c
         src/daemon/common.h
         src/daemon/daemon.c
         src/daemon/daemon.h
@@ -1017,7 +1017,7 @@ set(H2O_FILES
         src/web/server/h2o/http_server.h
         src/web/server/h2o/h2o_utils.c
         src/web/server/h2o/h2o_utils.h
-        src/web/server/h2o/streaming.c
+        src/web/server/h2o/rrdpush.c
         src/web/server/h2o/streaming.h
         src/web/server/h2o/connlist.c
         src/web/server/h2o/connlist.h
@@ -1371,35 +1371,49 @@ set(SYSTEMD_JOURNAL_PLUGIN_FILES
 )
 
 set(STREAMING_PLUGIN_FILES
-        src/streaming/rrdpush.c
         src/streaming/rrdpush.h
-        src/streaming/compression.c
-        src/streaming/compression.h
-        src/streaming/compression_brotli.c
-        src/streaming/compression_brotli.h
-        src/streaming/compression_gzip.c
-        src/streaming/compression_gzip.h
-        src/streaming/compression_lz4.c
-        src/streaming/compression_lz4.h
-        src/streaming/compression_zstd.c
-        src/streaming/compression_zstd.h
+        src/streaming/stream-compression/compression.c
+        src/streaming/stream-compression/compression.h
+        src/streaming/stream-compression/brotli.c
+        src/streaming/stream-compression/brotli.h
+        src/streaming/stream-compression/gzip.c
+        src/streaming/stream-compression/gzip.h
+        src/streaming/stream-compression/lz4.c
+        src/streaming/stream-compression/lz4.h
+        src/streaming/stream-compression/zstd.c
+        src/streaming/stream-compression/zstd.h
         src/streaming/receiver.c
         src/streaming/sender.c
         src/streaming/replication.c
         src/streaming/replication.h
-        src/streaming/common.h
+        src/streaming/h2o-common.h
         src/streaming/protocol/command-nodeid.c
         src/streaming/protocol/commands.c
         src/streaming/protocol/commands.h
         src/streaming/protocol/command-claimed_id.c
-        src/streaming/stream_path.c
-        src/streaming/stream_path.h
-        src/streaming/stream_capabilities.c
-        src/streaming/stream_capabilities.h
-        src/streaming/sender_connect.c
-        src/streaming/sender_internals.h
-        src/streaming/sender_execute.c
-        src/streaming/sender_commit.c
+        src/streaming/stream-path.c
+        src/streaming/stream-path.h
+        src/streaming/stream-capabilities.c
+        src/streaming/stream-capabilities.h
+        src/streaming/sender-connect.c
+        src/streaming/sender-internals.h
+        src/streaming/sender-execute.c
+        src/streaming/sender-commit.c
+        src/streaming/sender-destinations.c
+        src/streaming/stream-handshake.c
+        src/streaming/protocol/command-function.c
+        src/streaming/protocol/command-host-labels.c
+        src/streaming/protocol/command-chart-definition.c
+        src/streaming/protocol/command-begin-set-end.c
+        src/streaming/protocol/command-host-variables.c
+        src/streaming/stream-conf.c
+        src/streaming/stream-conf.h
+        src/streaming/stream-handshake.h
+        src/streaming/sender.h
+        src/streaming/sender-destinations.h
+        src/streaming/rrdhost-status.c
+        src/streaming/rrdhost-status.h
+        src/streaming/receiver.h
 )
 
 set(WEB_PLUGIN_FILES

src/daemon/analytics.c

@@ -522,7 +522,7 @@ void analytics_gather_mutable_meta_data(void)
     analytics_alarms_notifications();
 
     analytics_set_data(
-        &analytics_data.netdata_config_is_parent, (rrdhost_hosts_available() > 1 || configured_as_parent()) ? "true" : "false");
+        &analytics_data.netdata_config_is_parent, (rrdhost_hosts_available() > 1 || stream_conf_configured_as_parent()) ? "true" : "false");
 
     analytics_set_data(&analytics_data.netdata_host_agent_claimed, is_agent_claimed() ? "true" : "false");
 
@@ -619,7 +619,7 @@ cleanup:
  */
 void set_late_analytics_variables(struct rrdhost_system_info *system_info)
 {
-    analytics_set_data(&analytics_data.netdata_config_stream_enabled, default_rrdpush_enabled ? "true" : "false");
+    analytics_set_data(&analytics_data.netdata_config_stream_enabled, stream_conf_send_enabled ? "true" : "false");
     analytics_set_data_str(&analytics_data.netdata_config_memory_mode, (char *)rrd_memory_mode_name(default_rrd_memory_mode));
     analytics_set_data(&analytics_data.netdata_host_cloud_enabled, "true");
 

src/daemon/analytics.h

@@ -76,6 +76,7 @@ struct analytics_data {
     bool exporting_enabled;
 };
 
+struct rrdhost_system_info;
 void set_late_analytics_variables(struct rrdhost_system_info *system_info);
 void analytics_free_data(void);
 void analytics_log_shell(void);

src/daemon/common.h

@@ -32,7 +32,6 @@
 // streaming metrics between netdata servers
 #include "streaming/rrdpush.h"
 
-
 // anomaly detection
 #include "ml/ml.h"
 

src/daemon/main.c

@@ -1503,7 +1503,7 @@ int unittest_prepare_rrd(const char **user) {
         fprintf(stderr, "rrd_init failed for unittest\n");
         return 1;
     }
-    default_rrdpush_enabled = 0;
+    stream_conf_send_enabled = 0;
 
     return 0;
 }

src/database/contexts/api_v2_contexts.c

@@ -422,7 +422,7 @@ static void rrdcontext_to_json_v2_rrdhost(BUFFER *wb, RRDHOST *host, struct rrdc
             // stale        - connected but not having live data
             // reachable    - connected with live data
             // pruned       - not connected for some time and has been removed
-            buffer_json_member_add_string(wb, "state", rrdhost_state_cloud_emulation(host) ? "reachable" : "stale");
+            buffer_json_member_add_string(wb, "state", rrdhost_is_online(host) ? "reachable" : "stale");
 
             rrdhost_health_to_json_v2(wb, "health", &s);
             agent_capabilities_to_json(wb, host, "capabilities");

src/database/contexts/api_v2_contexts_agents.c

@@ -44,7 +44,7 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
                     sending++;
 
                 if(host != localhost) {
-                    if (rrdhost_state_cloud_emulation(host))
+                    if (rrdhost_is_online(host))
                         receiving++;
                     else
                         archived++;

src/database/engine/dbengine-stresstest.c

@@ -22,13 +22,13 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name) {
         default_rrd_history_entries,
         RRD_MEMORY_MODE_DBENGINE,
         health_plugin_enabled(),
-        default_rrdpush_enabled,
-        default_rrdpush_destination,
-        default_rrdpush_api_key,
-        default_rrdpush_send_charts_matching,
-        default_rrdpush_enable_replication,
-        default_rrdpush_seconds_to_replicate,
-        default_rrdpush_replication_step,
+        stream_conf_send_enabled,
+        stream_conf_send_destination,
+        stream_conf_send_api_key,
+        stream_conf_send_charts_matching,
+        stream_conf_replication_enabled,
+        stream_conf_replication_period,
+        stream_conf_replication_step,
         NULL,
         0
     );

src/database/engine/dbengine-unittest.c

@@ -108,13 +108,13 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name) {
         default_rrd_history_entries,
         RRD_MEMORY_MODE_DBENGINE,
         health_plugin_enabled(),
-        default_rrdpush_enabled,
-        default_rrdpush_destination,
-        default_rrdpush_api_key,
-        default_rrdpush_send_charts_matching,
-        default_rrdpush_enable_replication,
-        default_rrdpush_seconds_to_replicate,
-        default_rrdpush_replication_step,
+        stream_conf_send_enabled,
+        stream_conf_send_destination,
+        stream_conf_send_api_key,
+        stream_conf_send_charts_matching,
+        stream_conf_replication_enabled,
+        stream_conf_replication_period,
+        stream_conf_replication_step,
         NULL,
         0
         );