initial minimal h2o webserver integration (#14585)

Introduces h2o based web server as an alternative

.github/codeql/c-cpp-config.yml

@@ -0,0 +1,2 @@
+paths-ignore:
+  - httpd/h2o

.github/workflows/codeql.yml

@@ -84,6 +84,7 @@ jobs:
         uses: github/codeql-action/init@v2
         with:
           languages: cpp
+          config-file: ./.github/codeql/c-cpp-config.yml
       - name: Prepare environment
         run: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
       - name: Build netdata

.gitmodules

@@ -9,3 +9,7 @@
 	url = https://github.com/davisking/dlib.git
 	shallow = true
 	ignore = dirty
+[submodule "httpd/h2o"]
+	path = httpd/h2o
+	url = https://github.com/h2o/h2o.git
+	ignore = untracked

CMakeLists.txt

@@ -513,6 +513,7 @@ set(LIBNETDATA_FILES
         libnetdata/worker_utilization/worker_utilization.h
 		libnetdata/parser/parser.h
 		libnetdata/parser/parser.c
+        libnetdata/http/http_defs.h
         )
 
 IF(ENABLE_PLUGIN_EBPF)

Makefile.am

@@ -83,6 +83,7 @@ dist_noinst_DATA = \
     packaging/protobuf.version \
     packaging/version \
     database/engine/journalfile_v2.ksy.in \
+    httpd/h2o \
     $(NULL)
 
 # until integrated within build
@@ -197,6 +198,7 @@ LIBNETDATA_FILES = \
     libnetdata/string/utf8.h \
     libnetdata/worker_utilization/worker_utilization.c \
     libnetdata/worker_utilization/worker_utilization.h \
+    libnetdata/http/http_defs.h \
     $(NULL)
 
 if ENABLE_PLUGIN_EBPF
@@ -749,7 +751,8 @@ libmqttwebsockets_a_SOURCES = \
     mqtt_websockets/c-rbuf/src/ringbuffer_internal.h \
     mqtt_websockets/c_rhash/src/c_rhash.c \
     mqtt_websockets/c_rhash/include/c_rhash.h \
-    mqtt_websockets/c_rhash/src/c_rhash_internal.h
+    mqtt_websockets/c_rhash/src/c_rhash_internal.h \
+    $(NULL)
 
 libmqttwebsockets_a_CFLAGS = $(CFLAGS) -DMQTT_WSS_CUSTOM_ALLOC -DRBUF_CUSTOM_MALLOC -DMQTT_WSS_CPUSTATS -I$(srcdir)/aclk/helpers -I$(srcdir)/mqtt_websockets/c_rhash/include
 
@@ -939,6 +942,123 @@ DAEMON_FILES = \
     daemon/unit_test.h \
     $(NULL)
 
+HTTPD_FILES = \
+    httpd/http_server.c \
+    httpd/http_server.h \
+    httpd/h2o_utils.c \
+    httpd/h2o_utils.h \
+    $(NULL)
+
+libh2o_dir = httpd/h2o
+
+libh2o_a_SOURCES = \
+    $(libh2o_dir)/deps/cloexec/cloexec.c \
+    $(libh2o_dir)/deps/libgkc/gkc.c \
+    $(libh2o_dir)/deps/libyrmcds/close.c \
+    $(libh2o_dir)/deps/libyrmcds/connect.c \
+    $(libh2o_dir)/deps/libyrmcds/recv.c \
+    $(libh2o_dir)/deps/libyrmcds/send.c \
+    $(libh2o_dir)/deps/libyrmcds/send_text.c \
+    $(libh2o_dir)/deps/libyrmcds/socket.c \
+    $(libh2o_dir)/deps/libyrmcds/strerror.c \
+    $(libh2o_dir)/deps/libyrmcds/text_mode.c \
+    $(libh2o_dir)/deps/picohttpparser/picohttpparser.c \
+    $(libh2o_dir)/lib/common/cache.c \
+    $(libh2o_dir)/lib/common/file.c \
+    $(libh2o_dir)/lib/common/filecache.c \
+    $(libh2o_dir)/lib/common/hostinfo.c \
+    $(libh2o_dir)/lib/common/http1client.c \
+    $(libh2o_dir)/lib/common/memcached.c \
+    $(libh2o_dir)/lib/common/memory.c \
+    $(libh2o_dir)/lib/common/multithread.c \
+    $(libh2o_dir)/lib/common/serverutil.c \
+    $(libh2o_dir)/lib/common/socket.c \
+    $(libh2o_dir)/lib/common/socketpool.c \
+    $(libh2o_dir)/lib/common/string.c \
+    $(libh2o_dir)/lib/common/time.c \
+    $(libh2o_dir)/lib/common/timeout.c \
+    $(libh2o_dir)/lib/common/url.c \
+    $(libh2o_dir)/lib/core/config.c \
+    $(libh2o_dir)/lib/core/configurator.c \
+    $(libh2o_dir)/lib/core/context.c \
+    $(libh2o_dir)/lib/core/headers.c \
+    $(libh2o_dir)/lib/core/logconf.c \
+    $(libh2o_dir)/lib/core/proxy.c \
+    $(libh2o_dir)/lib/core/request.c \
+    $(libh2o_dir)/lib/core/token.c \
+    $(libh2o_dir)/lib/core/util.c \
+    $(libh2o_dir)/lib/handler/access_log.c \
+    $(libh2o_dir)/lib/handler/chunked.c \
+    $(libh2o_dir)/lib/handler/compress.c \
+    $(libh2o_dir)/lib/handler/compress/gzip.c \
+    $(libh2o_dir)/lib/handler/errordoc.c \
+    $(libh2o_dir)/lib/handler/expires.c \
+    $(libh2o_dir)/lib/handler/fastcgi.c \
+    $(libh2o_dir)/lib/handler/file.c \
+    $(libh2o_dir)/lib/handler/headers.c \
+    $(libh2o_dir)/lib/handler/mimemap.c \
+    $(libh2o_dir)/lib/handler/proxy.c \
+    $(libh2o_dir)/lib/handler/redirect.c \
+    $(libh2o_dir)/lib/handler/reproxy.c \
+    $(libh2o_dir)/lib/handler/throttle_resp.c \
+    $(libh2o_dir)/lib/handler/status.c \
+    $(libh2o_dir)/lib/handler/headers_util.c \
+    $(libh2o_dir)/lib/handler/status/events.c \
+    $(libh2o_dir)/lib/handler/status/requests.c \
+    $(libh2o_dir)/lib/handler/http2_debug_state.c \
+    $(libh2o_dir)/lib/handler/status/durations.c \
+    $(libh2o_dir)/lib/handler/configurator/access_log.c \
+    $(libh2o_dir)/lib/handler/configurator/compress.c \
+    $(libh2o_dir)/lib/handler/configurator/errordoc.c \
+    $(libh2o_dir)/lib/handler/configurator/expires.c \
+    $(libh2o_dir)/lib/handler/configurator/fastcgi.c \
+    $(libh2o_dir)/lib/handler/configurator/file.c \
+    $(libh2o_dir)/lib/handler/configurator/headers.c \
+    $(libh2o_dir)/lib/handler/configurator/proxy.c \
+    $(libh2o_dir)/lib/handler/configurator/redirect.c \
+    $(libh2o_dir)/lib/handler/configurator/reproxy.c \
+    $(libh2o_dir)/lib/handler/configurator/throttle_resp.c \
+    $(libh2o_dir)/lib/handler/configurator/status.c \
+    $(libh2o_dir)/lib/handler/configurator/http2_debug_state.c \
+    $(libh2o_dir)/lib/handler/configurator/headers_util.c \
+    $(libh2o_dir)/lib/http1.c \
+    $(libh2o_dir)/lib/tunnel.c \
+    $(libh2o_dir)/lib/http2/cache_digests.c \
+    $(libh2o_dir)/lib/http2/casper.c \
+    $(libh2o_dir)/lib/http2/connection.c \
+    $(libh2o_dir)/lib/http2/frame.c \
+    $(libh2o_dir)/lib/http2/hpack.c \
+    $(libh2o_dir)/lib/http2/scheduler.c \
+    $(libh2o_dir)/lib/http2/stream.c \
+    $(libh2o_dir)/lib/http2/http2_debug_state.c \
+    $(NULL)
+
+libh2o_a_INCLUDES = \
+    -I$(srcdir)/$(libh2o_dir)/include \
+    -I$(srcdir)/$(libh2o_dir)/deps/cloexec \
+    -I$(srcdir)/$(libh2o_dir)/deps/brotli/enc \
+    -I$(srcdir)/$(libh2o_dir)/deps/golombset \
+    -I$(srcdir)/$(libh2o_dir)/deps/libgkc \
+    -I$(srcdir)/$(libh2o_dir)/deps/libyrmcds \
+    -I$(srcdir)/$(libh2o_dir)/deps/klib \
+    -I$(srcdir)/$(libh2o_dir)/deps/neverbleed \
+    -I$(srcdir)/$(libh2o_dir)/deps/picohttpparser \
+    -I$(srcdir)/$(libh2o_dir)/deps/picotest \
+    -I$(srcdir)/$(libh2o_dir)/deps/yaml/include \
+    -I$(srcdir)/$(libh2o_dir)/deps/yoml \
+    $(NULL)
+
+if ENABLE_HTTPD
+noinst_LIBRARIES += libh2o.a
+
+# until h2o updates support for OpenSSL 3.0 we silence the warnings
+libh2o_a_CFLAGS = $(CFLAGS) -Wno-deprecated-declarations -Wno-unused-parameter -Wno-sign-compare -Wno-missing-field-initializers -DH2O_USE_LIBUV=0 $(libh2o_a_INCLUDES)
+
+if LINUX
+    libh2o_a_CFLAGS += -D_GNU_SOURCE
+endif
+endif #ENABLE_HTTPD
+
 NETDATA_FILES = \
     collectors/all.h \
     $(DAEMON_FILES) \
@@ -1007,6 +1127,11 @@ if ENABLE_ACLK
     NETDATA_COMMON_LIBS += libmqttwebsockets.a
 endif
 
+if ENABLE_HTTPD
+    NETDATA_FILES += $(HTTPD_FILES)
+    NETDATA_COMMON_LIBS += libh2o.a
+endif
+
 if LINK_STATIC_JSONC
     NETDATA_COMMON_LIBS += $(abs_top_srcdir)/externaldeps/jsonc/libjson-c.a
 endif

configure.ac

@@ -213,6 +213,12 @@ AC_ARG_ENABLE(
     [aclk_ssl_debug="yes"],
     [aclk_ssl_debug="no"]
 )
+AC_ARG_ENABLE(
+    [httpd],
+    [AS_HELP_STRING([--disable-httpd], [Disable webserver (h2o based) @<:@default autodetect@:>@])],
+    ,
+    [enable_httpd="detect"]
+)
 
 # -----------------------------------------------------------------------------
 # Enforce building with C99, bail early if we can't.
@@ -799,6 +805,38 @@ fi
 AC_MSG_RESULT([${with_libcap}])
 AM_CONDITIONAL([ENABLE_CAPABILITY], [test "${with_libcap}" = "yes"])
 
+# -----------------------------------------------------------------------------
+# HTTPD and h2o related
+
+can_build_httpd="no"
+if test "${enable_httpd}" != "no"; then
+    can_build_httpd="yes"
+    AC_MSG_CHECKING([can build HTTPD])
+    if test -z "${UV_LIBS}"; then
+        can_build_httpd="no"
+    fi
+    if test -n "${SSL_LIBS}"; then
+        OPTIONAL_SSL_CFLAGS="${SSL_CFLAGS}"
+        OPTIONAL_SSL_LIBS="${SSL_LIBS}"
+    else
+        can_build_httpd="no"
+    fi
+    if test "${with_zlib}" != "yes"; then
+        can_build_httpd="no"
+    fi
+    AC_MSG_RESULT([${can_build_httpd}])
+
+    if test "${can_build_httpd}" = "no" -a "${enable_httpd}" = "yes"; then
+        AC_MSG_ERROR([HTTPD was requested but it cannot be built])
+    fi
+
+    if test "${can_build_httpd}" = "yes"; then
+        AC_DEFINE([ENABLE_HTTPD], [1], [HTTPD (h2o based web server)])
+        HTTPD_CFLAGS="-I\$(abs_top_srcdir)/httpd/h2o/include -I\$(abs_top_srcdir)/httpd/h2o/deps/picotls/include -I\$(abs_top_srcdir)/httpd/h2o/deps/quicly/include -DH2O_USE_LIBUV=0"
+    fi
+fi
+AM_CONDITIONAL([ENABLE_HTTPD], [test "${can_build_httpd}" = "yes"])
+
 # -----------------------------------------------------------------------------
 # ACLK
 
@@ -1689,7 +1727,7 @@ CFLAGS="${originalCFLAGS} ${OPTIONAL_LTO_CFLAGS} ${OPTIONAL_PROTOBUF_CFLAGS} ${O
     ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} \
     ${OPTIONAL_KINESIS_CFLAGS} ${OPTIONAL_PUBSUB_CFLAGS} ${OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS} \
     ${OPTIONAL_MONGOC_CFLAGS} ${LWS_CFLAGS} ${OPTIONAL_JSONC_STATIC_CFLAGS} ${OPTIONAL_YAML_STATIC_CFLAGS} ${OPTIONAL_BPF_CFLAGS} ${JUDY_CFLAGS} \
-    ${OPTIONAL_ACLK_CFLAGS} ${OPTIONAL_ML_CFLAGS} ${OPTIONAL_OS_DEP_CFLAGS}"
+    ${OPTIONAL_ACLK_CFLAGS} ${OPTIONAL_ML_CFLAGS} ${OPTIONAL_OS_DEP_CFLAGS} ${HTTPD_CFLAGS}"
 
 CXXFLAGS="${CFLAGS} ${OPTIONAL_KINESIS_CXXFLAGS} ${CPP_STD_FLAG}"
 

daemon/buildinfo.c

@@ -20,6 +20,12 @@
 #endif
 #endif
 
+#ifdef ENABLE_HTTPD
+#define FEAT_HTTPD 1
+#else
+#define FEAT_HTTPD 0
+#endif
+
 #ifdef ENABLE_DBENGINE
 #define FEAT_DBENGINE 1
 #else
@@ -275,6 +281,7 @@ void print_build_info(void) {
     printf("    TLS Host Verification:      %s\n", FEAT_YES_NO(FEAT_TLS_HOST_VERIFY));
     printf("    Machine Learning:           %s\n", FEAT_YES_NO(FEAT_ML));
     printf("    Stream Compression:         %s\n", FEAT_YES_NO(FEAT_STREAM_COMPRESSION));
+    printf("    HTTPD (h2o):                %s\n", FEAT_YES_NO(FEAT_HTTPD));
 
     printf("Libraries:\n");
     printf("    protobuf:                %s%s\n", FEAT_YES_NO(FEAT_PROTOBUF), FEAT_PROTOBUF_BUNDLED);
@@ -328,7 +335,8 @@ void print_build_info_json(void) {
 
     printf("    \"tls-host-verify\": %s,\n",   FEAT_JSON_BOOL(FEAT_TLS_HOST_VERIFY));
     printf("    \"machine-learning\": %s\n",   FEAT_JSON_BOOL(FEAT_ML));
-    printf("    \"stream-compression\": %s\n",   FEAT_JSON_BOOL(FEAT_STREAM_COMPRESSION));
+    printf("    \"stream-compression\": %s\n", FEAT_JSON_BOOL(FEAT_STREAM_COMPRESSION));
+    printf("    \"httpd-h2o\": %s\n",          FEAT_JSON_BOOL(FEAT_HTTPD));
     printf("  },\n");
 
     printf("  \"libs\": {\n");

daemon/common.h

@@ -41,6 +41,11 @@
 // the netdata webserver(s)
 #include "web/server/web_server.h"
 
+// the new h2o based netdata webserver
+#ifdef ENABLE_HTTPD
+#include "httpd/http_server.h"
+#endif
+
 // streaming metrics between netdata servers
 #include "streaming/rrdpush.h"
 

daemon/main.c

@@ -170,6 +170,8 @@ static void service_to_buffer(BUFFER *wb, SERVICE_TYPE service) {
         buffer_strcat(wb, "ANALYTICS ");
     if(service & SERVICE_EXPORTERS)
         buffer_strcat(wb, "EXPORTERS ");
+    if(service & SERVICE_HTTPD)
+        buffer_strcat(wb, "HTTPD ");
 }
 
 static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {
@@ -365,6 +367,7 @@ void netdata_cleanup_and_exit(int ret) {
             | SERVICE_EXPORTERS
             | SERVICE_HEALTH
             | SERVICE_WEB_SERVER
+            | SERVICE_HTTPD
             , 3 * USEC_PER_SEC);
 
     delta_shutdown_time("stop collectors and streaming threads");
@@ -1979,6 +1982,14 @@ int main(int argc, char **argv) {
 
         if(web_server_mode != WEB_SERVER_MODE_NONE)
             api_listen_sockets_setup();
+
+#ifdef ENABLE_HTTPD
+        delta_startup_time("initialize httpd server");
+        for (int i = 0; static_threads[i].name; i++) {
+            if (static_threads[i].start_routine == httpd_main)
+                static_threads[i].enabled = httpd_is_enabled();
+        }
+#endif
     }
 
     delta_startup_time("set resource limits");

daemon/main.h

@@ -41,7 +41,8 @@ typedef enum {
     SERVICE_CONTEXT               = (1 << 10),
     SERVICE_ANALYTICS             = (1 << 11),
     SERVICE_EXPORTERS             = (1 << 12),
-    SERVICE_ACLKSYNC              = (1 << 13)
+    SERVICE_ACLKSYNC              = (1 << 13),
+    SERVICE_HTTPD                 = (1 << 14)
 } SERVICE_TYPE;
 
 typedef enum {