Browse Source

Code reorg and cleanup - enrichment of /api/v2 (#15294)

* claim script now accepts the same params as the kickstart

* rewrote buildinfo to unify all methods

* added cloud unavailable in cloud status

* added all exporters

* renamed httpd to h2o

* rename ENABLE_COMPRESSION to ENABLE_LZ4

* rename global variable

* rename ENABLE_HTTPS to ENABLE_OPENSSL

* fix coverity-scan for openssl

* add lz4 to coverity-scan

* added all plugins and most of the features

* added all plugins and most of the features

* generalize bitmap code so that we can have any size of bitmaps

* cleanup

* fix compilation without protobuf

* fix compilation with others allocators

* fix bitmap

* comprehensive bitmaps unit test

* bitmap as macros

* added developer mode

* added system info to build info

* cloud available/unavailable

* added /api/v2/info

* added units and ni to transitions

* when showing instances and transitions, show only the instances that have transitions

* cleanup

* add missing quotes

* add anchor to transitions

* added more to build info

* calculate retention per tier and expose it to /api/v2/info

* added currently collected metrics

* do not show space and retention when no numbers are available

* fix impossible overflow

* Add function for transitions and execute callback

* In case of error, reset and try next dictionary entry

* Fix error message

* simpler logic to maintain retention per tier

* /api/v2/alert_transitions

* Handle case of recipient null
Convert after and before to usec

* Add classification, type and component

* working /api/v2/alert_transitions

* Fix query to properly handle context and alert name

* cleanup

* Add search with transition

* accept transition in /api/v2/alert_transitions

* totaly dynamic facets

* fixed debug info

* restructured facets

* cleanup; removal of options=transitions

* updated alert entries flags

* method to exec

* Return also exec run timestamp
Temp table cleanup only when we don't execute with a transition

* cleanup obsolete anchor parameter

* Add sql_get_alert_configuration function

* added options=config to alert_transitions

* added /api/v2/alert_config

* preliminary work for /api/v2/claim

* initialize variables; do not expose expected retention if no disk space info is available; do not report aclk as initializing when not claimed

* fix claim session key filename

* put a newline into the session key file

* more progress on claiming

* final /api/v2/claim endpoint

* after claiming, refresh our state at the output

* Fix query to fetch config

* Remove debug log

* add configuration objects

* add configuration objects - fixed

* respect the NETDATA_DISABLE_CLOUD env variable

* NETDATA_DISABLE_CLOUD env variable sets the default, but the config sets the final value

* use a new claimed_id on every claiming

* regenerate random key on claiming and wait for online status

* ignore write() return value when writing a newline

* dont show cloud status disabled when claimed_id is missing

* added ctx to alert instances

* cleanup config and transitions from /api/v2/alerts

* fix unused variable

* in /api/v2/alert_config show 1 config without an array

* show alert values conditionally, by appending options=values

* When storing host info if the key value is empty, store unknown

* added options=summary to control when the alerts summary is shown

* increased http_api_v2 to version 5

* claming random key file is now not world readable

* added local-listeners binary that detects all the listening ports, their IPs and their command lines

---------

Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>
Costa Tsaousis 1 year ago
parent
commit
c74bf56ee2
10 changed files with 443 additions and 142 deletions
  1. 3 0
      .gitignore
  2. 2 2
      .gitmodules
  3. 1 1
      CMakeLists.txt
  4. 112 101
      Makefile.am
  5. 3 3
      aclk/aclk.c
  6. 1 1
      aclk/aclk.h
  7. 1 1
      aclk/aclk_capas.c
  8. 272 14
      claim/claim.c
  9. 17 1
      claim/claim.h
  10. 31 18
      claim/netdata-claim.sh.in

+ 3 - 0
.gitignore

@@ -69,6 +69,9 @@ slabinfo.plugin
 cgroup-network
 !cgroup-network/
 
+local-listeners
+!local-listeners/
+
 ebpf.plugin
 collectors/ebpf.plugin/reset_netdata_trace.sh
 !ebpf.plugin/

+ 2 - 2
.gitmodules

@@ -9,7 +9,7 @@
 	url = https://github.com/davisking/dlib.git
 	shallow = true
 	ignore = dirty
-[submodule "httpd/h2o"]
-	path = httpd/h2o
+[submodule "web/server/h2o/libh2o"]
+	path = web/server/h2o/libh2o
 	url = https://github.com/h2o/h2o.git
 	ignore = untracked

+ 1 - 1
CMakeLists.txt

@@ -102,7 +102,7 @@ pkg_check_modules(LIBLZ4 REQUIRED liblz4)
 set(NETDATA_COMMON_CFLAGS ${NETDATA_COMMON_CFLAGS} ${LIBLZ4_CFLAGS_OTHER})
 set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} ${LIBLZ4_LIBRARIES})
 set(NETDATA_COMMON_INCLUDE_DIRS ${NETDATA_COMMON_INCLUDE_DIRS} ${LIBLZ4_INCLUDE_DIRS})
-# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DENABLE_COMPRESSION=1")
+# set(NETDATA_REQUIRED_DEFINES "${NETDATA_REQUIRED_DEFINES} -DENABLE_LZ4=1")
 
 # -----------------------------------------------------------------------------
 # Judy General purpose dynamic array

+ 112 - 101
Makefile.am

@@ -84,7 +84,7 @@ dist_noinst_DATA = \
     packaging/protobuf.version \
     packaging/version \
     database/engine/journalfile_v2.ksy.in \
-    httpd/h2o \
+    web/server/h2o/libh2o \
     $(NULL)
 
 # until integrated within build
@@ -281,6 +281,10 @@ CGROUP_NETWORK_FILES = \
     $(LIBNETDATA_FILES) \
     $(NULL)
 
+LOCAL_LISTENERS_FILES = \
+    collectors/plugins.d/local_listeners.c \
+    $(NULL)
+
 DISKSPACE_PLUGIN_FILES = \
     collectors/diskspace.plugin/plugin_diskspace.c \
     $(NULL)
@@ -945,111 +949,111 @@ DAEMON_FILES = \
     daemon/unit_test.h \
     $(NULL)
 
-HTTPD_FILES = \
-    httpd/http_server.c \
-    httpd/http_server.h \
-    httpd/h2o_utils.c \
-    httpd/h2o_utils.h \
+H2O_FILES = \
+    web/server/h2o/http_server.c \
+    web/server/h2o/http_server.h \
+    web/server/h2o/h2o_utils.c \
+    web/server/h2o/h2o_utils.h \
     $(NULL)
 
 libh2o_a_SOURCES = \
-    httpd/h2o/deps/cloexec/cloexec.c \
-    httpd/h2o/deps/libgkc/gkc.c \
-    httpd/h2o/deps/libyrmcds/close.c \
-    httpd/h2o/deps/libyrmcds/connect.c \
-    httpd/h2o/deps/libyrmcds/recv.c \
-    httpd/h2o/deps/libyrmcds/send.c \
-    httpd/h2o/deps/libyrmcds/send_text.c \
-    httpd/h2o/deps/libyrmcds/socket.c \
-    httpd/h2o/deps/libyrmcds/strerror.c \
-    httpd/h2o/deps/libyrmcds/text_mode.c \
-    httpd/h2o/deps/picohttpparser/picohttpparser.c \
-    httpd/h2o/lib/common/cache.c \
-    httpd/h2o/lib/common/file.c \
-    httpd/h2o/lib/common/filecache.c \
-    httpd/h2o/lib/common/hostinfo.c \
-    httpd/h2o/lib/common/http1client.c \
-    httpd/h2o/lib/common/memcached.c \
-    httpd/h2o/lib/common/memory.c \
-    httpd/h2o/lib/common/multithread.c \
-    httpd/h2o/lib/common/serverutil.c \
-    httpd/h2o/lib/common/socket.c \
-    httpd/h2o/lib/common/socketpool.c \
-    httpd/h2o/lib/common/string.c \
-    httpd/h2o/lib/common/time.c \
-    httpd/h2o/lib/common/timeout.c \
-    httpd/h2o/lib/common/url.c \
-    httpd/h2o/lib/core/config.c \
-    httpd/h2o/lib/core/configurator.c \
-    httpd/h2o/lib/core/context.c \
-    httpd/h2o/lib/core/headers.c \
-    httpd/h2o/lib/core/logconf.c \
-    httpd/h2o/lib/core/proxy.c \
-    httpd/h2o/lib/core/request.c \
-    httpd/h2o/lib/core/token.c \
-    httpd/h2o/lib/core/util.c \
-    httpd/h2o/lib/handler/access_log.c \
-    httpd/h2o/lib/handler/chunked.c \
-    httpd/h2o/lib/handler/compress.c \
-    httpd/h2o/lib/handler/compress/gzip.c \
-    httpd/h2o/lib/handler/errordoc.c \
-    httpd/h2o/lib/handler/expires.c \
-    httpd/h2o/lib/handler/fastcgi.c \
-    httpd/h2o/lib/handler/file.c \
-    httpd/h2o/lib/handler/headers.c \
-    httpd/h2o/lib/handler/mimemap.c \
-    httpd/h2o/lib/handler/proxy.c \
-    httpd/h2o/lib/handler/redirect.c \
-    httpd/h2o/lib/handler/reproxy.c \
-    httpd/h2o/lib/handler/throttle_resp.c \
-    httpd/h2o/lib/handler/status.c \
-    httpd/h2o/lib/handler/headers_util.c \
-    httpd/h2o/lib/handler/status/events.c \
-    httpd/h2o/lib/handler/status/requests.c \
-    httpd/h2o/lib/handler/http2_debug_state.c \
-    httpd/h2o/lib/handler/status/durations.c \
-    httpd/h2o/lib/handler/configurator/access_log.c \
-    httpd/h2o/lib/handler/configurator/compress.c \
-    httpd/h2o/lib/handler/configurator/errordoc.c \
-    httpd/h2o/lib/handler/configurator/expires.c \
-    httpd/h2o/lib/handler/configurator/fastcgi.c \
-    httpd/h2o/lib/handler/configurator/file.c \
-    httpd/h2o/lib/handler/configurator/headers.c \
-    httpd/h2o/lib/handler/configurator/proxy.c \
-    httpd/h2o/lib/handler/configurator/redirect.c \
-    httpd/h2o/lib/handler/configurator/reproxy.c \
-    httpd/h2o/lib/handler/configurator/throttle_resp.c \
-    httpd/h2o/lib/handler/configurator/status.c \
-    httpd/h2o/lib/handler/configurator/http2_debug_state.c \
-    httpd/h2o/lib/handler/configurator/headers_util.c \
-    httpd/h2o/lib/http1.c \
-    httpd/h2o/lib/tunnel.c \
-    httpd/h2o/lib/http2/cache_digests.c \
-    httpd/h2o/lib/http2/casper.c \
-    httpd/h2o/lib/http2/connection.c \
-    httpd/h2o/lib/http2/frame.c \
-    httpd/h2o/lib/http2/hpack.c \
-    httpd/h2o/lib/http2/scheduler.c \
-    httpd/h2o/lib/http2/stream.c \
-    httpd/h2o/lib/http2/http2_debug_state.c \
+    web/server/h2o/libh2o/deps/cloexec/cloexec.c \
+    web/server/h2o/libh2o/deps/libgkc/gkc.c \
+    web/server/h2o/libh2o/deps/libyrmcds/close.c \
+    web/server/h2o/libh2o/deps/libyrmcds/connect.c \
+    web/server/h2o/libh2o/deps/libyrmcds/recv.c \
+    web/server/h2o/libh2o/deps/libyrmcds/send.c \
+    web/server/h2o/libh2o/deps/libyrmcds/send_text.c \
+    web/server/h2o/libh2o/deps/libyrmcds/socket.c \
+    web/server/h2o/libh2o/deps/libyrmcds/strerror.c \
+    web/server/h2o/libh2o/deps/libyrmcds/text_mode.c \
+    web/server/h2o/libh2o/deps/picohttpparser/picohttpparser.c \
+    web/server/h2o/libh2o/lib/common/cache.c \
+    web/server/h2o/libh2o/lib/common/file.c \
+    web/server/h2o/libh2o/lib/common/filecache.c \
+    web/server/h2o/libh2o/lib/common/hostinfo.c \
+    web/server/h2o/libh2o/lib/common/http1client.c \
+    web/server/h2o/libh2o/lib/common/memcached.c \
+    web/server/h2o/libh2o/lib/common/memory.c \
+    web/server/h2o/libh2o/lib/common/multithread.c \
+    web/server/h2o/libh2o/lib/common/serverutil.c \
+    web/server/h2o/libh2o/lib/common/socket.c \
+    web/server/h2o/libh2o/lib/common/socketpool.c \
+    web/server/h2o/libh2o/lib/common/string.c \
+    web/server/h2o/libh2o/lib/common/time.c \
+    web/server/h2o/libh2o/lib/common/timeout.c \
+    web/server/h2o/libh2o/lib/common/url.c \
+    web/server/h2o/libh2o/lib/core/config.c \
+    web/server/h2o/libh2o/lib/core/configurator.c \
+    web/server/h2o/libh2o/lib/core/context.c \
+    web/server/h2o/libh2o/lib/core/headers.c \
+    web/server/h2o/libh2o/lib/core/logconf.c \
+    web/server/h2o/libh2o/lib/core/proxy.c \
+    web/server/h2o/libh2o/lib/core/request.c \
+    web/server/h2o/libh2o/lib/core/token.c \
+    web/server/h2o/libh2o/lib/core/util.c \
+    web/server/h2o/libh2o/lib/handler/access_log.c \
+    web/server/h2o/libh2o/lib/handler/chunked.c \
+    web/server/h2o/libh2o/lib/handler/compress.c \
+    web/server/h2o/libh2o/lib/handler/compress/gzip.c \
+    web/server/h2o/libh2o/lib/handler/errordoc.c \
+    web/server/h2o/libh2o/lib/handler/expires.c \
+    web/server/h2o/libh2o/lib/handler/fastcgi.c \
+    web/server/h2o/libh2o/lib/handler/file.c \
+    web/server/h2o/libh2o/lib/handler/headers.c \
+    web/server/h2o/libh2o/lib/handler/mimemap.c \
+    web/server/h2o/libh2o/lib/handler/proxy.c \
+    web/server/h2o/libh2o/lib/handler/redirect.c \
+    web/server/h2o/libh2o/lib/handler/reproxy.c \
+    web/server/h2o/libh2o/lib/handler/throttle_resp.c \
+    web/server/h2o/libh2o/lib/handler/status.c \
+    web/server/h2o/libh2o/lib/handler/headers_util.c \
+    web/server/h2o/libh2o/lib/handler/status/events.c \
+    web/server/h2o/libh2o/lib/handler/status/requests.c \
+    web/server/h2o/libh2o/lib/handler/http2_debug_state.c \
+    web/server/h2o/libh2o/lib/handler/status/durations.c \
+    web/server/h2o/libh2o/lib/handler/configurator/access_log.c \
+    web/server/h2o/libh2o/lib/handler/configurator/compress.c \
+    web/server/h2o/libh2o/lib/handler/configurator/errordoc.c \
+    web/server/h2o/libh2o/lib/handler/configurator/expires.c \
+    web/server/h2o/libh2o/lib/handler/configurator/fastcgi.c \
+    web/server/h2o/libh2o/lib/handler/configurator/file.c \
+    web/server/h2o/libh2o/lib/handler/configurator/headers.c \
+    web/server/h2o/libh2o/lib/handler/configurator/proxy.c \
+    web/server/h2o/libh2o/lib/handler/configurator/redirect.c \
+    web/server/h2o/libh2o/lib/handler/configurator/reproxy.c \
+    web/server/h2o/libh2o/lib/handler/configurator/throttle_resp.c \
+    web/server/h2o/libh2o/lib/handler/configurator/status.c \
+    web/server/h2o/libh2o/lib/handler/configurator/http2_debug_state.c \
+    web/server/h2o/libh2o/lib/handler/configurator/headers_util.c \
+    web/server/h2o/libh2o/lib/http1.c \
+    web/server/h2o/libh2o/lib/tunnel.c \
+    web/server/h2o/libh2o/lib/http2/cache_digests.c \
+    web/server/h2o/libh2o/lib/http2/casper.c \
+    web/server/h2o/libh2o/lib/http2/connection.c \
+    web/server/h2o/libh2o/lib/http2/frame.c \
+    web/server/h2o/libh2o/lib/http2/hpack.c \
+    web/server/h2o/libh2o/lib/http2/scheduler.c \
+    web/server/h2o/libh2o/lib/http2/stream.c \
+    web/server/h2o/libh2o/lib/http2/http2_debug_state.c \
     $(NULL)
 
 libh2o_a_INCLUDES = \
-    -I$(srcdir)/httpd/h2o/include \
-    -I$(srcdir)/httpd/h2o/deps/cloexec \
-    -I$(srcdir)/httpd/h2o/deps/brotli/enc \
-    -I$(srcdir)/httpd/h2o/deps/golombset \
-    -I$(srcdir)/httpd/h2o/deps/libgkc \
-    -I$(srcdir)/httpd/h2o/deps/libyrmcds \
-    -I$(srcdir)/httpd/h2o/deps/klib \
-    -I$(srcdir)/httpd/h2o/deps/neverbleed \
-    -I$(srcdir)/httpd/h2o/deps/picohttpparser \
-    -I$(srcdir)/httpd/h2o/deps/picotest \
-    -I$(srcdir)/httpd/h2o/deps/yaml/include \
-    -I$(srcdir)/httpd/h2o/deps/yoml \
+    -I$(srcdir)/web/server/h2o/libh2o/include \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/cloexec \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/brotli/enc \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/golombset \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/libgkc \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/libyrmcds \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/klib \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/neverbleed \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/picohttpparser \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/picotest \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/yaml/include \
+    -I$(srcdir)/web/server/h2o/libh2o/deps/yoml \
     $(NULL)
 
-if ENABLE_HTTPD
+if ENABLE_H2O
 noinst_LIBRARIES += libh2o.a
 
 # until h2o updates support for OpenSSL 3.0 we silence the warnings
@@ -1058,7 +1062,7 @@ libh2o_a_CFLAGS = $(CFLAGS) -Wno-deprecated-declarations -Wno-unused-parameter -
 if LINUX
     libh2o_a_CFLAGS += -D_GNU_SOURCE
 endif
-endif #ENABLE_HTTPD
+endif #ENABLE_H2O
 
 NETDATA_FILES = \
     collectors/all.h \
@@ -1129,8 +1133,8 @@ if ENABLE_ACLK
     NETDATA_COMMON_LIBS += libmqttwebsockets.a
 endif
 
-if ENABLE_HTTPD
-    NETDATA_FILES += $(HTTPD_FILES)
+if ENABLE_H2O
+    NETDATA_FILES += $(H2O_FILES)
     NETDATA_COMMON_LIBS += libh2o.a
 endif
 
@@ -1205,6 +1209,13 @@ if ENABLE_PLUGIN_CGROUP_NETWORK
         $(NULL)
 endif
 
+if ENABLE_PLUGIN_LOCAL_LISTENERS
+    plugins_PROGRAMS += local-listeners
+    local_listeners_SOURCES = $(LOCAL_LISTENERS_FILES)
+    local_listeners_LDADD = \
+        $(NULL)
+endif
+
 if ENABLE_PLUGIN_FREEIPMI
     plugins_PROGRAMS += freeipmi.plugin
     freeipmi_plugin_SOURCES = $(FREEIPMI_PLUGIN_FILES)

+ 3 - 3
aclk/aclk.c

@@ -489,15 +489,15 @@ static int aclk_get_transport_idx(aclk_env_t *env) {
 }
 #endif
 
-ACLK_STATUS aclk_status = ACLK_STATUS_INITIALIZING;
+ACLK_STATUS aclk_status = ACLK_STATUS_NONE;
 
 const char *aclk_status_to_string(void) {
     switch(aclk_status) {
         case ACLK_STATUS_CONNECTED:
             return "connected";
 
-        case ACLK_STATUS_INITIALIZING:
-            return "initializing";
+        case ACLK_STATUS_NONE:
+            return "none";
 
         case ACLK_STATUS_DISABLED:
             return "disabled";

+ 1 - 1
aclk/aclk.h

@@ -15,7 +15,7 @@
 
 typedef enum __attribute__((packed)) {
     ACLK_STATUS_CONNECTED = 0,
-    ACLK_STATUS_INITIALIZING,
+    ACLK_STATUS_NONE,
     ACLK_STATUS_DISABLED,
     ACLK_STATUS_NO_CLOUD_URL,
     ACLK_STATUS_INVALID_CLOUD_URL,

+ 1 - 1
aclk/aclk_capas.c

@@ -4,7 +4,7 @@
 
 #include "ml/ml.h"
 
-#define HTTP_API_V2_VERSION 4
+#define HTTP_API_V2_VERSION 5
 
 const struct capability *aclk_get_agent_capas()
 {

+ 272 - 14
claim/claim.c

@@ -42,16 +42,16 @@ char *get_agent_claimid()
 }
 
 #define CLAIMING_COMMAND_LENGTH 16384
-#define CLAIMING_PROXY_LENGTH CLAIMING_COMMAND_LENGTH/4
+#define CLAIMING_PROXY_LENGTH (CLAIMING_COMMAND_LENGTH/4)
 
 extern struct registry registry;
 
 /* rrd_init() and post_conf_load() must have been called before this function */
-void claim_agent(char *claiming_arguments)
+CLAIM_AGENT_RESPONSE claim_agent(const char *claiming_arguments, bool force, const char **msg)
 {
-    if (!netdata_cloud_enabled) {
+    if (!force || !netdata_cloud_enabled) {
         error("Refusing to claim agent -> cloud functionality has been disabled");
-        return;
+        return CLAIM_AGENT_CLOUD_DISABLED;
     }
 
 #ifndef DISABLE_CLOUD
@@ -62,8 +62,11 @@ void claim_agent(char *claiming_arguments)
 
     // This is guaranteed to be set early in main via post_conf_load()
     char *cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", NULL);
-    if (cloud_base_url == NULL)
-        fatal("Do not move the cloud base url out of post_conf_load!!");
+    if (cloud_base_url == NULL) {
+        internal_fatal(true, "Do not move the cloud base url out of post_conf_load!!");
+        return CLAIM_AGENT_NO_CLOUD_URL;
+    }
+
     const char *proxy_str;
     ACLK_PROXY_TYPE proxy_type;
     char proxy_flag[CLAIMING_PROXY_LENGTH] = "-noproxy";
@@ -76,7 +79,6 @@ void claim_agent(char *claiming_arguments)
     snprintfz(command_buffer,
               CLAIMING_COMMAND_LENGTH,
               "exec netdata-claim.sh %s -hostname=%s -id=%s -url=%s -noreload %s",
-
               proxy_flag,
               netdata_configured_hostname,
               localhost->machine_guid,
@@ -87,7 +89,7 @@ void claim_agent(char *claiming_arguments)
     fp_child_output = netdata_popen(command_buffer, &command_pid, &fp_child_input);
     if(!fp_child_output) {
         error("Cannot popen(\"%s\").", command_buffer);
-        return;
+        return CLAIM_AGENT_CANNOT_EXECUTE_CLAIM_SCRIPT;
     }
     netdata_log_info("Waiting for claiming command to finish.");
     while (fgets(command_buffer, CLAIMING_COMMAND_LENGTH, fp_child_output) != NULL) {;}
@@ -95,25 +97,31 @@ void claim_agent(char *claiming_arguments)
     netdata_log_info("Agent claiming command returned with code %d", exit_code);
     if (0 == exit_code) {
         load_claiming_state();
-        return;
+        return CLAIM_AGENT_OK;
     }
     if (exit_code < 0) {
         error("Agent claiming command failed to complete its run.");
-        return;
+        return CLAIM_AGENT_CLAIM_SCRIPT_FAILED;
     }
     errno = 0;
     unsigned maximum_known_exit_code = sizeof(claiming_errors) / sizeof(claiming_errors[0]) - 1;
 
     if ((unsigned)exit_code > maximum_known_exit_code) {
         error("Agent failed to be claimed with an unknown error.");
-        return;
+        return CLAIM_AGENT_CLAIM_SCRIPT_RETURNED_INVALID_CODE;
     }
+
     error("Agent failed to be claimed with the following error message:");
     error("\"%s\"", claiming_errors[exit_code]);
+
+    if(msg) *msg = claiming_errors[exit_code];
+
 #else
     UNUSED(claiming_arguments);
     UNUSED(claiming_errors);
 #endif
+
+    return CLAIM_AGENT_FAILED_WITH_MESSAGE;
 }
 
 #ifdef ENABLE_ACLK
@@ -181,7 +189,7 @@ void load_claiming_state(void)
     freez(claimed_id);
 
     netdata_log_info("File '%s' was found. Setting state to AGENT_CLAIMED.", filename);
-    netdata_cloud_enabled = appconfig_get_boolean(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", 1);
+    netdata_cloud_enabled = appconfig_get_boolean_ondemand(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", netdata_cloud_enabled);
 #endif
 }
 
@@ -193,6 +201,10 @@ struct config cloud_config = { .first_section = NULL,
 
 void load_cloud_conf(int silent)
 {
+    char *nd_disable_cloud = getenv("NETDATA_DISABLE_CLOUD");
+    if (nd_disable_cloud && !strncmp(nd_disable_cloud, "1", 1))
+        netdata_cloud_enabled = CONFIG_BOOLEAN_NO;
+
     char *filename;
     errno = 0;
 
@@ -201,8 +213,254 @@ void load_cloud_conf(int silent)
     filename = strdupz_path_subpath(netdata_configured_varlib_dir, "cloud.d/cloud.conf");
 
     ret = appconfig_load(&cloud_config, filename, 1, NULL);
-    if(!ret && !silent) {
+    if(!ret && !silent)
         netdata_log_info("CONFIG: cannot load cloud config '%s'. Running with internal defaults.", filename);
-    }
+
     freez(filename);
+
+    // --------------------------------------------------------------------
+    // Check if the cloud is enabled
+
+#if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK )
+    netdata_cloud_enabled = CONFIG_BOOLEAN_NO;
+#else
+    netdata_cloud_enabled = appconfig_get_boolean_ondemand(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", netdata_cloud_enabled);
+#endif
+
+    // This must be set before any point in the code that accesses it. Do not move it from this function.
+    appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", DEFAULT_CLOUD_BASE_URL);
+}
+
+static char *netdata_random_session_id_filename = NULL;
+static uuid_t netdata_random_session_id = { 0 };
+
+bool netdata_random_session_id_generate(void) {
+    static char guid[UUID_STR_LEN] = "";
+
+    uuid_generate_random(netdata_random_session_id);
+    uuid_unparse_lower(netdata_random_session_id, guid);
+
+    char filename[FILENAME_MAX + 1];
+    snprintfz(filename, FILENAME_MAX, "%s/netdata_random_session_id", netdata_configured_varlib_dir);
+
+    bool ret = true;
+
+    (void)unlink(filename);
+
+    // save it
+    int fd = open(filename, O_WRONLY|O_CREAT|O_TRUNC, 640);
+    if(fd == -1) {
+        error("Cannot create random session id file '%s'.", filename);
+        ret = false;
+    }
+
+    if(write(fd, guid, UUID_STR_LEN - 1) != UUID_STR_LEN - 1) {
+        error("Cannot write the random session id file '%s'.", filename);
+        ret = false;
+    }
+
+    ssize_t rc = write(fd, "\n", 1);
+    (void)rc;
+
+    close(fd);
+
+    if(ret && (!netdata_random_session_id_filename || strcmp(netdata_random_session_id_filename, filename) != 0)) {
+        freez(netdata_random_session_id_filename);
+        netdata_random_session_id_filename = strdupz(filename);
+    }
+
+    return ret;
+}
+
+const char *netdata_random_session_id_get_filename(void) {
+    if(!netdata_random_session_id_filename)
+        netdata_random_session_id_generate();
+
+    return netdata_random_session_id_filename;
+}
+
+bool netdata_random_session_id_matches(const char *guid) {
+    if(uuid_is_null(netdata_random_session_id))
+        return false;
+
+    uuid_t uuid;
+
+    if(uuid_parse(guid, uuid))
+        return false;
+
+    if(uuid_compare(netdata_random_session_id, uuid) == 0)
+        return true;
+
+    return false;
+}
+
+static bool check_claim_param(const char *s) {
+    if(!s || !*s) return true;
+
+    do {
+        if(isalnum(*s) || *s == '.' || *s == ',' || *s == '-' || *s == ':' || *s == '/' || *s == '_')
+            ;
+        else
+            return false;
+
+    } while(*++s);
+
+    return true;
+}
+
+void claim_reload_all(void) {
+    error_log_limit_unlimited();
+    load_claiming_state();
+    registry_update_cloud_base_url();
+    rrdpush_send_claimed_id(localhost);
+    error_log_limit_reset();
+}
+
+int api_v2_claim(struct web_client *w, char *url) {
+    char *key = NULL;
+    char *token = NULL;
+    char *rooms = NULL;
+    char *base_url = NULL;
+
+    while (url) {
+        char *value = strsep_skip_consecutive_separators(&url, "&");
+        if (!value || !*value) continue;
+
+        char *name = strsep_skip_consecutive_separators(&value, "=");
+        if (!name || !*name) continue;
+        if (!value || !*value) continue;
+
+        if(!strcmp(name, "key"))
+            key = value;
+        else if(!strcmp(name, "token"))
+            token = value;
+        else if(!strcmp(name, "rooms"))
+            rooms = value;
+        else if(!strcmp(name, "url"))
+            base_url = value;
+    }
+
+    BUFFER *wb = w->response.data;
+    buffer_flush(wb);
+    buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+
+    time_t now_s = now_realtime_sec();
+    CLOUD_STATUS status = buffer_json_cloud_status(wb, now_s);
+
+    bool can_be_claimed = false;
+    switch(status) {
+        case CLOUD_STATUS_AVAILABLE:
+        case CLOUD_STATUS_DISABLED:
+        case CLOUD_STATUS_OFFLINE:
+            can_be_claimed = true;
+            break;
+
+        case CLOUD_STATUS_UNAVAILABLE:
+        case CLOUD_STATUS_BANNED:
+        case CLOUD_STATUS_ONLINE:
+            can_be_claimed = false;
+            break;
+    }
+
+    buffer_json_member_add_boolean(wb, "can_be_claimed", can_be_claimed);
+
+    if(can_be_claimed && key) {
+        if(!netdata_random_session_id_matches(key)) {
+            buffer_reset(wb);
+            buffer_strcat(wb, "invalid key");
+            netdata_random_session_id_generate(); // generate a new key, to avoid an attack to find it
+            return HTTP_RESP_FORBIDDEN;
+        }
+
+        if(!token || !base_url || !check_claim_param(token) || !check_claim_param(base_url) || (rooms && !check_claim_param(rooms))) {
+            buffer_reset(wb);
+            buffer_strcat(wb, "invalid parameters");
+            netdata_random_session_id_generate(); // generate a new key, to avoid an attack to find it
+            return HTTP_RESP_BAD_REQUEST;
+        }
+
+        netdata_random_session_id_generate(); // generate a new key, to avoid an attack to find it
+
+        netdata_cloud_enabled = CONFIG_BOOLEAN_AUTO;
+        appconfig_set_boolean(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", CONFIG_BOOLEAN_AUTO);
+        appconfig_set(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", base_url);
+
+        uuid_t claimed_id;
+        uuid_generate_random(claimed_id);
+        char claimed_id_str[UUID_STR_LEN];
+        uuid_unparse_lower(claimed_id, claimed_id_str);
+
+        BUFFER *t = buffer_create(1024, NULL);
+        if(rooms)
+            buffer_sprintf(t, "-id=%s -token=%s -rooms=%s", claimed_id_str, token, rooms);
+        else
+            buffer_sprintf(t, "-id=%s -token=%s", claimed_id_str, token);
+
+        bool success = false;
+        const char *msg = NULL;
+        CLAIM_AGENT_RESPONSE rc = claim_agent(buffer_tostring(t), true, &msg);
+        switch(rc) {
+            case CLAIM_AGENT_OK:
+                msg = "ok";
+                success = true;
+                can_be_claimed = false;
+                claim_reload_all();
+                {
+                    int ms = 0;
+                    do {
+                        status = cloud_status();
+                        if (status == CLOUD_STATUS_ONLINE)
+                            break;
+
+                        sleep_usec(100 * USEC_PER_MS);
+                        ms += 100;
+                    } while (ms < 5000);
+                }
+                break;
+
+            case CLAIM_AGENT_NO_CLOUD_URL:
+                msg = "No Netdata Cloud URL.";
+                break;
+
+            case CLAIM_AGENT_CLAIM_SCRIPT_FAILED:
+                msg = "Claiming script failed.";
+                break;
+
+            case CLAIM_AGENT_CLOUD_DISABLED:
+                msg = "Netdata Cloud is disabled on this agent.";
+                break;
+
+            case CLAIM_AGENT_CANNOT_EXECUTE_CLAIM_SCRIPT:
+                msg = "Failed to execute claiming script.";
+                break;
+
+            case CLAIM_AGENT_CLAIM_SCRIPT_RETURNED_INVALID_CODE:
+                msg = "Claiming script returned invalid code.";
+                break;
+
+            default:
+            case CLAIM_AGENT_FAILED_WITH_MESSAGE:
+                if(!msg)
+                    msg = "Unknown error";
+                break;
+        }
+
+        // our status may have changed
+        // refresh the status in our output
+        buffer_flush(wb);
+        buffer_json_initialize(wb, "\"", "\"", 0, true, false);
+        now_s = now_realtime_sec();
+        buffer_json_cloud_status(wb, now_s);
+
+        // and this is the status of the claiming command we run
+        buffer_json_member_add_boolean(wb, "success", success);
+        buffer_json_member_add_string(wb, "message", msg);
+    }
+
+    if(can_be_claimed)
+        buffer_json_member_add_string(wb, "key_filename", netdata_random_session_id_get_filename());
+
+    buffer_json_finalize(wb);
+
+    return HTTP_RESP_OK;
 }

+ 17 - 1
claim/claim.h

@@ -8,9 +8,25 @@
 extern char *claiming_pending_arguments;
 extern struct config cloud_config;
 
-void claim_agent(char *claiming_arguments);
+typedef enum __attribute__((packed)) {
+    CLAIM_AGENT_OK,
+    CLAIM_AGENT_CLOUD_DISABLED,
+    CLAIM_AGENT_NO_CLOUD_URL,
+    CLAIM_AGENT_CANNOT_EXECUTE_CLAIM_SCRIPT,
+    CLAIM_AGENT_CLAIM_SCRIPT_FAILED,
+    CLAIM_AGENT_CLAIM_SCRIPT_RETURNED_INVALID_CODE,
+    CLAIM_AGENT_FAILED_WITH_MESSAGE,
+} CLAIM_AGENT_RESPONSE;
+
+CLAIM_AGENT_RESPONSE claim_agent(const char *claiming_arguments, bool force, const char **msg);
 char *get_agent_claimid(void);
 void load_claiming_state(void);
 void load_cloud_conf(int silent);
+void claim_reload_all(void);
+
+bool netdata_random_session_id_generate(void);
+const char *netdata_random_session_id_get_filename(void);
+bool netdata_random_session_id_matches(const char *guid);
+int api_v2_claim(struct web_client *w, char *url);
 
 #endif //NETDATA_CLAIM_H

+ 31 - 18
claim/netdata-claim.sh.in

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # netdata
 # real-time performance and health monitoring, done right!
-# (C) 2017 Costa Tsaousis <costa@tsaousis.gr>
+# (C) 2023 Netdata Inc.
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 # Exit code: 0 - Success
@@ -194,24 +194,37 @@ if [ -r "${CLAIMING_DIR}/rooms" ]; then
         ROOMS="$(cat "${CLAIMING_DIR}/rooms")"
 fi
 
+variable_to_set=
 for arg in "$@"
 do
-        case $arg in
-                -token=*) TOKEN=${arg:7} ;;
-                -url=*) [ -n "${arg:5}" ] && URL_BASE=${arg:5} ;;
-                -id=*) ID=$(echo "${arg:4}" | tr '[:upper:]' '[:lower:]');;
-                -rooms=*) ROOMS=${arg:7} ;;
-                -hostname=*) HOSTNAME=${arg:10} ;;
-                -verbose) VERBOSE=1 ;;
-                -insecure) INSECURE=1 ;;
-                -proxy=*) PROXY=${arg:7} ;;
-                -noproxy) NOPROXY=yes ;;
-                -noreload) RELOAD=0 ;;
-                -user=*) NETDATA_USER=${arg:6} ;;
-                -daemon-not-running) NETDATA_RUNNING=0 ;;
-                *)  echo >&2 "Unknown argument ${arg}"
-                    exit 1 ;;
-        esac
+        if [ -z "$variable_to_set" ]; then
+              case $arg in
+                      --claim-token) variable_to_set="TOKEN" ;;
+                      --claim-rooms) variable_to_set="ROOMS" ;;
+                      --claim-url) variable_to_set="URL_BASE" ;;
+                      -token=*) TOKEN=${arg:7} ;;
+                      -url=*) [ -n "${arg:5}" ] && URL_BASE=${arg:5} ;;
+                      -id=*) ID=$(echo "${arg:4}" | tr '[:upper:]' '[:lower:]');;
+                      -rooms=*) ROOMS=${arg:7} ;;
+                      -hostname=*) HOSTNAME=${arg:10} ;;
+                      -verbose) VERBOSE=1 ;;
+                      -insecure) INSECURE=1 ;;
+                      -proxy=*) PROXY=${arg:7} ;;
+                      -noproxy) NOPROXY=yes ;;
+                      -noreload) RELOAD=0 ;;
+                      -user=*) NETDATA_USER=${arg:6} ;;
+                      -daemon-not-running) NETDATA_RUNNING=0 ;;
+                      *)  echo >&2 "Unknown argument ${arg}"
+                          exit 1 ;;
+              esac
+        else
+              case "$variable_to_set" in
+                      TOKEN) TOKEN="$arg" ;;
+                      ROOMS) ROOMS="$arg" ;;
+                      URL_BASE) URL_BASE="$arg" ;;
+              esac
+              variable_to_set=
+        fi
         shift 1
 done
 
@@ -402,7 +415,7 @@ if [ "${HTTP_STATUS_CODE}" = "204" ] || [ "${ERROR_KEY}" = "ErrAlreadyClaimed" ]
   cloud base url = $URL_BASE
 HERE_DOC
         if [ "$EUID" == "0" ]; then
-            chown -R "${NETDATA_USER}:${NETDATA_USER}" ${CLAIMING_DIR} || (echo >&2 "Claiming failed"; set -e; exit 2)
+            chown -R "${NETDATA_USER}:${NETDATA_USER}" "${CLAIMING_DIR}" || (echo >&2 "Claiming failed"; set -e; exit 2)
         fi
         if [ "${RELOAD}" == "0" ] ; then
             exit $EXIT_CODE

Some files were not shown because too many files changed in this diff