Adds ACLK-NG as fallback(#10315)

* adds a new implementation of ACLK written almost from scratch
* external dependencies only OpenSSL and JSON-C
* fallback for systems where ACLK Legacy can't build (for technical or philosophical reasons)
* can be forced to build by giving "--aclk-ng" to the installer

.github/workflows/build-and-install.yml

@@ -12,6 +12,8 @@ jobs:
     steps:
       - name: Git clone repository
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - run: |
           git fetch --prune --unshallow --tags
       - name: Build
@@ -98,6 +100,8 @@ jobs:
     steps:
       - name: Git clone repository
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: install-required-packages.sh on ${{ matrix.distro }}
         env:
           PRE: ${{ matrix.pre }}
@@ -183,6 +187,8 @@ jobs:
     steps:
       - name: Git clone repository
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: install-required-packages.sh on ${{ matrix.distro }}
         env:
           PRE: ${{ matrix.pre }}

.github/workflows/checks.yml

@@ -12,6 +12,8 @@ jobs:
     steps:
       - name: Git clone repository
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Run checksum checks on kickstart files
         env:
           LOCAL_ONLY: "true"
@@ -23,6 +25,8 @@ jobs:
     steps:
       - name: Git clone repository
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Install required packages
         run: |
           ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
@@ -43,6 +47,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Build
         run: >
           docker run -v "$PWD":/netdata -w /netdata alpine:latest /bin/sh -c
@@ -68,6 +74,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Prepare environment
         run: |
           ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
@@ -98,6 +106,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Prepare environment
         run: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
       - name: Build netdata

.github/workflows/coverity.yml

@@ -15,6 +15,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Prepare environment
         env:
           DEBIAN_FRONTEND: 'noninteractive'

.github/workflows/docker.yml

@@ -26,6 +26,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Determine if we should push changes and which tags to use
         if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly'
         run: |

.github/workflows/docs.yml

@@ -16,6 +16,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Run link check
         uses: gaurav-nelson/github-action-markdown-link-check@v1
         with:

.github/workflows/review.yml

@@ -16,6 +16,7 @@ jobs:
       - name: Git clone repository
         uses: actions/checkout@v2
         with:
+          submodules: recursive
           fetch-depth: 0
       - name: Check files
         run: |
@@ -57,6 +58,7 @@ jobs:
       - name: Git clone repository
         uses: actions/checkout@v2
         with:
+          submodules: recursive
           fetch-depth: 0
       - name: Check files
         run: |
@@ -80,6 +82,7 @@ jobs:
       - name: Git clone repository
         uses: actions/checkout@v2
         with:
+          submodules: recursive
           fetch-depth: 0
       - name: Check files
         run: |

.github/workflows/tests.yml

@@ -21,6 +21,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Prepare environment
         run: |
           ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
@@ -39,6 +41,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Prepare environment
         run: |
           ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
@@ -48,7 +52,7 @@ jobs:
       - name: Configure
         run: |
           autoreconf -ivf
-          ./configure
+          ./configure --without-aclk-ng
       # XXX: Work-around for bug with libbson-1.0 in Ubuntu 18.04
       # See: https://bugs.launchpad.net/ubuntu/+source/libmongoc/+bug/1790771
       #      https://jira.mongodb.org/browse/CDRIVER-2818

.github/workflows/updater.yml

@@ -34,6 +34,8 @@ jobs:
     steps:
       - name: Git clone repository
         uses: actions/checkout@v2
+        with:
+          submodules: recursive
       - name: Install required packages & build tarball
         run: |
           ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "mqtt_websockets"]
+	path = mqtt_websockets
+	url = https://github.com/underhood/mqtt_websockets.git

Makefile.am

@@ -116,10 +116,18 @@ SUBDIRS += \
     web \
     claim \
     parser \
-    aclk/legacy \
     spawn \
     $(NULL)
 
+if ACLK_NG
+SUBDIRS += \
+    mqtt_websockets \
+    $(NULL)
+else
+SUBDIRS += \
+    aclk/legacy \
+    $(NULL)
+endif
 
 AM_CFLAGS = \
     $(OPTIONAL_MATH_CFLAGS) \
@@ -525,6 +533,28 @@ PARSER_FILES = \
     parser/parser.h \
     $(NULL)
 
+if ACLK_NG
+ACLK_FILES = \
+    aclk/aclk.c \
+    aclk/aclk.h \
+    aclk/aclk_util.c \
+    aclk/aclk_util.h \
+    aclk/aclk_stats.c \
+    aclk/aclk_stats.h \
+    aclk/aclk_query.c \
+    aclk/aclk_query.h \
+    aclk/aclk_query_queue.c \
+    aclk/aclk_query_queue.h \
+    aclk/aclk_collector_list.c \
+    aclk/aclk_collector_list.h \
+    aclk/aclk_otp.c \
+    aclk/aclk_otp.h \
+    aclk/aclk_tx_msgs.c \
+    aclk/aclk_tx_msgs.h \
+    aclk/aclk_rx_msgs.c \
+    aclk/aclk_rx_msgs.h \
+    $(NULL)
+else #ACLK_NG
 ACLK_FILES = \
     aclk/legacy/aclk_rrdhost_state.h \
     aclk/legacy/aclk_common.c \
@@ -548,9 +578,8 @@ ACLK_FILES += \
     aclk/legacy/aclk_lws_https_client.c \
     aclk/legacy/aclk_lws_https_client.h \
     $(NULL)
-endif
-
-
+endif #ENABLE_ACLK
+endif #ACLK_NG
 
 SPAWN_PLUGIN_FILES = \
     spawn/spawn.c \
@@ -714,6 +743,12 @@ NETDATACLI_FILES = \
 sbin_PROGRAMS += netdata
 netdata_SOURCES = $(NETDATA_FILES)
 
+if ACLK_NG
+netdata_LDADD = \
+    mqtt_websockets/libmqttwebsockets.a \
+    $(NETDATA_COMMON_LIBS) \
+    $(NULL)
+else #ACLK_NG
 if ENABLE_ACLK
 netdata_LDADD = \
     externaldeps/mosquitto/libmosquitto.a \
@@ -721,11 +756,12 @@ netdata_LDADD = \
     $(OPTIONAL_LWS_LIBS) \
     $(NETDATA_COMMON_LIBS) \
     $(NULL)
-else
+else #ENABLE_ACLK
 netdata_LDADD = \
     $(NETDATA_COMMON_LIBS) \
     $(NULL)
-endif
+endif #ENABLE_ACLK
+endif #ACLK_NG
 
 if ENABLE_CXX_LINKER
     netdata_LINK = $(CXXLD) $(CXXFLAGS) $(LDFLAGS) -o $@