Fix bugs in streaming and enable support for gap filling (#9214)

This PR adds (inactive) support that we will use to fill the gaps on chart when a receiving agent goes offline and the sender reconnects. The streaming component has been reworked to make the connection bi-directional and fix several outstanding bugs in the area. 

* Fixed an incorrect case of version negotiation. Removed fatal() on exhaustion of fds. 
* Fixed cases that fell through to polling the socket after closing. 
* Fixed locking of data related to sender and receiver in the host structure. 
* Added fine-grained locks to reduce contention.
* Added circular buffer to sender to prevent starvation in high-latency conditions. 
* Fixed case where agent is a proxy and negotiated different streaming versions with sender and receiver. 
* Changed interface to new parser to put the buffering code in streaming. 
* Fixed the bug that stopped senders from reconnecting after their socket times out - this was part of the scaling fixes that provide an early shortcut path for rejecting connections without lock contention. 
* Uses fine-grained locking and a different approach to thread shutdown instead. 
* Added liveness detection to connections to allow selection of the best connection.

Makefile.am

@@ -138,6 +138,8 @@ LIBNETDATA_FILES = \
     libnetdata/avl/avl.h \
     libnetdata/buffer/buffer.c \
     libnetdata/buffer/buffer.h \
+    libnetdata/circular_buffer/circular_buffer.c \
+    libnetdata/circular_buffer/circular_buffer.h \
     libnetdata/clocks/clocks.c \
     libnetdata/clocks/clocks.h \
     libnetdata/dictionary/dictionary.c \
@@ -429,6 +431,8 @@ API_PLUGIN_FILES = \
 
 STREAMING_PLUGIN_FILES = \
     streaming/rrdpush.c \
+    streaming/sender.c \
+    streaming/receiver.c \
     streaming/rrdpush.h \
     $(NULL)
 

build_external/clean-install-arch-debug.Dockerfile

@@ -0,0 +1,62 @@
+FROM archlinux/base:latest
+
+# There is some redundancy between this file and the archlinux Dockerfile in the helper images
+# repo and also with the clean-install.Dockefile. Once the help image is availabled on Docker
+# Hub this file can be deleted.
+RUN echo sdlsjdkls
+RUN pacman -Syyu --noconfirm
+RUN pacman --noconfirm --needed -S autoconf \
+                                   autoconf-archive \
+                                   autogen \
+                                   automake \
+                                   gcc \
+                                   make \
+                                   git \
+                                   libuv \
+                                   lz4 \
+                                   netcat \
+                                   openssl \
+                                   pkgconfig \
+                                   python \
+                                   libvirt \
+                                   cmake \
+                                   valgrind \
+                                   gdb
+
+ARG EXTRA_CFLAGS
+COPY . /opt/netdata/source
+WORKDIR /opt/netdata/source
+
+RUN git config --global user.email "root@container"
+RUN git config --global user.name "Fake root"
+
+# RUN make distclean   -> not safe if tree state changed on host since last config
+# Kill everything that is not in .gitignore preserving any fresh changes, i.e. untracked changes will be
+# deleted but local changes to tracked files will be preserved.
+RUN if git status --porcelain | grep '^[MADRC]'; then \
+        git stash && git clean -dxf && (git stash apply || true) \
+    else \
+        git clean -dxf ; \
+    fi
+
+# Not everybody is updating distclean properly - fix.
+RUN find . -name '*.Po' -exec rm \{\} \;
+RUN rm -rf autom4te.cache
+RUN rm -rf .git/
+RUN find . -type f >/opt/netdata/manifest
+
+RUN CFLAGS="-Og -g -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -DNETDATA_INTERNAL_CHECKS=1\
+    -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1 ${EXTRA_CFLAGS}" ./netdata-installer.sh --require-cloud --disable-lto
+
+RUN ln -sf /dev/stdout /var/log/netdata/access.log
+RUN ln -sf /dev/stdout /var/log/netdata/debug.log
+RUN ln -sf /dev/stderr /var/log/netdata/error.log
+
+RUN printf >/opt/netdata/source/gdb_batch '\
+set args -D \n\
+handle SIG32 nostop \n\
+run \n\
+bt'
+
+#CMD ["/usr/sbin/valgrind", "--leak-check=full", "/usr/sbin/netdata", "-D"]
+CMD ["/usr/bin/gdb", "-x", "/opt/netdata/source/gdb_batch", "/usr/sbin/netdata"]

build_external/clean-install-arch-extras.Dockerfile

@@ -3,7 +3,7 @@ FROM archlinux/base:latest
 # There is some redundancy between this file and the archlinux Dockerfile in the helper images
 # repo and also with the clean-install.Dockefile. Once the help image is availabled on Docker
 # Hub this file can be deleted.
-
+RUN echo sdlsjdkls
 RUN pacman -Syyu --noconfirm
 RUN pacman --noconfirm --needed -S autoconf \
                                    autoconf-archive \
@@ -20,7 +20,8 @@ RUN pacman --noconfirm --needed -S autoconf \
                                    python \
                                    libvirt \
                                    cmake \
-                                   valgrind
+                                   valgrind \
+                                   gdb
 
 ARG EXTRA_CFLAGS
 COPY . /opt/netdata/source
@@ -44,12 +45,14 @@ RUN rm -rf autom4te.cache
 RUN rm -rf .git/
 RUN find . -type f >/opt/netdata/manifest
 
-RUN CFLAGS="-O1 -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -DNETDATA_INTERNAL_CHECKS=1\
+RUN CFLAGS="-Og -g -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -DNETDATA_INTERNAL_CHECKS=1\
     -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1 ${EXTRA_CFLAGS}" ./netdata-installer.sh --require-cloud --disable-lto
 
 RUN ln -sf /dev/stdout /var/log/netdata/access.log
 RUN ln -sf /dev/stdout /var/log/netdata/debug.log
 RUN ln -sf /dev/stderr /var/log/netdata/error.log
 
+RUN rm /var/lib/netdata/registry/netdata.public.unique.id
+
 CMD ["/usr/sbin/valgrind", "--leak-check=full", "/usr/sbin/netdata", "-D"]
    

build_external/clean-install.Dockerfile

@@ -33,4 +33,7 @@ RUN ln -sf /dev/stdout /var/log/netdata/access.log
 RUN ln -sf /dev/stdout /var/log/netdata/debug.log
 RUN ln -sf /dev/stderr /var/log/netdata/error.log
 
-CMD ["/usr/sbin/netdata","-D"]
+RUN rm /var/lib/netdata/registry/netdata.public.unique.id
+
+CMD ["/usr/sbin/netdata","-D"]
+ENTRYPOINT []