Add Debian 11 (Bullseye) to CI. (#11334)

* Add Debian 11 (Bullseye) to CI.

Official feature freeze is in a couple of days, and the release is soon
to follow.

* Fix handling of Debian 11 package builds.

* Fix build dependencies for Debian 11.

`dh-systemd` has long since been deprecated, we should instead be
depending on the minimum version of `debhelper` that includes the
required code. We need to special-case Ubuntu 16.04 because it predates
the merge of `dh-systemd` into `debhelper`.

* Use correct Docker image for Debian 11.

* Add repoconfig package build for Debian 11.

* Simplify handling of package building and testing.

This should fix the CI issues we’re seeing with package builds, or at
least make them easier to diagnose.

packaging/scripts/install.sh → .github/scripts/pkg-test.sh

@@ -7,11 +7,10 @@ install_debian_like() {
   apt-get update
 
   # Install NetData
-  apt-get install -y "/packages/netdata_${VERSION}_${ARCH}.deb"
+  apt-get install -y /netdata/artifacts/netdata_"${VERSION}"_*.deb || exit 1
 
   # Install testing tools
-  apt-get install -y --no-install-recommends \
-    curl netcat jq
+  apt-get install -y --no-install-recommends curl netcat jq || exit 1
 }
 
 install_fedora_like() {
@@ -23,10 +22,10 @@ install_fedora_like() {
   pkg_version="$(echo "${VERSION}" | tr - .)"
 
   # Install NetData
-  "$PKGMGR" install -y /packages/netdata-"${pkg_version}"-*.rpm
+  "$PKGMGR" install -y /netdata/artifacts/netdata-"${pkg_version}"-*.rpm
 
   # Install testing tools
-  "$PKGMGR" install -y curl nc jq
+  "$PKGMGR" install -y curl nc jq || exit 1
 }
 
 install_centos() {
@@ -38,13 +37,13 @@ install_centos() {
   pkg_version="$(echo "${VERSION}" | tr - .)"
 
   # Install EPEL (needed for `jq`
-  "$PKGMGR" install -y epel-release
+  "$PKGMGR" install -y epel-release || exit 1
 
   # Install NetData
-  "$PKGMGR" install -y /packages/netdata-"${pkg_version}"-*.rpm
+  "$PKGMGR" install -y /netdata/artifacts/netdata-"${pkg_version}"-*.rpm
 
   # Install testing tools
-  "$PKGMGR" install -y curl nc jq
+  "$PKGMGR" install -y curl nc jq || exit 1
 }
 
 install_suse_like() {
@@ -54,13 +53,45 @@ install_suse_like() {
   pkg_version="$(echo "${VERSION}" | tr - .)"
 
   # Install NetData
-  # FIXME: Allow unsigned packages (for now) #7773
-  zypper install -y --allow-unsigned-rpm \
-    /packages/netdata-"${pkg_version}"-*.rpm
+  zypper install -y --allow-unsigned-rpm /netdata/artifacts/netdata-"${pkg_version}"-*.rpm
 
   # Install testing tools
-  zypper install -y --no-recommends \
-    curl netcat-openbsd jq
+  zypper install -y --no-recommends curl netcat-openbsd jq || exit 1
+}
+
+dump_log() {
+  cat ./netdata.log
+}
+
+wait_for() {
+  host="${1}"
+  port="${2}"
+  name="${3}"
+  timeout="30"
+
+  if command -v nc > /dev/null ; then
+    netcat="nc"
+  elif command -v netcat > /dev/null ; then
+    netcat="netcat"
+  else
+    printf "Unable to find a usable netcat command.\n"
+    return 1
+  fi
+
+  printf "Waiting for %s on %s:%s ... " "${name}" "${host}" "${port}"
+
+  sleep 30
+
+  i=0
+  while ! ${netcat} -z "${host}" "${port}"; do
+    sleep 1
+    if [ "$i" -gt "$timeout" ]; then
+      printf "Timed out!\n"
+      return 1
+    fi
+    i="$((i + 1))"
+  done
+  printf "OK\n"
 }
 
 case "${DISTRO}" in
@@ -81,3 +112,17 @@ case "${DISTRO}" in
     exit 1
     ;;
 esac
+
+trap dump_log EXIT
+
+/usr/sbin/netdata -D > ./netdata.log 2>&1 &
+
+wait_for localhost 19999 netdata || exit 1
+
+curl -sS http://127.0.0.1:19999/api/v1/info > ./response || exit 1
+
+cat ./response
+
+jq '.version' ./response || exit 1
+
+trap - EXIT

.github/workflows/build-and-install.yml

@@ -36,6 +36,7 @@ jobs:
           - 'archlinux:latest'
           - 'centos:8'
           - 'centos:7'
+          - 'debian:bullseye' # 11
           - 'debian:10'
           - 'debian:9'
           - 'fedora:34'
@@ -70,6 +71,9 @@ jobs:
           - distro: 'centos:8'
             rmjsonc: 'dnf remove -y json-c-devel'
 
+          - distro: 'debian:bullseye' # 11
+            pre: 'apt-get update'
+            rmjsonc: 'apt-get remove -y libjson-c-dev'
           - distro: 'debian:10'
             pre: 'apt-get update'
             rmjsonc: 'apt-get remove -y libjson-c-dev'

.github/workflows/packaging.yml

@@ -30,6 +30,8 @@ jobs:
           - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386}
           - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
           - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386}
+          - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64, alias: bullseye}
+          - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/i386, arch: i386, alias: bullseye}
           - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
           - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
           - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
@@ -53,14 +55,14 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0 # We need full history for versioning
-          submodules: true
+          submodules: recursive
       - name: Checkout Tag # Otherwise check out the tag that triggered this.
         if: github.event_name == 'workflow_dispatch'
         uses: actions/checkout@v2
         with:
           ref: ${{ github.event.ref }}
           fetch-depth: 0 # We need full history for versioning
-          submodules: true
+          submodules: recursive
       - name: Check Base Branch
         run: |
           if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
@@ -89,32 +91,27 @@ jobs:
       - name: Setup QEMU
         if: matrix.platform != 'linux/amd64'
         uses: docker/setup-qemu-action@v1
-      - name: Setup Buildx
-        uses: docker/setup-buildx-action@v1
       - name: Prepare Docker Environment
         shell: bash
         run: |
           echo '{"cgroup-parent": "/actions_job", "experimental": true}' | sudo tee /etc/docker/daemon.json 2>/dev/null
           sudo service docker restart
+      - name: Set Base Image Version
+        shell: bash
+        run: |
+          if [ -z "${{ matrix.alias }}" ] ; then
+            echo "version=${{ matrix.version }}" >> $GITHUB_ENV
+          else
+            echo "version=${{ matrix.alias }}" >> $GITHUB_ENV
+          fi
       - name: Build Packages
-        uses: docker/build-push-action@v2
-        with:
-          platforms: ${{ matrix.platform }}
-          file: packaging/Dockerfile.packager
-          tags: local/package-builder:${{ matrix.distro}}${{ matrix.version }}
-          push: false
-          load: true
-          build-args: |
-            ARCH=${{ matrix.arch }}
-            DISTRO=${{ matrix.distro }}
-            TEST_BASE=${{ matrix.base_image }}
-            DISTRO_VERSION=${{ matrix.version }}
-            PKG_VERSION=${{ env.pkg_version }}
-      - name: Extract Packages
         shell: bash
         run: |
-          mkdir -p artifacts
-          docker run -e DO_NOT_TRACK=1 --platform ${{ matrix.platform }} -v $PWD/artifacts:/artifacts local/package-builder:${{ matrix.distro }}${{ matrix.version }}
+          docker run -e DO_NOT_TRACK=1 -e VERSION=${{ env.pkg_version }} --platform=${{ matrix.platform }} -v $PWD:/netdata netdata/package-builders:${{ matrix.distro }}${{ matrix.version }}
+      - name: Test Packages
+        shell: bash
+        run: |
+          docker run -e DO_NOT_TRACK=1 -e DISTRO=${{ matrix.distro }} -e VERSION=${{ env.pkg_version }} -e DISTRO_VERSION=${{ env.version }} --platform=${{ matrix.platform }} -v $PWD:/netdata ${{ matrix.base_image }}:${{ env.version }} /netdata/.github/scripts/pkg-test.sh
       - name: Upload
         if: github.event_name == 'workflow_dispatch'
         shell: bash

.github/workflows/repoconfig-packages.yml

@@ -18,6 +18,7 @@ jobs:
         include:
           - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
           - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
+          - {distro: debian, version: "11", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
           - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
           - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
           - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}

.github/workflows/updater.yml

@@ -27,6 +27,7 @@ jobs:
           - 'centos:8'
           - 'debian:9'
           - 'debian:10'
+          - 'debian:bullseye' # 11
           - 'fedora:33'
           - 'fedora:34'
           - 'ubuntu:16.04'
@@ -44,6 +45,8 @@ jobs:
             pre: 'apt-get update'
           - distro: 'debian:10'
             pre: 'apt-get update'
+          - distro: 'debian:bullseye' # 11
+            pre: 'apt-get update'
           - distro: 'ubuntu:16.04'
             pre: 'apt-get update'
           - distro: 'ubuntu:18.04'

contrib/debian/control

@@ -1,7 +1,6 @@
 Source: netdata
-Build-Depends: debhelper (>= 9),
+Build-Depends: debhelper (>= 9.20160709),
                dh-autoreconf,
-               dh-systemd (>= 1.5),
                dpkg-dev (>= 1.13.19),
                zlib1g-dev,
                uuid-dev,

contrib/debian/control.xenial

@@ -0,0 +1,62 @@
+Source: netdata
+Build-Depends: debhelper (>= 9),
+               dh-autoreconf,
+               dh-systemd (>= 1.5),
+               dpkg-dev (>= 1.13.19),
+               zlib1g-dev,
+               uuid-dev,
+               libelf-dev,
+               libuv1-dev,
+               liblz4-dev,
+               libjudy-dev,
+               libssl-dev,
+               libmnl-dev,
+               libjson-c-dev,
+               libcups2-dev,
+               libipmimonitoring-dev,
+               libnetfilter-acct-dev,
+               libsnappy-dev,
+               libprotobuf-dev,
+               libprotoc-dev,
+               cmake,
+               autogen,
+               autoconf,
+               automake,
+               pkg-config,
+               curl,
+               protobuf-compiler
+Section: net
+Priority: optional
+Maintainer: Netdata Builder <bot@netdata.cloud>
+Standards-Version: 3.9.6
+Homepage: https://netdata.cloud
+
+Package: netdata
+Architecture: any
+Depends: adduser,
+         libcap2-bin (>= 1:2.0),
+         lsb-base (>= 3.1-23.2),
+         openssl,
+         ${misc:Depends},
+         ${shlibs:Depends}
+Pre-Depends: dpkg (>= 1.17.14)
+Description: real-time charts for system monitoring
+ Netdata is a daemon that collects data in realtime (per second)
+ and presents a web site to view and analyze them. The presentation
+ is also real-time and full of interactive charts that precisely
+ render all collected values.
+
+Package: netdata-plugin-cups
+Architecture: any
+Depends: cups,
+         netdata (>= ${source:Version})
+Description: The Common Unix Printing System plugin for metrics collection from cupsd
+
+Package: netdata-plugin-freeipmi
+Architecture: any
+Depends: freeipmi,
+         netdata (= ${source:Version})
+Description: FreeIPMI - The Intelligent Platform Management System.
+ The IPMI specification defines a set of interfaces for platform management.
+ It is implemented by a number vendors for system management. The features of IPMI that most users will be interested in
+ are sensor monitoring, system event monitoring, power control, and serial-over-LAN (SOL).

packaging/Dockerfile.packager

@@ -1,45 +0,0 @@
-ARG ARCH=amd64
-ARG DISTRO=debian
-ARG TEST_BASE=debian
-ARG DISTRO_VERSION=10
-ARG PKG_VERSION=0.1
-
-FROM netdata/package-builders:${DISTRO}${DISTRO_VERSION} AS build
-
-ARG ARCH
-ARG DISTRO
-ARG DISTRO_VERSION
-ARG PKG_VERSION
-
-ENV ARCH=$ARCH
-ENV DISTRO=$DISTRO
-ENV DISTRO_VERSION=$DISTRO_VERSION
-ENV VERSION=$PKG_VERSION
-ENV DO_NOT_TRACK=1
-
-WORKDIR /netdata
-COPY . .
-
-RUN /build.sh
-
-FROM ${TEST_BASE}:${DISTRO_VERSION} AS runtime
-
-ARG ARCH
-ARG DISTRO
-ARG DISTRO_VERSION
-ARG PKG_VERSION
-
-ENV ARCH=$ARCH
-ENV DISTRO=$DISTRO
-ENV DISTRO_VERSION=$DISTRO_VERSION
-ENV VERSION=$PKG_VERSION
-ENV DO_NOT_TRACK=1
-
-COPY ./packaging/scripts/install.sh /install.sh
-COPY ./packaging/scripts/test.sh /test.sh
-
-COPY --from=build /netdata/artifacts /packages
-
-RUN /install.sh
-
-CMD ["/test.sh"]

packaging/scripts/test.sh

@@ -1,52 +0,0 @@
-#!/bin/sh
-
-dump_log() {
-  cat ./netdata.log
-}
-
-trap dump_log EXIT
-
-wait_for() {
-  host="${1}"
-  port="${2}"
-  name="${3}"
-  timeout="30"
-
-  if command -v nc > /dev/null ; then
-    netcat="nc"
-  elif command -v netcat > /dev/null ; then
-    netcat="netcat"
-  else
-    printf "Unable to find a usable netcat command.\n"
-    return 1
-  fi
-
-  printf "Waiting for %s on %s:%s ... " "${name}" "${host}" "${port}"
-
-  sleep 30
-
-  i=0
-  while ! ${netcat} -z "${host}" "${port}"; do
-    sleep 1
-    if [ "$i" -gt "$timeout" ]; then
-      printf "Timed out!\n"
-      return 1
-    fi
-    i="$((i + 1))"
-  done
-  printf "OK\n"
-}
-
-/usr/sbin/netdata -D > ./netdata.log 2>&1 &
-
-wait_for localhost 19999 netdata || exit 1
-
-curl -sS http://127.0.0.1:19999/api/v1/info > ./response || exit 1
-
-cat ./response
-
-jq '.version' ./response || exit 1
-
-trap - EXIT
-
-cp -a /packages/* /artifacts