Consolidate build matrix across CI jobs. (#11807)

* Restructure CI build matrix handling for better consolidation.

* Update core packaging code to use new distro.yml file.

* Update repository config packaging code to use new distro.yml file.

.github/data/build-matrix.json

@@ -1,98 +0,0 @@
-{
-  "include": [
-    {
-      "distro": "alpine:edge",
-      "artifact_key": "alpine-edge",
-      "pre": "apk add -U bash",
-      "rmjsonc": "apk del json-c-dev"
-    },
-    {
-      "distro": "alpine:3.14",
-      "artifact_key": "alpine-3.14",
-      "pre": "apk add -U bash",
-      "rmjsonc": "apk del json-c-dev"
-    },
-    {
-      "distro": "alpine:3.13",
-      "artifact_key": "alpine-3.13",
-      "pre": "apk add -U bash",
-      "rmjsonc": "apk del json-c-dev"
-    },
-    {
-      "distro": "alpine:3.12",
-      "artifact_key": "alpine-3.12",
-      "pre": "apk add -U bash",
-      "rmjsonc": "apk del json-c-dev"
-    },
-    {
-      "distro": "archlinux:latest",
-      "artifact_key": "archlinux",
-      "pre": "pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi"
-    },
-    {
-      "distro": "centos:7",
-      "artifact_key": "centos7"
-    },
-    {
-      "distro": "rockylinux/rockylinux:8",
-      "artifact_key": "rockylinux8",
-      "rmjsonc": "dnf remove -y json-c-devel"
-    },
-    {
-      "distro": "debian:11",
-      "artifact_key": "debian11",
-      "pre": "apt-get update",
-      "rmjsonc": "apt-get remove -y libjson-c-dev"
-    },
-    {
-      "distro": "debian:10",
-      "artifact_key": "debian10",
-      "pre": "apt-get update",
-      "rmjsonc": "apt-get remove -y libjson-c-dev"
-    },
-    {
-      "distro": "debian:9",
-      "artifact_key": "debian9",
-      "pre": "apt-get update",
-      "rmjsonc": "apt-get remove -y libjson-c-dev"
-    },
-    {
-      "distro": "fedora:35",
-      "artifact_key": "fedora35",
-      "rmjsonc": "dnf remove -y json-c-devel"
-    },
-    {
-      "distro": "fedora:34",
-      "artifact_key": "fedora34",
-      "rmjsonc": "dnf remove -y json-c-devel"
-    },
-    {
-      "distro": "opensuse/leap:15.3",
-      "artifact_key": "opensuse15.3",
-      "rmjsonc": "zypper rm -y libjson-c-devel"
-    },
-    {
-      "distro": "oraclelinux:8",
-      "artifact_key": "oraclelinux8",
-      "rmjsonc": "dnf remove -y json-c-devel"
-    },
-    {
-      "distro": "ubuntu:21.10",
-      "artifact_key": "ubuntu21.10",
-      "pre": "rm -f /etc/apt/apt.conf.d/docker && apt-get update",
-      "rmjsonc": "apt-get remove -y libjson-c-dev"
-    },
-    {
-      "distro": "ubuntu:20.04",
-      "artifact_key": "ubuntu20.04",
-      "pre": "apt-get update",
-      "rmjsonc": "apt-get remove -y libjson-c-dev"
-    },
-    {
-      "distro": "ubuntu:18.04",
-      "artifact_key": "ubuntu18.04",
-      "pre": "apt-get update",
-      "rmjsonc": "apt-get remove -y libjson-c-dev"
-    }
-  ]
-}

.github/data/distros.yml

@@ -0,0 +1,145 @@
+# This defines the full set of distros we run CI on.
+---
+platform_map: # map packaging architectures to docker platforms
+  i386: linux/i386
+  amd64: linux/amd64
+  armhf: linux/arm/v7
+  arm64: linux/arm64/v8
+include:
+  - &alpine
+    distro: alpine
+    version: edge
+    env_prep: |
+      apk add -U bash
+    jsonc_removal: |
+      apk del json-c-dev
+  - <<: *alpine
+    version: "3.15"
+  - <<: *alpine
+    version: "3.14"
+  - <<: *alpine
+    version: "3.13"
+  - <<: *alpine
+    version: "3.12"
+
+  - distro: archlinux
+    version: latest
+    env_prep: |
+      pacman --noconfirm -Syu && pacman --noconfirm -Sy grep libffi
+
+  - distro: rockylinux
+    version: "8"
+    base_image: rockylinux/rockylinux
+    jsonc_removal: |
+      dnf remove -y json-c-devel
+    packages:
+      type: rpm
+      repo_distro: el/8
+      arches:
+        - amd64
+        - arm64
+  - distro: centos
+    version: "7"
+    packages:
+      type: rpm
+      repo_distro: el/7
+      arches:
+        - amd64
+
+  - &debian
+    distro: debian
+    version: "11"
+    env_prep: |
+      apt-get update
+    jsonc_removal: |
+      apt-get purge -y libjson-c-dev
+    packages: &debian_packages
+      type: deb
+      repo_distro: debian/bullseye
+      arches:
+        - i386
+        - amd64
+        - armhf
+        - arm64
+  - <<: *debian
+    version: "10"
+    packages:
+      <<: *debian_packages
+      repo_distro: debian/buster
+  - <<: *debian
+    version: "9"
+    packages:
+      <<: *debian_packages
+      repo_distro: debian/stretch
+
+  - &fedora
+    distro: fedora
+    version: "35"
+    jsonc_removal: |
+      dnf remove -y json-c-devel
+    packages: &fedora_packages
+      type: rpm
+      repo_distro: fedora/35
+      arches:
+        - amd64
+        - armhf
+        - arm64
+  - <<: *fedora
+    version: "34"
+    packages:
+      <<: *fedora_packages
+      repo_distro: fedora/34
+
+  - &opensuse
+    distro: opensuse
+    version: "15.3"
+    base_image: opensuse/leap
+    jsonc_removal: |
+      zypper rm -y libjson-c-devel
+    packages:
+      type: rpm
+      repo_distro: opensuse/15.3
+      arches:
+        - amd64
+        - arm64
+
+  - distro: oraclelinux
+    version: "8"
+    jsonc_removal: |
+      dnf remove -y json-c-devel
+    packages:
+      type: rpm
+      repo_distro: ol/8
+      arches:
+        - amd64
+        - arm64
+
+  - &ubuntu
+    distro: ubuntu
+    version: "21.10"
+    env_prep: |
+      rm -f /etc/apt/apt.conf.d/docker && apt-get update
+    jsonc_removal: |
+      apt-get remove -y libjson-c-dev
+    packages: &ubuntu_packages
+      type: deb
+      repo_distro: ubuntu/impish
+      arches:
+        - amd64
+        - armhf
+        - arm64
+  - <<: *ubuntu
+    version: "20.04"
+    packages:
+      <<: *ubuntu_packages
+      repo_distro: ubuntu/focal
+  - <<: *ubuntu
+    version: "18.04"
+    packages:
+      <<: *ubuntu_packages
+      repo_distro: ubuntu/bionic
+      arches:
+        - i386
+        - amd64
+        - armhf
+        - arm64

.github/workflows/build.yml

@@ -129,13 +129,29 @@ jobs:
         uses: actions/checkout@v2
       - name: Prepare tools
         run: |
-          sudo apt-get update && sudo apt-get install -y jq
+          sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml
       - name: Read build matrix
         id: set-matrix
+        shell: python3 {0}
         run: |
-          TASKS="$(jq -c . .github/data/build-matrix.json)"
-          echo "Generated Matrix: $TASKS"
-          echo "::set-output name=matrix::$TASKS"
+          from ruamel.yaml import YAML
+          import json
+          yaml = YAML(typ='safe')
+          with open('.github/data/distros.yml') as f:
+              data = yaml.load(f)
+          del data['platform_map']
+          for i, v in enumerate(data['include']):
+              data['include'][i]['artifact_key'] = data['include'][i]['distro'] + str(data['include'][i]['version']).replace('.', '')
+              if 'packages' in data['include'][i]:
+                  del data['include'][i]['packages']
+              if 'base_image' in data['include'][i]:
+                  data['include'][i]['distro'] = data['include'][i]['base_image']
+                  del data['include'][i]['base_image']
+              data['include'][i]['distro'] = ':'.join([data['include'][i]['distro'], str(data['include'][i]['version'])])
+              del data['include'][i]['version']
+          matrix = json.dumps(data, sort_keys=True)
+          print('Generated Matrix: ' + matrix)
+          print('::set-output name=matrix::' + matrix)
 
   prepare-test-images: # Prepare the test environments for our build checks. This also checks dependency handling code for each tested environment.
     name: Prepare Test Environments
@@ -160,8 +176,8 @@ jobs:
           file: .github/dockerfiles/Dockerfile.build_test
           build-args: |
             BASE=${{ matrix.distro }}
-            PRE=${{ matrix.pre }}
-            RMJSONC=${{ matrix.rmjsonc }}
+            PRE=${{ matrix.env_prep }}
+            RMJSONC=${{ matrix.jsonc_removal }}
           outputs: type=oci,dest=/tmp/image.tar
           tags: test:${{ matrix.artifact_key }}
       - name: Upload image artifact
@@ -224,7 +240,7 @@ jobs:
           docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \
               /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud'
       - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C
-        if: matrix.rmjsonc != ''
+        if: matrix.jsonc_removal != ''
         run: |
           docker run --security-opt seccomp=unconfined -w /netdata sha256:${{ steps.load.outputs.image }} \
               /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud'

.github/workflows/packaging.yml

@@ -21,49 +21,56 @@ concurrency:
   group: packages-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: true
 jobs:
+  matrix:
+    name: Prepare Build Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Prepare tools
+        run: |
+          sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml
+      - name: Read build matrix
+        id: set-matrix
+        shell: python3 {0}
+        run: |
+          from ruamel.yaml import YAML
+          import json
+          yaml = YAML(typ='safe')
+          entries = list()
+
+          with open('.github/data/distros.yml') as f:
+              data = yaml.load(f)
+
+          for i, v in enumerate(data['include']):
+              if 'packages' in data['include'][i]:
+                  for arch in data['include'][i]['packages']['arches']:
+                      entries.append({
+                          'distro': data['include'][i]['distro'],
+                          'version': data['include'][i]['version'],
+                          'pkgclouddistro': data['include'][i]['packages']['repo_distro'],
+                          'format': data['include'][i]['packages']['type'],
+                          'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
+                          'platform': data['platform_map'][arch],
+                          'arch': arch
+                      })
+
+          entries.sort(key=lambda k: (k['arch'], k['distro'], k['version']))
+          matrix = json.dumps({'include': entries}, sort_keys=True)
+          print('Generated Matrix: ' + matrix)
+          print('::set-output name=matrix::' + matrix)
+
   build:
     name: Build
     runs-on: ubuntu-latest
     env:
       DOCKER_CLI_EXPERIMENTAL: enabled
+    needs:
+      - matrix
     strategy:
-      matrix:
-        include:
-          - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
-          - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386}
-          - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf}
-          - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64}
-          - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
-          - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386}
-          - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf}
-          - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64}
-          - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64, alias: bullseye}
-          - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/i386, arch: i386, alias: bullseye}
-          - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm/v7, arch: armhf, alias: bullseye}
-          - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/arm64/v8, arch: arm64, alias: bullseye}
-          - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
-          - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf}
-          - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64}
-          - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf}
-          - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64}
-          - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm/v7, arch: armhf}
-          - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/arm64/v8, arch: arm64}
-          - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
-          - {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/amd64, arch: amd64}
-          - {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/arm64/v8, arch: arm64}
-          - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
-          - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf}
-          - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64}
-          - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
-          - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm/v7, arch: armhf}
-          - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/arm64/v8, arch: arm64}
-          - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
-          - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/arm64/v8, arch: arm64}
-          - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64}
-          - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/arm64/v8, arch: arm64}
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
       # We intentiaonally disable the fail-fast behavior so that a
       # build failure for one version doesn't prevent us from publishing
       # successfully built and tested packages for another version.

.github/workflows/repoconfig-packages.yml

@@ -16,28 +16,56 @@ on:
 env:
   DISABLE_TELEMETRY: 1
 jobs:
+  matrix:
+    name: Prepare Build Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Prepare tools
+        run: |
+          sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml
+      - name: Read build matrix
+        id: set-matrix
+        shell: python3 {0}
+        run: |
+          from ruamel.yaml import YAML
+          import json
+          yaml = YAML(typ='safe')
+          entries = list()
+
+          with open('.github/data/distros.yml') as f:
+              data = yaml.load(f)
+
+          for i, v in enumerate(data['include']):
+              if 'packages' in data['include'][i]:
+                  entries.append({
+                      'distro': data['include'][i]['distro'],
+                      'version': data['include'][i]['version'],
+                      'pkgclouddistro': data['include'][i]['packages']['repo_distro'],
+                      'format': data['include'][i]['packages']['type'],
+                      'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
+                      'platform': data['platform_map']['amd64'],
+                      'arch': 'amd64'
+                  })
+
+          entries.sort(key=lambda k: (k['arch'], k['distro'], k['version']))
+          matrix = json.dumps({'include': entries}, sort_keys=True)
+          print('Generated Matrix: ' + matrix)
+          print('::set-output name=matrix::' + matrix)
+
   build:
     name: Build
     runs-on: ubuntu-latest
     env:
       DISABLE_TELEMETRY: 1
       DOCKER_CLI_EXPERIMENTAL: enabled
+    needs:
+      - matrix
     strategy:
-      # This needs to be kept in sync with the matrix in packaging.yml, but should only include the AMD64 lines.
-      matrix:
-        include:
-          - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
-          - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
-          - {distro: debian, version: "11", pkgclouddistro: debian/bullseye, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "21.10", pkgclouddistro: ubuntu/impish, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
-          - {distro: rockylinux, version: "8", pkgclouddistro: el/8, format: rpm, base_image: rockylinux/rockylinux, platform: linux/amd64, arch: amd64}
-          - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
-          - {distro: fedora, version: "35", pkgclouddistro: fedora/35, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
-          - {distro: opensuse, version: "15.3", pkgclouddistro: opensuse/15.3, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
-          - {distro: oraclelinux, version: "8", pkgclouddistro: ol/8, format: rpm, base_image: oraclelinux, platform: linux/amd64, arch: amd64}
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
       # We intentiaonally disable the fail-fast behavior so that a
       # build failure for one version doesn't prevent us from publishing
       # successfully built and tested packages for another version.