Start using new GitHub hosted ARM runners for CI when appropriate. (#19427)

* Start using new GitHub hosted ARM runners for CI when appropriate.

* Fix matrix generation.

* Fix actionlint error.

* Don't fail fast for static builds.

* Bump OpenSSL, cURL, and Bash versions for static builds.

* Fix more actionlint errors.

* Fix 32-bit ARM OpenSSL build.

* Fix ppc64le static openssl build.

* Fix duplicate key.

* Fix openssl ppc64le build again.

* Third attempt at fixing POWER8+ static builds.

---------

Co-authored-by: Ilya Mashchenko <ilya@netdata.cloud>

.github/data/distros.yml

@@ -6,7 +6,10 @@ platform_map:  # map packaging architectures to docker platforms
   arm64: linux/arm64/v8
   armhf: linux/arm/v7
   armhfp: linux/arm/v7
+  armv6l: linux/arm/v6
+  armv7l: linux/arm/v7
   i386: linux/386
+  ppc64le: linux/ppc64le
   x86_64: linux/amd64
 arch_order:  # sort order for per-architecture jobs in CI
   - amd64
@@ -14,8 +17,42 @@ arch_order:  # sort order for per-architecture jobs in CI
   - i386
   - armhf
   - armhfp
+  - armv6l
+  - armv7l
   - arm64
   - aarch64
+  - ppc64le
+arch_data:  # Mapping of per-architecture matrix behavior.
+  amd64: &amd64
+    qemu: false
+    runner: ubuntu-24.04
+  x86_64: *amd64
+  i386: *amd64
+  armhf: &arm
+    qemu: false
+    runner: ubuntu-24.04-arm
+  armhfp: *arm
+  armv6l: *arm
+  armv7l: *arm
+  arm64: &arm64
+    qemu: false
+    runner: ubuntu-24.04-arm
+  aarch64: *arm64
+  ppc64le:
+    qemu: true
+    runner: ubuntu-24.04
+static_arches:  # Static build architectures
+  - x86_64
+  - armv6l
+  - armv7l
+  - aarch64
+  - ppc64le
+docker_arches:  # Docker build archtiectures
+  - amd64
+  - i386
+  - armv7l
+  - arm64
+  - ppc64le
 default_sentry: &default_sentry # Default configuration for Sentry usage
   amd64: false
   x86_64: false

.github/scripts/gen-matrix-docker.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+import json
+
+from ruamel.yaml import YAML
+
+yaml = YAML(typ='safe')
+entries = list()
+
+with open('.github/data/distros.yml') as f:
+    data = yaml.load(f)
+
+for arch in data['docker_arches']:
+    entries.append({
+        'arch': arch,
+        'platform': data['platform_map'][arch],
+        'runner': data['arch_data'][arch]['runner'],
+        'qemu': data['arch_data'][arch]['qemu'],
+    })
+
+entries.sort(key=lambda k: data['arch_order'].index(k['arch']))
+matrix = json.dumps({'include': entries}, sort_keys=True)
+print(matrix)

.github/scripts/gen-matrix-packaging.py

@@ -30,7 +30,9 @@ for i, v in enumerate(data['include']):
                     'builder_rev': data['include'][i]['packages']['builder_rev'],
                     'platform': data['platform_map'][arch],
                     'bundle_sentry': data['include'][i]['bundle_sentry'][arch],
-                    'arch': arch
+                    'arch': arch,
+                    'runner': data['arch_data'][arch]['runner'],
+                    'qemu': data['arch_data'][arch]['qemu'],
                 })
 
 entries.sort(key=lambda k: (data['arch_order'].index(k['arch']), k['distro'], k['version']))

.github/scripts/gen-matrix-static.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import json
+
+from ruamel.yaml import YAML
+
+yaml = YAML(typ='safe')
+entries = list()
+
+with open('.github/data/distros.yml') as f:
+    data = yaml.load(f)
+
+for arch in data['static_arches']:
+    entries.append({
+        'arch': arch,
+        'runner': data['arch_data'][arch]['runner'],
+        'qemu': data['arch_data'][arch]['qemu'],
+    })
+
+entries.sort(key=lambda k: data['arch_order'].index(k['arch']))
+matrix = json.dumps({'include': entries}, sort_keys=True)
+print(matrix)

.github/workflows/build.yml

@@ -65,6 +65,7 @@ jobs:
             .github/workflows/build.yml
             .github/scripts/build-static.sh
             .github/scripts/get-static-cache-key.sh
+            .github/scripts/gen-matrix-static.py
             .github/scripts/gen-matrix-build.py
             .github/scripts/run-updater-check.sh
             packaging/cmake/
@@ -181,20 +182,58 @@ jobs:
             && needs.file-check.outputs.run == 'true'
           }}
 
+  static-matrix: # Generate the static build matrix.
+    name: Prepare Build Matrix
+    runs-on: ubuntu-latest
+    if: github.event_name != 'workflow_dispatch'
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@v4
+      - name: Prepare tools
+        id: prepare
+        run: |
+          sudo apt-get update || true
+          sudo apt-get install -y python3-ruamel.yaml
+      - name: Read build matrix
+        id: set-matrix
+        run: |
+          matrix="$(.github/scripts/gen-matrix-static.py)"
+          echo "Generated matrix: ${matrix}"
+          echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"
+      - name: Failure Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_COLOR: 'danger'
+          SLACK_FOOTER: ''
+          SLACK_ICON_EMOJI: ':github-actions:'
+          SLACK_TITLE: 'Static build matrix preparation failed:'
+          SLACK_USERNAME: 'GitHub Actions'
+          SLACK_MESSAGE: |-
+              ${{ github.repository }}: Failed to prepare build matrix for build checks.
+              Checkout: ${{ steps.checkout.outcome }}
+              Prepare tools: ${{ steps.prepare.outcome }}
+              Read build matrix: ${{ steps.set-matrix.outcome }}
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: >-
+          ${{
+            failure()
+            && startsWith(github.ref, 'refs/heads/master')
+            && github.event_name != 'pull_request'
+            && github.repository == 'netdata/netdata'
+          }}
+
   build-static: # Build the static binary archives, and store them as artifacts.
     name: Build Static
-    runs-on: ubuntu-latest
     needs:
       - file-check
+      - static-matrix
     strategy:
-      matrix:
-        arch:
-          - x86_64
-          - armv6l
-          - armv7l
-          - aarch64
-          - ppc64le
       fail-fast: false
+      matrix: ${{ fromJson(needs.static-matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: Skip Check
         id: skip
@@ -230,7 +269,7 @@ jobs:
           key: ${{ steps.cache-key.outputs.key }}
       - name: Set up QEMU
         id: qemu
-        if: needs.file-check.outputs.run == 'true'
+        if: matrix.qemu && needs.file-check.outputs.run == 'true'
         run: |
           sudo apt-get update
           sudo apt-get upgrade -y
@@ -239,7 +278,7 @@ jobs:
         if: github.event_name != 'workflow_dispatch' && needs.file-check.outputs.run == 'true' # Don’t use retries on PRs.
         run: |
           export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
-          export SKIP_EMULATION=1
+          [ "${{ matrix.qemu }}" == "true" ] || export SKIP_EMULATION=1
           .github/scripts/build-static.sh ${{ matrix.arch }}
       - name: Build
         if: github.event_name == 'workflow_dispatch' && needs.file-check.outputs.run == 'true'
@@ -250,7 +289,7 @@ jobs:
           max_attempts: 3
           command: |
             export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
-            export SKIP_EMULATION=1
+            [ "${{ matrix.qemu }}" == "true" ] || export SKIP_EMULATION=1
             .github/scripts/build-static.sh ${{ matrix.arch }}
       - name: Store
         id: store
@@ -374,7 +413,7 @@ jobs:
           SLACK_TITLE: 'Windows build failed:'
           SLACK_USERNAME: 'GitHub Actions'
           SLACK_MESSAGE: |-
-              ${{ github.repository }}: Updater checks for ${{ matrix.distro }} failed.
+              ${{ github.repository }}: Windows build failed.
               Checkout: ${{ steps.checkout.outcome }}
               Set Up Dependencies: ${{ steps.deps.outcome }}
               Build Netdata: ${{ steps.build.outcome }}
@@ -672,10 +711,10 @@ jobs:
           SLACK_COLOR: 'danger'
           SLACK_FOOTER: ''
           SLACK_ICON_EMOJI: ':github-actions:'
-          SLACK_TITLE: 'Updater checks for ${{ matrix.distro }} failed:'
+          SLACK_TITLE: 'Updater checks  failed:'
           SLACK_USERNAME: 'GitHub Actions'
           SLACK_MESSAGE: |-
-              ${{ github.repository }}: Updater checks for ${{ matrix.distro }} failed.
+              ${{ github.repository }}: Updater checks for failed.
               Checkout: ${{ steps.checkout.outcome }}
               Fetch artifacts: ${{ steps.fetch-artifacts.outcome }}
               Prepare artifact directory: ${{ steps.prepare.outcome }}
@@ -893,7 +932,7 @@ jobs:
 
   # Remaining jobs are only used for CI checks, and not as part of the release process
 
-  matrix: # Generate the shared build matrix for our Linux build tests.
+  src-matrix: # Generate the shared build matrix for our Linux build tests.
     name: Prepare Build Matrix
     runs-on: ubuntu-latest
     if: github.event_name != 'workflow_dispatch'
@@ -941,12 +980,12 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name != 'workflow_dispatch'
     needs:
-      - matrix
+      - src-matrix
       - file-check
     strategy:
       fail-fast: false
       max-parallel: 8
-      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
+      matrix: ${{ fromJson(needs.src-matrix.outputs.matrix) }}
     steps:
       - name: Skip Check
         id: skip
@@ -995,7 +1034,7 @@ jobs:
               ${{ github.repository }}: Build tests for ${{ matrix.distro }} failed.
               Checkout: ${{ steps.checkout.outcome }}
               Set up Buildx: ${{ steps.buildx.outcome }}
-              Build test environment: ${{ steps.build1.outcome }}
+              Build test environment: ${{ steps.build.outcome }}
               netdata-installer: ${{ steps.build-cloud.outcome }}
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
         if: >-

.github/workflows/docker.yml

@@ -72,6 +72,7 @@ jobs:
             netdata-installer.sh
             .github/workflows/docker.yml
             .github/scripts/docker-test.sh
+            .github/scripts/gen-matrix-docker.py
             .github/scripts/gen-docker-tags.py
             .github/scripts/gen-docker-imagetool-args.py
             packaging/cmake/
@@ -116,19 +117,57 @@ jobs:
             echo 'skip-go=' >> "${GITHUB_OUTPUT}"
           fi
 
+  matrix:
+    name: Generate Docker Build Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@v4
+      - name: Prepare tools
+        id: prepare
+        run: |
+          sudo apt-get update || true
+          sudo apt-get install -y python3-ruamel.yaml
+      - name: Read build matrix
+        id: set-matrix
+        run: |
+          matrix="$(.github/scripts/gen-matrix-docker.py)"
+          echo "Generated matrix: ${matrix}"
+          echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"
+      - name: Failure Notification
+        uses: rtCamp/action-slack-notify@v2
+        env:
+          SLACK_COLOR: 'danger'
+          SLACK_FOOTER: ''
+          SLACK_ICON_EMOJI: ':github-actions:'
+          SLACK_TITLE: 'Docker build matrix preparation failed:'
+          SLACK_USERNAME: 'GitHub Actions'
+          SLACK_MESSAGE: |-
+              ${{ github.repository }}: Failed to prepare build matrix for build checks.
+              Checkout: ${{ steps.checkout.outcome }}
+              Prepare tools: ${{ steps.prepare.outcome }}
+              Read build matrix: ${{ steps.set-matrix.outcome }}
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
+        if: >-
+          ${{
+            failure()
+            && startsWith(github.ref, 'refs/heads/master')
+            && github.event_name != 'pull_request'
+            && github.repository == 'netdata/netdata'
+          }}
+
   build-images:
     name: Build Docker Images
     needs:
       - file-check
-    runs-on: ubuntu-latest
+      - matrix
+    runs-on: ${{ matrix.runner }}
     strategy:
-      matrix:
-        platform:
-          - linux/amd64
-          - linux/i386
-          - linux/arm/v7
-          - linux/arm64
-          - linux/ppc64le
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
+      # Fail fast on releases, but run everything to completion on other triggers.
       fail-fast: false
     steps:
       - name: Skip Check
@@ -152,7 +191,7 @@ jobs:
         run: echo "OFFICIAL_IMAGE=true" >> "${GITHUB_ENV}"
       - name: Setup QEMU
         id: qemu
-        if: matrix.platform != 'linux/i386' && matrix.platform != 'linux/amd64' && needs.file-check.outputs.run == 'true'
+        if: matrix.qemu && needs.file-check.outputs.run == 'true'
         run: |
           sudo apt-get update
           sudo apt-get upgrade -y
@@ -237,15 +276,10 @@ jobs:
     needs:
       - build-images
       - gen-tags
+      - matrix
     strategy:
-      matrix:
-        platform:
-          - linux/amd64
-          - linux/i386
-          - linux/arm/v7
-          - linux/arm64
-          - linux/ppc64le
-    runs-on: ubuntu-latest
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: Checkout
         id: checkout
@@ -268,7 +302,7 @@ jobs:
         run: echo "OFFICIAL_IMAGE=true" >> "${GITHUB_ENV}"
       - name: Setup QEMU
         id: qemu
-        if: matrix.platform != 'linux/i386' && matrix.platform != 'linux/amd64'
+        if: matrix.qemu
         uses: docker/setup-qemu-action@v3
       - name: Setup Buildx
         id: prepare
@@ -391,15 +425,10 @@ jobs:
     needs:
       - build-images
       - gen-tags
+      - matrix
     strategy:
-      matrix:
-        platform:
-          - linux/amd64
-          - linux/i386
-          - linux/arm/v7
-          - linux/arm64
-          - linux/ppc64le
-    runs-on: ubuntu-latest
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: Checkout
         id: checkout
@@ -422,7 +451,7 @@ jobs:
         run: echo "OFFICIAL_IMAGE=true" >> "${GITHUB_ENV}"
       - name: Setup QEMU
         id: qemu
-        if: matrix.platform != 'linux/i386' && matrix.platform != 'linux/amd64'
+        if: matrix.qemu
         uses: docker/setup-qemu-action@v3
       - name: Setup Buildx
         id: prepare
@@ -547,15 +576,10 @@ jobs:
     needs:
       - build-images
       - gen-tags
+      - matrix
     strategy:
-      matrix:
-        platform:
-          - linux/amd64
-          - linux/i386
-          - linux/arm/v7
-          - linux/arm64
-          - linux/ppc64le
-    runs-on: ubuntu-latest
+      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
+    runs-on: ${{ matrix.runner }}
     steps:
       - name: Checkout
         id: checkout
@@ -578,7 +602,7 @@ jobs:
         run: echo "OFFICIAL_IMAGE=true" >> "${GITHUB_ENV}"
       - name: Setup QEMU
         id: qemu
-        if: matrix.platform != 'linux/i386' && matrix.platform != 'linux/amd64'
+        if: matrix.qemu
         uses: docker/setup-qemu-action@v3
       - name: Setup Buildx
         id: prepare

.github/workflows/packaging.yml

@@ -190,7 +190,7 @@ jobs:
 
   build:
     name: Build
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.runner }}
     env:
       DOCKER_CLI_EXPERIMENTAL: enabled
     needs:
@@ -230,7 +230,7 @@ jobs:
           fi
       - name: Setup QEMU
         id: qemu
-        if: matrix.platform != 'linux/i386' && matrix.platform != 'linux/amd64' && needs.file-check.outputs.run == 'true'
+        if: matrix.qemu && needs.file-check.outputs.run == 'true'
         run: |
           sudo apt-get update
           sudo apt-get upgrade -y
@@ -274,14 +274,6 @@ jobs:
                      -e VERSION=${{ needs.version-check.outputs.version }} -e DISTRO_VERSION=${{ matrix.version }} \
                      --platform=${{ matrix.platform }} -v "$PWD":/netdata ${{ matrix.base_image }} \
                      /netdata/.github/scripts/pkg-test.sh
-      - name: SSH setup
-        id: ssh-setup
-        if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true'
-        uses: shimataro/ssh-key-action@v2
-        with:
-          key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }}
-          name: id_ecdsa
-          known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }}
       - name: Import GPG Keys
         id: import-keys
         if: needs.file-check.outputs.run == 'true' && matrix.format == 'deb' && github.event_name != 'pull_request'
@@ -293,6 +285,14 @@ jobs:
         if: needs.file-check.outputs.run == 'true' && matrix.format == 'deb' && github.event_name != 'pull_request'
         shell: bash
         run: .github/scripts/deb-sign.sh artifacts ${{ steps.import-keys.outputs.fingerprint }}
+      - name: SSH setup
+        id: ssh-setup
+        if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && needs.file-check.outputs.run == 'true'
+        uses: shimataro/ssh-key-action@v2
+        with:
+          key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }}
+          name: id_ecdsa
+          known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }}
       - name: Upload to packages.netdata.cloud
         id: package-upload
         continue-on-error: true
@@ -325,15 +325,13 @@ jobs:
               ${{ github.repository }}: ${{ matrix.repo_distro }} ${{ matrix.version }} package build for ${{ matrix.arch }} failed.
               Checkout: ${{ steps.checkout.outcome }}
               Setup QEMU: ${{ steps.qemu.outcome }}
-              Setup Docker: ${{ steps.docker-config.outcome }}
               Fetch images: ${{ steps.fetch-images.outcome }}
               Build: ${{ steps.build.outcome }}
               Test: ${{ steps.test.outcome }}
-              Publish to PackageCloud: ${{ steps.upload.outcome }}
-              Import SSH Key: ${{ steps.ssh-setup.outcome }}
-              Publish to packages.netdata.cloud: ${{ steps.package-upload.outcome }}
               Import GPG Keys: ${{ steps.import-keys.outcome }}
               Sign DEB Packages: ${{ steps.sign-deb.outcome }}
+              Import SSH Key: ${{ steps.ssh-setup.outcome }}
+              Publish to packages.netdata.cloud: ${{ steps.package-upload.outcome }}
               Publish to packages2.netdata.cloud: ${{ steps.package2-upload.outcome }}
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
         if: >-

packaging/makeself/bundled-packages.version

@@ -1,13 +1,13 @@
 # Source of truth for all the packages we bundle in static builds
 PACKAGES=("OPENSSL" "CURL" "BASH" "IOPING" "LIBNETFILTER_ACT")
 SOURCE_TYPES=("GH_REPO_CLONE" "GH_REPO_CLONE" "DW_TARBALL" "GH_REPO_SOURCE" "DW_TARBALL")
-OPENSSL_VERSION="openssl-3.3.2"
+OPENSSL_VERSION="openssl-3.4.0"
 OPENSSL_SOURCE="https://github.com/openssl/openssl"
-CURL_VERSION="curl-8_10_1"
+CURL_VERSION="curl-8_11_1"
 CURL_SOURCE="https://github.com/curl/curl"
-BASH_VERSION="5.1.16"
+BASH_VERSION="5.2.37"
 BASH_ARTIFACT_SOURCE="http://ftp.gnu.org/gnu/bash"
-BASH_ARTIFACT_SHA256="5bac17218d3911834520dad13cd1f85ab944e1c09ae1aba55906be1f8192f558"
+BASH_ARTIFACT_SHA256="9599b22ecd1d5787ad7d3b7bf0c59f312b3396d1e281175dd1f8a4014da621ff"
 IOPING_VERSION="1.3"
 IOPING_SOURCE="https://github.com/koct9i/ioping"
 IOPING_ARTIFACT_SHA256="7aa48e70aaa766bc112dea57ebbe56700626871052380709df3a26f46766e8c8"

packaging/makeself/jobs/20-openssl.install.sh

@@ -36,9 +36,17 @@ fi
 cd "${NETDATA_MAKESELF_PATH}/tmp/openssl" || exit 1
 
 if [ "${CACHE_HIT:-0}" -eq 0 ]; then
-    sed -i "s/disable('static', 'pic', 'threads');/disable('static', 'pic');/" Configure
-    run ./config -static threads no-tests --prefix=/openssl-static --openssldir=/opt/netdata/etc/ssl
-    run make -j "$(nproc)"
+  COMMON_CONFIG="-static threads no-tests --prefix=/openssl-static --openssldir=/opt/netdata/etc/ssl"
+
+  sed -i "s/disable('static', 'pic', 'threads');/disable('static', 'pic');/" Configure
+
+  # shellcheck disable=SC2086
+  case "${BUILDARCH}" in
+    armv6l|armv7l) run ./config ${COMMON_CONFIG} linux-armv4 ;;
+    *) run ./config ${COMMON_CONFIG} ;;
+  esac
+
+  run make -j "$(nproc)"
 fi
 
 run make -j "$(nproc)" install_sw