netdata/.github/workflows/build.yml

---
# CI code for building release artifacts.
name: Build
on:
  push: # Master branch checks only validate the build and generate artifacts for testing.
    branches:
      - master
  pull_request: null # PR checks only validate the build and generate artifacts for testing.
  workflow_dispatch: # Dispatch runs build and does limited validation, then pushes to the appropriate storage location.
    inputs:
      type:
        description: Build Type
        default: nightly
        required: true
      version:
        description: Version Tag
        default: nightly
        required: true
concurrency: # This keeps multiple instances of the job from running concurrently for the same ref and event type.
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: true
jobs:
  file-check: # Check what files changed if we’re being run in a PR or on a push.
    name: Check Modified Files
    runs-on: ubuntu-latest
    outputs:
      run: ${{ steps.check-run.outputs.run }}
      skip-go: ${{ steps.check-go.outputs.skip-go }}
    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Check source files
        id: check-source-files
        uses: tj-actions/changed-files@v44
        with:
          since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
          files: |
            **/*.c
            **/*.cc
            **/*.h
            **/*.hh
            **/*.in
            **/*.patch
            src/aclk/aclk-schemas/
            src/ml/dlib/
            src/fluent-bit/
            src/web/server/h2o/libh2o/
          files_ignore: |
            netdata.spec.in
            **/*.md
      - name: Check build files
        id: check-build-files
        uses: tj-actions/changed-files@v44
        with:
          since_last_remote_commit: ${{ github.event_name != 'pull_request' }}
          files: |
            **/*.cmake
            CMakeLists.txt
            netdata-installer.sh
            .github/data/distros.yml
            .github/workflows/build.yml
            .github/scripts/build-static.sh
            .github/scripts/get-static-cache-key.sh
            .github/scripts/gen-matrix-build.py
            .github/scripts/run-updater-check.sh
            packaging/cmake/
            packaging/makeself/
            packaging/installer/
            packaging/*.sh
            packaging/*.version
            packaging/*.checksums
          files_ignore: |
            **/*.md
            packaging/repoconfig/
      - name: List all changed files in pattern
        continue-on-error: true
        env:
          CHANGED_SOURCE_FILES: ${{ steps.check-source-files.outputs.all_changed_files }}
          CHANGED_BUILD_FILES: ${{ steps.check-build-files.outputs.all_changed_files }}
        run: |
          for file in ${CHANGED_SOURCE_FILES} ${CHANGED_BUILD_FILES} ; do
            echo "$file was changed"
          done
      - name: Check Run
        id: check-run
        run: |
          if [ "${{ steps.check-source-files.outputs.any_modified }}" == "true" ] || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ] || [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
            echo 'run=true' >> "${GITHUB_OUTPUT}"
          else
            echo 'run=false' >> "${GITHUB_OUTPUT}"
          fi
      - name: Check Go
        id: check-go
        env:
          OTHER_CHANGED_FILES: ${{ steps.check-source-files.outputs.other_changed_files }}
        run: |
          if [ '${{ github.event_name }}' == 'pull_request' ]; then
            if echo "${OTHER_CHANGED_FILES}" | grep -q '.*/(.*\.go|go\.mod|go\.sum)$' || [ "${{ steps.check-build-files.outputs.any_modified }}" == "true" ]; then
              echo 'skip-go=' >> "${GITHUB_OUTPUT}"
            else
              echo 'skip-go=--disable-go' >> "${GITHUB_OUTPUT}"
            fi
          else
            echo 'skip-go=' >> "${GITHUB_OUTPUT}"
          fi

  build-dist: # Build the distribution tarball and store it as an artifact.
    name: Build Distribution Tarball
    runs-on: ubuntu-latest
    needs:
      - file-check
    outputs:
      distfile: ${{ steps.build.outputs.distfile }}
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Fix tags
        id: fix-tags
        if: github.event_name != 'push' && needs.file-check.outputs.run == 'true'
        run: |
          git fetch --tags --force
      - name: Mark Stable
        id: channel
        if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' && needs.file-check.outputs.run == 'true'
        run: |
          sed -i 's/^RELEASE_CHANNEL="nightly"/RELEASE_CHANNEL="stable"/' netdata-installer.sh
      - name: Build
        id: build
        if: needs.file-check.outputs.run == 'true'
        run: |
          mkdir -p artifacts/
          tar --create --file "artifacts/netdata-$(git describe).tar.gz" \
              --sort=name --posix --auto-compress --exclude=artifacts/ --exclude=.git \
              --exclude=.gitignore --exclude=.gitattributes --exclude=.gitmodules \
              --transform "s/^\\.\\//netdata-$(git describe)\\//" --verbose .
          cd artifacts/
          echo "distfile=$(find . -name 'netdata-*.tar.gz')" >> "${GITHUB_OUTPUT}"
      - name: Store
        id: store
        if: needs.file-check.outputs.run == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: dist-tarball
          path: artifacts/*.tar.gz
          retention-days: 30
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Distribution tarball creation failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to create source tarball for distribution.
              Checkout: ${{ steps.checkout.outcome }}
              Fix Tags: ${{ steps.fix-tags.outcome }}
              Mark stable: ${{ steps.channel.outcome }}
              Build: ${{ steps.build.outcome }}
              Store: ${{ steps.store.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}

  build-static: # Build the static binary archives, and store them as artifacts.
    name: Build Static
    runs-on: ubuntu-latest
    needs:
      - file-check
    strategy:
      matrix:
        arch:
          - x86_64
          - armv6l
          - armv7l
          - aarch64
          - ppc64le
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          submodules: recursive
      - name: Fix tags
        id: fix-tags
        if: github.event_name != 'push' && needs.file-check.outputs.run == 'true'
        run: |
          git fetch --tags --force
      - name: Mark Stable
        id: channel
        if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly' && needs.file-check.outputs.run == 'true'
        run: |
          sed -i 's/^RELEASE_CHANNEL="nightly"/RELEASE_CHANNEL="stable"/' netdata-installer.sh packaging/makeself/install-or-update.sh
      - name: Get Cache Key
        if: (github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache')) && needs.file-check.outputs.run == 'true'
        id: cache-key
        run: .github/scripts/get-static-cache-key.sh ${{ matrix.arch }} "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache') }}"
      - name: Cache
        if: (github.event_name != 'pull_request' || ! contains(github.event.pull_request.labels.*.name, 'run-ci/no-cache')) && needs.file-check.outputs.run == 'true'
        id: cache
        uses: actions/cache@v4
        with:
          path: artifacts/cache
          key: ${{ steps.cache-key.outputs.key }}
      - name: Build
        if: github.event_name != 'workflow_dispatch' && needs.file-check.outputs.run == 'true' # Don’t use retries on PRs.
        run: |
          export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
          .github/scripts/build-static.sh ${{ matrix.arch }}
      - name: Build
        if: github.event_name == 'workflow_dispatch' && needs.file-check.outputs.run == 'true'
        id: build
        uses: nick-fields/retry@v3
        with:
          timeout_minutes: 180
          max_attempts: 3
          command: |
            export EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }}
            .github/scripts/build-static.sh ${{ matrix.arch }}
      - name: Store
        id: store
        if: needs.file-check.outputs.run == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: dist-static-${{ matrix.arch }}
          path: artifacts/*.gz.run
          retention-days: 30
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Static build failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to create static installer archive for ${{ matrix.arch }}.
              Checkout: ${{ steps.checkout.outcome }}
              Fix Tags: ${{ steps.fix-tags.outcome }}
              Mark stable: ${{ steps.channel.outcome }}
              Build: ${{ steps.build.outcome }}
              Store: ${{ steps.store.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}

  prepare-upload: # Consolidate the artifacts for uploading or releasing.
    name: Prepare Artifacts
    runs-on: ubuntu-latest
    needs:
      - build-dist
      - build-static
      - file-check
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
      - name: Prepare Environment
        id: prepare
        if: needs.file-check.outputs.run == 'true'
        run: mkdir -p artifacts
      - name: Retrieve Build Artifacts
        id: fetch-dist
        if: needs.file-check.outputs.run == 'true'
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            pattern: dist-*
            path: dist-artifacts
            merge-multiple: true
          attempt_limit: 3
          attempt_delay: 2000
      - name: Prepare Artifacts
        id: consolidate
        if: needs.file-check.outputs.run == 'true'
        working-directory: ./artifacts/
        run: |
          mv ../dist-artifacts/* . || exit 1
          ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1
          cp ../packaging/version ./latest-version.txt || exit 1
          cp ../integrations/integrations.js ./integrations.js || exit 1
          sha256sum -b ./* > sha256sums.txt || exit 1
          cat sha256sums.txt
      - name: Store Artifacts
        id: store
        if: needs.file-check.outputs.run == 'true'
        uses: actions/upload-artifact@v4
        with:
          name: final-artifacts
          path: artifacts/*
          retention-days: 30
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Failed to prepare release artifacts for upload:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to prepare release artifacts for upload.
              Checkout: ${{ steps.checkout.outcome }}
              Prepare environment: ${{ steps.prepare.outcome }}
              Fetch dist tarball: ${{ steps.fetch-dist.outcome }}
              Consolidate artifacts: ${{ steps.consolidate.outcome }}
              Store: ${{ steps.store.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}

  artifact-verification-dist: # Verify the regular installer works with the consolidated artifacts.
    name: Test Consolidated Artifacts (Source)
    runs-on: ubuntu-latest
    needs:
      - prepare-upload
      - file-check
    services:
      apache: # This gets used to serve the dist tarball for the updater script.
        image: httpd:2.4
        ports:
          - 8080:80
        volumes:
          - ${{ github.workspace }}:/usr/local/apache2/htdocs/
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
      - name: Fetch artifacts
        id: fetch
        if: needs.file-check.outputs.run == 'true'
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: final-artifacts
            path: artifacts
          attempt_limit: 3
          attempt_delay: 2000
      - name: Prepare artifacts directory
        id: prepare
        if: needs.file-check.outputs.run == 'true'
        run: |
          mkdir -p download/latest
          mv artifacts/* download/latest
          ls -al download/latest
      - name: Verify that artifacts work with installer
        id: verify
        if: needs.file-check.outputs.run == 'true'
        env:
          NETDATA_TARBALL_BASEURL: http://localhost:8080/
        run: packaging/installer/kickstart.sh --build-only --dont-start-it --disable-telemetry --dont-wait
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Artifact verification for source tarball failed.'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Artifact verification for source tarball failed.
              Checkout: ${{ steps.checkout.outcome }}
              Fetch artifacts: ${{ steps.fetch.outcome }}
              Verify artifacts: ${{ steps.verify.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}

  artifact-verification-static: # Verify the static installer works with the consolidated artifacts.
    name: Test Consolidated Artifacts (Static)
    runs-on: ubuntu-latest
    needs:
      - prepare-upload
      - file-check
    services:
      apache: # This gets used to serve the static archives.
        image: httpd:2.4
        ports:
          - 8080:80
        volumes:
          - ${{ github.workspace }}:/usr/local/apache2/htdocs/
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
      - name: Fetch artifacts
        id: fetch-artifacts
        if: needs.file-check.outputs.run == 'true'
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: final-artifacts
            path: artifacts
          attempt_limit: 3
          attempt_delay: 2000
      - name: Prepare artifacts directory
        id: prepare
        if: needs.file-check.outputs.run == 'true'
        run: |
          mkdir -p download/latest
          mv artifacts/* download/latest
          ls -al download/latest
      - name: Verify that artifacts work with installer
        id: verify
        if: needs.file-check.outputs.run == 'true'
        env:
          NETDATA_TARBALL_BASEURL: http://localhost:8080/
        run: packaging/installer/kickstart.sh --static-only --dont-start-it --disable-telemetry
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Artifact verification for static build failed.'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Artifact verification for static build failed.
              Checkout:  ${{ steps.checkout.outcome }}
              Fetch artifacts: ${{ steps.fetch-artifacts.outcome }}
              Verify artifacts: ${{ steps.verify.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}

  upload-nightly: # Upload the nightly build artifacts to GCS.
    name: Upload Nightly Artifacts
    runs-on: ubuntu-latest
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' && github.repository == 'netdata/netdata'
    needs:
      - artifact-verification-dist
      - artifact-verification-static
    steps:
      - name: Retrieve Artifacts
        id: fetch
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: final-artifacts
            path: final-artifacts
          attempt_limit: 3
          attempt_delay: 2000
      - name: Authenticate to GCS
        id: gcs-auth
        uses: google-github-actions/auth@v2
        with:
          project_id: ${{ secrets.GCP_NIGHTLY_STORAGE_PROJECT }}
          credentials_json: ${{ secrets.GCS_STORAGE_SERVICE_KEY_JSON }}
      - name: Setup GCS
        id: gcs-setup
        uses: google-github-actions/setup-gcloud@v2.1.0
      - name: Upload Artifacts
        id: upload
        uses: google-github-actions/upload-cloud-storage@v2.1.0
        with:
          destination: ${{ secrets.GCP_NIGHTLY_STORAGE_BUCKET }}
          gzip: false
          path: ./final-artifacts/latest-version.txt
          parent: false
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Failed to upload nightly release artifacts:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to upload nightly release artifacts.
              Fetch artifacts: ${{ steps.fetch.outcome }}
              Authenticatie GCS: ${{ steps.gcs-auth.outcome }}
              Setup GCS: ${{ steps.gcs-setup.outcome }}
              Upload artifacts: ${{ steps.upload.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
          }}

  create-nightly: # Create a nightly build release in netdata/netdata-nightlies
    name: Create Nightly Release
    runs-on: ubuntu-latest
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'nightly' && github.repository == 'netdata/netdata'
    needs:
      - artifact-verification-dist
      - artifact-verification-static
    steps:
      - name: Checkout Main Repo
        id: checkout-main
        uses: actions/checkout@v4
        with:
          path: main
      - name: Checkout Nightly Repo
        id: checkout-nightly
        uses: actions/checkout@v4
        with:
          repository: netdata/netdata-nightlies
          path: nightlies
          token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
      - name: Retrieve Artifacts
        id: fetch
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: final-artifacts
            path: final-artifacts
          attempt_limit: 3
          attempt_delay: 2000
      - name: Prepare version info
        id: version
        run: |
          # shellcheck disable=SC2129
          echo "version=$(cat main/packaging/version)" >> "${GITHUB_OUTPUT}"
          echo "commit=$(cd nightlies && git rev-parse HEAD)" >> "${GITHUB_OUTPUT}"
          echo "date=$(date +%F)" >> "${GITHUB_OUTPUT}"
      - name: Create Release
        id: create-release
        uses: ncipollo/release-action@v1
        with:
          allowUpdates: false
          artifactErrorsFailBuild: true
          artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/integrations.js'
          owner: netdata
          repo: netdata-nightlies
          body: Netdata nightly build for ${{ steps.version.outputs.date }}.
          commit: ${{ steps.version.outputs.commit }}
          makeLatest: true
          tag: ${{ steps.version.outputs.version }}
          token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
      - name: Checkout netdata main Repo # Checkout back to netdata/netdata repo to the update latest packaged versions
        id: checkout-netdata
        uses: actions/checkout@v4
        with:
          token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
      - name: Init python environment for publish release metadata
        uses: actions/setup-python@v5
        id: init-python
        with:
          python-version: "3.12"
      - name: Setup python environment
        id: setup-python
        run: |
          pip install -r .github/scripts/modules/requirements.txt
      - name: Check if the version is latest and published
        id: check-latest-version
        run: |
          python .github/scripts/check_latest_versions.py  ${{ steps.version.outputs.version }}
      - name: SSH setup
        id: ssh-setup
        if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true'
        uses: shimataro/ssh-key-action@v2
        with:
          key: ${{ secrets.NETDATABOT_PACKAGES_SSH_KEY }}
          name: id_ecdsa
          known_hosts: ${{ secrets.PACKAGES_KNOWN_HOSTS }}
      - name: Sync release info to packages.netdata.cloud
        id: sync-releases
        continue-on-error: true
        if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true'
        run: |
          .github/scripts/upload-new-version-tags.sh packages.netdata.cloud
      - name: Sync release info to packages2.netdata.cloud
        id: sync-releases2
        if: github.event_name == 'workflow_dispatch' && github.repository == 'netdata/netdata' && steps.check-latest-version.outputs.versions_needs_update == 'true'
        run: |
          .github/scripts/upload-new-version-tags.sh packages.netdata.cloud
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Failed to draft release:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to create nightly release or attach artifacts.
              Checkout netdata/netdata: ${{ steps.checkout-main.outcome }}
              Checkout netdata/netdata-nightlies: ${{ steps.checkout-nightly.outcome }}
              Fetch artifacts: ${{ steps.fetch.outcome }}
              Prepare version info: ${{ steps.version.outcome }}
              Create release: ${{ steps.create-release.outcome }}
              Checkout back netdata/netdata: ${{ steps.checkout-netdata.outcome }}
              Init python environment: ${{ steps.init-python.outcome }}
              Setup python environment: ${{ steps.setup-python.outcome }}
              Check the nearly published release against the advertised: ${{ steps.check-latest-version.outcome }}
              Setup ssh: ${{ steps.ssh-setup.outcome }}
              Sync release info to packages.netdata.cloud: ${{ steps.sync-releases.outcome }}
              Sync release info to packages2.netdata.cloud: ${{ steps.sync-releases2.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && github.event_name == 'workflow_dispatch'
          }}

  normalize-tag: # Fix the release tag if needed
    name: Normalize Release Tag
    runs-on: ubuntu-latest
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release'
    outputs:
      tag: ${{ steps.tag.outputs.tag }}
    steps:
      - name: Normalize Tag
        id: tag
        run: |
          if echo ${{ github.event.inputs.version }} | grep -qE '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$'; then
            echo "tag=v${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
          else
            echo "tag=${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
          fi

  upload-release: # Create the draft release and upload the build artifacts.
    name: Create Release Draft
    runs-on: ubuntu-latest
    if: github.event_name == 'workflow_dispatch' && github.event.inputs.type == 'release' && github.repository == 'netdata/netdata'
    needs:
      - artifact-verification-dist
      - artifact-verification-static
      - normalize-tag
    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@v4
      - name: Retrieve Artifacts
        id: fetch
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: final-artifacts
            path: final-artifacts
          attempt_limit: 3
          attempt_delay: 2000
      - name: Create Release
        id: create-release
        uses: ncipollo/release-action@v1
        with:
          allowUpdates: false
          artifactErrorsFailBuild: true
          artifacts: 'final-artifacts/sha256sums.txt,final-artifacts/netdata-*.tar.gz,final-artifacts/netdata-*.gz.run,final-artifacts/integrations.js'
          draft: true
          tag: ${{ needs.normalize-tag.outputs.tag }}
          token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Failed to draft release:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to create draft release or attach artifacts.
              Checkout: ${{ steps.checkout.outcome }}
              Fetch artifacts: ${{ steps.fetch.outcome }}
              Create draft release: ${{ steps.create-release.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && github.event_name == 'workflow_dispatch'
          }}
      - name: Success Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'good'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Created agent draft release:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: "${{ github.repository }}: ${{ steps.create-release.outputs.html_url }}"
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            success()
            && github.event_name == 'workflow_dispatch'
          }}

  # Remaining jobs are only used for CI checks, and not as part of the release process

  matrix: # Generate the shared build matrix for our Linux build tests.
    name: Prepare Build Matrix
    runs-on: ubuntu-latest
    if: github.event_name != 'workflow_dispatch'
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@v4
      - name: Prepare tools
        id: prepare
        run: |
          sudo apt-get update || true
          sudo apt-get install -y python3-ruamel.yaml
      - name: Read build matrix
        id: set-matrix
        run: |
          matrix="$(.github/scripts/gen-matrix-build.py)"
          echo "Generated matrix: ${matrix}"
          echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Build matrix preparation failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Failed to prepare build matrix for build checks.
              Checkout: ${{ steps.checkout.outcome }}
              Prepare tools: ${{ steps.prepare.outcome }}
              Read build matrix: ${{ steps.set-matrix.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
          }}

  prepare-test-images: # Prepare the test environments for our build checks. This also checks dependency handling code for each tested environment.
    name: Prepare Test Environments
    runs-on: ubuntu-latest
    if: github.event_name != 'workflow_dispatch'
    needs:
      - matrix
    env:
      RETRY_DELAY: 300
    strategy:
      # Unlike the actual build tests, this completes _very_ fast (average of about 3 minutes for each job), so we
      # just run everything in parallel instead lof limiting job concurrency.
      fail-fast: false
      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@v4
      - name: Setup Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3
      - name: Build test environment
        id: build1
        uses: docker/build-push-action@v6
        continue-on-error: true # We retry 3 times at 5 minute intervals if there is a failure here.
        with:
          push: false
          load: false
          file: .github/dockerfiles/Dockerfile.build_test
          build-args: |
            BASE=${{ matrix.distro }}
            PRE=${{ matrix.env_prep }}
            RMJSONC=${{ matrix.jsonc_removal }}
          outputs: type=docker,dest=/tmp/image.tar
          tags: test:${{ matrix.artifact_key }}
      - name: Retry delay
        if: ${{ steps.build1.outcome == 'failure' }}
        run: sleep "${RETRY_DELAY}"
      - name: Build test environment (attempt 2)
        if: ${{ steps.build1.outcome == 'failure' }}
        id: build2
        uses: docker/build-push-action@v6
        continue-on-error: true # We retry 3 times at 5 minute intervals if there is a failure here.
        with:
          push: false
          load: false
          file: .github/dockerfiles/Dockerfile.build_test
          build-args: |
            BASE=${{ matrix.distro }}
            PRE=${{ matrix.env_prep }}
            RMJSONC=${{ matrix.jsonc_removal }}
          outputs: type=docker,dest=/tmp/image.tar
          tags: test:${{ matrix.artifact_key }}
      - name: Retry delay
        if: ${{ steps.build1.outcome == 'failure' && steps.build2.outcome == 'failure' }}
        run: sleep "${RETRY_DELAY}"
      - name: Build test environment (attempt 3)
        if: ${{ steps.build1.outcome == 'failure' && steps.build2.outcome == 'failure' }}
        id: build3
        uses: docker/build-push-action@v6
        with:
          push: false
          load: false
          file: .github/dockerfiles/Dockerfile.build_test
          build-args: |
            BASE=${{ matrix.distro }}
            PRE=${{ matrix.env_prep }}
            RMJSONC=${{ matrix.jsonc_removal }}
          outputs: type=docker,dest=/tmp/image.tar
          tags: test:${{ matrix.artifact_key }}
      - name: Upload image artifact
        id: upload
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.artifact_key }}-test-env
          path: /tmp/image.tar
          retention-days: 30
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Test environment preparation for ${{ matrix.distro }} failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Test environment preparation for ${{ matrix.distro }} failed.
              Checkout: ${{ steps.checkout.outcome }}
              Set up Buildx: ${{ steps.buildx.outcome }}
              Build test environment: ${{ steps.build1.outcome }}
              Build test environment (attempt 2): ${{ steps.build2.outcome }}
              Build test environment (attempt 3): ${{ steps.build3.outcome }}
              Upload: ${{ steps.upload.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
          }}

  source-build: # Test various source build arrangements.
    name: Test Source Build
    runs-on: ubuntu-latest
    if: github.event_name != 'workflow_dispatch'
    needs:
      - matrix
      - prepare-test-images
      - file-check
    strategy:
      fail-fast: false
      max-parallel: 8
      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Fetch test environment
        id: fetch
        if: needs.file-check.outputs.run == 'true'
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: ${{ matrix.artifact_key }}-test-env
            path: .
          attempt_limit: 3
          attempt_delay: 2000
      - name: Load test environment
        id: load
        if: needs.file-check.outputs.run == 'true'
        run: docker load --input image.tar
      - name: netdata-installer on ${{ matrix.distro }}, disable cloud
        id: build-no-cloud
        if: needs.file-check.outputs.run == 'true'
        run: |
          docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
              /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
      - name: netdata-installer on ${{ matrix.distro }}, require cloud
        id: build-cloud
        if: needs.file-check.outputs.run == 'true'
        run: |
          docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
              /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
      - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C
        id: build-no-jsonc
        if: matrix.jsonc_removal != '' && needs.file-check.outputs.run == 'true'
        run: |
          docker run --security-opt seccomp=unconfined -w /netdata test:${{ matrix.artifact_key }} \
              /bin/sh -c '/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud --one-time-build ${{ needs.file-check.outputs.skip-go }}'
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Build tests for ${{ matrix.distro }} failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Build tests for ${{ matrix.distro }} failed.
              Checkout: ${{ steps.checkout.outcome }}
              Fetch test environment: ${{ steps.fetch.outcome }}
              Load test environment: ${{ steps.load.outcome }}
              netdata-installer, disable cloud: ${{ steps.build-no-cloud.outcome }}
              netdata-installer, require cloud: ${{ steps.build-cloud.outcome }}
              netdata-installer, no JSON-C: ${{ steps.build-no-jsonc.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}

  macos-build: # Test building on macOS
    runs-on: ${{ matrix.runner }}
    if: github.event_name != 'workflow_dispatch'
    needs:
      - file-check
    strategy:
      fail-fast: false
      max-parallel: 8
      matrix:
        include:
          - name: macos-12
            runner: macos-12
          - name: macos-13
            runner: macos-13
          - name: macos-14-M1
            runner: macos-14
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - uses: actions/checkout@v4
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        with:
          submodules: recursive
      - name: Install latest bash
        id: install-bash
        if: needs.file-check.outputs.run == 'true'
        run: |
          brew install bash
      - name: Install netdata dependencies
        id: install-nd-dep
        if: needs.file-check.outputs.run == 'true'
        run: |
          bash ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
      - name: Build from source
        id: build-source
        if: needs.file-check.outputs.run == 'true'
        run: |
          sudo bash ./netdata-installer.sh --install-no-prefix /usr/local/netdata --dont-wait --dont-start-it --require-cloud --one-time-build
      - name: Test Agent start up
        id: test-agent
        if: needs.file-check.outputs.run == 'true'
        run: |
          /usr/local/netdata/usr/sbin/netdata -D > ./netdata.log 2>&1 &
          ./packaging/runtime-check.sh
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Build & test from source macOS failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: macOS Build and test.
              Checkout: ${{ steps.checkout.outcome }}
              Setup runner: ${{ steps.install-bash.outcome }}
              Install netdata required packages: ${{ steps.install-nd-dep.outcome }}
              Build from source: ${{ steps.build-source.outcome }}
              Test Agent runtime: ${{ steps.test-agent.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
          }}

  updater-check: # Test the generated dist archive using the updater code.
    name: Test Generated Distfile and Updater Code
    runs-on: ubuntu-latest
    if: github.event_name != 'workflow_dispatch'
    needs:
      - build-dist
      - matrix
      - prepare-test-images
      - file-check
    strategy:
      fail-fast: false
      max-parallel: 8
      matrix: ${{ fromJson(needs.matrix.outputs.matrix) }}
    services:
      apache: # This gets used to serve the dist tarball for the updater script.
        image: httpd:2.4
        ports:
          - 8080:80
        volumes:
          - ${{ github.workspace }}:/usr/local/apache2/htdocs/
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        id: checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
      - name: Fetch dist tarball artifacts
        id: fetch-tarball
        if: needs.file-check.outputs.run == 'true'
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: dist-tarball
            path: dist-tarball
          attempt_limit: 3
          attempt_delay: 2000
      - name: Prepare artifact directory
        id: prepare
        if: needs.file-check.outputs.run == 'true'
        run: |
          mkdir -p artifacts/download/v9999.0.0 || exit 1
          mkdir -p artifacts/latest || exit 1
          echo "v9999.0.0" > artifacts/latest/latest-version.txt || exit 1
          cp dist-tarball/* artifacts/download/v9999.0.0 || exit 1
          cd artifacts/download/v9999.0.0 || exit 1
          ln -s ${{ needs.build-dist.outputs.distfile }} netdata-latest.tar.gz || exit 1
          ls -lFh
          sha256sum -b ./* > "sha256sums.txt" || exit 1
          cat sha256sums.txt
          cd ../.. || exit 1
          ls -lR
      - name: Fetch test environment
        id: fetch-test-environment
        if: needs.file-check.outputs.run == 'true'
        uses: Wandalen/wretry.action@v3
        with:
          action: actions/download-artifact@v4
          with: |
            name: ${{ matrix.artifact_key }}-test-env
            path: .
          attempt_limit: 3
          attempt_delay: 2000
      - name: Load test environment
        id: load
        if: needs.file-check.outputs.run == 'true'
        run: docker load --input image.tar
      - name: Install netdata and run the updater on ${{ matrix.distro }}
        id: updater-check
        if: needs.file-check.outputs.run == 'true'
        run: |
          docker run --security-opt seccomp=unconfined -e DISABLE_TELEMETRY=1 --network host -w /netdata \
                 -e EXTRA_INSTALL_FLAGS=${{ needs.file-check.outputs.skip-go }} \
                 test:${{ matrix.artifact_key }} /netdata/.github/scripts/run-updater-check.sh
      - name: Failure Notification
        uses: rtCamp/action-slack-notify@v2
        env:
          SLACK_COLOR: 'danger'
          SLACK_FOOTER: ''
          SLACK_ICON_EMOJI: ':github-actions:'
          SLACK_TITLE: 'Updater checks for ${{ matrix.distro }} failed:'
          SLACK_USERNAME: 'GitHub Actions'
          SLACK_MESSAGE: |-
              ${{ github.repository }}: Updater checks for ${{ matrix.distro }} failed.
              Checkout: ${{ steps.checkout.outcome }}
              Fetch dist tarball: ${{ steps.fetch-tarball.outcome }}
              Prepare artifact directory: ${{ steps.prepare.outcome }}
              Fetch test environment: ${{ steps.fetch-test-environment.outcome }}
              Load test environment: ${{ steps.load.outcome }}
              Updater check: ${{ steps.updater-check.outcome }}
          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
        if: >-
          ${{
            failure()
            && startsWith(github.ref, 'refs/heads/master')
            && github.event_name != 'pull_request'
            && github.repository == 'netdata/netdata'
            && needs.file-check.outputs.run == 'true'
          }}

  gitignore-check: # Verify that the build process does not make any changes to the source tree.
    name: .gitignore
    needs:
      - file-check
    runs-on: ubuntu-latest
    steps:
      - name: Skip Check
        id: skip
        if: needs.file-check.outputs.run != 'true'
        run: echo "SKIPPED"
      - name: Checkout
        if: needs.file-check.outputs.run == 'true'
        uses: actions/checkout@v4
        with:
          submodules: recursive
      - name: Prepare environment
        if: needs.file-check.outputs.run == 'true'
        run: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
      - name: Build netdata
        if: needs.file-check.outputs.run == 'true'
        run: ./netdata-installer.sh --dont-start-it --disable-telemetry --dont-wait --install-prefix /tmp/install --one-time-build ${{ needs.file-check.outputs.skip-go }}
      - name: Check that repo is clean
        if: needs.file-check.outputs.run == 'true'
        run: |
          git status --porcelain=v1 > /tmp/porcelain
          if [ -s /tmp/porcelain ]; then
            cat /tmp/porcelain
            exit 1
          fi