Update workflows to use $GITHUB_OUTPUT instead of ::set-output:: (#13960)

* Update workflows to use $GITHUB_OUTPUT instead of ::set-output::

* Fix python code.

* Fix handling of python-based build matrix generation.

.github/scripts/gen-matrix-build.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import json
+
+from ruamel.yaml import YAML
+
+yaml = YAML(typ='safe')
+entries = []
+
+with open('.github/data/distros.yml') as f:
+    data = yaml.load(f)
+
+for i, v in enumerate(data['include']):
+    e = {
+      'artifact_key': v['distro'] + str(v['version']).replace('.', ''),
+      'version': v['version'],
+    }
+
+    if 'base_image' in v:
+        e['distro'] = ':'.join([v['base_image'], str(v['version'])])
+    else:
+        e['distro'] = ':'.join([v['distro'], str(v['version'])])
+
+    if 'env_prep' in v:
+        e['env_prep'] = v['env_prep']
+
+    if 'jsonc_removal' in v:
+        e['jsonc_removal'] = v['jsonc_removal']
+
+    entries.append(e)
+
+entries.sort(key=lambda k: k['distro'])
+matrix = json.dumps({'include': entries}, sort_keys=True)
+print(matrix)

.github/scripts/gen-matrix-packaging.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+
+from ruamel.yaml import YAML
+
+ALWAYS_RUN_ARCHES = ["amd64", "x86_64"]
+SHORT_RUN = sys.argv[1]
+yaml = YAML(typ='safe')
+entries = list()
+run_limited = False
+
+with open('.github/data/distros.yml') as f:
+    data = yaml.load(f)
+
+if "${{ github.event_name }}" == "pull_request" and bool(int(SHORT_RUN)):
+    run_limited = True
+
+for i, v in enumerate(data['include']):
+    if 'packages' in data['include'][i]:
+        for arch in data['include'][i]['packages']['arches']:
+            if arch in ALWAYS_RUN_ARCHES or not run_limited:
+                entries.append({
+                    'distro': data['include'][i]['distro'],
+                    'version': data['include'][i]['version'],
+                    'repo_distro': data['include'][i]['packages']['repo_distro'],
+                    'format': data['include'][i]['packages']['type'],
+                    'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
+                    'platform': data['platform_map'][arch],
+                    'arch': arch
+                })
+
+entries.sort(key=lambda k: (data['arch_order'].index(k['arch']), k['distro'], k['version']))
+matrix = json.dumps({'include': entries}, sort_keys=True)
+print(matrix)

.github/scripts/gen-matrix-repoconfig.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+import json
+
+from ruamel.yaml import YAML
+
+yaml = YAML(typ='safe')
+entries = list()
+
+with open('.github/data/distros.yml') as f:
+    data = yaml.load(f)
+
+for i, v in enumerate(data['include']):
+    if 'packages' in data['include'][i]:
+        entries.append({
+            'distro': data['include'][i]['distro'],
+            'version': data['include'][i]['version'],
+            'pkgclouddistro': data['include'][i]['packages']['repo_distro'],
+            'format': data['include'][i]['packages']['type'],
+            'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
+            'platform': data['platform_map']['amd64']
+        })
+
+entries.sort(key=lambda k: (k['distro'], k['version']))
+matrix = json.dumps({'include': entries}, sort_keys=True)
+print(matrix)

.github/scripts/get-static-cache-key.sh

@@ -12,4 +12,4 @@ docker run -it --rm --platform "${platform}" netdata/static-builder sh -c 'apk l
 
 h="$(sha256sum /tmp/static-cache-key-data | cut -f 1 -d ' ')"
 
-echo "::set-output name=key::static-${arch}-${h}"
+echo "key=static-${arch}-${h}" >> "${GITHUB_OUTPUT}"

.github/scripts/prepare-release-base.sh

@@ -97,7 +97,7 @@ git config user.email "bot@netdata.cloud"
 
 if [ "${REPO}" != "netdata/netdata" ] && [ -z "${RELEASE_TEST}" ]; then
     echo "::notice::Not running in the netdata/netdata repository, not queueing a release build."
-    echo "::set-output name=run::false"
+    echo "run=false" >> "${GITHUB_OUTPUT}"
 elif [ "${EVENT_NAME}" = 'schedule' ] || [ "${EVENT_TYPE}" = 'nightly' ]; then
     echo "::notice::Preparing a nightly release build."
     LAST_TAG=$(git describe --abbrev=0 --tags)
@@ -107,15 +107,16 @@ elif [ "${EVENT_NAME}" = 'schedule' ] || [ "${EVENT_TYPE}" = 'nightly' ]; then
     HEAD_COMMIT="$(git rev-parse HEAD)"
     if [ "${EVENT_NAME}" = 'schedule' ] && [ "${LAST_VERSION_COMMIT}" = "${HEAD_COMMIT}" ] && grep -qE '.*-nightly$' packaging/version; then
         echo "::notice::No commits since last nightly build, not publishing a new nightly build."
-        echo "::set-output name=run::false"
+        echo "run=false" >> "${GITHUB_OUTPUT}"
     else
         echo "${NEW_VERSION}" > packaging/version || exit 1
-        echo "::set-output name=run::true"
-        echo "::set-output name=message::Update changelog and version for nightly build: ${NEW_VERSION}."
-        echo "::set-output name=ref::master"
-        echo "::set-output name=type::nightly"
-        echo "::set-output name=branch::master"
-        echo "::set-output name=version::nightly"
+        # shellcheck disable=SC2129
+        echo "run=true" >> "${GITHUB_OUTPUT}"
+        echo "message=Update changelog and version for nightly build: ${NEW_VERSION}." >> "${GITHUB_OUTPUT}"
+        echo "ref=master" >> "${GITHUB_OUTPUT}"
+        echo "type=nightly" >> "${GITHUB_OUTPUT}"
+        echo "branch=master" >> "${GITHUB_OUTPUT}"
+        echo "version=nightly" >> "${GITHUB_OUTPUT}"
     fi
 elif [ "${EVENT_TYPE}" = 'patch' ] && [ "${EVENT_VERSION}" != "nightly" ]; then
     echo "::notice::Preparing a patch release build."
@@ -130,12 +131,13 @@ elif [ "${EVENT_TYPE}" = 'patch' ] && [ "${EVENT_VERSION}" != "nightly" ]; then
     major_matches || exit 1
     check_newer_patch_version || exit 1
     echo "${EVENT_VERSION}" > packaging/version || exit 1
-    echo "::set-output name=run::true"
-    echo "::set-output name=message::Patch release ${EVENT_VERSION}."
-    echo "::set-output name=ref::${EVENT_VERSION}"
-    echo "::set-output name=type::release"
-    echo "::set-output name=branch::${branch_name}"
-    echo "::set-output name=version::$(tr -d 'v' < packaging/version)"
+    # shellcheck disable=SC2129
+    echo "run=true" >> "${GITHUB_OUTPUT}"
+    echo "message=Patch release ${EVENT_VERSION}." >> "${GITHUB_OUTPUT}"
+    echo "ref=${EVENT_VERSION}" >> "${GITHUB_OUTPUT}"
+    echo "type=release" >> "${GITHUB_OUTPUT}"
+    echo "branch=${branch_name}" >> "${GITHUB_OUTPUT}"
+    echo "version=$(tr -d 'v' < packaging/version)" >> "${GITHUB_OUTPUT}"
 elif [ "${EVENT_TYPE}" = 'minor' ] && [ "${EVENT_VERSION}" != "nightly" ]; then
     echo "::notice::Preparing a minor release build."
     check_version_format || exit 1
@@ -149,13 +151,14 @@ elif [ "${EVENT_TYPE}" = 'minor' ] && [ "${EVENT_VERSION}" != "nightly" ]; then
         exit 1
     fi
     echo "${EVENT_VERSION}" > packaging/version || exit 1
-    echo "::set-output name=run::true"
-    echo "::set-output name=message::Minor release ${EVENT_VERSION}."
-    echo "::set-output name=ref::${EVENT_VERSION}"
-    echo "::set-output name=type::release"
-    echo "::set-output name=branch::master"
-    echo "::set-output name=new-branch::${branch_name}"
-    echo "::set-output name=version::$(tr -d 'v' < packaging/version)"
+    # shellcheck disable=SC2129
+    echo "run=true" >> "${GITHUB_OUTPUT}"
+    echo "message=Minor release ${EVENT_VERSION}." >> "${GITHUB_OUTPUT}"
+    echo "ref=${EVENT_VERSION}" >> "${GITHUB_OUTPUT}"
+    echo "type=release" >> "${GITHUB_OUTPUT}"
+    echo "branch=master" >> "${GITHUB_OUTPUT}"
+    echo "new-branch=${branch_name}" >> "${GITHUB_OUTPUT}"
+    echo "version=$(tr -d 'v' < packaging/version)" >> "${GITHUB_OUTPUT}"
 elif [ "${EVENT_TYPE}" = 'major' ] && [ "${EVENT_VERSION}" != "nightly" ]; then
     echo "::notice::Preparing a major release build."
     check_version_format || exit 1
@@ -164,12 +167,13 @@ elif [ "${EVENT_TYPE}" = 'major' ] && [ "${EVENT_VERSION}" != "nightly" ]; then
     check_newer_major_version || exit 1
     check_for_existing_tag || exit 1
     echo "${EVENT_VERSION}" > packaging/version || exit 1
-    echo "::set-output name=run::true"
-    echo "::set-output name=message::Major release ${EVENT_VERSION}"
-    echo "::set-output name=ref::${EVENT_VERSION}"
-    echo "::set-output name=type::release"
-    echo "::set-output name=branch::master"
-    echo "::set-output name=version::$(tr -d 'v' < packaging/version)"
+    # shellcheck disable=SC2129
+    echo "run=true" >> "${GITHUB_OUTPUT}"
+    echo "message=Major release ${EVENT_VERSION}" >> "${GITHUB_OUTPUT}"
+    echo "ref=${EVENT_VERSION}" >> "${GITHUB_OUTPUT}"
+    echo "type=release" >> "${GITHUB_OUTPUT}"
+    echo "branch=master" >> "${GITHUB_OUTPUT}"
+    echo "version=$(tr -d 'v' < packaging/version)" >> "${GITHUB_OUTPUT}"
 else
     echo '::error::Unrecognized release type or invalid version.'
     exit 1

.github/workflows/build.yml

@@ -57,7 +57,7 @@ jobs:
                       --with-math \
                       --with-user=netdata
           make dist
-          echo "::set-output name=distfile::$(find . -name 'netdata-*.tar.gz')"
+          echo "distfile=$(find . -name 'netdata-*.tar.gz')" >> "${GITHUB_OUTPUT}"
           cp netdata-*.tar.gz artifacts/
       - name: Store
         id: store
@@ -183,39 +183,10 @@ jobs:
           sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml
       - name: Read build matrix
         id: set-matrix
-        shell: python3 {0}
         run: |
-          from ruamel.yaml import YAML
-          import json
-          yaml = YAML(typ='safe')
-          entries = list()
-
-          with open('.github/data/distros.yml') as f:
-              data = yaml.load(f)
-
-          for i, v in enumerate(data['include']):
-              e = {
-                'artifact_key': v['distro'] + str(v['version']).replace('.', ''),
-                'version': v['version'],
-              }
-
-              if 'base_image' in v:
-                e['distro'] = ':'.join([v['base_image'], str(v['version'])])
-              else:
-                e['distro'] = ':'.join([v['distro'], str(v['version'])])
-
-              if 'env_prep' in v:
-                e['env_prep'] = v['env_prep']
-
-              if 'jsonc_removal' in v:
-                e['jsonc_removal'] = v['jsonc_removal']
-
-              entries.append(e)
-
-          entries.sort(key=lambda k: k['distro'])
-          matrix = json.dumps({'include': entries}, sort_keys=True)
-          print('Generated Matrix: ' + matrix)
-          print('::set-output name=matrix::' + matrix)
+          matrix="$(.github/scripts/gen-matrix-build.py)"
+          echo "Generated matrix: ${matrix}"
+          echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"
       - name: Failure Notification
         uses: rtCamp/action-slack-notify@v2
         env:
@@ -363,7 +334,7 @@ jobs:
         id: load
         run: |
           docker load --input image.tar | tee image-info.txt
-          echo "::set-output name=image::$(cut -d ':' -f 3 image-info.txt)"
+          echo "image=$(cut -d ':' -f 3 image-info.txt)" >> "${GITHUB_OUTPUT}"
       - name: Regular build on ${{ matrix.distro }}
         id: build-basic
         run: |
@@ -458,7 +429,7 @@ jobs:
         id: load
         run: |
           docker load --input image.tar | tee image-info.txt
-          echo "::set-output name=image::$(cut -d ':' -f 3 image-info.txt)"
+          echo "image=$(cut -d ':' -f 3 image-info.txt)" >> "${GITHUB_OUTPUT}"
       - name: Install netdata and run the updater on ${{ matrix.distro }}
         id: updater-check
         run: |
@@ -790,9 +761,9 @@ jobs:
         id: tag
         run: |
           if echo ${{ github.event.inputs.version }} | grep -qE '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$'; then
-            echo "::set-output name=tag::v${{ github.event.inputs.version }}"
+            echo "tag=v${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
           else
-            echo "::set-output name=tag::${{ github.event.inputs.version }}"
+            echo "tag=${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
           fi
 
   upload-release: # Create the draft release and upload the build artifacts.

.github/workflows/cloud_regression.yml

@@ -33,10 +33,10 @@ jobs:
             NETDATA_CUSTOM_PR_NUMBER=""
             NETDATA_CUSTOM_COMMIT_HASH="${{ github.sha }}"
           fi
-            echo "netdata_repo=${NETDATA_CUSTOM_REPO}" >> $GITHUB_OUTPUT
-            echo "netdata_branch=${NETDATA_CUSTOM_BRANCH}" >> $GITHUB_OUTPUT
-            echo "netdata_pr_number=${NETDATA_CUSTOM_PR_NUMBER}" >> $GITHUB_OUTPUT
-            echo "netdata_commit_hash=${NETDATA_CUSTOM_COMMIT_HASH}" >> $GITHUB_OUTPUT
+          echo "netdata_repo=${NETDATA_CUSTOM_REPO}" >> $GITHUB_OUTPUT
+          echo "netdata_branch=${NETDATA_CUSTOM_BRANCH}" >> $GITHUB_OUTPUT
+          echo "netdata_pr_number=${NETDATA_CUSTOM_PR_NUMBER}" >> $GITHUB_OUTPUT
+          echo "netdata_commit_hash=${NETDATA_CUSTOM_COMMIT_HASH}" >> $GITHUB_OUTPUT
 
       - name: Trigger Full Cloud Regression
         uses: aurelien-baudet/workflow-dispatch@v2

.github/workflows/codeql.yml

@@ -32,39 +32,39 @@ jobs:
         run: |
           if [ "${{ github.event_name }}" = "pull_request" ]; then
             if [ "${{ contains(github.event.pull_request.labels.*.name, 'run-ci/codeql') }}" = "true" ]; then
-              echo '::set-output name=run::true'
+              echo "run=true" >> "${GITHUB_OUTPUT}"
               echo '::notice::Found ci/codeql label, unconditionally running all CodeQL checks.'
             else
-              echo '::set-output name=run::false'
+              echo "run=false" >> "${GITHUB_OUTPUT}"
             fi
           else
-            echo '::set-output name=run::true'
+            echo "run=true" >> "${GITHUB_OUTPUT}"
           fi
       - name: Check for C/C++ changes
         id: cpp
         run: |
           if [ "${{ steps.always.outputs.run }}" = "false" ]; then
             if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq '.*\.[ch](xx|\+\+)?' ; then
-              echo '::set-output name=run::true'
+              echo "run=true" >> "${GITHUB_OUTPUT}"
               echo '::notice::C/C++ code has changed, need to run CodeQL.'
             else
-              echo '::set-output name=run::false'
+              echo "run=false" >> "${GITHUB_OUTPUT}"
             fi
           else
-            echo '::set-output name=run::true'
+            echo "run=true" >> "${GITHUB_OUTPUT}"
           fi
       - name: Check for python changes
         id: python
         run: |
           if [ "${{ steps.always.outputs.run }}" = "false" ]; then
             if git diff --name-only origin/${{ github.base_ref }} HEAD | grep -Eq 'collectors/python.d.plugin/.*\.py' ; then
-              echo '::set-output name=run::true'
+              echo "run=true" >> "${GITHUB_OUTPUT}"
               echo '::notice::Python code has changed, need to run CodeQL.'
             else
-              echo '::set-output name=run::false'
+              echo "run=false" >> "${GITHUB_OUTPUT}"
             fi
           else
-            echo '::set-output name=run::true'
+            echo "run=true" >> "${GITHUB_OUTPUT}"
           fi
 
   analyze-cpp:

.github/workflows/docker.yml

@@ -129,9 +129,9 @@ jobs:
         id: tag
         run: |
           if echo ${{ github.event.inputs.version }} | grep -qE '^[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+$'; then
-            echo "::set-output name=tag::v${{ github.event.inputs.version }}"
+            echo "tag=v${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
           else
-            echo "::set-output name=tag::${{ github.event.inputs.version }}"
+            echo "tag=${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
           fi
 
   docker-publish:

.github/workflows/packaging.yml

@@ -44,41 +44,15 @@ jobs:
           sudo apt-get update && sudo apt-get install -y python3-ruamel.yaml
       - name: Read build matrix
         id: set-matrix
-        shell: python3 {0}
         run: |
-          from ruamel.yaml import YAML
-          import json
-          import re
-          import os
-          ALWAYS_RUN_ARCHES = ["amd64", "x86_64"]
-          yaml = YAML(typ='safe')
-          entries = list()
-          run_limited = False
-
-          with open('.github/data/distros.yml') as f:
-              data = yaml.load(f)
-
-          if "${{ github.event_name }}" == "pull_request" and "${{ !contains(github.event.pull_request.labels.*.name, 'run-ci/packaging') }}":
-              run_limited = True
-
-          for i, v in enumerate(data['include']):
-              if 'packages' in data['include'][i]:
-                  for arch in data['include'][i]['packages']['arches']:
-                      if arch in ALWAYS_RUN_ARCHES or not run_limited:
-                          entries.append({
-                              'distro': data['include'][i]['distro'],
-                              'version': data['include'][i]['version'],
-                              'repo_distro': data['include'][i]['packages']['repo_distro'],
-                              'format': data['include'][i]['packages']['type'],
-                              'base_image': data['include'][i]['base_image'] if 'base_image' in data['include'][i] else data['include'][i]['distro'],
-                              'platform': data['platform_map'][arch],
-                              'arch': arch
-                          })
-
-          entries.sort(key=lambda k: (data['arch_order'].index(k['arch']), k['distro'], k['version']))
-          matrix = json.dumps({'include': entries}, sort_keys=True)
-          print('Generated Matrix: ' + matrix)
-          print('::set-output name=matrix::' + matrix)
+          if [ "${{ github.event_name }}" = "pull_request" ] && \
+             [ "${{ !contains(github.event.pull_request.labels.*.name, 'run-ci/packaging') }}" = "true" ]; then
+            matrix="$(.github/scripts/gen-matrix-packaging.py 1)"
+          else
+            matrix="$(.github/scripts/gen-matrix-packaging.py 0)"
+          fi
+          echo "Generated matrix: ${matrix}"
+          echo "matrix=${matrix}" >> "${GITHUB_OUTPUT}"
       - name: Failure Notification
         uses: rtCamp/action-slack-notify@v2
         env:
@@ -117,24 +91,24 @@ jobs:
           if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
             case "${{ github.event.inputs.type }}" in
               "release")
-                echo "::set-output name=repo::${REPO_PREFIX}"
-                echo "::set-output name=version::${{ github.event.inputs.version }}"
-                echo "::set-output name=retention::365"
+                echo "repo=${REPO_PREFIX}" >> "${GITHUB_OUTPUT}"
+                echo "version=${{ github.event.inputs.version }}" >> "${GITHUB_OUTPUT}"
+                echo "retention=365" >> "${GITHUB_OUTPUT}"
                 ;;
               "nightly")
-                echo "::set-output name=repo::${REPO_PREFIX}-edge"
-                echo "::set-output name=version::$(tr -d 'v' < packaging/version)"
-                echo "::set-output name=retention::30"
+                echo "repo=${REPO_PREFIX}-edge" >> "${GITHUB_OUTPUT}"
+                echo "version=$(tr -d 'v' < packaging/version)" >> "${GITHUB_OUTPUT}"
+                echo "retention=30" >> "${GITHUB_OUTPUT}"
                 ;;
               *)
-                echo "::set-output name=repo::${REPO_PREFIX}-devel"
-                echo "::set-output name=version::0.${GITHUB_SHA}"
-                echo "::set-output name=retention::30"
+                echo "repo=${REPO_PREFIX}-devel" >> "${GITHUB_OUTPUT}"
+                echo "version=0.${GITHUB_SHA}" >> "${GITHUB_OUTPUT}"
+                echo "retention=30" >> "${GITHUB_OUTPUT}"
                 ;;
             esac
           else
-            echo "::set-output name=version::$(cut -d'-' -f 1 packaging/version | tr -d 'v')"
-            echo "::set-output name=retention::0"
+            echo "version=$(cut -d'-' -f 1 packaging/version | tr -d 'v')" >> "${GITHUB_OUTPUT}"
+            echo "retention=0" >> "${GITHUB_OUTPUT}"
           fi
       - name: Failure Notification
         uses: rtCamp/action-slack-notify@v2