Fix handling of binary package uploads. (#10860)

.github/scripts/package_cloud_wrapper.sh

@@ -29,7 +29,7 @@ fi
 # Install dependency if not there
 if ! command -v package_cloud > /dev/null 2>&1; then
 	echo "No package cloud gem found, installing"
-	gem install -V package_cloud || (echo "Package cloud installation failed. you might want to check if required dependencies are there (ruby gcc gcc-c++ ruby-devel)" && exit 1)
+	sudo gem install -V package_cloud || (echo "Package cloud installation failed. you might want to check if required dependencies are there (ruby gcc gcc-c++ ruby-devel)" && exit 1)
 else
 	echo "Found package_cloud gem, continuing"
 fi

.github/scripts/parse_packagecloud_dist_id.py

@@ -1,39 +0,0 @@
-#!/usr/bin/env python3
-'''
-Parse the PackageCloud distributions JSON data to get a dist ID for uploads.
-
-This takes the JSON distributions data from Packagecloud on stdin and
-the package format, distribution name and version as arguments, and
-prints either an error message or the parsed distribution ID based on
-the arguments.
-'''
-
-import json
-import sys
-
-fmt = sys.argv[1]      # The package format ('deb' or 'rpm')
-distro = sys.argv[2]   # The distro name
-version = sys.argv[3]  # The distro version
-print(fmt)
-print(distro)
-print(version)
-
-data = json.load(sys.stdin)
-versions = []
-
-for entry in data[fmt]:
-    if entry['display_name'] == distro:
-        versions = entry['versions']
-        break
-
-if not versions:
-    print('Could not find version information for the requested distribution.')
-    sys.exit(-1)
-
-for entry in versions:
-    if entry['version_number'] == version:
-        print(entry['id'])
-        sys.exit(0)
-
-print('Unable to find id for requested version.')
-sys.exit(-1)

.github/workflows/packaging.yml

@@ -24,23 +24,23 @@ jobs:
     strategy:
       matrix:
         include:
-          - {distro: debian, version: "9", pkgclouddistro: Debian, pkgcloudversion: "9.0", format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
-          - {distro: debian, version: "9", pkgclouddistro: Debian, pkgcloudversion: "9.0", format: deb, base_image: debian, platform: linux/i386, arch: i386}
-          - {distro: debian, version: "10", pkgclouddistro: Debian, pkgcloudversion: "10.0", format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
-          - {distro: debian, version: "10", pkgclouddistro: Debian, pkgcloudversion: "10.0", format: deb, base_image: debian, platform: linux/i386, arch: i386}
-          - {distro: ubuntu, version: "16.04", pkgclouddistro: Ubuntu, pkgcloudversion: "16.04", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "16.04", pkgclouddistro: Ubuntu, pkgcloudversion: "16.04", format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
-          - {distro: ubuntu, version: "18.04", pkgclouddistro: Ubuntu, pkgcloudversion: "18.04", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "18.04", pkgclouddistro: Ubuntu, pkgcloudversion: "18.04", format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
-          - {distro: ubuntu, version: "20.04", pkgclouddistro: Ubuntu, pkgcloudversion: "20.04", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "20.10", pkgclouddistro: Ubuntu, pkgcloudversion: "20.10", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: ubuntu, version: "21.04", pkgclouddistro: Ubuntu, pkgcloudversion: "20.10", format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
-          - {distro: centos, version: "7", pkgclouddistro: Enterprise Linux, pkgcloudversion: "7.0", format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
-          - {distro: centos, version: "8", pkgclouddistro: Enterprise Linux, pkgcloudversion: "8.0", format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
-          - {distro: fedora, version: "32", pkgclouddistro: Fedora, pkgcloudversion: "32.0", format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
-          - {distro: fedora, version: "33", pkgclouddistro: Fedora, pkgcloudversion: "33.0", format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
-          - {distro: fedora, version: "34", pkgclouddistro: Fedora, pkgcloudversion: "33.0", format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
-          - {distro: opensuse, version: "15.2", pkgclouddistro: openSUSE, pkgcloudversion: "15.2", format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
+          - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
+          - {distro: debian, version: "9", pkgclouddistro: debian/stretch, format: deb, base_image: debian, platform: linux/i386, arch: i386}
+          - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/amd64, arch: amd64}
+          - {distro: debian, version: "10", pkgclouddistro: debian/buster, format: deb, base_image: debian, platform: linux/i386, arch: i386}
+          - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "16.04", pkgclouddistro: ubuntu/xenial, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
+          - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "18.04", pkgclouddistro: ubuntu/bionic, format: deb, base_image: ubuntu, platform: linux/i386, arch: i386}
+          - {distro: ubuntu, version: "20.04", pkgclouddistro: ubuntu/focal, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "20.10", pkgclouddistro: ubuntu/groovy, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: ubuntu, version: "21.04", pkgclouddistro: ubuntu/hirsute, format: deb, base_image: ubuntu, platform: linux/amd64, arch: amd64}
+          - {distro: centos, version: "7", pkgclouddistro: el/7, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
+          - {distro: centos, version: "8", pkgclouddistro: el/8, format: rpm, base_image: centos, platform: linux/amd64, arch: amd64}
+          - {distro: fedora, version: "32", pkgclouddistro: fedora/32, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
+          - {distro: fedora, version: "33", pkgclouddistro: fedora/33, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
+          - {distro: fedora, version: "34", pkgclouddistro: fedora/34, format: rpm, base_image: fedora, platform: linux/amd64, arch: amd64}
+          - {distro: opensuse, version: "15.2", pkgclouddistro: opensuse/15.2, format: rpm, base_image: opensuse/leap, platform: linux/amd64, arch: amd64}
       # We intentiaonally disable the fail-fast behavior so that a
       # build failure for one version doesn't prevent us from publishing
       # successfully built and tested packages for another version.
@@ -51,12 +51,14 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0 # We need full history for versioning
+          submodules: true
       - name: Checkout Tag # Otherwise check out the tag that triggered this.
-        if: github.event_name == 'wrokflow_dispatch'
+        if: github.event_name == 'workflow_dispatch'
         uses: actions/checkout@v2
         with:
-          refs: ${{ github.event.ref }}
+          ref: ${{ github.event.ref }}
           fetch-depth: 0 # We need full history for versioning
+          submodules: true
       - name: Check Base Branch
         run: |
           if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
@@ -74,7 +76,7 @@ jobs:
                 ;;
               *)
                 echo "repo=${{ secrets.PACKAGE_CLOUD_REPO }}-devel" >> $GITHUB_ENV
-                echo "pkg_version=$(git describe | sed -e 's/^v//')" >> $GITHUB_ENV
+                echo "pkg_version=0.${GITHUB_SHA}" >> $GITHUB_ENV
                 echo "pkg_retention_days=30" >> $GITHUB_ENV
                 ;;
             esac
@@ -114,13 +116,13 @@ jobs:
       - name: Upload
         if: github.event_name == 'workflow_dispatch'
         shell: bash
+        env:
+          PKG_CLOUD_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_KEY }}
         run: |
-          # This figures out the distribution ID for the upload.
-          DIST_ID="$(curl https://${{ secrets.PACKAGE_CLOUD_API_KEY }}:@packagecloud.io/api/v1/distributions.json | ${GITHUB_WORKSPACE}/.github/scripts/parse_packagecloud_dist_id.py ${{ matrix.format }} ${{ matrix.pkgclouddistro }} ${{ matrix.pkgcloudversion }})"
-          for pkgfile in artifacts/*.${FORMAT} ; do
-            curl -F "package[distro_version_id]=${BUILD_ID}" \
-                 -F "package[package_file]=@${pkgfile}" \
-                 https://${{ secrets.PACKAGE_CLOUD_API_KEY }}:@packagecloud.io/api/v1/repos/${{ env.repo }}/packages.json || exit 1
+          echo "Packages to upload:\n$(ls artifacts/*.${{ matrix.format }})"
+          for pkgfile in artifacts/*.${{ matrix.format }} ; do
+            .github/scripts/package_cloud_wrapper.sh push ${{ env.repo }}/${{ matrix.pkgclouddistro }} ${pkgfile}
+          done
       - name: Clean
         if: github.event_name == 'workflow_dispatch'
         shell: bash

packaging/Dockerfile.packager

@@ -36,7 +36,7 @@ ENV VERSION=$PKG_VERSION
 COPY ./packaging/scripts/install.sh /install.sh
 COPY ./packaging/scripts/test.sh /test.sh
 
-COPY --from=build /netdata/artifacts /artifacts
+COPY --from=build /netdata/artifacts /packages
 
 RUN /install.sh
 

packaging/scripts/install.sh

@@ -7,7 +7,7 @@ install_debian_like() {
   apt-get update
 
   # Install NetData
-  apt-get install -y "/artifacts/netdata_${VERSION}_${ARCH}.deb"
+  apt-get install -y "/packages/netdata_${VERSION}_${ARCH}.deb"
 
   # Install testing tools
   apt-get install -y --no-install-recommends \
@@ -23,7 +23,7 @@ install_fedora_like() {
   pkg_version="$(echo "${VERSION}" | tr - .)"
 
   # Install NetData
-  "$PKGMGR" install -y /artifacts/netdata-"${pkg_version}"-*.rpm
+  "$PKGMGR" install -y /packages/netdata-"${pkg_version}"-*.rpm
 
   # Install testing tools
   "$PKGMGR" install -y curl nc jq
@@ -41,7 +41,7 @@ install_centos() {
   "$PKGMGR" install -y epel-release
 
   # Install NetData
-  "$PKGMGR" install -y /artifacts/netdata-"${pkg_version}"-*.rpm
+  "$PKGMGR" install -y /packages/netdata-"${pkg_version}"-*.rpm
 
   # Install testing tools
   "$PKGMGR" install -y curl nc jq
@@ -56,7 +56,7 @@ install_suse_like() {
   # Install NetData
   # FIXME: Allow unsigned packages (for now) #7773
   zypper install -y --allow-unsigned-rpm \
-    /artifacts/netdata-"${pkg_version}"-*.rpm
+    /packages/netdata-"${pkg_version}"-*.rpm
 
   # Install testing tools
   zypper install -y --no-recommends \

packaging/scripts/test.sh

@@ -39,3 +39,5 @@ cat ./response
 jq '.version' ./response || exit 1
 
 trap - EXIT
+
+cp -a /packages/* /artifacts