Cache invariant components in static builds to reduce build times. (#12877)

* Add basic build caching support to static builds.

Cache is store din `artifacts/cache/${BUILDARCH}`. Each third-party
component utilizes a separate build cache. Invalidation is only done for
version changes (more rigorous invalidation is expected to be handled
externally).

* Integrate static build caching with CI.

* Fix fping cache handling.

* Test caching in CI.

* Properly skip rebuilds on cache hits.

* Remove static build container when done with it.

* Reuse existing image automatically if it’s for the correct platform.

* Test CI build caching.

* Fix static build job names.

.github/scripts/get-static-cache-key.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+arch="${1}"
+platform="$(packaging/makeself/uname2platform.sh "${arch}")"
+
+docker pull --platform "${platform}" netdata/static-builder
+
+# shellcheck disable=SC2046
+cat $(find packaging/makeself/jobs -type f ! -regex '.*\(netdata\|-makeself\).*') > /tmp/static-cache-key-data
+
+docker run -it --rm --platform "${platform}" netdata/static-builder sh -c 'apk list -I 2>/dev/null' >> /tmp/static-cache-key-data
+
+h="$(sha256sum /tmp/static-cache-key-data | cut -f 1 -d ' ')"
+
+echo "::set-output name=key::static-${arch}-${h}"

.github/workflows/build.yml

@@ -88,10 +88,10 @@ jobs:
     strategy:
       matrix:
         arch:
-          - 'x86_64'
-          - 'armv7l'
-          - 'aarch64'
-          - 'ppc64le'
+          - x86_64
+          - armv7l
+          - aarch64
+          - ppc64le
     steps:
       - name: Checkout
         id: checkout
@@ -104,6 +104,15 @@ jobs:
         if: github.event_name == 'workflow_dispatch' && github.event.inputs.type != 'nightly'
         run: |
           sed -i 's/^RELEASE_CHANNEL="nightly" *#/RELEASE_CHANNEL="stable" #/' netdata-installer.sh packaging/makeself/install-or-update.sh
+      - name: Get Cache Key
+        id: cache-key
+        run: .github/scripts/get-static-cache-key.sh ${{ matrix.arch }}
+      - name: Cache
+        id: cache
+        uses: actions/cache@v3
+        with:
+          path: artifacts/cache
+          key: ${{ steps.cache-key.outputs.key }}
       - name: Build
         if: github.event_name != 'workflow_dispatch' # Don’t use retries on PRs.
         run: .github/scripts/build-static.sh ${{ matrix.arch }}

packaging/makeself/build-static.sh

@@ -9,16 +9,11 @@ BUILDARCH="${1}"
 
 set -e
 
-case ${BUILDARCH} in
-  x86_64) platform=linux/amd64 ;;
-  armv7l) platform=linux/arm/v7 ;;
-  aarch64) platform=linux/arm64/v8 ;;
-  ppc64le) platform=linux/ppc64le ;;
-  *)
-    echo "Unknown target architecture '${BUILDARCH}'."
+platform="$("$(dirname "${0}")/uname2platform.sh" "${BUILDARCH}")"
+
+if [ -z "${platform}" ]; then
     exit 1
-    ;;
-esac
+fi
 
 DOCKER_IMAGE_NAME="netdata/static-builder"
 
@@ -27,18 +22,23 @@ if [ "${BUILDARCH}" != "$(uname -m)" ] && [ "$(uname -m)" = 'x86_64' ] && [ -z "
 fi
 
 if docker inspect "${DOCKER_IMAGE_NAME}" > /dev/null 2>&1; then
-    docker image rm "${DOCKER_IMAGE_NAME}" || exit 1
+    img_platform="$(docker image inspect netdata/static-builder --format '{{.Os}}/{{.Architecture}}/{{.Variant}}')"
+    if [ "${img_platform%'/'}" != "${platform}" ]; then
+        docker image rm "${DOCKER_IMAGE_NAME}" || exit 1
+    fi
 fi
 
-docker pull --platform "${platform}" "${DOCKER_IMAGE_NAME}"
+if ! docker inspect "${DOCKER_IMAGE_NAME}" > /dev/null 2>&1; then
+    docker pull --platform "${platform}" "${DOCKER_IMAGE_NAME}"
+fi
 
 # Run the build script inside the container
 if [ -t 1 ]; then
-  run docker run -e BUILDARCH="${BUILDARCH}" -a stdin -a stdout -a stderr -i -t -v "$(pwd)":/netdata:rw \
+  run docker run --rm -e BUILDARCH="${BUILDARCH}" -a stdin -a stdout -a stderr -i -t -v "$(pwd)":/netdata:rw \
     "${DOCKER_IMAGE_NAME}" \
     /bin/sh /netdata/packaging/makeself/build.sh "${@}"
 else
-  run docker run -e BUILDARCH="${BUILDARCH}" -v "$(pwd)":/netdata:rw \
+  run docker run --rm -e BUILDARCH="${BUILDARCH}" -v "$(pwd)":/netdata:rw \
     -e GITHUB_ACTIONS="${GITHUB_ACTIONS}" "${DOCKER_IMAGE_NAME}" \
     /bin/sh /netdata/packaging/makeself/build.sh "${@}"
 fi

packaging/makeself/functions.sh

@@ -29,34 +29,64 @@ set -euo pipefail
 # -----------------------------------------------------------------------------
 
 fetch() {
-  local dir="${1}" url="${2}" sha256="${3}"
+  local dir="${1}" url="${2}" sha256="${3}" key="${4}"
   local tar="${dir}.tar.gz"
+  local cache="${NETDATA_SOURCE_PATH}/artifacts/cache/${BUILDARCH}/${key}"
 
-  if [ ! -f "${NETDATA_MAKESELF_PATH}/tmp/${tar}" ]; then
-    run wget -O "${NETDATA_MAKESELF_PATH}/tmp/${tar}" "${url}"
+  if [ -d "${NETDATA_MAKESELF_PATH}/tmp/${dir}" ]; then
+    rm -rf "${NETDATA_MAKESELF_PATH}/tmp/${dir}"
   fi
 
-  # Check SHA256 of gzip'd tar file (apparently alpine's sha256sum requires
-  # two empty spaces between the checksum and the file's path)
-  set +e
-  echo "${sha256}  ${NETDATA_MAKESELF_PATH}/tmp/${tar}" | sha256sum -c -s
-  local rc=$?
-  if [ ${rc} -ne 0 ]; then
-      echo >&2 "SHA256 verification of tar file ${tar} failed (rc=${rc})"
-      echo >&2 "expected: ${sha256}, got $(sha256sum "${NETDATA_MAKESELF_PATH}/tmp/${tar}")"
-      exit 1
-  fi
-  set -e
+  if [ -d "${cache}/${dir}" ]; then
+    echo "Found cached copy of build directory for ${key}, using it."
+    cp -a "${cache}/${dir}" "${NETDATA_MAKESELF_PATH}/tmp/"
+    CACHE_HIT=1
+  else
+    echo "No cached copy of build directory for ${key} found, fetching sources instead."
+
+    if [ ! -f "${NETDATA_MAKESELF_PATH}/tmp/${tar}" ]; then
+      run wget -O "${NETDATA_MAKESELF_PATH}/tmp/${tar}" "${url}"
+    fi
+
+    # Check SHA256 of gzip'd tar file (apparently alpine's sha256sum requires
+    # two empty spaces between the checksum and the file's path)
+    set +e
+    echo "${sha256}  ${NETDATA_MAKESELF_PATH}/tmp/${tar}" | sha256sum -c -s
+    local rc=$?
+    if [ ${rc} -ne 0 ]; then
+        echo >&2 "SHA256 verification of tar file ${tar} failed (rc=${rc})"
+        echo >&2 "expected: ${sha256}, got $(sha256sum "${NETDATA_MAKESELF_PATH}/tmp/${tar}")"
+        exit 1
+    fi
+    set -e
 
-  if [ ! -d "${NETDATA_MAKESELF_PATH}/tmp/${dir}" ]; then
     cd "${NETDATA_MAKESELF_PATH}/tmp"
     run tar -zxpf "${tar}"
     cd -
+
+    CACHE_HIT=0
   fi
 
   run cd "${NETDATA_MAKESELF_PATH}/tmp/${dir}"
 }
 
+store_cache() {
+    key="${1}"
+    src="${2}"
+
+    cache="${NETDATA_SOURCE_PATH}/artifacts/cache/${BUILDARCH}/${key}"
+
+    if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+        if [ -d "${cache}" ]; then
+            rm -rf "${cache}"
+        fi
+
+        mkdir -p "${cache}"
+
+        cp -a "${src}" "${cache}"
+    fi
+}
+
 # -----------------------------------------------------------------------------
 
 # load the functions of the netdata-installer.sh

packaging/makeself/jobs/20-openssl.install.sh

@@ -13,15 +13,36 @@ export CFLAGS='-fno-lto -pipe'
 export LDFLAGS='-static'
 export PKG_CONFIG="pkg-config --static"
 
-# Might be bind-mounted
-if [ ! -d "${NETDATA_MAKESELF_PATH}/tmp/openssl" ]; then
+if [ -d "${NETDATA_MAKESELF_PATH}/tmp/openssl" ]; then
+  rm -rf "${NETDATA_MAKESELF_PATH}/tmp/openssl"
+fi
+
+if [ -d "${NETDATA_MAKESELF_PATH}/tmp/openssl" ]; then
+  rm -rf "${NETDATA_MAKESELF_PATH}/tmp/openssl"
+fi
+
+cache="${NETDATA_SOURCE_PATH}/artifacts/cache/${BUILDARCH}/openssl"
+
+if [ -d "${cache}" ]; then
+  echo "Found cached copy of build directory for openssl, using it."
+  cp -a "${cache}/openssl" "${NETDATA_MAKESELF_PATH}/tmp/"
+  CACHE_HIT=1
+else
+  echo "No cached copy of build directory for openssl found, fetching sources instead."
   run git clone --branch "${version}" --single-branch --depth 1 git://git.openssl.org/openssl.git "${NETDATA_MAKESELF_PATH}/tmp/openssl"
+  CACHE_HIT=0
 fi
+
 cd "${NETDATA_MAKESELF_PATH}/tmp/openssl" || exit 1
 
-run ./config -static no-tests --prefix=/openssl-static --openssldir=/opt/netdata/etc/ssl
-run make -j "$(nproc)"
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+    run ./config -static no-tests --prefix=/openssl-static --openssldir=/opt/netdata/etc/ssl
+    run make -j "$(nproc)"
+fi
+
 run make -j "$(nproc)" install_sw
 
+store_cache openssl "${NETDATA_MAKESELF_PATH}/tmp/openssl"
+
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::endgroup::" || true

packaging/makeself/jobs/50-bash-5.1.16.install.sh

@@ -4,37 +4,43 @@
 # shellcheck source=packaging/makeself/functions.sh
 . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1
 
+version="5.1.16"
+
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::building bash" || true
 
-fetch "bash-5.1.16" "http://ftp.gnu.org/gnu/bash/bash-5.1.16.tar.gz" \
-    5bac17218d3911834520dad13cd1f85ab944e1c09ae1aba55906be1f8192f558
+fetch "bash-${version}" "http://ftp.gnu.org/gnu/bash/bash-${version}.tar.gz" \
+    5bac17218d3911834520dad13cd1f85ab944e1c09ae1aba55906be1f8192f558 bash
 
 export CFLAGS="-pipe"
 export PKG_CONFIG_PATH="/openssl-static/lib/pkgconfig"
 
-run ./configure \
-  --prefix="${NETDATA_INSTALL_PATH}" \
-  --without-bash-malloc \
-  --enable-static-link \
-  --enable-net-redirections \
-  --enable-array-variables \
-  --disable-progcomp \
-  --disable-profiling \
-  --disable-nls \
-  --disable-dependency-tracking
-
-run make clean
-run make -j "$(nproc)"
-
-cat > examples/loadables/Makefile << EOF
-all:
-clean:
-install:
-EOF
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+    run ./configure \
+        --prefix="${NETDATA_INSTALL_PATH}" \
+        --without-bash-malloc \
+        --enable-static-link \
+        --enable-net-redirections \
+        --enable-array-variables \
+        --disable-progcomp \
+        --disable-profiling \
+        --disable-nls \
+        --disable-dependency-tracking
+
+    run make clean
+    run make -j "$(nproc)"
+
+    cat > examples/loadables/Makefile <<-EOF
+	all:
+	clean:
+	install:
+	EOF
+fi
 
 run make install
 
+store_cache bash "${NETDATA_MAKESELF_PATH}/tmp/bash-${version}"
+
 if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
   run strip "${NETDATA_INSTALL_PATH}"/bin/bash
 fi

packaging/makeself/jobs/50-curl-7.82.0.install.sh

@@ -4,49 +4,56 @@
 # shellcheck source=packaging/makeself/functions.sh
 . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1
 
+version="7.82.0"
+
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building cURL" || true
 
-fetch "curl-7.82.0" "https://curl.haxx.se/download/curl-7.82.0.tar.gz" \
-    910cc5fe279dc36e2cca534172c94364cf3fcf7d6494ba56e6c61a390881ddce
+fetch "curl-${version}" "https://curl.haxx.se/download/curl-${version}.tar.gz" \
+    910cc5fe279dc36e2cca534172c94364cf3fcf7d6494ba56e6c61a390881ddce curl
 
 export CFLAGS="-I/openssl-static/include -pipe"
 export LDFLAGS="-static -L/openssl-static/lib"
 export PKG_CONFIG="pkg-config --static"
 export PKG_CONFIG_PATH="/openssl-static/lib/pkgconfig"
 
-run autoreconf -fi
-
-run ./configure \
-  --prefix="${NETDATA_INSTALL_PATH}" \
-  --enable-optimize \
-  --disable-shared \
-  --enable-static \
-  --enable-http \
-  --disable-ldap \
-  --disable-ldaps \
-  --enable-proxy \
-  --disable-dict \
-  --disable-telnet \
-  --disable-tftp \
-  --disable-pop3 \
-  --disable-imap \
-  --disable-smb \
-  --disable-smtp \
-  --disable-gopher \
-  --enable-ipv6 \
-  --enable-cookies \
-  --with-ca-fallback \
-  --with-openssl \
-  --disable-dependency-tracking
-
-# Curl autoconf does not honour the curl_LDFLAGS environment variable
-run sed -i -e "s/LDFLAGS =/LDFLAGS = -all-static/" src/Makefile
-
-run make clean
-run make -j "$(nproc)"
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+    run autoreconf -fi
+
+    run ./configure \
+        --prefix="${NETDATA_INSTALL_PATH}" \
+        --enable-optimize \
+        --disable-shared \
+        --enable-static \
+        --enable-http \
+        --disable-ldap \
+        --disable-ldaps \
+        --enable-proxy \
+        --disable-dict \
+        --disable-telnet \
+        --disable-tftp \
+        --disable-pop3 \
+        --disable-imap \
+        --disable-smb \
+        --disable-smtp \
+        --disable-gopher \
+        --enable-ipv6 \
+        --enable-cookies \
+        --with-ca-fallback \
+        --with-openssl \
+        --disable-dependency-tracking
+
+    # Curl autoconf does not honour the curl_LDFLAGS environment variable
+    run sed -i -e "s/LDFLAGS =/LDFLAGS = -all-static/" src/Makefile
+
+    run make clean
+    run make -j "$(nproc)"
+fi
+
 run make install
 
+store_cache curl "${NETDATA_MAKESELF_PATH}/tmp/curl-${version}"
+
 if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
   run strip "${NETDATA_INSTALL_PATH}"/bin/curl
 fi

packaging/makeself/jobs/50-fping-5.1.install.sh

@@ -4,32 +4,39 @@
 # shellcheck source=packaging/makeself/functions.sh
 . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1
 
+version="5.1"
+
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building fping" || true
 
-fetch "fping-5.1" "https://fping.org/dist/fping-5.1.tar.gz" \
-    1ee5268c063d76646af2b4426052e7d81a42b657e6a77d8e7d3d2e60fd7409fe
+fetch "fping-${version}" "https://fping.org/dist/fping-${version}.tar.gz" \
+    1ee5268c063d76646af2b4426052e7d81a42b657e6a77d8e7d3d2e60fd7409fe fping
 
 export CFLAGS="-static -I/openssl-static/include -pipe"
 export LDFLAGS="-static -L/openssl-static/lib"
 export PKG_CONFIG_PATH="/openssl-static/lib/pkgconfig"
 
-run ./configure \
-  --prefix="${NETDATA_INSTALL_PATH}" \
-  --enable-ipv4 \
-  --enable-ipv6 \
-  --disable-dependency-tracking
-
-cat > doc/Makefile << EOF
-all:
-clean:
-install:
-EOF
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+    run ./configure \
+        --prefix="${NETDATA_INSTALL_PATH}" \
+        --enable-ipv4 \
+        --enable-ipv6 \
+        --disable-dependency-tracking
+
+    cat > doc/Makefile <<-EOF
+	all:
+	clean:
+	install:
+	EOF
+
+    run make clean
+    run make -j "$(nproc)"
+fi
 
-run make clean
-run make -j "$(nproc)"
 run make install
 
+store_cache fping "${NETDATA_MAKESELF_PATH}/tmp/fping-${version}"
+
 if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
   run strip "${NETDATA_INSTALL_PATH}"/bin/fping
 fi

packaging/makeself/jobs/50-ioping-1.2.install.sh

@@ -4,19 +4,26 @@
 # shellcheck source=packaging/makeself/functions.sh
 . "$(dirname "${0}")/../functions.sh" "${@}" || exit 1
 
+version='1.2'
+
 # shellcheck disable=SC2015
 [ "${GITHUB_ACTIONS}" = "true" ] && echo "::group::Building ioping" || true
 
-fetch "ioping-1.2" "https://github.com/koct9i/ioping/archive/v1.2.tar.gz" \
-    d3e4497c653a1e96df67c72ce2b70da18e9f5e3b93179a5bb57a6e30ceacfa75
+fetch "ioping-${version}" "https://github.com/koct9i/ioping/archive/v${version}.tar.gz" \
+    d3e4497c653a1e96df67c72ce2b70da18e9f5e3b93179a5bb57a6e30ceacfa75 ioping
 
 export CFLAGS="-static -pipe"
 
-run make clean
-run make -j "$(nproc)"
+if [ "${CACHE_HIT:-0}" -eq 0 ]; then
+    run make clean
+    run make -j "$(nproc)"
+fi
+
 run mkdir -p "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/
 run install -o root -g root -m 4750 ioping "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/
 
+store_cache ioping "${NETDATA_MAKESELF_PATH}/tmp/ioping-${version}"
+
 if [ "${NETDATA_BUILD_WITH_DEBUG}" -eq 0 ]; then
   run strip "${NETDATA_INSTALL_PATH}"/usr/libexec/netdata/plugins.d/ioping
 fi

packaging/makeself/uname2platform.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+set -e
+
+BUILDARCH="${1}"
+
+case "${BUILDARCH}" in
+  x86_64) echo "linux/amd64" ;;
+  armv7l) echo "linux/arm/v7" ;;
+  aarch64) echo "linux/arm64/v8" ;;
+  ppc64le) echo "linux/ppc64le" ;;
+  *)
+    echo "Unknown target architecture '${BUILDARCH}'." >&2
+    exit 1
+    ;;
+esac