Bundle the react dashboard code into the agent repo directly. (#11139)

* Remove code for bundling the dashoard on install.

* Bundle the dashboard code directly into the agent repo.

This diffstat looks huge, but it’s actually relatively simple. The only
_actual_ changes are in the Makefiles, `configure.ac`, and the addition of
`generate_dashboard_makefile.py`. Everything else consists of removing
files that are included in the dashboard tarball, and extracting the
contents of the tarball into `web/gui/dashboard`.

* CI cleanup.

* Automate bundling of the dashboard code.

This replaces the makefile generator script with one that handles
bundling of the dashboard code in it’s entirety, and updates the GHA
workflow used for generating dashboard PRs to use that instead of the
existing shell commands.

It also removes the packaging/dashboard.* files, as they are no longer
needed.

.eslintignore

@@ -1 +1,2 @@
 **/*{.,-}min.js
+web/gui/dashboard/*

.github/workflows/checks.yml

@@ -19,28 +19,6 @@ jobs:
           LOCAL_ONLY: "true"
         run: |
           ./tests/installer/checksums.sh
-  dashboard-checks:
-    name: Dashboard
-    runs-on: ubuntu-latest
-    steps:
-      - name: Git clone repository
-        uses: actions/checkout@v2
-        with:
-          submodules: recursive
-      - name: Install required packages
-        run: |
-          ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
-      - name: Backup dashboard.js
-        run: |
-          cp web/gui/dashboard.js /tmp/dashboard.js
-      - name: Regenerate dashboard.js
-        run: |
-          autoreconf -ivf
-          ./configure --enable-maintainer-mode
-          make dist
-      - name: Compare generated Dashboard vs. Backed up Dashboard
-        run: |
-          diff -sNrdu /tmp/dashboard.js web/gui/dashboard.js
   libressl-checks:
     name: LibreSSL
     runs-on: ubuntu-latest

.github/workflows/dashboard-pr.yml

@@ -27,14 +27,11 @@ jobs:
           git push -u origin dashboard-${{ github.event.inputs.dashboard_version }}
       - name: Update Files
         run: |
-          curl -o dashboard.tar.gz https://github.com/netdata/dashboard/releases/download/${{ github.event.inputs.dashboard_version }}/dashboard.tar.gz
-          echo ${{ github.event.inputs.dashboard_version }} > packaging/dashboard.version
-          sha256sum dashboard.tar.gz > packaging/dashboard.checksums
-          rm dashboard.tar.gz
+          web/gui/bundle_dashboard.py ${{ github.event.inputs.dashboard_version }}
       - name: Commit Changes
         uses: swinton/commit@v2.x
         env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GH_TOKEN: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}
         with:
           files: |
             packaging/dashboard.version
@@ -47,4 +44,4 @@ jobs:
           source_branch: dashboard-${{ github.event.inputs.dashboard_version }}
           pr_title: 'Update dashboard to version ${{ github.event.inputs.dashboard_version }}.'
           pr_body: 'See https://github.com/netdata/dashboard/releases/tag/${{ github.event.inputs.dashboard_version }} for changes.'
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_token: ${{ secrets.NETDATABOT_GITHUB_TOKEN }}

.gitignore

@@ -98,6 +98,7 @@ TODO.txt
 
 web/gui/chart-info/
 web/gui/control.html
+web/gui/dashboard.js
 web/gui/datasource.css
 web/gui/gadget.xml
 web/gui/index_new.html

Makefile.am

@@ -57,15 +57,12 @@ dist_noinst_DATA = \
     netdata.cppcheck \
     netdata.spec \
     package.json \
-    packaging/bundle-dashboard.sh \
     packaging/bundle-ebpf.sh \
     packaging/bundle-judy.sh \
     packaging/bundle-libbpf.sh \
     packaging/bundle-lws.sh \
     packaging/bundle-mosquitto.sh \
     packaging/check-kernel-config.sh \
-    packaging/dashboard.checksums \
-    packaging/dashboard.version \
     packaging/ebpf.checksums \
     packaging/ebpf.version \
     packaging/go.d.checksums \

configure.ac

@@ -1667,6 +1667,7 @@ AC_CONFIG_FILES([
     web/api/queries/sum/Makefile
     web/api/health/Makefile
     web/gui/Makefile
+    web/gui/dashboard/Makefile
     web/server/Makefile
     web/server/static/Makefile
     claim/Makefile

contrib/debian/rules

@@ -70,7 +70,6 @@ override_dh_install:
 
 	# Move files that local user shouldn't be editing to /usr/share/netdata
 	#
-	packaging/bundle-dashboard.sh . ${TOP}/var/lib/netdata/www
 	mkdir -p "$(TOP)/usr/share/netdata/www"
 	for D in $$(find "$(TOP)/var/lib/netdata/www/" -maxdepth 1 -type d -printf '%f '); do \
 		echo Relocating $$D; \

netdata-installer.sh

@@ -1362,40 +1362,6 @@ fi
 
 # -----------------------------------------------------------------------------
 
-copy_react_dashboard() {
-  run cp -a $(find ${1} -mindepth 1 -maxdepth 1) "${NETDATA_WEB_DIR}"
-  run chown -R "${NETDATA_WEB_USER}:${NETDATA_WEB_GROUP}" "${NETDATA_WEB_DIR}"
-}
-
-install_react_dashboard() {
-  progress "Fetching and installing dashboard"
-
-  DASHBOARD_PACKAGE_VERSION="$(cat packaging/dashboard.version)"
-
-  tmp="$(mktemp -d -t netdata-dashboard-XXXXXX)"
-  DASHBOARD_PACKAGE_BASENAME="dashboard.tar.gz"
-
-  if fetch_and_verify "dashboard" \
-    "https://github.com/netdata/dashboard/releases/download/${DASHBOARD_PACKAGE_VERSION}/${DASHBOARD_PACKAGE_BASENAME}" \
-    "${DASHBOARD_PACKAGE_BASENAME}" \
-    "${tmp}" \
-    "${NETDATA_LOCAL_TARBALL_OVERRIDE_DASHBOARD}"; then
-    if run tar -xf "${tmp}/${DASHBOARD_PACKAGE_BASENAME}" -C "${tmp}" &&
-      copy_react_dashboard "${tmp}/build" &&
-      rm -rf "${tmp}"; then
-      run_ok "React dashboard installed."
-    else
-      run_failed "Failed to install React dashboard. The install process will continue, but you will not be able to use the new dashboard."
-    fi
-  else
-    run_failed "Unable to fetch React dashboard. The install process will continue, but you will not be able to use the new dashboard."
-  fi
-}
-
-install_react_dashboard
-
-# -----------------------------------------------------------------------------
-
 # govercomp compares go.d.plugin versions. Exit codes:
 # 0 - version1 == version2
 # 1 - version1 > version2

netdata.spec.in

@@ -411,7 +411,6 @@ install_go() {
 install_go
 install -m 0640 -p go.d.plugin "${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d/go.d.plugin"
 
-${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-dashboard.sh ${RPM_BUILD_DIR}/%{name}-%{version} ${RPM_BUILD_ROOT}%{_datadir}/%{name}/web
 %if 0%{?have_bpf}
 ${RPM_BUILD_DIR}/%{name}-%{version}/packaging/bundle-ebpf.sh ${RPM_BUILD_DIR}/%{name}-%{version} ${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d
 %endif

packaging/bundle-dashboard.sh

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-SRCDIR="${1}"
-WEBDIR="${2}"
-
-DASHBOARD_TARBALL="dashboard.tar.gz"
-DASHBOARD_VERSION="$(cat "${SRCDIR}/packaging/dashboard.version")"
-
-mkdir -p "${SRCDIR}/tmp/dashboard"
-curl -sSL --connect-timeout 10 --retry 3 "https://github.com/netdata/dashboard/releases/download/${DASHBOARD_VERSION}/${DASHBOARD_TARBALL}" > "${DASHBOARD_TARBALL}" || exit 1
-sha256sum -c "${SRCDIR}/packaging/dashboard.checksums" || exit 1
-tar -xzf "${DASHBOARD_TARBALL}" -C "${SRCDIR}/tmp/dashboard" || exit 1
-# shellcheck disable=SC2046
-cp -a $(find "${SRCDIR}/tmp/dashboard/build" -mindepth 1 -maxdepth 1) "${WEBDIR}"