netdata/packaging/ci: Create manual nightly deployment tool (#5899)

* netdata/packaging/ci: Create manual nightly deployment tool

1. Alter docker tools to shebang with /usr/bin/env, safer to detect the right BASH (>4.0)
2. Generate script packaging/manual_nightly_deployment.sh, that runs only from TLD of netdata GIT
   and provides the means to deploy to docker and GCS the latest nightly build of netdata.
   Only option passed to the script is whether you want to deploy in docker, gcs or both.
   Note: for GCS deployment the user has to define which bucket to update.
   Note2: this tool has a hard dependency on gsutil to be properly setup on the development environment

* netdata/packaging/ci: Do not change global config for mail and user

Since we are now starting to use these scripts externally, we need to make sure we dont mess developer config.
It seems that we are touching global config, in an attempt to modify a single commit message.
Alter this and instead use the --author option for the single commit that we care about instead of altering the universe

Cheers

.travis/create_artifacts.sh

@@ -14,10 +14,6 @@ if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
 fi;
 
 echo "--- Initialize git configuration ---"
-export GIT_MAIL="bot@netdata.cloud"
-export GIT_USER="netdatabot"
-git config user.email "${GIT_MAIL}"
-git config user.name "${GIT_USER}"
 git checkout master
 git pull
 

.travis/draft_release.sh

@@ -23,10 +23,6 @@ if [ ! -f .gitignore ]; then
 fi
 
 echo "--- Initialize git configuration ---"
-export GIT_MAIL="bot@netdata.cloud"
-export GIT_USER="netdatabot"
-git config user.email "${GIT_MAIL}"
-git config user.name "${GIT_USER}"
 git checkout master
 git pull
 

.travis/generate_changelog_and_tag_release.sh

@@ -48,8 +48,6 @@ fi
 echo "--- Initialize git configuration ---"
 export GIT_MAIL="bot@netdata.cloud"
 export GIT_USER="netdatabot"
-git config user.email "${GIT_MAIL}"
-git config user.name "${GIT_USER}"
 git checkout master
 git pull
 
@@ -62,7 +60,7 @@ echo "---- GENERATE CHANGELOG -----"
 git add CHANGELOG.md
 
 echo "---- COMMIT AND PUSH CHANGES ----"
-git commit -m "[ci skip] release $GIT_TAG"
+git commit -m "[ci skip] release $GIT_TAG" --author "${GIT_USER} <${GIT_MAIL}>"
 git tag "$GIT_TAG" -a -m "Automatic tag generation for travis build no. $TRAVIS_BUILD_NUMBER"
 git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')"
 git push "https://${GITHUB_TOKEN}:@$(git config --get remote.origin.url | sed -e 's/^https:\/\///')" --tags

.travis/generate_changelog_for_nightlies.sh

@@ -38,8 +38,6 @@ if [ ! "${TRAVIS_REPO_SLUG}" == "netdata/netdata" ]; then
 	exit 0
 fi
 
-git config user.email "${GIT_MAIL}"
-git config user.name "${GIT_USER}"
 git checkout master
 git pull
 
@@ -58,7 +56,7 @@ echo "Changelog created! Adding packaging/version(${NEW_VERSION}) and CHANGELOG.
 echo "${NEW_VERSION}" > packaging/version
 git add packaging/version && echo "1) Added packaging/version to repository" || FAIL=1
 git add CHANGELOG.md && echo "2) Added changelog file to repository" || FAIL=1
-git commit -m '[ci skip] create nightly packages and update changelog' && echo "3) Committed changes to repository" || FAIL=1
+git commit -m '[ci skip] create nightly packages and update changelog' --author "${GIT_USER} <${GIT_MAIL}>" && echo "3) Committed changes to repository" || FAIL=1
 git push "https://${GITHUB_TOKEN}:@${PUSH_URL}" && echo "4) Pushed changes to remote ${PUSH_URL}" || FAIL=1
 
 # In case of a failure, wrap it up and bail out cleanly

packaging/docker/build.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Cross-arch docker build helper script
+#
 #
 # Copyright: SPDX-License-Identifier: GPL-3.0-or-later
 #

packaging/docker/publish.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+#
 # Cross-arch docker publish helper script
 # Needs docker in version >18.02 due to usage of manifests
 #

packaging/manual_nightly_deployment.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+#
+# This tool allows netdata team to manually deploy nightlies
+# It emulates the nightly operations required for a new version to be published for our users
+#
+# Copyright: SPDX-License-Identifier: GPL-3.0-or-later
+#
+# Author  : Pavlos Emm. Katsoulakis <paul@netdata.cloud>
+#
+set -e
+
+# If we are not in netdata git repo, at the top level directory, fail
+TOP_LEVEL=$(basename "$(git rev-parse --show-toplevel)")
+CWD=$(git rev-parse --show-cdup || echo "")
+if [ -n "${CWD}" ] || [ ! "${TOP_LEVEL}" == "netdata" ]; then
+    echo "Run as .travis/$(basename "$0") from top level directory of netdata git repository"
+    echo "Changelog generation process aborted"
+    exit 1
+fi
+
+if [ $# -lt 1 ] || [ $# -gt 2 ]; then
+	echo "Run as ./$(basename "$0") [docker|gcs]|all]  from the top level directory of netdata GIT repository"
+	exit 1
+fi
+
+GSUTIL_BINARY=$(command -v gsutil 2> /dev/null)
+if [ -z "${GSUTIL_BINARY}" ]; then
+	echo "No gsutil utility available, you need gsutil deployed to manually deploy to GCS"
+	exit 1
+fi;
+
+# Function declarations
+publish_docker() {
+
+	# Ensure REPOSITORY present
+	if [ -z "${REPOSITORY}" ]; then
+		echo "Please provide the repository to deploy the containers:"
+		read -r REPOSITORY
+		export REPOSITORY
+	else
+		echo "Docker publishing to ${REPOSITORY}"
+	fi
+
+	# Ensure DOCKER_USERNAME present
+	if [ -z "${DOCKER_USERNAME}" ]; then
+		echo "For repository ${REPOSITORY}, Please provide the docker USERNAME to use:"
+		read -r DOCKER_USERNAME
+		export DOCKER_USERNAME
+	else
+		echo "Using docker username ${DOCKER_USERNAME}"
+	fi
+
+	# Ensure DOCKER_PASS present
+	if [ -z "${DOCKER_PASS}" ]; then
+		echo "Username ${DOCKER_USERNAME} received, now give me the password:"
+		read -r -s DOCKER_PASS
+		export DOCKER_PASS
+	else
+		echo "Docker password has already been set to env, using that"
+	fi
+
+	echo "Building Docker images.."
+	packaging/docker/build.sh
+
+	echo "Publishing Docker images.."
+	packaging/docker/publish.sh
+}
+
+publish_nightly_binaries() {
+	echo "Publishing nightly binaries to GCS"
+
+	echo "Please select the bucket to sync, from the ones available to you:"
+	bucket_list=$(${GSUTIL_BINARY} list | tr '\n' ' ')
+	declare -A buckets
+	idx=0
+	for abucket in ${bucket_list}; do
+		echo "${idx}. ${abucket}"
+		buckets["${idx}"]=${abucket}
+		((idx=idx+1))
+	done
+	read -p"Selection>" -r -n 1 selected_bucket
+
+	echo "Ok!"
+	echo "Syncing artifacts directory contents with GCS bucket: ${buckets[${selected_bucket}]}"
+	if [ -d artifacts ]; then
+		${GSUTIL_BINARY} -m rsync -r artifacts "${buckets["${selected_bucket}"]}"
+		echo "GCS Sync complete!"
+	else
+		echo "Directory artifacts does not exist, nothing to do on GCS"
+	fi
+}
+
+prepare_and_publish_gcs() {
+	# Prepare the artifacts directory
+	echo "Preparing artifacts directory contents"
+	.travis/create_artifacts.sh
+
+	# Publish it to GCS
+	publish_nightly_binaries
+
+	# Clean up
+	echo "Cleaning up repository"
+	make clean || echo "Nothing to clean"
+	make distclean || echo "Nothing to distclean"
+	rm -rf artifacts
+}
+
+# Mandatory variable declarations
+export TRAVIS_REPO_SLUG="netdata/netdata"
+
+echo "Manual nightly deployment procedure started"
+case "$1" in
+	"docker")
+		publish_docker
+		;;
+	"gcs")
+		prepare_and_publish_gcs
+		;;
+	"all")
+		publish_docker
+		prepare_and_publish_gcs
+		;;
+	*)
+		echo "ERROR: Invalid request parameter $1. Valid values are: docker, gcs, all"
+		;;
+esac
+echo "Manual nightly deployment completed!"