simplify CI system configuration (#4534)

[minor]

.travis.yml

@@ -1,78 +1,62 @@
----
 dist: trusty
 sudo: true
-
-# C includes autotools and make by default
 language: c
 services:
-  - docker
+- docker
 
 stages:
-  - lint
-  - test
-  - build
-  - name: release
-    if: branch = master AND type != pull_request
-  - name: package
-    if: branch = master AND type != pull_request
+- test
+- build
+- name: packaging
+  if: branch = master AND type != pull_request
 
 jobs:
   include:
-    - stage: "lint"
-      name: "python"
-      install: skip
-      script: docker run -ti --rm -v $(pwd):/code alpine/flake8:3.5.0 --max-line-length=120 --exclude=third_party,urllib3,pyyaml3,pyyaml2 /code/collectors/python.d.plugin/python.d.plugin.in /code/
-    - name: "css"
-      install: skip
-      script: docker run --rm -v $(pwd):/code eeacms/csslint sh -c "csslint /code/web/gui/*.css" || echo "OK"
-    - name: "bash"
-      install: skip
-      script: docker run --rm --volume "$(pwd)":/code:ro --entrypoint sh koalaman/shellcheck-alpine:v0.5.0 -c 'for file in $(find /code/ -type f -name "*.sh" ! -path "*/makeself/*" ); do if ! shellcheck --format=gcc $file; then export FAILED=true; fi; done; if [ "$FAILED" != "" ]; then exit 1; fi'
-    - stage: "test"
-      name: "C"
-      install: sudo apt-get install -y libcap2-bin zlib1g-dev uuid-dev fakeroot libipmimonitoring-dev libmnl-dev libnetfilter-acct-dev
-      script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it --enable-plugin-nfacct --enable-plugin-freeipmi --disable-lto && $HOME/netdata/usr/sbin/netdata -W unittest
-      env: CFLAGS='-O1 -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1'
-#      env: CFLAGS='-O1 -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -DNETDATA_INTERNAL_CHECKS=1 -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1'
-#    - name: "Node.js"
-#      script: tests/run_nodejs.sh
-    - name: "coverity"
-      install: sudo apt-get install -y zlib1g-dev uuid-dev libipmimonitoring-dev libmnl-dev libnetfilter-acct-dev
-      script: ./coverity-scan.sh || echo "Coverity failed :("
-      env: INSTALL_COVERITY="true"
-      if: type = cron
-
-    - stage: "build"
-      name: "OSX"
-      install: brew install fakeroot ossp-uuid
-      script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it
-      os: osx
-    - name: "ubuntu 14.04 (not containerized)"
-      install: sudo apt-get install -y libcap2-bin zlib1g-dev uuid-dev fakeroot
-      script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME 
-    - name: "ubuntu 18.04"
-      script: ./.travis/containerized_build.sh ubuntu1804
-    - name: "CentOS 7"
-      script: ./.travis/containerized_build.sh centos7
-    - name: "CentOS 6"
-      script: ./.travis/containerized_build.sh centos6
-    - name: "alpine"
-      script: ./.travis/containerized_build.sh alpine
+  - stage: test
+    name: C
+    install: sudo apt-get install -y libcap2-bin zlib1g-dev uuid-dev fakeroot libipmimonitoring-dev libmnl-dev libnetfilter-acct-dev
+    script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it --enable-plugin-nfacct --enable-plugin-freeipmi --disable-lto && $HOME/netdata/usr/sbin/netdata -W unittest
+    env: CFLAGS='-O1 -DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_VERIFY_LOCKS=1'
+  - name: coverity
+    install: sudo apt-get install -y zlib1g-dev uuid-dev libipmimonitoring-dev libmnl-dev libnetfilter-acct-dev
+    script: ./coverity-scan.sh || echo "Coverity failed :("
+    env: INSTALL_COVERITY="true"
+    if: type = cron
 
-    - stage: "release"
-      name: "Create release"
-      install: sudo apt-get install -y gnupg libcap2-bin zlib1g-dev uuid-dev fakeroot
-      script: .travis/releaser.sh
+  - stage: build
+    name: OSX
+    install: brew install fakeroot ossp-uuid
+    script: fakeroot ./netdata-installer.sh --install $HOME --dont-wait --dont-start-it
+    os: osx
+  - name: ubuntu 14.04 (not containerized)
+    install: sudo apt-get install -y libcap2-bin zlib1g-dev uuid-dev fakeroot
+    script: fakeroot ./netdata-installer.sh --dont-wait --dont-start-it --install $HOME
+  - name: ubuntu 18.04
+    script: ".travis/containerized_build.sh ubuntu1804"
+  - name: CentOS 7
+    script: ".travis/containerized_build.sh centos7"
+  - name: CentOS 6
+    script: ".travis/containerized_build.sh centos6"
+  - name: alpine
+    script: ".travis/containerized_build.sh alpine"
 
-    - stage: "package"
-      name: "Archive and self-extractor"
-      install: sudo apt-get install -y gnupg libcap2-bin zlib1g-dev uuid-dev fakeroot
-      script: .travis/create_artifacts.sh
-    - name: "Docker images"
-      install: sudo apt update -y && sudo apt install -y --only-upgrade docker-ce && docker info
-      script: docker/build.sh
-      env: REPOSITORY="netdata/netdata"
-      if: type = cron OR tag =~ ^v
+  - stage: packaging
+    name: Update Changelog/Create release
+    install: sudo apt-get install -y gnupg libcap2-bin zlib1g-dev uuid-dev fakeroot
+    script: ".travis/releaser.sh"
+  - name: Nightly tarball and self-extractor build
+    install: sudo apt-get install -y gnupg libcap2-bin zlib1g-dev uuid-dev fakeroot
+    script: ".travis/firehol_create_artifacts.sh"
+    if: type = cron
+  - name: Nightly docker images
+    install: sudo apt update -y && sudo apt install -y --only-upgrade docker-ce &&  docker info
+    script: "docker/build.sh"
+    env: REPOSITORY="netdata/netdata"
+    if: type = cron
 
 notifications:
   webhooks: https://app.fossa.io/hooks/travisci
+  slack:
+    secure: Ntl1GNjgGDdmMQsudDi/4+co6zRV8znJXOB0joLVHNV1icHvUSMKShgpY08AoWoxNYMDVHXOUzEGwiZd6MI4MleNu9gwtACwgXe+sDjgBIDvQYC6JQS7XlANJfzBlZnrGnaiVVlBmI/LajgABCsrTAYV8gVrq2zA3u56UrNwS3qRJnbaqGgH64SnHbdllHQ7l7Qcc/27voy6Op32oJhDZ5bVY5CRZUR42oPvlDx9PSD0yqSkLwFrIE0Qg+mTVDZ4Watrv8iv91ghc6GwwPvNGShcYI04P8hn08FZ4eGH47K4AZf97IDargmxPsEwkOhgcl3w4J5KHZzmVb7hsEs6+iZz4KulD2u4D/xy5BGUcRZKMuyiO99dmZO61wP0Ra0Wzfe0exG+LFxAmC0jiYS67xL0YWs2SKkO7iftthI8NX1Lus1PGleBpJ28LKQdZjaKz3w2xIRlJ45eEdwBgtnNkW68NsmJCflrNkAlaVW6+zCPMpT4rT41rpkh0Fl+YbvOhH/S3VfiTY7OqTU5B9ttgJuN/m8PWi8g5MKO2C1TWkK3cV4O2PnpugtioHuTuj2X/AKOrh7hlefedDxPZPIxFmLi3OKtgHBzwzQk79p6V9ffbbVfsO+p8kiKJuc8s0v6d++Rt2FLXwUquJGDKC0hz8cAMSUiFaP5t1TlhTk7LJ4=
+    on_success: change
+    on_pull_requests: false

.travis/README.md

@@ -27,6 +27,7 @@ installations of netdata. Jobs are run on following operating systems:
   - ubuntu 16.04 (containerized)
   - CentOS 6 (containerized)
   - CentOS 7 (containerized)
+  - alpine (containerized)
 
 ### Release
 
@@ -41,15 +42,33 @@ Releases are generated by searching for a keyword in last commit message. Keywor
 All keywords MUST be surrounded with square braces.
 Alternative is to push a tag to master branch.
 
-### Packages
+### Packaging
 
-As a name might suggest, this stage is responsible for creating netdata packages such as:
+This stage is executed only on "master" branch and it is separated into 3 jobs:
+  - Update Changelog/Create release
+  - Nightly tarball and self-extractor build
+  - Nightly docker images
+
+##### Update Changelog/Create release
+
+This job is running one script called `releaser.sh`, which is responsible for a couple of things. First of all it 
+automatically updates our CHANGELOG.md file based on GitHub features (mostly labels and pull requests). Apart from 
+that it can also create a new git tag and a github draft release connected to that tag.
+Releases are generated by searching for a keyword in last commit message. Keywords are:
+ - `[patch]` or `[fix]` to bump patch number
+ - `[minor]`, `[feature]` or `[feat]` to bump minor number
+ - `[major]` or `[breaking change]` to bump major number
+All keywords MUST be surrounded with square brackets.
+
+Alternatively new release can be also created by pushing new tag to master branch.
+
+##### Nightly tarball and self-extractor build AND Nightly docker images
+
+As names might suggest those two jobs are responsible for nightly netdata package creation and are run every day (in
+cron). Combined they produce:
   - docker images
-  - tar repository (soon to be removed)
+  - tar.gz archive (soon to be removed)
   - self-extracting package
 
-This stage is separated into 2 parallel jobs. One creating docker images and second one creating github artifacts.
-Whole stage is executed only on master branch, but there are also additional parameters assigned to jobs:
-  - docker images - execution only on cron job or when there is a tag assigned starting with `v`
-  - artifacts - script responsible for creating artifacts doesn't have special conditions, but deployment to GitHub
-                releases is done only when there is tag assigned
+Currently "Nightly tarball and self-extractor build" is using old firehol script and it is planed to be replaced with
+new design.

.travis/create_artifacts.sh

@@ -7,20 +7,25 @@ then
   exit 1
 fi
 
-eval "$(ssh-agent -s)"
-./.travis/decrypt-if-have-key decb6f6387c4
-export KEYSERVER=ipv4.pool.sks-keyservers.net
-./packaging/gpg-recv-key phil@firehol.org "0762 9FF7 89EA 6156 012F  9F50 C406 9602 1359 9237"
-./packaging/gpg-recv-key costa@tsaousis.gr "4DFF 624A E564 3B51 2872  1F40 29CA 3358 89B9 A863"
-# Run the commit hooks in case the developer didn't
-git diff 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | ./packaging/check-files -
-fakeroot ./packaging/git-build
 # Make sure stdout is in blocking mode. If we don't, then conda create will barf during downloads.
 # See https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 for details.
 python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
-# make self-extractor
+echo "--- Create tarball ---"
+make dist
+echo "--- Create self-extractor ---"
 ./makeself/build-x86_64-static.sh
-for i in *.tar.gz; do sha512sum -b "$i" > "$i.sha"; done
-for i in *.gz.run; do sha512sum -b "$i" > "$i.sha"; done
-sha256sum -b ./*.tar.gz ./*.gz.run > "sha256sums.txt"
-./.travis/deploy-if-have-key
+
+echo "--- Create checksums ---"
+GIT_TAG=$(git tag --points-at)
+if [ "${GIT_TAG}" != "" ]; then
+    ln -s netdata-latest.gz.run "netdata-${GIT_TAG}.gz.run"
+    ln -s netdata-*.tar.gz "netdata-${GIT_TAG}.tar.gz"
+    sha256sum -b "netdata-${GIT_TAG}.gz.run" "netdata-${GIT_TAG}.tar.gz" > "sha256sums.txt"
+else
+    sha256sum -b ./*.tar.gz ./*.gz.run > "sha256sums.txt"
+fi
+
+echo "checksums:"
+cat sha256sums.txt
+
+

.travis/firehol_create_artifacts.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# shellcheck disable=SC2230
+
+# WARNING: This script is deprecated and placed here until @paulfantom figures out how to fully replace it
+
+if [ ! -f .gitignore ]
+then
+  echo "Run as ./travis/$(basename "$0") from top level directory of git repository"
+  exit 1
+fi
+
+eval "$(ssh-agent -s)"
+./.travis/decrypt-if-have-key decb6f6387c4
+export KEYSERVER=ipv4.pool.sks-keyservers.net
+./packaging/gpg-recv-key phil@firehol.org "0762 9FF7 89EA 6156 012F  9F50 C406 9602 1359 9237"
+./packaging/gpg-recv-key costa@tsaousis.gr "4DFF 624A E564 3B51 2872  1F40 29CA 3358 89B9 A863"
+# Run the commit hooks in case the developer didn't
+git diff 4b825dc642cb6eb9a060e54bf8d69288fbee4904 | ./packaging/check-files -
+fakeroot ./packaging/git-build
+# Make sure stdout is in blocking mode. If we don't, then conda create will barf during downloads.
+# See https://github.com/travis-ci/travis-ci/issues/4704#issuecomment-348435959 for details.
+python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
+echo "--- Create tarball ---"
+make dist
+echo "--- Create self-extractor ---"
+./makeself/build-x86_64-static.sh
+echo "--- Create checksums ---"
+for i in *.tar.gz; do sha512sum -b "$i" > "$i.sha"; done #FIXME remove?
+for i in *.gz.run; do sha512sum -b "$i" > "$i.sha"; done #FIXME remove?
+sha256sum -b ./*.tar.gz ./*.gz.run > "sha256sums.txt"
+./.travis/deploy-if-have-key

.travis/releaser.sh

@@ -94,14 +94,11 @@ if [ "${GIT_TAG}" == "HEAD" ]; then
     exit 0
 fi
 
-echo "---- CREATING RELEASE ARTIFACTS  -----"
-python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
+echo "---- CREATING TAGGED DOCKER CONTAINERS ----"
+./docker/build.sh
 
-./makeself/build-x86_64-static.sh
-make dist
-ln -s netdata-latest.gz.run "netdata-${GIT_TAG}.gz.run"
-ln -s netdata-*.tar.gz "netdata-${GIT_TAG}.tar.gz"
-sha256sum -b "netdata-${GIT_TAG}.gz.run" "netdata-${GIT_TAG}.tar.gz" > "sha256sums.txt"
+echo "---- CREATING RELEASE ARTIFACTS -----"
+./.travis/create_artifacts.sh
 
 echo "---- CREATING RELEASE DRAFT WITH ASSETS -----"
 # Download hub