Anonymous statistics (#5113)

* Added shell and dashboard anonymous statistics

* Check for environment var NETDATA_REGISTRY_UNIQUE_ID

* Fix indentation

* Removed health-cmdapi-test

* docs/anonymous-statistics.md

.gitignore

@@ -94,6 +94,8 @@ system/netdata.plist
 system/netdata-freebsd
 system/edit-config
 
+daemon/anonymous-statistics.sh
+
 health/notifications/alarm-notify.sh
 collectors/cgroups.plugin/cgroup-name.sh
 collectors/tc.plugin/tc-qos-helper.sh

collectors/apps.plugin/apps_plugin.c

@@ -15,6 +15,9 @@ void netdata_cleanup_and_exit(int ret) {
     exit(ret);
 }
 
+void send_statistics( const char *action, const char *action_result, const char *action_data) {
+    return;
+}
 // callbacks required by popen()
 void signals_block(void) {};
 void signals_unblock(void) {};

collectors/cgroups.plugin/cgroup-network.c

@@ -24,6 +24,10 @@ void netdata_cleanup_and_exit(int ret) {
     exit(ret);
 }
 
+void send_statistics( const char *action, const char *action_result, const char *action_data) {
+    return;
+}
+
 // callbacks required by popen()
 void signals_block(void) {};
 void signals_unblock(void) {};

collectors/freeipmi.plugin/freeipmi_plugin.c

@@ -35,6 +35,10 @@ void netdata_cleanup_and_exit(int ret) {
     exit(ret);
 }
 
+void send_statistics( const char *action, const char *action_result, const char *action_data) {
+    return;
+}
+
 // callbacks required by popen()
 void signals_block(void) {};
 void signals_unblock(void) {};

daemon/Makefile.am

@@ -2,8 +2,19 @@
 
 AUTOMAKE_OPTIONS = subdir-objects
 MAINTAINERCLEANFILES= $(srcdir)/Makefile.in
+CLEANFILES = \
+	anonymous-statistics.sh \
+	$(NULL)
+
+include $(top_srcdir)/build/subst.inc
+SUFFIXES = .in
 
 dist_noinst_DATA = \
 	README.md \
 	config/README.md \
+	anonymous-statistics.sh.in \
+	$(NULL)
+
+dist_plugins_SCRIPTS = \
+	anonymous-statistics.sh \
 	$(NULL)

daemon/anonymous-statistics.sh.in

@@ -0,0 +1,194 @@
+#!/usr/bin/env sh
+
+# Valid actions:
+
+# - FATAL       - netdata exited due to a fatal condition
+#      ACTION_RESULT  -- program name and thread tag
+#      ACTION_DATA    -- fmt, args passed to fatal
+# - START       - netdata started
+#      ACTION_DATA     -- nan
+# - EXIT     - installation action
+#      ACTION_DATA    -- ret value of
+
+ACTION="${1}"
+ACTION_RESULT="${2}"
+ACTION_DATA="${3}"
+ACTION_DATA=$(echo "${ACTION_DATA}" | tr '"' "'")
+
+# -------------------------------------------------------------------------------------------------
+# check opt-out
+
+if [ -f "@configdir_POST@/.opt-out-from-anonymous-statistics" ]; then
+	exit 0
+fi
+
+# -------------------------------------------------------------------------------------------------
+# detect the operating system
+
+OS_DETECTION="unknown"
+NAME="unknown"
+VERSION="unknown"
+VERSION_ID="unknown"
+ID="unknown"
+ID_LIKE="unknown"
+
+if [ -f "/etc/os-release" ]; then
+	OS_DETECTION="/etc/os-release"
+	eval "$(grep -E "^(NAME|ID|ID_LIKE|VERSION|VERSION_ID)=" </etc/os-release)"
+fi
+
+if [ "${NAME}" = "unknown" ] || [ "${VERSION}" = "unknown" ] || [ "${ID}" = "unknown" ]; then
+	if [ -f "/etc/lsb-release" ]; then
+		if [ "${OS_DETECTION}" = "unknown" ]; then OS_DETECTION="/etc/lsb-release"; else OS_DETECTION="Mixed"; fi
+		DISTRIB_ID="unknown"
+		DISTRIB_RELEASE="unknown"
+		DISTRIB_CODENAME="unknown"
+		eval "$(grep -E "^(DISTRIB_ID|DISTRIB_RELEASE|DISTRIB_CODENAME)=" </etc/lsb-release)"
+		if [ "${NAME}" = "unknown" ]; then NAME="${DISTRIB_ID}"; fi
+		if [ "${VERSION}" = "unknown" ]; then VERSION="${DISTRIB_RELEASE}"; fi
+		if [ "${ID}" = "unknown" ]; then ID="${DISTRIB_CODENAME}"; fi
+	elif [ -n "$(command -v lsb_release 2>/dev/null)" ]; then
+		if [ "${OS_DETECTION}" = "unknown" ]; then OS_DETECTION="lsb_release"; else OS_DETECTION="Mixed"; fi
+		if [ "${NAME}" = "unknown" ]; then NAME="$(lsb_release -is 2>/dev/null)"; fi
+		if [ "${VERSION}" = "unknown" ]; then VERSION="$(lsb_release -rs 2>/dev/null)"; fi
+		if [ "${ID}" = "unknown" ]; then ID="$(lsb_release -cs 2>/dev/null)"; fi
+	fi
+fi
+
+# -------------------------------------------------------------------------------------------------
+# detect the kernel
+
+KERNEL_NAME="$(uname -s)"
+KERNEL_VERSION="$(uname -r)"
+ARCHITECTURE="$(uname -m)"
+
+# -------------------------------------------------------------------------------------------------
+# detect the virtualization
+
+VIRTUALIZATION="unknown"
+VIRT_DETECTION="none"
+
+if [ -n "$(command -v systemd-detect-virt 2>/dev/null)" ]; then
+	VIRTUALIZATION="$(systemd-detect-virt)"
+	VIRT_DETECTION="systemd-detect-virt"
+else
+	if grep -q "^flags.*hypervisor" /proc/cpuinfo 2>/dev/null; then
+		VIRTUALIZATION="hypervisor"
+		VIRT_DETECTION="/proc/cpuinfo"
+	fi
+fi
+
+# -------------------------------------------------------------------------------------------------
+# detect containers
+
+CONTAINER="none"
+CONT_DETECTION="none"
+
+IFS='(, ' read -r process _ </proc/1/sched
+if [ "${process}" = "netdata" ]; then
+	CONTAINER="container"
+	CONT_DETECTION="process"
+fi
+
+# ubuntu and debian supply /bin/running-in-container
+# https://www.apt-browse.org/browse/ubuntu/trusty/main/i386/upstart/1.12.1-0ubuntu4/file/bin/running-in-container
+if /bin/running-in-container >/dev/null 2>&1; then
+	CONTAINER="container"
+	CONT_DETECTION="/bin/running-in-container"
+fi
+
+# lxc sets environment variable 'container'
+#shellcheck disable=SC2154
+if [ -n "${container}" ]; then
+	CONTAINER="lxc"
+	CONT_DETECTION="containerenv"
+fi
+
+# docker creates /.dockerenv
+# http://stackoverflow.com/a/25518345
+if [ -f "/.dockerenv" ]; then
+	CONTAINER="docker"
+	CONT_DETECTION="dockerenv"
+fi
+
+# -------------------------------------------------------------------------------------------------
+# check netdata version
+
+if [ -z "${NETDATA_VERSION}" ]; then
+	NETDATA_VERSION="uknown"
+	netdata -V >/dev/null 2>&1 && NETDATA_VERSION="$(netdata -V 2>&1 | cut -d ' ' -f 2)"
+fi
+
+# -------------------------------------------------------------------------------------------------
+# check netdata unique id
+if [ -z "${NETDATA_REGISTRY_UNIQUE_ID}" ] ; then
+	if [ -f "@registrydir_POST@/netdata.public.unique.id" ]; then
+		NETDATA_REGISTRY_UNIQUE_ID="$(cat "@registrydir_POST@/netdata.public.unique.id")"
+	else
+		NETDATA_REGISTRY_UNIQUE_ID="unknown"
+	fi
+fi
+
+
+# -------------------------------------------------------------------------------------------------
+# send the anonymous statistics to GA
+# https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters
+if [ -n "$(command -v curl 2>/dev/null)" ]; then
+	curl -X POST -Ss --max-time 2 \
+		--data "v=1" \
+		--data "tid=UA-64295674-3" \
+		--data "aip=1" \
+		--data "ds=shell" \
+		--data-urlencode "cid=${NETDATA_REGISTRY_UNIQUE_ID}" \
+		--data-urlencode "cs=${NETDATA_REGISTRY_UNIQUE_ID}" \
+		--data "t=event" \
+		--data "ni=1" \
+		--data "an=anonymous-statistics" \
+		--data-urlencode "av=${NETDATA_VERSION}" \
+		--data-urlencode "ec=${ACTION}" \
+		--data-urlencode "ea=${ACTION_RESULT}" \
+		--data-urlencode "el=${ACTION_DATA}" \
+		--data-urlencode "cd1=${NAME}" \
+		--data-urlencode "cd2=${ID}" \
+		--data-urlencode "cd3=${ID_LIKE}" \
+		--data-urlencode "cd4=${VERSION}" \
+		--data-urlencode "cd5=${VERSION_ID}" \
+		--data-urlencode "cd6=${OS_DETECTION}" \
+		--data-urlencode "cd7=${KERNEL_NAME}" \
+		--data-urlencode "cd8=${KERNEL_VERSION}" \
+		--data-urlencode "cd9=${ARCHITECTURE}" \
+		--data-urlencode "cd10=${VIRTUALIZATION}" \
+		--data-urlencode "cd11=${VIRT_DETECTION}" \
+		--data-urlencode "cd12=${CONTAINER}" \
+		--data-urlencode "cd13=${CONT_DETECTION}" \
+		"https://www.google-analytics.com/collect" >/dev/null 2>&1
+else
+	wget -q -O - --timeout=1 "https://www.google-analytics.com/collect?\
+&v=1\
+&tid=UA-64295674-3\
+&aip=1\
+&ds=shell\
+&cid=${NETDATA_REGISTRY_UNIQUE_ID}\
+&cs=${NETDATA_REGISTRY_UNIQUE_ID}\
+&t=event\
+&ni=1\
+&an=anonymous-statistics\
+&av=${NETDATA_VERSION}\
+&ec=${ACTION}\
+&ea=${ACTION_RESULT}\
+&el=${ACTION_DATA}\
+&cd1=${NAME}\
+&cd2=${ID}\
+&cd3=${ID_LIKE}\
+&cd4=${VERSION}\
+&cd5=${VERSION_ID}\
+&cd6=${OS_DETECTION}\
+&cd7=${KERNEL_NAME}\
+&cd8=${KERNEL_VERSION}\
+&cd9=${ARCHITECTURE}\
+&cd10=${VIRTUALIZATION}\
+&cd11=${VIRT_DETECTION}\
+&cd12=${CONTAINER}\
+&cd13=${CONT_DETECTION}\
+" > /dev/null 2>&1
+fi

daemon/common.h

@@ -78,5 +78,6 @@ extern char *netdata_configured_varlib_dir;
 extern char *netdata_configured_home_dir;
 extern char *netdata_configured_host_prefix;
 extern char *netdata_configured_timezone;
+extern int netdata_anonymous_statistics_enabled;
 
 #endif /* NETDATA_COMMON_H */

daemon/daemon.h

@@ -8,6 +8,7 @@ extern int become_user(const char *username, int pid_fd);
 extern int become_daemon(int dont_fork, const char *user);
 
 extern void netdata_cleanup_and_exit(int i);
+extern void send_statistics(const char *action, const char *action_result, const char *action_data);
 
 extern char pidfile[];
 

daemon/main.c

@@ -2,6 +2,8 @@
 
 #include "common.h"
 
+int netdata_anonymous_statistics_enabled;
+
 struct config netdata_config = {
         .sections = NULL,
         .mutex = NETDATA_MUTEX_INITIALIZER,
@@ -22,6 +24,8 @@ void netdata_cleanup_and_exit(int ret) {
     error_log_limit_unlimited();
     info("EXIT: netdata prepares to exit with code %d...", ret);
 
+	send_statistics("EXIT", ret?"ERROR":"OK","-");
+
     // cleanup/save the database and exit
     info("EXIT: cleaning up the database...");
     rrdhost_cleanup_all();
@@ -458,6 +462,7 @@ static void get_netdata_configured_variables() {
         netdata_configured_plugins_dir_base = strdupz(config_get(CONFIG_SECTION_GLOBAL, "plugins directory",  plugins_dirs));
         quoted_strings_splitter(netdata_configured_plugins_dir_base, plugin_directories, PLUGINSD_MAX_DIRECTORIES, config_isspace);
         netdata_configured_plugins_dir = plugin_directories[0];
+
     }
 
     // ------------------------------------------------------------------------
@@ -581,6 +586,7 @@ void set_global_environment() {
         setenv("NETDATA_UPDATE_EVERY", b, 1);
     }
 
+    setenv("NETDATA_VERSION"          , program_version, 1);
     setenv("NETDATA_HOSTNAME"         , netdata_configured_hostname, 1);
     setenv("NETDATA_CONFIG_DIR"       , verify_required_directory(netdata_configured_user_config_dir),  1);
     setenv("NETDATA_USER_CONFIG_DIR"  , verify_required_directory(netdata_configured_user_config_dir),  1);
@@ -643,6 +649,47 @@ static int load_netdata_conf(char *filename, char overwrite_used) {
     return ret;
 }
 
+
+void send_statistics( const char *action, const char *action_result, const char *action_data) {
+    static char *as_script;
+    if (netdata_anonymous_statistics_enabled == -1) {
+        char *optout_file = mallocz(sizeof(char) * (strlen(netdata_configured_user_config_dir) +strlen(".opt-out-from-anonymous-statistics") + 2));
+        sprintf(optout_file, "%s/%s", netdata_configured_user_config_dir, ".opt-out-from-anonymous-statistics");
+        if (likely(access(optout_file, R_OK) != 0)) {
+            as_script = mallocz(sizeof(char) * (strlen(netdata_configured_plugins_dir) + strlen("anonymous-statistics.sh") + 2));
+            sprintf(as_script, "%s/%s", netdata_configured_plugins_dir, "anonymous-statistics.sh");
+			if (unlikely(access(as_script, R_OK) != 0)) {
+				netdata_anonymous_statistics_enabled=0;
+				info("Anonymous statistics script %s not found.",as_script);
+				freez(as_script);
+			} else {
+				netdata_anonymous_statistics_enabled=1;
+			}
+		} else {
+            netdata_anonymous_statistics_enabled = 0;
+            as_script = NULL;
+        }
+        freez(optout_file);
+    }
+	if(!netdata_anonymous_statistics_enabled) return;
+    if (!action) return;
+    if (!action_result) action_result="";
+    if (!action_data) action_data="";
+    char *command_to_run=mallocz(sizeof(char) * (strlen(action) + strlen(action_result) + strlen(action_data) + strlen(as_script) + 10));
+    pid_t command_pid;
+
+    sprintf(command_to_run,"%s '%s' '%s' '%s'", as_script, action, action_result, action_data);
+    info("%s", command_to_run);
+
+    FILE *fp = mypopen(command_to_run, &command_pid);
+    if(fp) {
+        char buffer[100 + 1];
+        while (fgets(buffer, 100, fp) != NULL);
+        mypclose(fp, command_pid);
+    }
+    freez(command_to_run);
+}
+
 int main(int argc, char **argv) {
     int i;
     int config_loaded = 0;
@@ -915,6 +962,9 @@ int main(int argc, char **argv) {
         get_netdata_configured_variables();
         set_global_environment();
 
+        netdata_anonymous_statistics_enabled=-1;
+        send_statistics("START","-", "-");
+
         // work while we are cd into config_dir
         // to allow the plugins refer to their config
         // files using relative filenames

daemon/main.h

@@ -43,5 +43,6 @@ struct netdata_static_thread {
 extern void cancel_main_threads(void);
 extern int killpid(pid_t pid, int signal);
 extern void netdata_cleanup_and_exit(int ret) NORETURN;
+extern void send_statistics(const char *action, const char *action_result, const char *action_data);
 
 #endif /* NETDATA_MAIN_H */