Assorted build cleanup for external data collection plugins. (#18501)

* Make Python collectors optional at build time.

Given the general shift of most data collectors to the Go plugin, it’s
no-longer the case that users ‘always’ need to have the Python plugin
included.

This also makes them consistent with the other external collectors.

* Make charts.d collectors optional at build time.

Same reasoning as for the Python collectors.

Also, this fixes up loopsleepms.sh to only be handled if a collector
that needs it is enabled.

* Properly tie config directories to their plugin components.

The Go and Python plugin config directories were being installed
unconditionally as part of the main Netdata component. This is wrong,
they should be installed as part of the plugin packages/components
instead, as they are only needed if those plugins are installed.

CMakeLists.txt

@@ -156,8 +156,10 @@ option(ENABLE_DBENGINE "Enable dbengine metrics storage" True)
 
 # Data collection plugins
 option(ENABLE_PLUGIN_GO "Enable metric collectors written in Go" ${DEFAULT_FEATURE_STATE})
+option(ENABLE_PLUGIN_PYTHON "Enable metric collectors written in Python" ${DEFAULT_FEATURE_STATE})
 
 cmake_dependent_option(ENABLE_PLUGIN_APPS "Enable per-process resource usage monitoring" ${DEFAULT_FEATURE_STATE} "NOT OS_WINDOWS" False)
+cmake_dependent_option(ENABLE_PLUGIN_CHARTS "Enable metric collectors written in Bash" ${DEFAULT_FEATURE_STATE} "NOT OS_WINDOWS" False)
 cmake_dependent_option(ENABLE_PLUGIN_CUPS "Enable CUPS monitoring" ${DEFAULT_FEATURE_STATE} "NOT OS_WINDOWS" False)
 
 cmake_dependent_option(ENABLE_PLUGIN_FREEIPMI "Enable IPMI monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX OR OS_FREEBSD" False)
@@ -2482,9 +2484,7 @@ install(DIRECTORY COMPONENT netdata DESTINATION var/lib/netdata/cloud.d)
 install(DIRECTORY COMPONENT netdata DESTINATION var/run/netdata)
 install(DIRECTORY COMPONENT netdata DESTINATION etc/netdata)
 install(DIRECTORY COMPONENT netdata DESTINATION etc/netdata/custom-plugins.d)
-install(DIRECTORY COMPONENT netdata DESTINATION etc/netdata/go.d)
 install(DIRECTORY COMPONENT netdata DESTINATION etc/netdata/health.d)
-install(DIRECTORY COMPONENT netdata DESTINATION etc/netdata/python.d)
 install(DIRECTORY COMPONENT netdata DESTINATION etc/netdata/ssl)
 install(DIRECTORY COMPONENT netdata DESTINATION etc/netdata/statsd.d)
 install(DIRECTORY COMPONENT netdata DESTINATION usr/lib/netdata/conf.d)
@@ -2839,60 +2839,66 @@ endif()
 # charts.d plugin
 #
 
-install(DIRECTORY COMPONENT plugin-chartsd DESTINATION etc/netdata/charts.d)
+if(ENABLE_PLUGIN_CHARTS)
+  install(DIRECTORY COMPONENT plugin-chartsd DESTINATION etc/netdata/charts.d)
 
-configure_file(src/collectors/charts.d.plugin/charts.d.plugin.in src/collectors/charts.d.plugin/charts.d.plugin @ONLY)
-install(PROGRAMS
-        ${CMAKE_BINARY_DIR}/src/collectors/charts.d.plugin/charts.d.plugin
-        COMPONENT plugin-chartsd
-        DESTINATION usr/libexec/netdata/plugins.d)
+  configure_file(src/collectors/charts.d.plugin/charts.d.plugin.in src/collectors/charts.d.plugin/charts.d.plugin @ONLY)
+  install(PROGRAMS
+          ${CMAKE_BINARY_DIR}/src/collectors/charts.d.plugin/charts.d.plugin
+          COMPONENT plugin-chartsd
+          DESTINATION usr/libexec/netdata/plugins.d)
 
-install(PROGRAMS
-        src/collectors/charts.d.plugin/charts.d.dryrun-helper.sh
-        COMPONENT plugin-chartsd
-        DESTINATION usr/libexec/netdata/plugins.d)
+  install(PROGRAMS
+          src/collectors/charts.d.plugin/charts.d.dryrun-helper.sh
+          COMPONENT plugin-chartsd
+          DESTINATION usr/libexec/netdata/plugins.d)
 
-# loopsleepms is used by the tc.plugin -> ship it in the netdata component
-install(FILES
-        src/collectors/charts.d.plugin/loopsleepms.sh.inc
-        COMPONENT netdata
-        DESTINATION usr/libexec/netdata/plugins.d)
+  install(FILES
+          src/collectors/charts.d.plugin/charts.d.conf
+          COMPONENT plugin-chartsd
+          DESTINATION usr/lib/netdata/conf.d)
 
-install(FILES
-        src/collectors/charts.d.plugin/charts.d.conf
-        COMPONENT plugin-chartsd
-        DESTINATION usr/lib/netdata/conf.d)
+  install(PROGRAMS
+          src/collectors/charts.d.plugin/example/example.chart.sh
+          src/collectors/charts.d.plugin/libreswan/libreswan.chart.sh
+          src/collectors/charts.d.plugin/opensips/opensips.chart.sh
+          COMPONENT plugin-chartsd
+          DESTINATION usr/libexec/netdata/charts.d)
 
-install(PROGRAMS
-        src/collectors/charts.d.plugin/example/example.chart.sh
-        src/collectors/charts.d.plugin/libreswan/libreswan.chart.sh
-        src/collectors/charts.d.plugin/opensips/opensips.chart.sh
-        COMPONENT plugin-chartsd
-        DESTINATION usr/libexec/netdata/charts.d)
+  install(FILES
+          src/collectors/charts.d.plugin/example/example.conf
+          src/collectors/charts.d.plugin/libreswan/libreswan.conf
+          src/collectors/charts.d.plugin/opensips/opensips.conf
+          COMPONENT plugin-chartsd
+          DESTINATION usr/lib/netdata/conf.d/charts.d)
 
-install(FILES
-        src/collectors/charts.d.plugin/example/example.conf
-        src/collectors/charts.d.plugin/libreswan/libreswan.conf
-        src/collectors/charts.d.plugin/opensips/opensips.conf
-        COMPONENT plugin-chartsd
-        DESTINATION usr/lib/netdata/conf.d/charts.d)
+  if(BUILD_FOR_PACKAGING)
+    install(FILES
+            ${PKG_FILES_PATH}/copyright
+            COMPONENT plugin-chartsd
+            DESTINATION usr/share/doc/netdata-plugin-chartsd)
+  endif()
+endif()
 
-if(BUILD_FOR_PACKAGING)
-        install(FILES
-                ${PKG_FILES_PATH}/copyright
-                COMPONENT plugin-chartsd
-                DESTINATION usr/share/doc/netdata-plugin-chartsd)
+# This is needed both by the TC plugin (which only gets built on Linux) and the charts plugin.
+if(OS_LINUX OR ENABLE_PLUGIN_CHARTS)
+  install(FILES
+          src/collectors/charts.d.plugin/loopsleepms.sh.inc
+          COMPONENT netdata
+          DESTINATION usr/libexec/netdata/plugins.d)
 endif()
 
 #
 # tc-qos-helper
 #
 
-configure_file(src/collectors/tc.plugin/tc-qos-helper.sh.in src/collectors/tc.plugin/tc-qos-helper.sh @ONLY)
-install(PROGRAMS
-        ${CMAKE_BINARY_DIR}/src/collectors/tc.plugin/tc-qos-helper.sh
-        COMPONENT netdata
-        DESTINATION usr/libexec/netdata/plugins.d)
+if(OS_LINUX)
+  configure_file(src/collectors/tc.plugin/tc-qos-helper.sh.in src/collectors/tc.plugin/tc-qos-helper.sh @ONLY)
+  install(PROGRAMS
+          ${CMAKE_BINARY_DIR}/src/collectors/tc.plugin/tc-qos-helper.sh
+          COMPONENT netdata
+          DESTINATION usr/libexec/netdata/plugins.d)
+endif()
 
 # confs
 install(FILES
@@ -2905,52 +2911,56 @@ install(FILES
 # python.d plugin
 #
 
-configure_file(src/collectors/python.d.plugin/python.d.plugin.in src/collectors/python.d.plugin/python.d.plugin @ONLY)
-install(PROGRAMS ${CMAKE_BINARY_DIR}/src/collectors/python.d.plugin/python.d.plugin
-        COMPONENT plugin-pythond
-        DESTINATION usr/libexec/netdata/plugins.d)
+if(ENABLE_PLUGIN_PYTHON)
+  install(DIRECTORY COMPONENT plugin-pythond DESTINATION etc/netdata/python.d)
 
-install(DIRECTORY src/collectors/python.d.plugin/python_modules
-        COMPONENT plugin-pythond
-        DESTINATION usr/libexec/netdata/python.d)
+  configure_file(src/collectors/python.d.plugin/python.d.plugin.in src/collectors/python.d.plugin/python.d.plugin @ONLY)
+  install(PROGRAMS ${CMAKE_BINARY_DIR}/src/collectors/python.d.plugin/python.d.plugin
+          COMPONENT plugin-pythond
+          DESTINATION usr/libexec/netdata/plugins.d)
 
-install(FILES src/collectors/python.d.plugin/python.d.conf
-        COMPONENT plugin-pythond
-        DESTINATION usr/lib/netdata/conf.d)
+  install(DIRECTORY src/collectors/python.d.plugin/python_modules
+          COMPONENT plugin-pythond
+          DESTINATION usr/libexec/netdata/python.d)
 
-install(FILES
-        src/collectors/python.d.plugin/am2320/am2320.conf
-        src/collectors/python.d.plugin/ceph/ceph.conf
-        src/collectors/python.d.plugin/go_expvar/go_expvar.conf
-        src/collectors/python.d.plugin/haproxy/haproxy.conf
-        src/collectors/python.d.plugin/openldap/openldap.conf
-        src/collectors/python.d.plugin/oracledb/oracledb.conf
-        src/collectors/python.d.plugin/pandas/pandas.conf
-        src/collectors/python.d.plugin/spigotmc/spigotmc.conf
-        src/collectors/python.d.plugin/traefik/traefik.conf
-        src/collectors/python.d.plugin/zscores/zscores.conf
-        COMPONENT plugin-pythond
-        DESTINATION usr/lib/netdata/conf.d/python.d)
+  install(FILES src/collectors/python.d.plugin/python.d.conf
+          COMPONENT plugin-pythond
+          DESTINATION usr/lib/netdata/conf.d)
 
-install(FILES
-        src/collectors/python.d.plugin/am2320/am2320.chart.py
-        src/collectors/python.d.plugin/ceph/ceph.chart.py
-        src/collectors/python.d.plugin/go_expvar/go_expvar.chart.py
-        src/collectors/python.d.plugin/haproxy/haproxy.chart.py
-        src/collectors/python.d.plugin/openldap/openldap.chart.py
-        src/collectors/python.d.plugin/oracledb/oracledb.chart.py
-        src/collectors/python.d.plugin/pandas/pandas.chart.py
-        src/collectors/python.d.plugin/spigotmc/spigotmc.chart.py
-        src/collectors/python.d.plugin/traefik/traefik.chart.py
-        src/collectors/python.d.plugin/zscores/zscores.chart.py
-        COMPONENT plugin-pythond
-        DESTINATION usr/libexec/netdata/python.d)
+  install(FILES
+          src/collectors/python.d.plugin/am2320/am2320.conf
+          src/collectors/python.d.plugin/ceph/ceph.conf
+          src/collectors/python.d.plugin/go_expvar/go_expvar.conf
+          src/collectors/python.d.plugin/haproxy/haproxy.conf
+          src/collectors/python.d.plugin/openldap/openldap.conf
+          src/collectors/python.d.plugin/oracledb/oracledb.conf
+          src/collectors/python.d.plugin/pandas/pandas.conf
+          src/collectors/python.d.plugin/spigotmc/spigotmc.conf
+          src/collectors/python.d.plugin/traefik/traefik.conf
+          src/collectors/python.d.plugin/zscores/zscores.conf
+          COMPONENT plugin-pythond
+          DESTINATION usr/lib/netdata/conf.d/python.d)
 
-if(BUILD_FOR_PACKAGING)
-        install(FILES
-                ${PKG_FILES_PATH}/copyright
-                COMPONENT plugin-pythond
-                DESTINATION usr/share/doc/netdata-plugin-pythond)
+  install(FILES
+          src/collectors/python.d.plugin/am2320/am2320.chart.py
+          src/collectors/python.d.plugin/ceph/ceph.chart.py
+          src/collectors/python.d.plugin/go_expvar/go_expvar.chart.py
+          src/collectors/python.d.plugin/haproxy/haproxy.chart.py
+          src/collectors/python.d.plugin/openldap/openldap.chart.py
+          src/collectors/python.d.plugin/oracledb/oracledb.chart.py
+          src/collectors/python.d.plugin/pandas/pandas.chart.py
+          src/collectors/python.d.plugin/spigotmc/spigotmc.chart.py
+          src/collectors/python.d.plugin/traefik/traefik.chart.py
+          src/collectors/python.d.plugin/zscores/zscores.chart.py
+          COMPONENT plugin-pythond
+          DESTINATION usr/libexec/netdata/python.d)
+
+  if(BUILD_FOR_PACKAGING)
+    install(FILES
+            ${PKG_FILES_PATH}/copyright
+            COMPONENT plugin-pythond
+            DESTINATION usr/share/doc/netdata-plugin-pythond)
+  endif()
 endif()
 
 #
@@ -2966,6 +2976,8 @@ install(PROGRAMS ${CMAKE_BINARY_DIR}/src/collectors/ioping.plugin/ioping.plugin
 # go.d.plugin
 #
 if(ENABLE_PLUGIN_GO)
+    install(DIRECTORY COMPONENT plugin-go DESTINATION etc/netdata/go.d)
+
     install(FILES src/go/plugin/go.d/config/go.d.conf
             COMPONENT plugin-go
             DESTINATION usr/lib/netdata/conf.d)

netdata-installer.sh

@@ -253,6 +253,8 @@ LIBS_ARE_HERE=0
 NETDATA_ENABLE_ML=""
 ENABLE_DBENGINE=1
 ENABLE_GO=1
+ENABLE_PYTHON=1
+ENABLE_CHARTS=1
 ENABLE_H2O=1
 FORCE_LEGACY_CXX=0
 NETDATA_CMAKE_OPTIONS="${NETDATA_CMAKE_OPTIONS-}"
@@ -281,6 +283,10 @@ while [ -n "${1}" ]; do
     "--enable-plugin-go") ENABLE_GO=1 ;;
     "--disable-plugin-go") ENABLE_GO=0 ;;
     "--disable-go") ENABLE_GO=0 ;;
+    "--enable-plugin-python") ENABLE_PYTHON=1 ;;
+    "--disable-plugin-python") ENABLE_PYTHON=0 ;;
+    "--enable-plugin-charts") ENABLE_CHARTS=1 ;;
+    "--disable-plugin-charts") ENABLE_CHARTS=0 ;;
     "--enable-plugin-nfacct") ENABLE_NFACCT=1 ;;
     "--disable-plugin-nfacct") ENABLE_NFACCT=0 ;;
     "--enable-plugin-xenstat") ENABLE_XENSTAT=1 ;;

netdata.spec.in

@@ -394,6 +394,8 @@ happened, on your systems and applications.
 	-DENABLE_PLUGIN_CGROUP_NETWORK=On \
 	-DENABLE_PLUGIN_DEBUGFS=On \
 	-DENABLE_PLUGIN_GO=On \
+	-DENABLE_PLUGIN_PYTHON=On \
+	-DENABLE_PLUGIN_CHARTS=On \
 	-DENABLE_PLUGIN_LOCAL_LISTENERS=On \
 	-DENABLE_PLUGIN_PERF=On \
 	-DENABLE_PLUGIN_SLABINFO=On \

packaging/build-package.sh

@@ -35,6 +35,8 @@ add_cmake_option ENABLE_PLUGIN_CGROUP_NETWORK On
 add_cmake_option ENABLE_PLUGIN_DEBUGFS On
 add_cmake_option ENABLE_PLUGIN_FREEIPMI On
 add_cmake_option ENABLE_PLUGIN_GO On
+add_cmake_option ENABLE_PLUGIN_PYTHON On
+add_cmake_option ENABLE_PLUGIN_CHARTS On
 add_cmake_option ENABLE_PLUGIN_LOCAL_LISTENERS On
 add_cmake_option ENABLE_PLUGIN_NFACCT On
 add_cmake_option ENABLE_PLUGIN_PERF On

packaging/cmake/Modules/Packaging.cmake

@@ -63,8 +63,13 @@ netdata-plugin-network-viewer")
 set(CPACK_DEBIAN_NETDATA_PACKAGE_CONFLICTS
 		"netdata-core, netdata-plugins-bash, netdata-plugins-python, netdata-web")
 
-list(APPEND _main_deps "netdata-plugin-chartsd")
-list(APPEND _main_deps "netdata-plugin-pythond")
+if(ENABLE_PLUGIN_CHARTS)
+  list(APPEND _main_deps "netdata-plugin-chartsd")
+endif()
+
+if(ENABLE_PLUGIN_PYTHON)
+  list(APPEND _main_deps "netdata-plugin-pythond")
+endif()
 
 if(ENABLE_PLUGIN_APPS)
         list(APPEND _main_deps "netdata-plugin-apps")
@@ -437,7 +442,9 @@ list(APPEND CPACK_COMPONENTS_ALL "netdata")
 if(ENABLE_PLUGIN_APPS)
         list(APPEND CPACK_COMPONENTS_ALL "plugin-apps")
 endif()
-list(APPEND CPACK_COMPONENTS_ALL "plugin-chartsd")
+if(ENABLE_PLUGIN_CHARTS)
+  list(APPEND CPACK_COMPONENTS_ALL "plugin-chartsd")
+endif()
 if(ENABLE_PLUGIN_CUPS)
         list(APPEND CPACK_COMPONENTS_ALL "plugin-cups")
 endif()
@@ -465,7 +472,9 @@ endif()
 if(ENABLE_PLUGIN_PERF)
         list(APPEND CPACK_COMPONENTS_ALL "plugin-perf")
 endif()
-list(APPEND CPACK_COMPONENTS_ALL "plugin-pythond")
+if(ENABLE_PLUGIN_PYTHON)
+  list(APPEND CPACK_COMPONENTS_ALL "plugin-pythond")
+endif()
 if(ENABLE_PLUGIN_SLABINFO)
         list(APPEND CPACK_COMPONENTS_ALL "plugin-slabinfo")
 endif()

packaging/installer/functions.sh

@@ -303,6 +303,18 @@ prepare_cmake_options() {
     enable_feature PLUGIN_GO 0
   fi
 
+  if [ "${ENABLE_PYTHON:-1}" -eq 1 ]; then
+    enable_feature PLUGIN_PYTHON 1
+  else
+    enable_feature PLUGIN_PYTHON 0
+  fi
+
+  if [ "${ENABLE_CHARTS:-1}" -eq 1 ]; then
+    enable_feature PLUGIN_CHARTS 1
+  else
+    enable_feature PLUGIN_CHARTS 0
+  fi
+
   if [ "${USE_SYSTEM_PROTOBUF:-0}" -eq 1 ]; then
     enable_feature BUNDLED_PROTOBUF 0
   else