Fix (and improve) Coverity scanning. (#16605)

* Split feature detection code to functions.

This lets us reuse it in places like the coverity scan script.

* Add ninja to the coverity build environment.

* Restructure coverity scan script to use cmake.

* Fix incorrect function name.

* Add extra plugin deps for coverity scan environment.

* Fix new issues introduced in coverity script.

* Fix things broken by previous fix.

* Fix Docker image builds.

* Assorted fixes.

* Correctly handle common case of set but empty prefix.

.github/workflows/coverity.yml

@@ -33,7 +33,8 @@ jobs:
           sudo apt-get install -y libjson-c-dev libyaml-dev libipmimonitoring-dev \
                                   libcups2-dev libsnappy-dev libprotobuf-dev \
                                   libprotoc-dev libssl-dev protobuf-compiler \
-                                  libnetfilter-acct-dev
+                                  libnetfilter-acct-dev libmongoc-dev libxen-dev \
+                                  libsystemd-dev ninja-build
       - name: Run coverity-scan
         id: run
         env:

coverity-scan.sh

@@ -42,11 +42,19 @@ INSTALL_DIR="/opt"
 # the version of coverity to use
 COVERITY_BUILD_VERSION="${COVERITY_BUILD_VERSION:-cov-analysis-linux64-2023.6.2}"
 
-# TODO: For some reasons this does not fully load on Debian 10 (Haven't checked if it happens on other distros yet), it breaks
-source packaging/installer/functions.sh || echo "Failed to fully load the functions library"
+. packaging/installer/functions.sh
 
-cpus=$(find_processors)
-[ -z "${cpus}" ] && cpus=1
+JOBS=$(find_processors)
+[ -z "${JOBS}" ] && JOBS=1
+
+if command -v ninja 2>&1; then
+    ninja="$(command -v ninja)"
+fi
+
+CMAKE_OPTS="${ninja:+-G Ninja}"
+BUILD_OPTS="VERBOSE=1"
+[ -n "${ninja}" ] && BUILD_OPTS="-v"
+NETDATA_BUILD_DIR="${NETDATA_BUILD_DIR:-./cmake-build-release/}"
 
 if [ -f ".coverity-scan.conf" ]; then
   source ".coverity-scan.conf"
@@ -102,19 +110,20 @@ scanit() {
   progress "Working on netdata version: ${version}"
 
   progress "Cleaning up old builds..."
-  run make clean || echo >&2 "Nothing to clean"
+  rm -rf "${NETDATA_BUILD_DIR}"
 
   [ -d "cov-int" ] && rm -rf "cov-int"
 
   [ -f netdata-coverity-analysis.tgz ] && run rm netdata-coverity-analysis.tgz
 
   progress "Configuring netdata source..."
+  USE_SYSTEM_PROTOBUF=1
+  prepare_cmake_options
 
-  run autoreconf -ivf
-  run ./configure ${OTHER_OPTIONS}
+  run cmake ${NETDATA_CMAKE_OPTIONS}
 
   progress "Analyzing netdata..."
-  run "${covbuild}" --dir cov-int make -j${cpus}
+  run "${covbuild}" --dir cov-int cmake --build "${NETDATA_BUILD_DIR}" --parallel ${JOBS} -- ${BUILD_OPTS}
 
   echo >&2 "Compressing analysis..."
   run tar czvf netdata-coverity-analysis.tgz cov-int
@@ -170,21 +179,6 @@ installit() {
   return 0
 }
 
-OTHER_OPTIONS="--disable-lto"
-OTHER_OPTIONS+=" --with-zlib"
-OTHER_OPTIONS+=" --with-math"
-OTHER_OPTIONS+=" --enable-lz4"
-OTHER_OPTIONS+=" --enable-openssl"
-OTHER_OPTIONS+=" --enable-jsonc"
-OTHER_OPTIONS+=" --enable-plugin-nfacct"
-OTHER_OPTIONS+=" --enable-plugin-freeipmi"
-OTHER_OPTIONS+=" --enable-plugin-cups"
-OTHER_OPTIONS+=" --enable-exporting-prometheus-remote-write"
-# TODO: enable these plugins too
-#OTHER_OPTIONS+=" --enable-plugin-xenstat"
-#OTHER_OPTIONS+=" --enable-exporting-kinesis"
-#OTHER_OPTIONS+=" --enable-exporting-mongodb"
-
 FOUND_OPTS="NO"
 while [ -n "${1}" ]; do
   if [ "${1}" = "--with-install" ]; then

netdata-installer.sh

@@ -454,15 +454,6 @@ if [ "$(uname -s)" = "Linux" ] && [ -f /proc/meminfo ]; then
   fi
 fi
 
-enable_feature() {
-  NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed -e "s/-DENABLE_${1}=Off[[:space:]]*//g" -e "s/-DENABLE_${1}=On[[:space:]]*//g")"
-  if [ "${2}" -eq 1 ]; then
-    NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed "s/$/ -DENABLE_${1}=On/")"
-  else
-    NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed "s/$/ -DENABLE_${1}=Off/")"
-  fi
-}
-
 # set default make options
 if [ -z "${MAKEOPTS}" ]; then
   MAKEOPTS="-j$(find_processors)"
@@ -1104,35 +1095,6 @@ echo >&2
 NETDATA_BUILD_DIR="${NETDATA_BUILD_DIR:-./cmake-build-release/}"
 rm -rf "${NETDATA_BUILD_DIR}"
 
-check_for_module() {
-  if [ -z "${pkgconf}" ]; then
-    pkgconf="$(command -v pkgconf 2>/dev/null)"
-    [ -z "${pkgconf}" ] && pkgconf="$(command -v pkg-config 2>/dev/null)"
-    [ -z "${pkgconf}" ] && fatal "Unable to find a usable pkgconf/pkg-config command, cannot build Netdata." I0013
-  fi
-
-  "${pkgconf}" "${1}"
-  return "${?}"
-}
-
-check_for_feature() {
-  feature_name="${1}"
-  feature_state="${2}"
-  shift 2
-  feature_modules="${*}"
-
-  if [ -z "${feature_state}" ]; then
-    # shellcheck disable=SC2086
-    if check_for_module ${feature_modules}; then
-      enable_feature "${feature_name}" 1
-    else
-      enable_feature "${feature_name}" 0
-    fi
-  else
-    enable_feature "${feature_name}" "${feature_state}"
-  fi
-}
-
 # function to extract values from the config file
 config_option() {
   section="${1}"
@@ -1161,74 +1123,7 @@ NETDATA_GROUP="$(id -g -n "${NETDATA_USER}" 2> /dev/null)"
 [ -z "${NETDATA_GROUP}" ] && NETDATA_GROUP="${NETDATA_USER}"
 echo >&2 "Netdata user and group set to: ${NETDATA_USER}/${NETDATA_GROUP}"
 
-NETDATA_CMAKE_OPTIONS="-S ./ -B ${NETDATA_BUILD_DIR} ${CMAKE_OPTS} -DCMAKE_INSTALL_PREFIX=${NETDATA_PREFIX} ${NETDATA_USER:+-DNETDATA_USER=${NETDATA_USER}} ${NETDATA_CMAKE_OPTIONS} "
-
-# Feature autodetection code starts here
-
-if [ "${USE_SYSTEM_PROTOBUF}" -eq 1 ]; then
-  enable_feature BUNDLED_PROTOBUF 0
-else
-  enable_feature BUNDLED_PROTOBUF 1
-fi
-
-if [ -z "${ENABLE_SYSTEMD_PLUGIN}" ]; then
-    if check_for_module libsystemd; then
-        if check_for_module libelogind; then
-            ENABLE_SYSTEMD_JOURNAL=0
-        else
-            ENABLE_SYSTEMD_JOURNAL=1
-        fi
-    else
-        ENABLE_SYSTEMD_JOURNAL=0
-    fi
-fi
-
-enable_feature PLUGIN_SYSTEMD_JOURNAL "${ENABLE_SYSTEMD_JOURNAL}"
-
-[ -z "${NETDATA_ENABLE_ML}" ] && NETDATA_ENABLE_ML=1
-enable_feature ML "${NETDATA_ENABLE_ML}"
-
-if command -v cups-config >/dev/null 2>&1 || check_for_module libcups || check_for_module cups; then
-  ENABLE_CUPS=1
-else
-  ENABLE_CUPS=0
-fi
-
-enable_feature PLUGIN_CUPS "${ENABLE_CUPS}"
-
-IS_LINUX=0
-[ "$(uname -s)" = "Linux" ] && IS_LINUX=1
-enable_feature PLUGIN_DEBUGFS "${IS_LINUX}"
-enable_feature PLUGIN_PERF "${IS_LINUX}"
-enable_feature PLUGIN_SLABINFO "${IS_LINUX}"
-enable_feature PLUGIN_CGROUP_NETWORK "${IS_LINUX}"
-enable_feature PLUGIN_LOCAL_LISTENERS "${IS_LINUX}"
-enable_feature PLUGIN_LOGS_MANAGEMENT "${ENABLE_LOGS_MANAGEMENT}"
-enable_feature LOGS_MANAGEMENT_TESTS "${ENABLE_LOGS_MANAGEMENT_TESTS}"
-
-enable_feature ACLK "${ENABLE_CLOUD}"
-enable_feature CLOUD "${ENABLE_CLOUD}"
-enable_feature BUNDLED_JSONC "${NETDATA_BUILD_JSON_C}"
-enable_feature BUNDLED_YAML "${BUNDLE_YAML}"
-enable_feature DBENGINE "${ENABLE_DBENGINE}"
-enable_feature H2O "${ENABLE_H2O}"
-enable_feature PLUGIN_EBPF "${ENABLE_EBPF}"
-
-ENABLE_APPS=0
-
-if [ "${IS_LINUX}" = 1 ] || [ "$(uname -s)" = "FreeBSD" ]; then
-    ENABLE_APPS=1
-fi
-
-enable_feature PLUGIN_APPS "${ENABLE_APPS}"
-
-check_for_feature EXPORTER_PROMETHEUS_REMOTE_WRITE "${EXPORTER_PROMETHEUS}" snappy
-check_for_feature EXPORTER_MONGODB "${EXPORTER_MONGODB}" libmongoc-1.0
-check_for_feature PLUGIN_FREEIPMI "${ENABLE_FREEIPMI}" libipmimonitoring
-check_for_feature PLUGIN_NFACCT "${ENABLE_NFACCT}" libnetfilter_acct libnml
-check_for_feature PLUGIN_XENSTAT "${ENABLE_XENSTAT}" xenstat xenlight
-
-# End of feature autodetection code
+prepare_cmake_options
 
 if [ -n "${NETDATA_PREPARE_ONLY}" ]; then
     progress "Exiting before building Netdata as requested."

packaging/docker/Dockerfile

@@ -29,7 +29,7 @@ RUN chmod +x netdata-installer.sh && \
    cp -rp /deps/* /usr/local/ && \
    /bin/echo -e "INSTALL_TYPE='oci'\nPREBUILT_ARCH='$(uname -m)'" > ./system/.install-type && \
    CFLAGS="$(packaging/docker/gen-cflags.sh)" LDFLAGS="-Wl,--gc-sections" ./netdata-installer.sh --dont-wait --dont-start-it --use-system-protobuf \
-   ${EXTRA_INSTALL_OPTS} --disable-ebpf --one-time-build --enable-lto "$([ "$RELEASE_CHANNEL" = stable ] && echo --stable-channel)"
+   ${EXTRA_INSTALL_OPTS} --disable-ebpf --install-no-prefix / "$([ "$RELEASE_CHANNEL" = stable ] && echo --stable-channel)"
 
 # files to one directory
 RUN mkdir -p /app/usr/sbin/ \

packaging/installer/functions.sh

@@ -191,6 +191,112 @@ netdata_banner() {
   echo >&2
 }
 
+# -----------------------------------------------------------------------------
+# Feature management and configuration commands
+
+enable_feature() {
+  NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed -e "s/-DENABLE_${1}=Off[[:space:]]*//g" -e "s/-DENABLE_${1}=On[[:space:]]*//g")"
+  if [ "${2}" -eq 1 ]; then
+    NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed "s/$/ -DENABLE_${1}=On/")"
+  else
+    NETDATA_CMAKE_OPTIONS="$(echo "${NETDATA_CMAKE_OPTIONS}" | sed "s/$/ -DENABLE_${1}=Off/")"
+  fi
+}
+
+check_for_module() {
+  if [ -z "${pkgconf}" ]; then
+    pkgconf="$(command -v pkgconf 2>/dev/null)"
+    [ -z "${pkgconf}" ] && pkgconf="$(command -v pkg-config 2>/dev/null)"
+    [ -z "${pkgconf}" ] && fatal "Unable to find a usable pkgconf/pkg-config command, cannot build Netdata." I0013
+  fi
+
+  "${pkgconf}" "${1}"
+  return "${?}"
+}
+
+check_for_feature() {
+  feature_name="${1}"
+  feature_state="${2}"
+  shift 2
+  feature_modules="${*}"
+
+  if [ -z "${feature_state}" ]; then
+    # shellcheck disable=SC2086
+    if check_for_module ${feature_modules}; then
+      enable_feature "${feature_name}" 1
+    else
+      enable_feature "${feature_name}" 0
+    fi
+  else
+    enable_feature "${feature_name}" "${feature_state}"
+  fi
+}
+
+prepare_cmake_options() {
+  NETDATA_CMAKE_OPTIONS="-S ./ -B ${NETDATA_BUILD_DIR} ${CMAKE_OPTS} ${NETDATA_PREFIX+-DCMAKE_INSTALL_PREFIX="${NETDATA_PREFIX}"} ${NETDATA_USER:+-DNETDATA_USER=${NETDATA_USER}} ${NETDATA_CMAKE_OPTIONS} "
+
+  if [ "${USE_SYSTEM_PROTOBUF:-1}" -eq 1 ]; then
+    enable_feature BUNDLED_PROTOBUF 0
+  else
+    enable_feature BUNDLED_PROTOBUF 1
+  fi
+
+  if [ -z "${ENABLE_SYSTEMD_JOURNAL}" ]; then
+      if check_for_module libsystemd; then
+          if check_for_module libelogind; then
+              ENABLE_SYSTEMD_JOURNAL=0
+          else
+              ENABLE_SYSTEMD_JOURNAL=1
+          fi
+      else
+          ENABLE_SYSTEMD_JOURNAL=0
+      fi
+  fi
+
+  enable_feature PLUGIN_SYSTEMD_JOURNAL "${ENABLE_SYSTEMD_JOURNAL}"
+
+  if command -v cups-config >/dev/null 2>&1 || check_for_module libcups || check_for_module cups; then
+    ENABLE_CUPS=1
+  else
+    ENABLE_CUPS=0
+  fi
+
+  enable_feature PLUGIN_CUPS "${ENABLE_CUPS}"
+
+  IS_LINUX=0
+  [ "$(uname -s)" = "Linux" ] && IS_LINUX=1
+  enable_feature PLUGIN_DEBUGFS "${IS_LINUX}"
+  enable_feature PLUGIN_PERF "${IS_LINUX}"
+  enable_feature PLUGIN_SLABINFO "${IS_LINUX}"
+  enable_feature PLUGIN_CGROUP_NETWORK "${IS_LINUX}"
+  enable_feature PLUGIN_LOCAL_LISTENERS "${IS_LINUX}"
+  enable_feature PLUGIN_EBPF "${ENABLE_EBPF:-0}"
+  enable_feature PLUGIN_LOGS_MANAGEMENT "${ENABLE_LOGS_MANAGEMENT:-0}"
+  enable_feature LOGS_MANAGEMENT_TESTS "${ENABLE_LOGS_MANAGEMENT_TESTS:-0}"
+
+  enable_feature ACLK "${ENABLE_CLOUD:-1}"
+  enable_feature CLOUD "${ENABLE_CLOUD:-1}"
+  enable_feature BUNDLED_JSONC "${NETDATA_BUILD_JSON_C:-0}"
+  enable_feature BUNDLED_YAML "${BUNDLE_YAML:-0}"
+  enable_feature DBENGINE "${ENABLE_DBENGINE:-1}"
+  enable_feature H2O "${ENABLE_H2O:-1}"
+  enable_feature ML "${NETDATA_ENABLE_ML:-1}"
+
+  ENABLE_APPS=0
+
+  if [ "${IS_LINUX}" = 1 ] || [ "$(uname -s)" = "FreeBSD" ]; then
+      ENABLE_APPS=1
+  fi
+
+  enable_feature PLUGIN_APPS "${ENABLE_APPS}"
+
+  check_for_feature EXPORTER_PROMETHEUS_REMOTE_WRITE "${EXPORTER_PROMETHEUS}" snappy
+  check_for_feature EXPORTER_MONGODB "${EXPORTER_MONGODB}" libmongoc-1.0
+  check_for_feature PLUGIN_FREEIPMI "${ENABLE_FREEIPMI}" libipmimonitoring
+  check_for_feature PLUGIN_NFACCT "${ENABLE_NFACCT}" libnetfilter_acct libnml
+  check_for_feature PLUGIN_XENSTAT "${ENABLE_XENSTAT}" xenstat xenlight
+}
+
 # -----------------------------------------------------------------------------
 # portable service command