Add netdata-plugin-systemd-journal package. (#15733)

* Add netdata-plugin-systemd-journal package.

* Fix suggests usage in spec file.

* Skip systemd journal on old platforms in native packages.

* Actually build systemd-journal plugin everywhere.

* Fix spec file syntax.

* Install systemd-journal plugin by default on supported systems.

* Update packaging/installer/netdata-updater.sh

Co-authored-by: Ilya Mashchenko <ilya@netdata.cloud>

* Explicitly mark optional plugins as auto-installed when possible.

---------

Co-authored-by: Ilya Mashchenko <ilya@netdata.cloud>

contrib/debian/control

@@ -54,6 +54,7 @@ Conflicts: netdata-core,
            netdata-web
 Suggests: netdata-plugin-cups,
           netdata-plugin-freeipmi
+Recommends: netdata-plugin-systemd-journal
 Description: real-time charts for system monitoring
  Netdata is a daemon that collects data in realtime (per second)
  and presents a web site to view and analyze them. The presentation
@@ -182,10 +183,21 @@ Description: The perf metrics collector for the Netdata Agent
 
 Package: netdata-plugin-debugfs
 Architecture: any
-Depends: ${shlibs:Debends},
+Depends: ${shlibs:Depends},
          netdata (= ${source:Version})
 Pre-Depends: libcap2-bin, adduser
 Conflicts: netdata (<< ${source:Version})
 Description: The debugfs metrics collector for the Netdata Agent
  This plugin allows the Netdata Agent to collect Linux kernel metrics
  exposed through debugfs.
+
+Package: netdata-plugin-systemd-journal
+Architecture: any
+Depends: ${shlibs:Depends},
+         netdata (= ${source:Version})
+Pre-Depends: libcap2-bin, adduser
+Build-Depends: libsystemd-dev
+Conflicts: netdata (<< ${source:Version})
+Description: The systemd-journal collector for the Netdata Agent
+ This plugin allows the Netdata Agent to present logs from the systemd
+ journal on Netdata Cloud or the local Agent dashboard.

contrib/debian/control.xenial

@@ -1,61 +0,0 @@
-Source: netdata
-Build-Depends: debhelper (>= 9),
-               dh-autoreconf,
-               dh-systemd (>= 1.5),
-               dpkg-dev (>= 1.13.19),
-               zlib1g-dev,
-               uuid-dev,
-               libelf-dev,
-               libuv1-dev,
-               liblz4-dev,
-               libssl-dev,
-               libmnl-dev,
-               libjson-c-dev,
-               libcups2-dev,
-               libipmimonitoring-dev,
-               libnetfilter-acct-dev,
-               libsnappy-dev,
-               libprotobuf-dev,
-               libprotoc-dev,
-               cmake,
-               autogen,
-               autoconf,
-               automake,
-               pkg-config,
-               curl,
-               protobuf-compiler
-Section: net
-Priority: optional
-Maintainer: Netdata Builder <bot@netdata.cloud>
-Standards-Version: 3.9.6
-Homepage: https://netdata.cloud
-
-Package: netdata
-Architecture: any
-Depends: adduser,
-         libcap2-bin (>= 1:2.0),
-         lsb-base (>= 3.1-23.2),
-         openssl,
-         ${misc:Depends},
-         ${shlibs:Depends}
-Pre-Depends: dpkg (>= 1.17.14)
-Description: real-time charts for system monitoring
- Netdata is a daemon that collects data in realtime (per second)
- and presents a web site to view and analyze them. The presentation
- is also real-time and full of interactive charts that precisely
- render all collected values.
-
-Package: netdata-plugin-cups
-Architecture: any
-Depends: cups,
-         netdata (>= ${source:Version})
-Description: The Common Unix Printing System plugin for metrics collection from cupsd
-
-Package: netdata-plugin-freeipmi
-Architecture: any
-Depends: freeipmi,
-         netdata (= ${source:Version})
-Description: FreeIPMI - The Intelligent Platform Management System.
- The IPMI specification defines a set of interfaces for platform management.
- It is implemented by a number vendors for system management. The features of IPMI that most users will be interested in
- are sensor monitoring, system event monitoring, power control, and serial-over-LAN (SOL).

contrib/debian/netdata-plugin-systemd-journal.postinst

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+
+case "$1" in
+  configure|reconfigure)
+    chown root:netdata /usr/libexec/netdata/plugins.d/systemd-journal.plugin
+    setcap "cap_dac_read_search=eip" /usr/libexec/netdata/plugins.d/systemd-journal.plugin
+    ;;
+esac
+
+#DEBHELPER#
+
+exit 0

contrib/debian/netdata-plugin-systemd-journal.preinst

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+set -e
+
+case "$1" in
+  install)
+    if ! getent group netdata > /dev/null; then
+      addgroup --quiet --system netdata
+    fi
+    ;;
+esac
+
+#DEBHELPER#

contrib/debian/rules

@@ -125,6 +125,11 @@ override_dh_install:
 	mv -f $(TEMPTOP)/usr/libexec/netdata/plugins.d/debugfs.plugin \
 	$(TOP)-plugin-debugfs/usr/libexec/netdata/plugins.d/debugfs.plugin
 
+	# Add systemd-journal plugin install rules
+	mkdir -p $(TOP)-plugin-systemd-journal/usr/libexec/netdata/plugins.d/
+	mv -f $(TEMPTOP)/usr/libexec/netdata/plugins.d/systemd-journal.plugin \
+	$(TOP)-plugin-systemd-journal/usr/libexec/netdata/plugins.d/systemd-journal.plugin; \
+
 	# Set the rest of the software in the main package
 	#
 	cp -rp $(TEMPTOP)/usr $(TOP)
@@ -214,7 +219,7 @@ override_dh_fixperms:
 	chmod 4750 $(TOP)/usr/libexec/netdata/plugins.d/local-listeners
 
 	# systemd-journal
-	# chmod 4750 $(TOP)/usr/libexec/netdata/plugins.d/systemd-journal.plugin
+	chmod 4750 $(TOP)-plugin-systemd-journal/usr/libexec/netdata/plugins.d/systemd-journal.plugin
 
 override_dh_installlogrotate:
 	cp system/logrotate/netdata debian/netdata.logrotate

netdata.spec.in

@@ -158,13 +158,14 @@ Requires: netdata-plugin-chartsd
 Requires: netdata-plugin-slabinfo
 Requires: netdata-plugin-perf
 %if 0%{?_have_nfacct}
-Requires:   netdata-plugin-nfacct
+Requires: netdata-plugin-nfacct
 %endif
 %if 0%{?_have_freeipmi} && 0%{?centos_ver} != 6 && 0%{?centos_ver} != 7 && 0%{?amazon_linux} != 2
-Suggests:   netdata-plugin-freeipmi
+Suggests: netdata-plugin-freeipmi
 %endif
 %if 0%{?centos_ver} != 6 && 0%{?centos_ver} != 7 && 0%{?amazon_linux} != 2
-Suggests:   netdata-plugin-cups
+Suggests: netdata-plugin-cups
+Recommends: netdata-plugin-systemd-journal
 %endif
 
 
@@ -203,6 +204,10 @@ BuildRequires: cups-devel >= 1.7
 %endif
 # end - cups plugin dependencies
 
+# systemd-journal dependencies
+BuildRequires: systemd-devel
+# end - systemd-journal dependencies
+
 # Prometheus remote write dependencies
 BuildRequires: snappy-devel
 # end - prometheus remote write dependencies
@@ -248,13 +253,13 @@ export CFLAGS="${CFLAGS} -fPIC" && ${RPM_BUILD_DIR}/%{name}-%{version}/packaging
 autoreconf -ivf
 %configure \
 	%if 0%{!?_have_ebpf}
-	--disable-ebpf
+	--disable-ebpf \
 	%endif
 	%if 0%{!?_have_freeipmi}
-	--disable-plugin-freeipmi
+	--disable-plugin-freeipmi \
 	%endif
 	%if 0%{!?_have_nfacct}
-	--disable-plugin-nfacct
+	--disable-plugin-nfacct \
 	%endif
 	%if 0%{?centos_ver:1}
 	%if %{centos_ver} < 8
@@ -306,6 +311,10 @@ install -m 4750 -p apps.plugin "${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.
 # Install debugfs.plugin
 install -m 0750 -p debugfs.plugin "${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d/debugfs.plugin"
 
+# ###########################################################
+# Install systemd-journal.plugin
+install -m 4750 -p systemd-journal.plugin "${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d/systemd-journal.plugin"
+
 # ###########################################################
 # Install perf.plugin
 install -m 4750 -p perf.plugin "${RPM_BUILD_ROOT}%{_libexecdir}/%{name}/plugins.d/perf.plugin"
@@ -600,6 +609,9 @@ rm -rf "${RPM_BUILD_ROOT}"
 # perf belongs to a different sub-package
 %exclude %{_libexecdir}/%{name}/plugins.d/perf.plugin
 
+# systemd-journal belongs to a different sub-package
+%exclude %{_libexecdir}/%{name}/plugins.d/systemd-journal.plugin
+
 # CUPS belongs to a different sub package
 %if 0%{?centos_ver} != 6 && 0%{?centos_ver} != 7
 %exclude %{_libexecdir}/%{name}/plugins.d/cups.plugin
@@ -901,7 +913,30 @@ fi
 # CAP_DAC_READ_SEARCH required for data collection.
 %caps(cap_dac_read_search=ep) %attr(0750,root,netdata) %{_libexecdir}/%{name}/plugins.d/debugfs.plugin
 
+%package plugin-systemd-journal
+Summary: The systemd-journal plugin for the Netdata Agent
+Group: Applications/System
+Requires: netdata = %{version}
+Conflicts: netdata < %{version}
+
+%description plugin-systemd-journal
+ This plugin allows the Netdata Agent to present entries from the systemd
+ journal on Netdata Cloud or the local Agent Dashboard.
+
+%pre plugin-systemd-journal
+
+if ! getent group netdata > /dev/null; then
+  groupadd --system netdata
+fi
+
+%files plugin-systemd-journal
+%defattr(0750,root,netdata,0750)
+# CAP_DAC_READ_SEARCH required for data collection.
+%caps(cap_dac_read_search=ep) %attr(0750,root,netdata) %{_libexecdir}/%{name}/plugins.d/systemd-journal.plugin
+
 %changelog
+* Thu Aug 03 2023 Austin Hemmelgarn <austin@netdata.cloud> 0.0.0-21
+- Added systemd-journal plugin handling
 * Wed Jun 14 2023 Konstantin Shalygin <k0ste@k0ste.ru> 0.0.0-20
 - Added eBPF build dependency (again)
 * Fri Apr 07 2023 Austin Hemmelgarn <austin@netdata.cloud> 0.0.0-19

packaging/installer/UPDATE.md

@@ -182,3 +182,6 @@ The following configuration options are currently supported:
   as a scheduled task. This random delay helps avoid issues resulting from too many nodes trying to reconnect to
   the Cloud at the same time. The default value is 3600, which corresponds to one hour. Most users should not ever
   need to change this.
+- `NETDATA_NO_SYSTEMD_JOURNAL`: If set to a value other than 0, skip attempting to install the
+  `netdata-plugin-systemd-journal` package on supported systems on update. This optional package will be installed
+  by default on supported systems by the updater if this option is not set. Only affects systems using native packages.

packaging/installer/netdata-updater.sh

@@ -37,7 +37,9 @@ PACKAGES_SCRIPT="https://raw.githubusercontent.com/netdata/netdata/master/packag
 NETDATA_STABLE_BASE_URL="${NETDATA_BASE_URL:-https://github.com/netdata/netdata/releases}"
 NETDATA_NIGHTLY_BASE_URL="${NETDATA_BASE_URL:-https://github.com/netdata/netdata-nightlies/releases}"
 
+# Following variables are intended to be overridden by the updater config file.
 NETDATA_UPDATER_JITTER=3600
+NETDATA_NO_SYSTEMD_JOURNAL=0
 
 script_dir="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)"
 
@@ -758,9 +760,11 @@ update_binpkg() {
       fi
       pm_cmd="apt-get"
       repo_subcmd="update"
+      install_subcmd="install"
+      mark_auto_cmd="apt-mark auto"
       pkg_install_opts="${interactive_opts}"
       repo_update_opts="${interactive_opts}"
-      pkg_installed_check="dpkg -s"
+      pkg_installed_check="dpkg-query -s"
       INSTALL_TYPE="binpkg-deb"
       ;;
     centos|fedora|ol|amzn)
@@ -770,10 +774,13 @@ update_binpkg() {
       if command -v dnf > /dev/null; then
         pm_cmd="dnf"
         repo_subcmd="makecache"
+        mark_auto_cmd="dnf mark remove"
       else
         pm_cmd="yum"
+        mark_auto_cmd="yumdb set reason dep"
       fi
       upgrade_subcmd="upgrade"
+      install_subcmd="install"
       pkg_install_opts="${interactive_opts}"
       repo_update_opts="${interactive_opts}"
       pkg_installed_check="rpm -q"
@@ -787,6 +794,8 @@ update_binpkg() {
       fi
       pm_cmd="zypper"
       repo_subcmd="--gpg-auto-import-keys refresh"
+      install_subcmd="install"
+      mark_auto_cmd=""
       pkg_install_opts=""
       repo_update_opts=""
       pkg_installed_check="rpm -q"
@@ -816,6 +825,20 @@ update_binpkg() {
 
   # shellcheck disable=SC2086
   env ${env} ${pm_cmd} ${upgrade_subcmd} ${pkg_install_opts} netdata >&3 2>&3 || fatal "Failed to update Netdata package." U000F
+
+  if ${pkg_installed_check} systemd > /dev/null 2>&1; then
+    if [ "${NETDATA_NO_SYSTEMD_JOURNAL}" -eq 0 ]; then
+      if ! ${pkg_installed_check} netdata-plugin-systemd-journal > /dev/null 2>&1; then
+        env ${env} ${pm_cmd} ${install_subcmd} ${pkg_install_opts} netdata-plugin-systemd-journal >&3 2>&3
+
+        if [ -n "${mark_auto_cmd}" ]; then
+          # shellcheck disable=SC2086
+          env ${env} ${mark_auto_cmd} netdata-plugin-systemd-journal >&3 2>&3
+        fi
+      fi
+    fi
+  fi
+
   [ -n "${logfile}" ] && rm "${logfile}" && logfile=
   return 0
 }

system/netdata-updater.conf

@@ -5,3 +5,17 @@
 # actually running the update. The default is 3600 (one
 # hour). Most users should not need to change this.
 #NETDATA_UPDATER_JITTER="3600"
+
+# On systems using our native packages, the updater will by default
+# attempt to install optional plugin packages that would be installed by
+# default on clean installs if those packages are supported on the system.
+#
+# This behavior can be disabled on a per-package basis using the below
+# variables. Setting the variable to a value other than 0 will disable
+# the corresponding package (note that you still need to remove the package
+# yourself if you don0t want it, this just controls whether the updater
+# will try to ensure it’s installed or not).
+#
+# NETDATA_NO_SYSTEMD_JOURNAL controls the `netdata-plugin-systemd-journal`
+# package, which provides the systemd journal plugin for Netdata.
+#NETDATA_NO_SYSTEMD_JOURNAL="0"