Bundle libyaml (#14704)

* bundle libyaml

* remove comment

* .github/workflows/coverity.yml

* add to coverity and tests

* add deps

* add to netdata.spec.in

* add to contrib/debia/control

* remove extra gentoo libyaml

.github/workflows/coverity.yml

@@ -30,7 +30,7 @@ jobs:
         run: |
           ./packaging/installer/install-required-packages.sh \
             --dont-wait --non-interactive netdata
-          sudo apt-get install -y libjson-c-dev libipmimonitoring-dev \
+          sudo apt-get install -y libjson-c-dev libyaml-dev libipmimonitoring-dev \
                                   libcups2-dev libsnappy-dev libprotobuf-dev \
                                   libprotoc-dev libssl-dev protobuf-compiler \
                                   libnetfilter-acct-dev

.github/workflows/tests.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Prepare environment
         run: |
           ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
-          sudo apt-get install -y libjson-c-dev libipmimonitoring-dev libcups2-dev libsnappy-dev \
+          sudo apt-get install -y libjson-c-dev libyaml-dev libipmimonitoring-dev libcups2-dev libsnappy-dev \
                                   libprotobuf-dev libprotoc-dev libssl-dev protobuf-compiler \
                                   libnetfilter-acct-dev
       - name: Run ./tests/run-unit-tests.sh

Makefile.am

@@ -73,6 +73,8 @@ dist_noinst_DATA = \
     packaging/installer/UPDATE.md \
     packaging/jsonc.checksums \
     packaging/jsonc.version \
+    packaging/yaml.checksums \
+    packaging/yaml.version \
     packaging/libbpf.checksums \
     packaging/libbpf.version \
     packaging/protobuf.checksums \
@@ -981,6 +983,7 @@ NETDATA_COMMON_LIBS = \
     libjudy.a \
     $(OPTIONAL_SSL_LIBS) \
     $(OPTIONAL_JSONC_LIBS) \
+    $(OPTIONAL_YAML_LIBS) \
     $(OPTIONAL_ATOMIC_LIBS) \
     $(OPTIONAL_DL_LIBS) \
     $(NULL)
@@ -993,6 +996,10 @@ if LINK_STATIC_JSONC
     NETDATA_COMMON_LIBS += $(abs_top_srcdir)/externaldeps/jsonc/libjson-c.a
 endif
 
+if LINK_STATIC_YAML
+    NETDATA_COMMON_LIBS += $(abs_top_srcdir)/externaldeps/libyaml/libyaml.a
+endif
+
 NETDATACLI_FILES = \
     daemon/commands.h \
     $(LIBNETDATA_FILES) \

configure.ac

@@ -481,6 +481,17 @@ PKG_CHECK_MODULES([JSON],[json-c],AC_CHECK_LIB(
 
 OPTIONAL_JSONC_LIBS="${JSONC_LIBS}"
 
+# -----------------------------------------------------------------------------
+# YAML library
+
+AC_CHECK_LIB(
+    [yaml],
+    [yaml_parser_initialize],
+    [YAML_LIBS="-lyaml"]
+)
+
+OPTIONAL_YAML_LIBS="${YAML_LIBS}"
+
 # -----------------------------------------------------------------------------
 # DB engine and HTTPS
 test "${enable_dbengine}" = "yes" -a -z "${LZ4_LIBS}" && \
@@ -610,6 +621,38 @@ fi
 AC_MSG_RESULT([${enable_jsonc}])
 AM_CONDITIONAL([ENABLE_JSONC], [test "${enable_jsonc}" = "yes"])
 
+# -----------------------------------------------------------------------------
+# YAML
+
+if test -z "${YAML_LIBS}"; then
+    # Try and detect manual static build presence (from netdata-installer.sh)
+    AC_MSG_CHECKING([if statically built libyaml is present])
+    HAVE_libyaml_a="no"
+    if test -f "externaldeps/libyaml/libyaml.a"; then
+        LIBS_BKP="${LIBS}"
+        LIBS="externaldeps/libyaml/libyaml.a"
+        AC_LINK_IFELSE([AC_LANG_SOURCE([[#include "externaldeps/libyaml/yaml.h"
+                                         int main (int argc, char **argv) {
+                                             yaml_parser_t parser;
+                                             yaml_parser_initialize(&parser);
+                                         }]])],
+                                        [HAVE_libyaml_a="yes"],
+                                        [HAVE_libyaml_a="no"])
+        LIBS="${LIBS_BKP}"
+    fi
+
+	if test "${HAVE_libyaml_a}" = "yes"; then
+		AC_DEFINE([LINK_STATIC_YAML], [1], [static yaml should be used])
+		YAML_LIBS="static"
+		OPTIONAL_YAML_STATIC_CFLAGS="-I \$(abs_top_srcdir)/externaldeps/libyaml"
+	fi
+	AC_MSG_RESULT([${HAVE_libyaml_a}])
+fi
+AM_CONDITIONAL([LINK_STATIC_YAML], [test "${YAML_LIBS}" = "static"])
+
+test -z "${YAML_LIBS}" && \
+    AC_MSG_ERROR([LIBYAML required but not found. Try installing 'libyaml-dev'.])
+
 # -----------------------------------------------------------------------------
 # compiler options
 
@@ -1602,7 +1645,7 @@ CFLAGS="${originalCFLAGS} ${OPTIONAL_LTO_CFLAGS} ${OPTIONAL_PROTOBUF_CFLAGS} ${O
     ${OPTIONAL_ZLIB_CFLAGS} ${OPTIONAL_UUID_CFLAGS} \
     ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} \
     ${OPTIONAL_KINESIS_CFLAGS} ${OPTIONAL_PUBSUB_CFLAGS} ${OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS} \
-    ${OPTIONAL_MONGOC_CFLAGS} ${LWS_CFLAGS} ${OPTIONAL_JSONC_STATIC_CFLAGS} ${OPTIONAL_BPF_CFLAGS} ${JUDY_CFLAGS} \
+    ${OPTIONAL_MONGOC_CFLAGS} ${LWS_CFLAGS} ${OPTIONAL_JSONC_STATIC_CFLAGS} ${OPTIONAL_YAML_STATIC_CFLAGS} ${OPTIONAL_BPF_CFLAGS} ${JUDY_CFLAGS} \
     ${OPTIONAL_ACLK_CFLAGS} ${OPTIONAL_ML_CFLAGS} ${OPTIONAL_OS_DEP_CFLAGS}"
 
 CXXFLAGS="${CFLAGS} ${CXX11FLAG}"
@@ -1624,6 +1667,7 @@ AC_SUBST([OPTIONAL_UV_LIBS])
 AC_SUBST([OPTIONAL_LZ4_LIBS])
 AC_SUBST([OPTIONAL_SSL_LIBS])
 AC_SUBST([OPTIONAL_JSONC_LIBS])
+AC_SUBST([OPTIONAL_YAML_LIBS])
 AC_SUBST([OPTIONAL_NFACCT_CFLAGS])
 AC_SUBST([OPTIONAL_NFACCT_LIBS])
 AC_SUBST([OPTIONAL_ZLIB_CFLAGS])

contrib/debian/control

@@ -10,6 +10,7 @@ Build-Depends: debhelper (>= 9.20160709),
                libssl-dev,
                libmnl-dev,
                libjson-c-dev,
+               libyaml-dev,
                libcups2-dev,
                libipmimonitoring-dev,
                libnetfilter-acct-dev,

libnetdata/libnetdata.h

@@ -551,6 +551,7 @@ extern char *netdata_configured_host_prefix;
 #include "onewayalloc/onewayalloc.h"
 #include "worker_utilization/worker_utilization.h"
 #include "parser/parser.h"
+#include "yaml.h"
 
 // BEWARE: Outside of the C code this also exists in alarm-notify.sh
 #define DEFAULT_CLOUD_BASE_URL "https://api.netdata.cloud"

netdata-installer.sh

@@ -709,6 +709,70 @@ bundle_jsonc() {
 
 bundle_jsonc
 
+# -----------------------------------------------------------------------------
+build_yaml() {
+  env_cmd=''
+
+  if [ -z "${DONT_SCRUB_CFLAGS_EVEN_THOUGH_IT_MAY_BREAK_THINGS}" ]; then
+    env_cmd="env CFLAGS='-fPIC -pipe -Wno-unused-value' CXXFLAGS='-fPIC -pipe' LDFLAGS="
+  fi
+
+  cd "${1}" > /dev/null || return 1
+  run eval "${env_cmd} ./configure --disable-shared --disable-dependency-tracking --with-pic"
+  run eval "${env_cmd} ${make} ${MAKEOPTS}"
+  cd - > /dev/null || return 1
+}
+
+copy_yaml() {
+  target_dir="${PWD}/externaldeps/libyaml"
+
+  run mkdir -p "${target_dir}" || return 1
+
+  run cp "${1}/src/.libs/libyaml.a" "${target_dir}/libyaml.a" || return 1
+  run cp "${1}/include/yaml.h" "${target_dir}/" || return 1
+}
+
+bundle_yaml() {
+  if pkg-config yaml-0.1; then
+    return 0
+  fi
+
+  if [ -z "$(command -v cmake)" ]; then
+    run_failed "Could not find cmake, which is required to build YAML. Critical error."
+    return 0
+  fi
+
+  [ -n "${GITHUB_ACTIONS}" ] && echo "::group::Bundling YAML."
+
+  progress "Prepare YAML"
+
+  YAML_PACKAGE_VERSION="$(cat packaging/yaml.version)"
+
+  tmp="$(mktemp -d -t netdata-yaml-XXXXXX)"
+  YAML_PACKAGE_BASENAME="yaml-${YAML_PACKAGE_VERSION}.tar.gz"
+
+  if fetch_and_verify "yaml" \
+    "https://github.com/yaml/libyaml/releases/download/${YAML_PACKAGE_VERSION}/${YAML_PACKAGE_BASENAME}" \
+    "${YAML_PACKAGE_BASENAME}" \
+    "${tmp}" \
+    "${NETDATA_LOCAL_TARBALL_OVERRIDE_YAML}"; then
+    if run tar --no-same-owner -xf "${tmp}/${YAML_PACKAGE_BASENAME}" -C "${tmp}" &&
+      build_yaml "${tmp}/yaml-${YAML_PACKAGE_VERSION}" &&
+      copy_yaml "${tmp}/yaml-${YAML_PACKAGE_VERSION}" &&
+      rm -rf "${tmp}"; then
+      run_ok "YAML built and prepared."
+    else
+      run_failed "Failed to build YAML, critical error."
+    fi
+  else
+    run_failed "Unable to fetch sources for YAML, critical error."
+  fi
+
+  [ -n "${GITHUB_ACTIONS}" ] && echo "::endgroup::"
+}
+
+bundle_yaml
+
 # -----------------------------------------------------------------------------
 
 get_kernel_version() {

netdata.spec.in

@@ -145,12 +145,14 @@ BuildRequires: protobuf-devel
 BuildRequires: libprotobuf-c-devel
 BuildRequires: liblz4-devel
 BuildRequires: libjson-c-devel
+BuildRequires: libyaml-devel
 %else
 %if 0%{?fedora}
 BuildRequires: protobuf-devel
 BuildRequires: protobuf-c-devel
 BuildRequires: lz4-devel
 BuildRequires: json-c-devel
+BuildRequires: libyaml-devel
 %else
 %if 0%{?centos_ver} >= 8
 BuildRequires: protobuf-devel
@@ -158,6 +160,7 @@ BuildRequires: protobuf-c-devel
 %endif
 BuildRequires: lz4-devel
 BuildRequires: json-c-devel
+BuildRequires: libyaml-devel
 %endif
 %endif
 

packaging/installer/dependencies/alpine.sh

@@ -31,6 +31,7 @@ package_tree="
   util-linux-dev
   libmnl-dev
   json-c-dev
+  yaml-dev
   "
 
 usage() {

packaging/installer/dependencies/arch.sh

@@ -20,6 +20,7 @@ declare -a package_tree=(
   util-linux
   libmnl
   json-c
+  libyaml
   libuv
   lz4
   openssl