netdata/packaging/dag/nd.py

from typing import List

import enum
import os
import pathlib
import uuid

import dagger
import jinja2

import imageutils


class Platform:
    def __init__(self, platform: str):
        self.platform = dagger.Platform(platform)

    def escaped(self) -> str:
        return str(self.platform).removeprefix("linux/").replace("/", "_")

    def __eq__(self, other):
        if isinstance(other, Platform):
            return self.platform == other.platform
        elif isinstance(other, dagger.Platform):
            return self.platform == other
        else:
            return NotImplemented

    def __ne__(self, other):
        return not (self == other)

    def __hash__(self):
        return hash(self.platform)

    def __str__(self) -> str:
        return str(self.platform)


SUPPORTED_PLATFORMS = set(
    [
        Platform("linux/x86_64"),
        Platform("linux/arm64"),
        Platform("linux/i386"),
        Platform("linux/arm/v7"),
        Platform("linux/arm/v6"),
        Platform("linux/ppc64le"),
        Platform("linux/s390x"),
        Platform("linux/riscv64"),
    ]
)


SUPPORTED_DISTRIBUTIONS = set(
    [
        "alpine_3_18",
        "alpine_3_19",
        "amazonlinux2",
        "centos7",
        "centos-stream8",
        "centos-stream9",
        "debian10",
        "debian11",
        "debian12",
        "fedora37",
        "fedora38",
        "fedora39",
        "opensuse15.4",
        "opensuse15.5",
        "opensusetumbleweed",
        "oraclelinux8",
        "oraclelinux9",
        "rockylinux8",
        "rockylinux9",
        "ubuntu20.04",
        "ubuntu22.04",
        "ubuntu23.04",
        "ubuntu23.10",
    ]
)


class Distribution:
    def __init__(self, display_name):
        self.display_name = display_name

        if self.display_name == "alpine_3_18":
            self.docker_tag = "alpine:3.18"
            self.builder = imageutils.build_alpine_3_18
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "alpine_3_19":
            self.docker_tag = "alpine:3.19"
            self.builder = imageutils.build_alpine_3_19
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "amazonlinux2":
            self.docker_tag = "amazonlinux:2"
            self.builder = imageutils.build_amazon_linux_2
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "centos7":
            self.docker_tag = "centos:7"
            self.builder = imageutils.build_centos_7
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "centos-stream8":
            self.docker_tag = "quay.io/centos/centos:stream8"
            self.builder = imageutils.build_centos_stream_8
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "centos-stream9":
            self.docker_tag = "quay.io/centos/centos:stream9"
            self.builder = imageutils.build_centos_stream_9
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "debian10":
            self.docker_tag = "debian:10"
            self.builder = imageutils.build_debian_10
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "debian11":
            self.docker_tag = "debian:11"
            self.builder = imageutils.build_debian_11
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "debian12":
            self.docker_tag = "debian:12"
            self.builder = imageutils.build_debian_12
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "fedora37":
            self.docker_tag = "fedora:37"
            self.builder = imageutils.build_fedora_37
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "fedora38":
            self.docker_tag = "fedora:38"
            self.builder = imageutils.build_fedora_38
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "fedora39":
            self.docker_tag = "fedora:39"
            self.platforms = SUPPORTED_PLATFORMS
            self.builder = imageutils.build_fedora_39
        elif self.display_name == "opensuse15.4":
            self.docker_tag = "opensuse/leap:15.4"
            self.builder = imageutils.build_opensuse_15_4
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "opensuse15.5":
            self.docker_tag = "opensuse/leap:15.5"
            self.builder = imageutils.build_opensuse_15_5
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "opensusetumbleweed":
            self.docker_tag = "opensuse/tumbleweed:latest"
            self.builder = imageutils.build_opensuse_tumbleweed
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "oraclelinux8":
            self.docker_tag = "oraclelinux:8"
            self.builder = imageutils.build_oracle_linux_8
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "oraclelinux9":
            self.docker_tag = "oraclelinux:9"
            self.builder = imageutils.build_oracle_linux_9
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "rockylinux8":
            self.docker_tag = "rockylinux:8"
            self.builder = imageutils.build_rocky_linux_8
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "rockylinux9":
            self.docker_tag = "rockylinux:9"
            self.builder = imageutils.build_rocky_linux_9
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "ubuntu20.04":
            self.docker_tag = "ubuntu:20.04"
            self.builder = imageutils.build_ubuntu_20_04
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "ubuntu22.04":
            self.docker_tag = "ubuntu:22.04"
            self.builder = imageutils.build_ubuntu_22_04
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "ubuntu23.04":
            self.docker_tag = "ubuntu:23.04"
            self.builder = imageutils.build_ubuntu_23_04
            self.platforms = SUPPORTED_PLATFORMS
        elif self.display_name == "ubuntu23.10":
            self.docker_tag = "ubuntu:23.10"
            self.builder = imageutils.build_ubuntu_23_10
            self.platforms = SUPPORTED_PLATFORMS
        else:
            raise ValueError(f"Unknown distribution: {self.display_name}")

    def _cache_volume(
        self, client: dagger.Client, platform: dagger.Platform, path: str
    ) -> dagger.CacheVolume:
        tag = "_".join([self.display_name, Platform(platform).escaped()])
        return client.cache_volume(f"{path}-{tag}")

    def build(
        self, client: dagger.Client, platform: dagger.Platform
    ) -> dagger.Container:
        if platform not in self.platforms:
            raise ValueError(
                f"Building {self.display_name} is not supported on {platform}."
            )

        ctr = self.builder(client, platform)
        ctr = imageutils.install_cargo(ctr)

        return ctr


class FeatureFlags(enum.Flag):
    DBEngine = enum.auto()
    GoPlugin = enum.auto()
    ExtendedBPF = enum.auto()
    LogsManagement = enum.auto()
    MachineLearning = enum.auto()
    BundledProtobuf = enum.auto()


class NetdataInstaller:
    def __init__(
        self,
        platform: Platform,
        distro: Distribution,
        repo_root: pathlib.Path,
        prefix: pathlib.Path,
        features: FeatureFlags,
    ):
        self.platform = platform
        self.distro = distro
        self.repo_root = repo_root
        self.prefix = prefix
        self.features = features

    def _mount_repo(
        self, client: dagger.Client, ctr: dagger.Container, repo_root: pathlib.Path
    ) -> dagger.Container:
        host_repo_root = pathlib.Path(__file__).parent.parent.parent.as_posix()
        exclude_dirs = ["build", "fluent-bit/build", "packaging/dag"]

        # The installer builds/stores intermediate artifacts under externaldeps/
        # We add a volume to speed up rebuilds. The volume has to be unique
        # per platform/distro in order to avoid mixing unrelated artifacts
        # together.
        externaldeps = self.distro._cache_volume(client, self.platform, "externaldeps")

        ctr = (
            ctr.with_directory(
                self.repo_root.as_posix(), client.host().directory(host_repo_root)
            )
            .with_workdir(self.repo_root.as_posix())
            .with_mounted_cache(
                os.path.join(self.repo_root, "externaldeps"), externaldeps
            )
        )

        return ctr

    def install(self, client: dagger.Client, ctr: dagger.Container) -> dagger.Container:
        args = ["--dont-wait", "--dont-start-it", "--disable-telemetry"]

        if FeatureFlags.DBEngine not in self.features:
            args.append("--disable-dbengine")

        if FeatureFlags.GoPlugin not in self.features:
            args.append("--disable-go")

        if FeatureFlags.ExtendedBPF not in self.features:
            args.append("--disable-ebpf")

        if FeatureFlags.MachineLearning not in self.features:
            args.append("--disable-ml")

        if FeatureFlags.BundledProtobuf not in self.features:
            args.append("--use-system-protobuf")

        args.extend(["--install-prefix", self.prefix.parent.as_posix()])

        ctr = self._mount_repo(client, ctr, self.repo_root.as_posix())

        ctr = ctr.with_env_variable(
            "NETDATA_CMAKE_OPTIONS", "-DCMAKE_BUILD_TYPE=Debug"
        ).with_exec(["./netdata-installer.sh"] + args)

        return ctr


class Endpoint:
    def __init__(self, hostname: str, port: int):
        self.hostname = hostname
        self.port = port

    def __str__(self):
        return ":".join([self.hostname, str(self.port)])


class ChildStreamConf:
    def __init__(
        self,
        installer: NetdataInstaller,
        destinations: List[Endpoint],
        api_key: uuid.UUID,
    ):
        self.installer = installer
        self.substitutions = {
            "enabled": "yes",
            "destination": " ".join([str(dst) for dst in destinations]),
            "api_key": api_key,
            "timeout_seconds": 60,
            "default_port": 19999,
            "send_charts_matching": "*",
            "buffer_size_bytes": 1024 * 1024,
            "reconnect_delay_seconds": 5,
            "initial_clock_resync_iterations": 60,
        }

    def render(self) -> str:
        tmpl_path = pathlib.Path(__file__).parent / "files/child_stream.conf"
        with open(tmpl_path) as fp:
            tmpl = jinja2.Template(fp.read())

        return tmpl.render(**self.substitutions)


class ParentStreamConf:
    def __init__(self, installer: NetdataInstaller, api_key: uuid.UUID):
        self.installer = installer
        self.substitutions = {
            "api_key": str(api_key),
            "enabled": "yes",
            "allow_from": "*",
            "default_history": 3600,
            "health_enabled_by_default": "auto",
            "default_postpone_alarms_on_connect_seconds": 60,
            "multiple_connections": "allow",
        }

    def render(self) -> str:
        tmpl_path = pathlib.Path(__file__).parent / "files/parent_stream.conf"
        with open(tmpl_path) as fp:
            tmpl = jinja2.Template(fp.read())

        return tmpl.render(**self.substitutions)


class StreamConf:
    def __init__(self, child_conf: ChildStreamConf, parent_conf: ParentStreamConf):
        self.child_conf = child_conf
        self.parent_conf = parent_conf

    def render(self) -> str:
        child_section = self.child_conf.render() if self.child_conf else ""
        parent_section = self.parent_conf.render() if self.parent_conf else ""
        return "\n".join([child_section, parent_section])


class AgentContext:
    def __init__(
        self,
        client: dagger.Client,
        platform: dagger.Platform,
        distro: Distribution,
        installer: NetdataInstaller,
        endpoint: Endpoint,
        api_key: uuid.UUID,
        allow_children: bool,
    ):
        self.client = client
        self.platform = platform
        self.distro = distro
        self.installer = installer
        self.endpoint = endpoint
        self.api_key = api_key
        self.allow_children = allow_children

        self.parent_contexts = []

        self.built_distro = False
        self.built_agent = False

    def add_parent(self, parent_context: "AgentContext"):
        self.parent_contexts.append(parent_context)

    def build_container(self) -> dagger.Container:
        ctr = self.distro.build(self.client, self.platform)
        ctr = self.installer.install(self.client, ctr)

        if len(self.parent_contexts) == 0 and not self.allow_children:
            return ctr.with_exposed_port(self.endpoint.port)

        destinations = [parent_ctx.endpoint for parent_ctx in self.parent_contexts]
        child_stream_conf = ChildStreamConf(self.installer, destinations, self.api_key)

        parent_stream_conf = None
        if self.allow_children:
            parent_stream_conf = ParentStreamConf(self.installer, self.api_key)

        stream_conf = StreamConf(child_stream_conf, parent_stream_conf)

        # write the stream conf to localhost and cp it in the container
        host_stream_conf_path = pathlib.Path(
            f"/tmp/{self.endpoint.hostname}_stream.conf"
        )
        with open(host_stream_conf_path, "w") as fp:
            fp.write(stream_conf.render())

        ctr_stream_conf_path = self.installer.prefix / "etc/netdata/stream.conf"

        ctr = ctr.with_file(
            ctr_stream_conf_path.as_posix(),
            self.client.host().file(host_stream_conf_path.as_posix()),
        )

        ctr = ctr.with_exposed_port(self.endpoint.port)

        return ctr