ydb/generate_cmake

#!/usr/bin/env python3

import sys
import os
import shutil
import subprocess
import multiprocessing
import json
import stat
import filecmp
import urllib.request
from argparse import ArgumentParser


def mkdir(path):
    try:
        os.mkdir(path)
    except FileExistsError as e:
        pass


def remove_file(path):
    try:
        os.remove(path)
    except FileNotFoundError as e:
        pass


def compare_files(lhs_file_path, rhs_file_path):
    try:
        return filecmp.cmp(lhs_file_path, rhs_file_path)
    except FileNotFoundError as e:
        return False


def rmtree(path):
    try:
        shutil.rmtree(path)
    except FileNotFoundError as e:
        pass


def make_file_executable(file_path):
    st = os.stat(file_path)
    os.chmod(file_path, st.st_mode + stat.S_IEXEC)


def get_binary_id(resource_file_path):
    with open(resource_file_path) as file:
        ymake_file_json = json.load(file)

    linux_uri = str(ymake_file_json["by_platform"]["linux"]["uri"])
    return linux_uri.split(":")[1]


def download_binary(root_path, binary_name, binary_path):
    root_binary_resource_file_path = (
        f"{root_path}/build/external_resources/{binary_name}/resources.json"
    )
    tmp_binary_resource_file_path = binary_path + "_resources.json"

    if compare_files(root_binary_resource_file_path, tmp_binary_resource_file_path):
        print(f"Use {binary_name} binary from cache {binary_path}")
    else:
        binary_id = get_binary_id(root_binary_resource_file_path)
        devtools_registry_s3_url = "https://devtools-registry.s3.yandex.net"
        devtools_registry_s3_binary_url = f"{devtools_registry_s3_url}/{binary_id}"
        print(
            f"Download {binary_name} binary from {devtools_registry_s3_binary_url} into {binary_path}"
        )
        remove_file(binary_path)
        urllib.request.urlretrieve(devtools_registry_s3_binary_url, binary_path)
        make_file_executable(binary_path)

    shutil.copy(root_binary_resource_file_path, tmp_binary_resource_file_path)


def generate_graph_for_platform(generate_graph_for_platform):
    platform = generate_graph_for_platform[0]
    generate_graph_command = generate_graph_for_platform[1]

    output = subprocess.check_output(
        generate_graph_command, stderr=subprocess.STDOUT, shell=True
    ).decode("utf-8")

    allowed_error_patterns = [
        "to directory without ya.make: [[imp]]$S/build/platform/",
        "to missing directory: [[imp]]$S/build/platform/",
        "to directory without ya.make: [[imp]]$S/build/external_resources/",
        "to missing directory: [[imp]]$S/build/external_resources/",
        "could not resolve include file: [[imp]]openssl",
        "could not resolve include file: [[imp]]zlib",
        "could not resolve include file: [[imp]]ares.h",
        "in $B/contrib/libs/openssl/",
        "in $B/contrib/libs/zlib",
        "in $B/contrib/libs/c-ares",
        "in $B/contrib/libs/libc_compat/ubuntu_14/liblibs-libc_compat-ubuntu_14.a",
        "in $B/contrib/libs/linux-headers/libcontrib-libs-linux-headers.a",
        "in $B/contrib/libs/farmhash/",
        "in $B/contrib/libs/curl/",
        "in $B/contrib/libs/libxml/",
        "in $B/contrib/libs/apache/arrow/",
        "in $B/contrib/libs/grpc/",
        "in $S/contrib/tools/protoc/plugins/cpp_styleguide/ya.make",
        "in $S/contrib/tools/protoc/plugins/grpc_cpp",
        "in $B/contrib/restricted/boost/",
        "in $B/library/cpp/charset/",
        "in $B/library/cpp/uri/",
        "in $B/library/cpp/unicode/punycode/",
        "in $B/library/cpp/config/",
        "in $S/tools/rescompiler/bin/",
        # Fix
        "in $B/ydb/library/actors/dnsresolver/ut/library-cpp-actors-dnsresolver-ut",
        "in $B/ydb/library/pdisk_io/libydb-library-pdisk_io",
    ]

    if platform == "windows-x86_64":
        # Fix
        allowed_error_patterns.append("in $B/ydb/core/tx/tiering/core-tx-tiering")
        allowed_error_patterns.append(
            "in $B/ydb/library/yql/providers/s3/serializations/providers-s3-serializations"
        )

    result_errors = []
    for line in output.split("\n"):
        if not line.startswith("Error"):
            continue

        error_is_allowed = False
        for allowed_error_pattern in allowed_error_patterns:
            if allowed_error_pattern in line:
                error_is_allowed = True
                break

        if error_is_allowed:
            continue

        result_errors.append(line)

    return result_errors


if __name__ == "__main__":
    parser = ArgumentParser(description="Generate CMake files from Ya make files")
    parser.add_argument("--ymake_bin", help="Path to ymake binary")
    parser.add_argument("--yexport_bin", help="Path to yexport binary")
    parser.add_argument("--tmp", help="Path to tmp dir")
    parser.add_argument(
        "--debug", action="store_true", default=False, help="Run script in debug mode"
    )

    try:
        args = parser.parse_args()
    except Exception as e:
        print(e, file=sys.stderr)
        sys.exit(1)

    tmp_folder_path = args.tmp
    if tmp_folder_path is None:
        tmp_folder_path = "/tmp/ydb-generate-cmake"

    ymake_binary_path = args.ymake_bin
    yexport_binary_path = args.yexport_bin
    debug = args.debug

    ydb_tmp_folder_path = tmp_folder_path + "/ydb"
    ydb_metadata_folder_path = tmp_folder_path + "/metadata"
    ydb_bin_folder_path = tmp_folder_path + "/bin"
    plugins_folder_path = ydb_tmp_folder_path + "/build/plugins"

    mkdir(tmp_folder_path)
    mkdir(ydb_metadata_folder_path)
    mkdir(ydb_bin_folder_path)

    root_folder = os.getcwd()

    if ymake_binary_path is None:
        ymake_binary_path = ydb_bin_folder_path + "/ymake"
        download_binary(root_folder, "ymake", ymake_binary_path)

    if yexport_binary_path is None:
        yexport_binary_path = ydb_bin_folder_path + "/yexport"
        download_binary(root_folder, "yexport", yexport_binary_path)

    rmtree(ydb_tmp_folder_path)
    shutil.copytree(root_folder, ydb_tmp_folder_path)

    platforms = [
        ("linux-x86_64", "default-linux-x86_64"),
        ("linux-aarch64", "default-linux-aarch64"),
        ("darwin-x86_64", "default-darwin-x86_64"),
        ("windows-x86_64", "default-win-x86_64"),
    ]

    generate_graph_for_platform_commands = []

    for platform, target_platform in platforms:
        print(f"Platform {platform} target platform {target_platform}")

        dump_export_path = f"{ydb_metadata_folder_path}/{platform}.conf"
        graph_export_path = f"{ydb_metadata_folder_path}/sem.{platform}.json"

        generate_dump_command = f"{root_folder}/scripts/generate_dump.sh {platform} {target_platform} > {dump_export_path}"
        print(f"Generate dump command {generate_dump_command}")

        subprocess.check_output(generate_dump_command, shell=True)

        # In original script there are targets kikimr/docs/ru/docs_oss ydb ydb/tests/oss/launch library/cpp/actors tools/rescompiler/bin
        generate_graph_command = f'{ymake_binary_path} --build-root "{ydb_tmp_folder_path}" --config "{dump_export_path}"\
            --plugins-root "{plugins_folder_path}" --xs --xx --sem-graph --keep-going\
            ydb ydb/tests/oss/launch library/cpp/actors tools/rescompiler/bin > {graph_export_path}'
        print(f"Generate graph command {generate_graph_command}")

        generate_graph_for_platform_commands.append((platform, generate_graph_command))

    errors_for_platform = []
    with multiprocessing.Pool(len(generate_graph_for_platform_commands)) as pool:
        errors_for_platform = pool.map(
            generate_graph_for_platform, generate_graph_for_platform_commands
        )

    for index, (platform, target_platform) in enumerate(platforms):
        errors_for_platform_size = len(errors_for_platform[index])
        if errors_for_platform_size == 0:
            continue

        print(
            f"Found {errors_for_platform_size} errors for platform {platform}",
            file=sys.stderr,
        )
        for error in errors_for_platform[index]:
            print(error, file=sys.stderr)

        sys.exit(1)

    yexport_command = f"{yexport_binary_path} --export-root \"{ydb_tmp_folder_path}\" --target YDB \
            --semantic-graph \"{ydb_metadata_folder_path + '/sem.linux-x86_64.json'}\" --platforms linux-x86_64 \
            --semantic-graph \"{ydb_metadata_folder_path + '/sem.linux-aarch64.json'}\" --platforms linux-aarch64 \
            --semantic-graph \"{ydb_metadata_folder_path + '/sem.darwin-x86_64.json'}\" --platforms darwin-x86_64 \
            --semantic-graph \"{ydb_metadata_folder_path + '/sem.windows-x86_64.json'}\" --platforms windows-x86_64"
    print(f"yexport command {yexport_command}")

    yexport_output = subprocess.check_output(
        yexport_command, stderr=subprocess.STDOUT, shell=True
    ).decode("utf-8")

    if debug:
        print("yexport output")
        print(yexport_output)

    rsync_command = f'rsync --recursive --delete  --perms\
        --exclude .git --exclude contrib --exclude library/cpp/actors\
        "{ydb_tmp_folder_path}/" "{root_folder}/"'

    print(f"rsync command {rsync_command}")
    subprocess.check_output(rsync_command, shell=True)

    sys.exit(0)