ydb/generate_cmake

#!/usr/bin/env python3

import sys
import os
import shutil
import subprocess
import multiprocessing
import json
import stat
import filecmp
import urllib.request
from argparse import ArgumentParser


def mkdir(path):
    try:
        os.mkdir(path)
    except FileExistsError as e:
        pass


def remove_file(path):
    try:
        os.remove(path)
    except FileNotFoundError as e:
        pass


def compare_files(lhs_file_path, rhs_file_path):
    try:
        return filecmp.cmp(lhs_file_path, rhs_file_path)
    except FileNotFoundError as e:
        return False


def rmtree(path):
    try:
        shutil.rmtree(path)
    except FileNotFoundError as e:
        pass


def generate_graph_for_platform(generate_graph_for_platform):
    platform = generate_graph_for_platform[0]
    generate_graph_command = generate_graph_for_platform[1]

    output = subprocess.check_output(
        generate_graph_command, stderr=subprocess.STDOUT, shell=True
    ).decode("utf-8")

    allowed_error_patterns = [
        "to directory without ya.make: [[imp]]$S/build/platform/",
        "to missing directory: [[imp]]$S/build/platform/",
        "to directory without ya.make: [[imp]]$S/build/external_resources/",
        "to missing directory: [[imp]]$S/build/external_resources/",
        "could not resolve include file: [[imp]]openssl",
        "could not resolve include file: [[imp]]zlib",
        "could not resolve include file: [[imp]]ares.h",
        "in $B/contrib/libs/openssl/",
        "in $B/contrib/libs/zlib",
        "in $B/contrib/libs/c-ares",
        "in $B/contrib/libs/libc_compat/ubuntu_14/liblibs-libc_compat-ubuntu_14.a",
        "in $B/contrib/libs/linux-headers/libcontrib-libs-linux-headers.a",
        "in $B/contrib/libs/farmhash/",
        "in $B/contrib/libs/curl/",
        "in $B/contrib/libs/libxml/",
        "in $B/contrib/libs/apache/arrow/",
        "in $B/contrib/libs/grpc/",
        "in $S/contrib/tools/protoc/plugins/cpp_styleguide/ya.make",
        "in $S/contrib/tools/protoc/plugins/grpc_cpp",
        "in $B/contrib/restricted/boost/",
        "in $B/library/cpp/charset/",
        "in $B/library/cpp/uri/",
        "in $B/library/cpp/unicode/punycode/",
        "in $B/library/cpp/config/",
        "in $S/tools/rescompiler/bin/",
        # Fix
        "in $B/ydb/library/actors/dnsresolver/ut/library-cpp-actors-dnsresolver-ut",
        "in $B/ydb/library/pdisk_io/libydb-library-pdisk_io",
        "skip unknown statement: ADD_YTEST vector of size",
        "skip unknown statement: _REGISTER_NO_CHECK_IMPORTS vector of size",
        "skip unknown statement: CHECK_CONTRIB_CREDITS vector of size",
        "skip unknown statement: ASSERT vector of size",
        "skip unknown statement: FILES vector of size",
        "skip unknown statement: ADD_CHECK vector of size",
        "skip unknown statement: COPY vector of size",
        "skip unknown statement: PY_EXTRALIBS vector of size",
        "skip unknown statement: LLVM_BC vector of size",
        "skip unknown statement: SUPPRESSIONS vector of size",
        ". It is not intended for export.",
    ]

    if platform == "windows-x86_64":
        # Fix
        allowed_error_patterns.append("in $B/ydb/core/tx/tiering/core-tx-tiering")
        allowed_error_patterns.append(
            "in $B/ydb/library/yql/providers/s3/serializations/providers-s3-serializations"
        )

    result_errors = []
    for line in output.split("\n"):
        if not line.startswith("Error"):
            continue

        error_is_allowed = False
        for allowed_error_pattern in allowed_error_patterns:
            if allowed_error_pattern in line:
                error_is_allowed = True
                break

        if error_is_allowed:
            continue

        result_errors.append(line)

    return result_errors


if __name__ == "__main__":
    parser = ArgumentParser(description="Generate CMake files from Ya make files")
    parser.add_argument("--ymake_bin", help="Path to ymake binary")
    parser.add_argument("--yexport_bin", help="Path to yexport binary")
    parser.add_argument("--tmp", help="Path to tmp dir")
    parser.add_argument(
        "-k", "--keep-going", action="store_true", default=False, help="Ignore unknown build graph errors and try to perform export")
    parser.add_argument(
        "--debug", action="store_true", default=False, help="Run script in debug mode"
    )

    try:
        args = parser.parse_args()
    except Exception as e:
        print(e, file=sys.stderr)
        sys.exit(1)

    tmp_folder_path = args.tmp
    if tmp_folder_path is None:
        tmp_folder_path = "/tmp/ydb-generate-cmake"

    ymake_binary_path = args.ymake_bin
    yexport_binary_path = args.yexport_bin
    debug = args.debug
    keep_going = args.keep_going

    ydb_tmp_folder_path = tmp_folder_path + "/ydb"
    ydb_metadata_folder_path = tmp_folder_path + "/metadata"
    plugins_folder_path = ydb_tmp_folder_path + "/build/plugins"

    mkdir(tmp_folder_path)
    mkdir(ydb_metadata_folder_path)

    root_folder = os.getcwd()

    if ymake_binary_path is None:
        ymake_binary_path = root_folder + "/ya tool ymake"

    if yexport_binary_path is None:
        libiconv_path="contrib/libs/libiconv/dynamic"
        compile_libiconv_command = f"{root_folder}/ya make -r {libiconv_path}"
        print("Compliling libiconv...")
        subprocess.check_output(compile_libiconv_command, shell=True)
        yexport_binary_path = f"LD_LIBRARY_PATH={libiconv_path} {root_folder}/ya tool yexport"

    platforms = [
        ("linux-x86_64", "default-linux-x86_64"),
        ("linux-aarch64", "default-linux-aarch64"),
        ("darwin-x86_64", "default-darwin-x86_64"),
        ("windows-x86_64", "default-win-x86_64"),
        ("darwin-arm64", "default-darwin-arm64"),
    ]

    generate_graph_for_platform_commands = []

    for platform, target_platform in platforms:
        print(f"Platform {platform} target platform {target_platform}")

        dump_export_path = f"{ydb_metadata_folder_path}/{platform}.conf"
        graph_export_path = f"{ydb_metadata_folder_path}/sem.{platform}.json"

        generate_dump_command = f"{root_folder}/scripts/generate_dump.sh {platform} {target_platform} > {dump_export_path}"
        print(f"Generate dump command {generate_dump_command}")

        subprocess.check_output(generate_dump_command, shell=True)

        # In original script there are targets kikimr/docs/ru/docs_oss ydb ydb/tests/oss/launch library/cpp/actors tools/rescompiler/bin
        generate_graph_command = f'{ymake_binary_path} --build-root "{ydb_tmp_folder_path}" --config "{dump_export_path}"\
            --plugins-root "{plugins_folder_path}" --xs --xx --sem-graph --keep-going\
            ydb ydb/tests/oss/launch tools/rescompiler/bin > {graph_export_path}'
        print(f"Generate graph command {generate_graph_command}")

        generate_graph_for_platform_commands.append((platform, generate_graph_command))

    errors_for_platform = []
    with multiprocessing.Pool(len(generate_graph_for_platform_commands)) as pool:
        errors_for_platform = pool.map(
            generate_graph_for_platform, generate_graph_for_platform_commands
        )

    for index, (platform, target_platform) in enumerate(platforms):
        errors_for_platform_size = len(errors_for_platform[index])
        if errors_for_platform_size == 0:
            continue

        print(
            f"Found {errors_for_platform_size} errors for platform {platform}",
            file=sys.stderr,
        )
        for error in errors_for_platform[index]:
            print(error, file=sys.stderr)

        if not keep_going:
            sys.exit(1)

    yexport_command = f"{yexport_binary_path} --export-root \"{root_folder}\" --target YDB \
            --semantic-graph \"{ydb_metadata_folder_path + '/sem.linux-x86_64.json'}\" --platforms linux-x86_64 \
            --semantic-graph \"{ydb_metadata_folder_path + '/sem.linux-aarch64.json'}\" --platforms linux-aarch64 \
            --semantic-graph \"{ydb_metadata_folder_path + '/sem.darwin-x86_64.json'}\" --platforms darwin-x86_64 \
            --semantic-graph \"{ydb_metadata_folder_path + '/sem.darwin-arm64.json'}\" --platforms darwin-arm64 \
            --semantic-graph \"{ydb_metadata_folder_path + '/sem.windows-x86_64.json'}\" --platforms windows-x86_64"
#    yexport_command = f"{yexport_binary_path} --export-root \"{ydb_tmp_folder_path}\" --target YDB \
#            --semantic-graph \"{ydb_metadata_folder_path + '/sem.darwin-x86_64.json'}\" --platforms darwin-x86_64"
    print(f"yexport command {yexport_command}")

    yexport_output = subprocess.check_output(
        yexport_command, stderr=subprocess.STDOUT, shell=True
    ).decode("utf-8")

    if debug:
        print("yexport output")
        print(yexport_output)

    sys.exit(0)

    rsync_command = f'rsync --recursive --delete  --perms\
        --exclude .git --exclude contrib --exclude library/cpp/actors\
        "{ydb_tmp_folder_path}/" "{root_folder}/"'

    print(f"rsync command {rsync_command}")
    subprocess.check_output(rsync_command, shell=True)

    sys.exit(0)