#!/usr/bin/env python3 import sys import os import shutil import subprocess import multiprocessing import json import stat import filecmp import urllib.request from argparse import ArgumentParser def mkdir(path): try: os.mkdir(path) except FileExistsError as e: pass def remove_file(path): try: os.remove(path) except FileNotFoundError as e: pass def compare_files(lhs_file_path, rhs_file_path): try: return filecmp.cmp(lhs_file_path, rhs_file_path) except FileNotFoundError as e: return False def rmtree(path): try: shutil.rmtree(path) except FileNotFoundError as e: pass def generate_graph_for_platform(generate_graph_for_platform): platform = generate_graph_for_platform[0] generate_graph_command = generate_graph_for_platform[1] output = subprocess.check_output( generate_graph_command, stderr=subprocess.STDOUT, shell=True ).decode("utf-8") allowed_error_patterns = [ "to directory without ya.make: [[imp]]$S/build/platform/", "to missing directory: [[imp]]$S/build/platform/", "to directory without ya.make: [[imp]]$S/build/external_resources/", "to missing directory: [[imp]]$S/build/external_resources/", "could not resolve include file: [[imp]]openssl", "could not resolve include file: [[imp]]zlib", "could not resolve include file: [[imp]]ares.h", "in $B/contrib/libs/openssl/", "in $B/contrib/libs/zlib", "in $B/contrib/libs/c-ares", "in $B/contrib/libs/libc_compat/ubuntu_14/liblibs-libc_compat-ubuntu_14.a", "in $B/contrib/libs/linux-headers/libcontrib-libs-linux-headers.a", "in $B/contrib/libs/farmhash/", "in $B/contrib/libs/curl/", "in $B/contrib/libs/libxml/", "in $B/contrib/libs/apache/arrow/", "in $B/contrib/libs/grpc/", "in $S/contrib/tools/protoc/plugins/cpp_styleguide/ya.make", "in $S/contrib/tools/protoc/plugins/grpc_cpp", "in $B/contrib/restricted/boost/", "in $B/library/cpp/charset/", "in $B/library/cpp/uri/", "in $B/library/cpp/unicode/punycode/", "in $B/library/cpp/config/", "in $S/tools/rescompiler/bin/", # Fix "in $B/ydb/library/actors/dnsresolver/ut/library-cpp-actors-dnsresolver-ut", "in $B/ydb/library/pdisk_io/libydb-library-pdisk_io", "skip unknown statement: ADD_YTEST vector of size", "skip unknown statement: _REGISTER_NO_CHECK_IMPORTS vector of size", "skip unknown statement: CHECK_CONTRIB_CREDITS vector of size", "skip unknown statement: ASSERT vector of size", "skip unknown statement: FILES vector of size", "skip unknown statement: ADD_CHECK vector of size", "skip unknown statement: COPY vector of size", "skip unknown statement: PY_EXTRALIBS vector of size", "skip unknown statement: LLVM_BC vector of size", "skip unknown statement: SUPPRESSIONS vector of size", ". It is not intended for export.", ] if platform == "windows-x86_64": # Fix allowed_error_patterns.append("in $B/ydb/core/tx/tiering/core-tx-tiering") allowed_error_patterns.append( "in $B/ydb/library/yql/providers/s3/serializations/providers-s3-serializations" ) result_errors = [] for line in output.split("\n"): if not line.startswith("Error"): continue error_is_allowed = False for allowed_error_pattern in allowed_error_patterns: if allowed_error_pattern in line: error_is_allowed = True break if error_is_allowed: continue result_errors.append(line) return result_errors if __name__ == "__main__": parser = ArgumentParser(description="Generate CMake files from Ya make files") parser.add_argument("--ymake_bin", help="Path to ymake binary") parser.add_argument("--yexport_bin", help="Path to yexport binary") parser.add_argument("--tmp", help="Path to tmp dir") parser.add_argument( "-k", "--keep-going", action="store_true", default=False, help="Ignore unknown build graph errors and try to perform export") parser.add_argument( "--debug", action="store_true", default=False, help="Run script in debug mode" ) try: args = parser.parse_args() except Exception as e: print(e, file=sys.stderr) sys.exit(1) tmp_folder_path = args.tmp if tmp_folder_path is None: tmp_folder_path = "/tmp/ydb-generate-cmake" ymake_binary_path = args.ymake_bin yexport_binary_path = args.yexport_bin debug = args.debug keep_going = args.keep_going root_folder = os.getcwd() ydb_tmp_folder_path = tmp_folder_path + "/ydb" ydb_metadata_folder_path = tmp_folder_path + "/metadata" plugins_folder_path = root_folder + "/build/plugins" mkdir(tmp_folder_path) mkdir(ydb_metadata_folder_path) if ymake_binary_path is None: ymake_binary_path = root_folder + "/ya tool ymake" if yexport_binary_path is None: libiconv_path="contrib/libs/libiconv/dynamic" compile_libiconv_command = f"{root_folder}/ya make -r {libiconv_path}" print("Compliling libiconv...") subprocess.check_output(compile_libiconv_command, shell=True) yexport_binary_path = f"LD_LIBRARY_PATH={libiconv_path} {root_folder}/ya tool yexport" platforms = [ ("linux-x86_64", "default-linux-x86_64"), ("linux-aarch64", "default-linux-aarch64"), ("darwin-x86_64", "default-darwin-x86_64"), ("windows-x86_64", "default-win-x86_64"), ("darwin-arm64", "default-darwin-arm64"), ] generate_graph_for_platform_commands = [] for platform, target_platform in platforms: print(f"Platform {platform} target platform {target_platform}") dump_export_path = f"{ydb_metadata_folder_path}/{platform}.conf" graph_export_path = f"{ydb_metadata_folder_path}/sem.{platform}.json" generate_dump_command = f"{root_folder}/scripts/generate_dump.sh {platform} {target_platform} > {dump_export_path}" print(f"Generate dump command {generate_dump_command}") subprocess.check_output(generate_dump_command, shell=True) # In original script there are targets kikimr/docs/ru/docs_oss ydb ydb/tests/oss/launch library/cpp/actors tools/rescompiler/bin generate_graph_command = f'{ymake_binary_path} --build-root "{ydb_tmp_folder_path}" --config "{dump_export_path}"\ --plugins-root "{plugins_folder_path}" --xs --xx --sem-graph --keep-going\ ydb ydb/tests/oss/launch tools/rescompiler/bin > {graph_export_path}' print(f"Generate graph command {generate_graph_command}") generate_graph_for_platform_commands.append((platform, generate_graph_command)) errors_for_platform = [] with multiprocessing.Pool(len(generate_graph_for_platform_commands)) as pool: errors_for_platform = pool.map( generate_graph_for_platform, generate_graph_for_platform_commands ) for index, (platform, target_platform) in enumerate(platforms): errors_for_platform_size = len(errors_for_platform[index]) if errors_for_platform_size == 0: continue print( f"Found {errors_for_platform_size} errors for platform {platform}", file=sys.stderr, ) for error in errors_for_platform[index]: print(error, file=sys.stderr) if not keep_going: sys.exit(1) yexport_command = f"{yexport_binary_path} --export-root \"{root_folder}\" --target YDB \ --semantic-graph \"{ydb_metadata_folder_path + '/sem.linux-x86_64.json'}\" --platforms linux-x86_64 \ --semantic-graph \"{ydb_metadata_folder_path + '/sem.linux-aarch64.json'}\" --platforms linux-aarch64 \ --semantic-graph \"{ydb_metadata_folder_path + '/sem.darwin-x86_64.json'}\" --platforms darwin-x86_64 \ --semantic-graph \"{ydb_metadata_folder_path + '/sem.darwin-arm64.json'}\" --platforms darwin-arm64 \ --semantic-graph \"{ydb_metadata_folder_path + '/sem.windows-x86_64.json'}\" --platforms windows-x86_64 --generator cmake" # yexport_command = f"{yexport_binary_path} --export-root \"{ydb_tmp_folder_path}\" --target YDB \ # --semantic-graph \"{ydb_metadata_folder_path + '/sem.darwin-x86_64.json'}\" --platforms darwin-x86_64" print(f"yexport command {yexport_command}") yexport_output = subprocess.check_output( yexport_command, stderr=subprocess.STDOUT, shell=True ).decode("utf-8") if debug: print("yexport output") print(yexport_output) sys.exit(0) rsync_command = f'rsync --recursive --delete --perms\ --exclude .git --exclude contrib --exclude library/cpp/actors\ "{ydb_tmp_folder_path}/" "{root_folder}/"' print(f"rsync command {rsync_command}") subprocess.check_output(rsync_command, shell=True) sys.exit(0)