generate_cmake 8.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241
  1. #!/usr/bin/env python3
  2. import sys
  3. import os
  4. import shutil
  5. import subprocess
  6. import multiprocessing
  7. import json
  8. import stat
  9. import filecmp
  10. import urllib.request
  11. from argparse import ArgumentParser
  12. def mkdir(path):
  13. try:
  14. os.mkdir(path)
  15. except FileExistsError as e:
  16. pass
  17. def remove_file(path):
  18. try:
  19. os.remove(path)
  20. except FileNotFoundError as e:
  21. pass
  22. def compare_files(lhs_file_path, rhs_file_path):
  23. try:
  24. return filecmp.cmp(lhs_file_path, rhs_file_path)
  25. except FileNotFoundError as e:
  26. return False
  27. def rmtree(path):
  28. try:
  29. shutil.rmtree(path)
  30. except FileNotFoundError as e:
  31. pass
  32. def generate_graph_for_platform(generate_graph_for_platform):
  33. platform = generate_graph_for_platform[0]
  34. generate_graph_command = generate_graph_for_platform[1]
  35. output = subprocess.check_output(
  36. generate_graph_command, stderr=subprocess.STDOUT, shell=True
  37. ).decode("utf-8")
  38. allowed_error_patterns = [
  39. "to directory without ya.make: [[imp]]$S/build/platform/",
  40. "to missing directory: [[imp]]$S/build/platform/",
  41. "to directory without ya.make: [[imp]]$S/build/external_resources/",
  42. "to missing directory: [[imp]]$S/build/external_resources/",
  43. "could not resolve include file: [[imp]]openssl",
  44. "could not resolve include file: [[imp]]zlib",
  45. "could not resolve include file: [[imp]]ares.h",
  46. "in $B/contrib/libs/openssl/",
  47. "in $B/contrib/libs/zlib",
  48. "in $B/contrib/libs/c-ares",
  49. "in $B/contrib/libs/libc_compat/ubuntu_14/liblibs-libc_compat-ubuntu_14.a",
  50. "in $B/contrib/libs/linux-headers/libcontrib-libs-linux-headers.a",
  51. "in $B/contrib/libs/farmhash/",
  52. "in $B/contrib/libs/curl/",
  53. "in $B/contrib/libs/libxml/",
  54. "in $B/contrib/libs/apache/arrow/",
  55. "in $B/contrib/libs/grpc/",
  56. "in $S/contrib/tools/protoc/plugins/cpp_styleguide/ya.make",
  57. "in $S/contrib/tools/protoc/plugins/grpc_cpp",
  58. "in $B/contrib/restricted/boost/",
  59. "in $B/library/cpp/charset/",
  60. "in $B/library/cpp/uri/",
  61. "in $B/library/cpp/unicode/punycode/",
  62. "in $B/library/cpp/config/",
  63. "in $S/tools/rescompiler/bin/",
  64. # Fix
  65. "in $B/ydb/library/actors/dnsresolver/ut/library-cpp-actors-dnsresolver-ut",
  66. "in $B/ydb/library/pdisk_io/libydb-library-pdisk_io",
  67. "skip unknown statement: ADD_YTEST vector of size",
  68. "skip unknown statement: _REGISTER_NO_CHECK_IMPORTS vector of size",
  69. "skip unknown statement: CHECK_CONTRIB_CREDITS vector of size",
  70. "skip unknown statement: ASSERT vector of size",
  71. "skip unknown statement: FILES vector of size",
  72. "skip unknown statement: ADD_CHECK vector of size",
  73. "skip unknown statement: COPY vector of size",
  74. "skip unknown statement: PY_EXTRALIBS vector of size",
  75. "skip unknown statement: LLVM_BC vector of size",
  76. "skip unknown statement: SUPPRESSIONS vector of size",
  77. ". It is not intended for export.",
  78. ]
  79. if platform == "windows-x86_64":
  80. # Fix
  81. allowed_error_patterns.append("in $B/ydb/core/tx/tiering/core-tx-tiering")
  82. allowed_error_patterns.append(
  83. "in $B/ydb/library/yql/providers/s3/serializations/providers-s3-serializations"
  84. )
  85. result_errors = []
  86. for line in output.split("\n"):
  87. if not line.startswith("Error"):
  88. continue
  89. error_is_allowed = False
  90. for allowed_error_pattern in allowed_error_patterns:
  91. if allowed_error_pattern in line:
  92. error_is_allowed = True
  93. break
  94. if error_is_allowed:
  95. continue
  96. result_errors.append(line)
  97. return result_errors
  98. if __name__ == "__main__":
  99. parser = ArgumentParser(description="Generate CMake files from Ya make files")
  100. parser.add_argument("--ymake_bin", help="Path to ymake binary")
  101. parser.add_argument("--yexport_bin", help="Path to yexport binary")
  102. parser.add_argument("--tmp", help="Path to tmp dir")
  103. parser.add_argument(
  104. "-k", "--keep-going", action="store_true", default=False, help="Ignore unknown build graph errors and try to perform export")
  105. parser.add_argument(
  106. "--debug", action="store_true", default=False, help="Run script in debug mode"
  107. )
  108. try:
  109. args = parser.parse_args()
  110. except Exception as e:
  111. print(e, file=sys.stderr)
  112. sys.exit(1)
  113. tmp_folder_path = args.tmp
  114. if tmp_folder_path is None:
  115. tmp_folder_path = "/tmp/ydb-generate-cmake"
  116. ymake_binary_path = args.ymake_bin
  117. yexport_binary_path = args.yexport_bin
  118. debug = args.debug
  119. keep_going = args.keep_going
  120. ydb_tmp_folder_path = tmp_folder_path + "/ydb"
  121. ydb_metadata_folder_path = tmp_folder_path + "/metadata"
  122. plugins_folder_path = ydb_tmp_folder_path + "/build/plugins"
  123. mkdir(tmp_folder_path)
  124. mkdir(ydb_metadata_folder_path)
  125. root_folder = os.getcwd()
  126. if ymake_binary_path is None:
  127. ymake_binary_path = root_folder + "/ya tool ymake"
  128. if yexport_binary_path is None:
  129. libiconv_path="contrib/libs/libiconv/dynamic"
  130. compile_libiconv_command = f"{root_folder}/ya make -r {libiconv_path}"
  131. print("Compliling libiconv...")
  132. subprocess.check_output(compile_libiconv_command, shell=True)
  133. yexport_binary_path = f"LD_LIBRARY_PATH={libiconv_path} {root_folder}/ya tool yexport"
  134. platforms = [
  135. ("linux-x86_64", "default-linux-x86_64"),
  136. ("linux-aarch64", "default-linux-aarch64"),
  137. ("darwin-x86_64", "default-darwin-x86_64"),
  138. ("windows-x86_64", "default-win-x86_64"),
  139. ("darwin-arm64", "default-darwin-arm64"),
  140. ]
  141. generate_graph_for_platform_commands = []
  142. for platform, target_platform in platforms:
  143. print(f"Platform {platform} target platform {target_platform}")
  144. dump_export_path = f"{ydb_metadata_folder_path}/{platform}.conf"
  145. graph_export_path = f"{ydb_metadata_folder_path}/sem.{platform}.json"
  146. generate_dump_command = f"{root_folder}/scripts/generate_dump.sh {platform} {target_platform} > {dump_export_path}"
  147. print(f"Generate dump command {generate_dump_command}")
  148. subprocess.check_output(generate_dump_command, shell=True)
  149. # In original script there are targets kikimr/docs/ru/docs_oss ydb ydb/tests/oss/launch library/cpp/actors tools/rescompiler/bin
  150. generate_graph_command = f'{ymake_binary_path} --build-root "{ydb_tmp_folder_path}" --config "{dump_export_path}"\
  151. --plugins-root "{plugins_folder_path}" --xs --xx --sem-graph --keep-going\
  152. ydb ydb/tests/oss/launch tools/rescompiler/bin > {graph_export_path}'
  153. print(f"Generate graph command {generate_graph_command}")
  154. generate_graph_for_platform_commands.append((platform, generate_graph_command))
  155. errors_for_platform = []
  156. with multiprocessing.Pool(len(generate_graph_for_platform_commands)) as pool:
  157. errors_for_platform = pool.map(
  158. generate_graph_for_platform, generate_graph_for_platform_commands
  159. )
  160. for index, (platform, target_platform) in enumerate(platforms):
  161. errors_for_platform_size = len(errors_for_platform[index])
  162. if errors_for_platform_size == 0:
  163. continue
  164. print(
  165. f"Found {errors_for_platform_size} errors for platform {platform}",
  166. file=sys.stderr,
  167. )
  168. for error in errors_for_platform[index]:
  169. print(error, file=sys.stderr)
  170. if not keep_going:
  171. sys.exit(1)
  172. yexport_command = f"{yexport_binary_path} --export-root \"{root_folder}\" --target YDB \
  173. --semantic-graph \"{ydb_metadata_folder_path + '/sem.linux-x86_64.json'}\" --platforms linux-x86_64 \
  174. --semantic-graph \"{ydb_metadata_folder_path + '/sem.linux-aarch64.json'}\" --platforms linux-aarch64 \
  175. --semantic-graph \"{ydb_metadata_folder_path + '/sem.darwin-x86_64.json'}\" --platforms darwin-x86_64 \
  176. --semantic-graph \"{ydb_metadata_folder_path + '/sem.darwin-arm64.json'}\" --platforms darwin-arm64 \
  177. --semantic-graph \"{ydb_metadata_folder_path + '/sem.windows-x86_64.json'}\" --platforms windows-x86_64"
  178. # yexport_command = f"{yexport_binary_path} --export-root \"{ydb_tmp_folder_path}\" --target YDB \
  179. # --semantic-graph \"{ydb_metadata_folder_path + '/sem.darwin-x86_64.json'}\" --platforms darwin-x86_64"
  180. print(f"yexport command {yexport_command}")
  181. yexport_output = subprocess.check_output(
  182. yexport_command, stderr=subprocess.STDOUT, shell=True
  183. ).decode("utf-8")
  184. if debug:
  185. print("yexport output")
  186. print(yexport_output)
  187. sys.exit(0)
  188. rsync_command = f'rsync --recursive --delete --perms\
  189. --exclude .git --exclude contrib --exclude library/cpp/actors\
  190. "{ydb_tmp_folder_path}/" "{root_folder}/"'
  191. print(f"rsync command {rsync_command}")
  192. subprocess.check_output(rsync_command, shell=True)
  193. sys.exit(0)