Browse Source

Library import 7 (#937)

AlexSm 1 year ago
parent
commit
2e180154bd

+ 0 - 7
build/conf/go.conf

@@ -250,13 +250,11 @@ GO_LINK_EXE=
 GO_LINK_EXE+=$GENERATE_MF
 GO_LINK_EXE+=&& $GENERATE_VCS_C_INFO_NODEP
 GO_LINK_EXE+=&& $GENERATE_VCS_GO_INFO_NODEP
-GO_LINK_EXE+=&& $COPY_PROFILE_RUNTIME
 GO_LINK_EXE+=&& $_GO_LINK_EXE_IMPL($_GO_SRCS_VALUE CGO_FILES $_CGO_SRCS_VALUE EXTRA_TEXT_INPUTS $_GO_EMBED_INPUTS)
 
 GO_LINK_TEST=
 GO_LINK_TEST+=$GENERATE_VCS_C_INFO_NODEP
 GO_LINK_TEST+=&& $GENERATE_VCS_GO_INFO_NODEP
-GO_LINK_TEST+=&& $COPY_PROFILE_RUNTIME
 GO_LINK_TEST+=&& $_GO_LINK_TEST_IMPL($_GO_SRCS_VALUE CGO_FILES $_CGO_SRCS_VALUE EXTRA_TEXT_INPUTS $_GO_EMBED_INPUTS GO_TEST_FILES $_GO_TEST_SRCS_VALUE GO_XTEST_FILES $_GO_XTEST_SRCS_VALUE)
 
 GO_LINK_DLL=
@@ -873,11 +871,6 @@ module GO_PROGRAM: _GO_BASE_UNIT {
         _GO_BUILDMODE=++buildmode=pie
     }
 
-    when ($NEED_PROFILE_RUNTIME == "yes") {
-        LDFLAGS+=-resource-dir=contrib/libs/clang-rt
-        PEERDIR+=contrib/libs/clang${CLANG_VER}-rt/lib/profile
-    }
-
     when ($OPENSOURCE == "yes" && $AUTOCHECK == "yes") {
         # FIXME: Replace AUTOCHECK == yes with _not a host platform_ check after YMAKE-218
         MODULE_LICENSES_RESTRICTION_TYPES = ALLOW_ONLY

+ 3 - 3
build/external_resources/yexport/resources.json

@@ -1,13 +1,13 @@
 {
     "by_platform": {
         "darwin": {
-            "uri": "sbr:5562968267"
+            "uri": "sbr:5641379824"
         },
         "darwin-arm64": {
-            "uri": "sbr:5562965675"
+            "uri": "sbr:5641378801"
         },
         "linux": {
-            "uri": "sbr:5562947428"
+            "uri": "sbr:5641398951"
         }
     }
 }

+ 5 - 5
build/external_resources/ymake/public.resources.json

@@ -1,19 +1,19 @@
 {
     "by_platform": {
         "darwin": {
-            "uri": "sbr:5569062376"
+            "uri": "sbr:5622651213"
         },
         "darwin-arm64": {
-            "uri": "sbr:5569072879"
+            "uri": "sbr:5622657921"
         },
         "linux": {
-            "uri": "sbr:5569070627"
+            "uri": "sbr:5622653964"
         },
         "linux-aarch64": {
-            "uri": "sbr:5569095361"
+            "uri": "sbr:5622652638"
         },
         "win32-clang-cl": {
-            "uri": "sbr:5569066861"
+            "uri": "sbr:5622653810"
         }
     }
 }

+ 5 - 5
build/external_resources/ymake/resources.json

@@ -1,19 +1,19 @@
 {
     "by_platform": {
         "darwin": {
-            "uri": "sbr:5569055879"
+            "uri": "sbr:5622649543"
         },
         "darwin-arm64": {
-            "uri": "sbr:5569046692"
+            "uri": "sbr:5622648352"
         },
         "linux": {
-            "uri": "sbr:5569080477"
+            "uri": "sbr:5622649049"
         },
         "linux-aarch64": {
-            "uri": "sbr:5569058127"
+            "uri": "sbr:5622653320"
         },
         "win32-clang-cl": {
-            "uri": "sbr:5569069854"
+            "uri": "sbr:5622653040"
         }
     }
 }

+ 16 - 0
build/mapping.conf.json

@@ -127,34 +127,42 @@
         "5498749381": "https://devtools-registry.s3.yandex.net/5498749381",
         "5534092912": "https://devtools-registry.s3.yandex.net/5534092912",
         "5562971772": "https://devtools-registry.s3.yandex.net/5562971772",
+        "5641388522": "https://devtools-registry.s3.yandex.net/5641388522",
         "5486590469": "https://devtools-registry.s3.yandex.net/5486590469",
         "5498750509": "https://devtools-registry.s3.yandex.net/5498750509",
         "5534043120": "https://devtools-registry.s3.yandex.net/5534043120",
         "5562958333": "https://devtools-registry.s3.yandex.net/5562958333",
+        "5641392899": "https://devtools-registry.s3.yandex.net/5641392899",
         "5486590393": "https://devtools-registry.s3.yandex.net/5486590393",
         "5498735180": "https://devtools-registry.s3.yandex.net/5498735180",
         "5534059422": "https://devtools-registry.s3.yandex.net/5534059422",
         "5562961825": "https://devtools-registry.s3.yandex.net/5562961825",
+        "5641386025": "https://devtools-registry.s3.yandex.net/5641386025",
         "5476908047": "https://devtools-registry.s3.yandex.net/5476908047",
         "5509380757": "https://devtools-registry.s3.yandex.net/5509380757",
         "5550834592": "https://devtools-registry.s3.yandex.net/5550834592",
         "5569062376": "https://devtools-registry.s3.yandex.net/5569062376",
+        "5622651213": "https://devtools-registry.s3.yandex.net/5622651213",
         "5476891477": "https://devtools-registry.s3.yandex.net/5476891477",
         "5509369518": "https://devtools-registry.s3.yandex.net/5509369518",
         "5550838146": "https://devtools-registry.s3.yandex.net/5550838146",
         "5569072879": "https://devtools-registry.s3.yandex.net/5569072879",
+        "5622657921": "https://devtools-registry.s3.yandex.net/5622657921",
         "5476896849": "https://devtools-registry.s3.yandex.net/5476896849",
         "5509364024": "https://devtools-registry.s3.yandex.net/5509364024",
         "5550826871": "https://devtools-registry.s3.yandex.net/5550826871",
         "5569070627": "https://devtools-registry.s3.yandex.net/5569070627",
+        "5622653964": "https://devtools-registry.s3.yandex.net/5622653964",
         "5476895322": "https://devtools-registry.s3.yandex.net/5476895322",
         "5509360013": "https://devtools-registry.s3.yandex.net/5509360013",
         "5550843115": "https://devtools-registry.s3.yandex.net/5550843115",
         "5569095361": "https://devtools-registry.s3.yandex.net/5569095361",
+        "5622652638": "https://devtools-registry.s3.yandex.net/5622652638",
         "5476896707": "https://devtools-registry.s3.yandex.net/5476896707",
         "5509388614": "https://devtools-registry.s3.yandex.net/5509388614",
         "5550838970": "https://devtools-registry.s3.yandex.net/5550838970",
         "5569066861": "https://devtools-registry.s3.yandex.net/5569066861",
+        "5622653810": "https://devtools-registry.s3.yandex.net/5622653810",
         "2980468199": "https://devtools-registry.s3.yandex.net/2980468199",
         "5562224408": "https://devtools-registry.s3.yandex.net/5562224408"
     },
@@ -285,34 +293,42 @@
         "5498749381": "yexport for darwin",
         "5534092912": "yexport for darwin",
         "5562971772": "yexport for darwin",
+        "5641388522": "yexport for darwin",
         "5486590469": "yexport for darwin-arm64",
         "5498750509": "yexport for darwin-arm64",
         "5534043120": "yexport for darwin-arm64",
         "5562958333": "yexport for darwin-arm64",
+        "5641392899": "yexport for darwin-arm64",
         "5486590393": "yexport for linux",
         "5498735180": "yexport for linux",
         "5534059422": "yexport for linux",
         "5562961825": "yexport for linux",
+        "5641386025": "yexport for linux",
         "5476908047": "ymake for darwin",
         "5509380757": "ymake for darwin",
         "5550834592": "ymake for darwin",
         "5569062376": "ymake for darwin",
+        "5622651213": "ymake for darwin",
         "5476891477": "ymake for darwin-arm64",
         "5509369518": "ymake for darwin-arm64",
         "5550838146": "ymake for darwin-arm64",
         "5569072879": "ymake for darwin-arm64",
+        "5622657921": "ymake for darwin-arm64",
         "5476896849": "ymake for linux",
         "5509364024": "ymake for linux",
         "5550826871": "ymake for linux",
         "5569070627": "ymake for linux",
+        "5622653964": "ymake for linux",
         "5476895322": "ymake for linux-aarch64",
         "5509360013": "ymake for linux-aarch64",
         "5550843115": "ymake for linux-aarch64",
         "5569095361": "ymake for linux-aarch64",
+        "5622652638": "ymake for linux-aarch64",
         "5476896707": "ymake.exe for win32-clang-cl",
         "5509388614": "ymake.exe for win32-clang-cl",
         "5550838970": "ymake.exe for win32-clang-cl",
         "5569066861": "ymake.exe for win32-clang-cl",
+        "5622653810": "ymake.exe for win32-clang-cl",
         "2980468199": "ytexec for linux",
         "5562224408": "ytexec for linux"
     },

+ 1 - 0
build/plugins/lib/test_const/__init__.py

@@ -139,6 +139,7 @@ CLANG_COVERAGE_TEST_TYPES = (
     "coverage_extractor",
     "exectest",
     "gtest",
+    "go_test",
     # java tests might use shared libraries
     "java",
     "py2test",

+ 1 - 1
build/plugins/yql_python_udf.py

@@ -44,7 +44,7 @@ def onregister_yql_python_udf(unit, *args):
 
     path = name + '.yql_python_udf.cpp'
     libra_flag = '1' if add_libra_modules else '0'
-    unit.onpython(
+    unit.onrun_python2(
         [
             'build/scripts/gen_yql_python_udf.py',
             flavor,

+ 7 - 3
build/scripts/clang_static_analyzer.py

@@ -3,9 +3,9 @@ import sys
 import os
 import re
 import argparse
-import yaml
+import json
 
-CLANG_SA_CONFIG='static_analyzer.yaml'
+CLANG_SA_CONFIG='static_analyzer.json'
 
 def parse_args():
     parser = argparse.ArgumentParser()
@@ -31,7 +31,7 @@ def parse_config(config_file):
     conf = None
     try:
         with open(config_file, 'r') as afile:
-            conf = yaml.safe_load(afile)
+            conf = json.load(afile)
     except:
         conf = None
     return conf
@@ -61,6 +61,10 @@ def main():
     config_file = find_config(args.config_file)
     conf = parse_config(config_file)
 
+    # Ensure we can read config
+    if not conf:
+        raise ValueError(f"Cant parse config file, check its syntax: {config_file}")
+
     # Ensure we have at least one check
     if ('checks' not in conf) or (not conf['checks']):
         raise ValueError("There are no checks in the config file")

+ 201 - 0
build/scripts/create_recursive_library_for_cmake.py

@@ -0,0 +1,201 @@
+# Custom script is necessary because CMake does not yet support creating static libraries combined with dependencies
+# https://gitlab.kitware.com/cmake/cmake/-/issues/22975
+#
+# This script is intended to be used set as a CXX_LINKER_LAUNCHER property for recursive library targets.
+# It parses the linking command and transforms it to archiving commands combining static libraries from dependencies.
+
+import argparse
+import os
+import shlex
+import subprocess
+import sys
+import tempfile
+
+
+class Opts(object):
+    def __init__(self, args):
+        argparser = argparse.ArgumentParser(allow_abbrev=False)
+        argparser.add_argument('--cmake-binary-dir', required=True)
+        argparser.add_argument('--cmake-ar', required=True)
+        argparser.add_argument('--cmake-ranlib', required=True)
+        argparser.add_argument('--cmake-host-system-name', required=True)
+        argparser.add_argument('--cmake-cxx-standard-libraries')
+        argparser.add_argument('--global-part-suffix', required=True)
+        self.parsed_args, other_args = argparser.parse_known_args(args=args)
+
+        if len(other_args) < 2:
+            # must contain at least '--linking-cmdline' and orginal linking tool name
+            raise Exception('not enough arguments')
+        if other_args[0] != '--linking-cmdline':
+            raise Exception("expected '--linking-cmdline' arg, got {}".format(other_args[0]))
+
+        self.is_msvc_linker = other_args[1].endswith('\\link.exe')
+
+        is_host_system_windows = self.parsed_args.cmake_host_system_name == 'Windows'
+        std_libraries_to_exclude_from_input = (
+            set(self.parsed_args.cmake_cxx_standard_libraries.split())
+            if self.parsed_args.cmake_cxx_standard_libraries is not None
+            else set()
+        )
+        msvc_preserved_option_prefixes = [
+            'errorreport',
+            'machine:',
+            'nodefaultlib',
+            'nologo',
+            'subsystem',
+        ]
+
+        self.preserved_options = []
+
+        # these variables can contain paths absolute or relative to CMAKE_BINARY_DIR
+        self.global_libs_and_objects_input = []
+        self.non_global_libs_input = []
+        self.output = None
+
+        def is_external_library(path):
+            """
+            Check whether this library has been built in this CMake project or came from Conan-provided dependencies
+            (these use absolute paths).
+            If it is a library that is added from some other path (like CUDA) return True
+            """
+            return not (os.path.exists(path) or os.path.exists(os.path.join(self.parsed_args.cmake_binary_dir, path)))
+
+        def process_input(args):
+            i = 0
+            is_in_whole_archive = False
+
+            while i < len(args):
+                arg = args[i]
+                if is_host_system_windows and ((arg[0] == '/') or (arg[0] == '-')):
+                    arg_wo_specifier_lower = arg[1:].lower()
+                    if arg_wo_specifier_lower.startswith('out:'):
+                        self.output = arg[len('/out:') :]
+                    elif arg_wo_specifier_lower.startswith('wholearchive:'):
+                        lib_path = arg[len('/wholearchive:') :]
+                        if not is_external_library(lib_path):
+                            self.global_libs_and_objects_input.append(lib_path)
+                    else:
+                        for preserved_option_prefix in msvc_preserved_option_prefixes:
+                            if arg_wo_specifier_lower.startswith(preserved_option_prefix):
+                                self.preserved_options.append(arg)
+                                break
+                    # other flags are non-linking related and just ignored
+                elif arg[0] == '-':
+                    if arg == '-o':
+                        if (i + 1) >= len(args):
+                            raise Exception('-o flag without an argument')
+                        self.output = args[i + 1]
+                        i += 1
+                    elif arg == '-Wl,--whole-archive':
+                        is_in_whole_archive = True
+                    elif arg == '-Wl,--no-whole-archive':
+                        is_in_whole_archive = False
+                    elif arg.startswith('-Wl,-force_load,'):
+                        lib_path = arg[len('-Wl,-force_load,') :]
+                        if not is_external_library(lib_path):
+                            self.global_libs_and_objects_input.append(lib_path)
+                    elif arg == '-isysroot':
+                        i += 1
+                    # other flags are non-linking related and just ignored
+                elif arg[0] == '@':
+                    # response file with args
+                    with open(arg[1:]) as response_file:
+                        parsed_args = shlex.shlex(response_file, posix=False, punctuation_chars=False)
+                        parsed_args.whitespace_split = True
+                        args_in_response_file = list(arg.strip('"') for arg in parsed_args)
+                        process_input(args_in_response_file)
+                elif not is_external_library(arg):
+                    if is_in_whole_archive or arg.endswith('.o') or arg.endswith('.obj'):
+                        self.global_libs_and_objects_input.append(arg)
+                    elif arg not in std_libraries_to_exclude_from_input:
+                        self.non_global_libs_input.append(arg)
+                i += 1
+
+        process_input(other_args[2:])
+
+        if self.output is None:
+            raise Exception("No output specified")
+
+        if (len(self.global_libs_and_objects_input) == 0) and (len(self.non_global_libs_input) == 0):
+            raise Exception("List of input objects and libraries is empty")
+
+
+class FilesCombiner(object):
+    def __init__(self, opts):
+        self.opts = opts
+
+        archiver_tool_path = opts.parsed_args.cmake_ar
+        if sys.platform.startswith('darwin'):
+            # force LIBTOOL even if CMAKE_AR is defined because 'ar' under Darwin does not contain the necessary options
+            arch_type = 'LIBTOOL'
+            archiver_tool_path = 'libtool'
+        elif opts.is_msvc_linker:
+            arch_type = 'LIB'
+        elif opts.parsed_args.cmake_ar.endswith('llvm-ar'):
+            arch_type = 'LLVM_AR'
+        elif opts.parsed_args.cmake_ar.endswith('ar'):
+            arch_type = 'GNU_AR'
+        else:
+            raise Exception('Unsupported arch type for CMAKE_AR={}'.format(opts.parsed_args.cmake_ar))
+
+        self.archiving_cmd_prefix = [
+            sys.executable,
+            os.path.join(os.path.dirname(os.path.abspath(__file__)), 'link_lib.py'),
+            archiver_tool_path,
+            arch_type,
+            'gnu',  # llvm_ar_format, used only if arch_type == 'LLVM_AR'
+            opts.parsed_args.cmake_binary_dir,
+            'None',  # plugin. Unused for now
+        ]
+        # the remaining archiving cmd args are [output, .. input .. ]
+
+    def do(self, output, input_list):
+        input_file_path = None
+        try:
+            if self.opts.is_msvc_linker:
+                # use response file for input (because of Windows cmdline length limitations)
+
+                # can't use NamedTemporaryFile because of permissions issues on Windows
+                input_file_fd, input_file_path = tempfile.mkstemp()
+                try:
+                    input_file = os.fdopen(input_file_fd, 'w')
+                    for input in input_list:
+                        if ' ' in input:
+                            input_file.write('"{}" '.format(input))
+                        else:
+                            input_file.write('{} '.format(input))
+                    input_file.flush()
+                finally:
+                    os.close(input_file_fd)
+                input_args = ['@' + input_file_path]
+            else:
+                input_args = input_list
+
+            cmd = self.archiving_cmd_prefix + [output] + self.opts.preserved_options + input_args
+            subprocess.check_call(cmd)
+        finally:
+            if input_file_path is not None:
+                os.remove(input_file_path)
+
+        if not self.opts.is_msvc_linker:
+            subprocess.check_call([self.opts.parsed_args.cmake_ranlib, output])
+
+
+if __name__ == "__main__":
+    opts = Opts(sys.argv[1:])
+
+    output_prefix, output_ext = os.path.splitext(opts.output)
+    globals_output = output_prefix + opts.parsed_args.global_part_suffix + output_ext
+
+    if os.path.exists(globals_output):
+        os.remove(globals_output)
+    if os.path.exists(opts.output):
+        os.remove(opts.output)
+
+    files_combiner = FilesCombiner(opts)
+
+    if len(opts.global_libs_and_objects_input) > 0:
+        files_combiner.do(globals_output, opts.global_libs_and_objects_input)
+
+    if len(opts.non_global_libs_input) > 0:
+        files_combiner.do(opts.output, opts.non_global_libs_input)

+ 3 - 0
build/scripts/gather_swig_java.cmake

@@ -0,0 +1,3 @@
+file(GLOB JAVA_FILES ${JAVA_SRC_DIR}/*.java)
+list(JOIN JAVA_FILES "\n" JAVA_LST_CONTENT)
+file(WRITE ${JAVA_LST} ${JAVA_LST_CONTENT})

Some files were not shown because too many files changed in this diff