from __future__ import print_function import sys import os import subprocess import tempfile import collections import optparse import pipes import thinlto_cache import link_exe from process_whole_archive_option import ProcessWholeArchiveOption from fix_py2_protobuf import fix_py2 def shlex_join(cmd): # equivalent to shlex.join() in python 3 return ' '.join(pipes.quote(part) for part in cmd) def parse_export_file(p): with open(p, 'r') as f: for l in f: l = l.strip() if l and '#' not in l: words = l.split() if len(words) == 2 and words[0] == 'linux_version': yield {'linux_version': words[1]} elif len(words) == 2: yield {'lang': words[0], 'sym': words[1]} elif len(words) == 1: yield {'lang': 'C', 'sym': words[0]} else: raise Exception('unsupported exports line: ' + l) def to_c(sym): symbols = collections.deque(sym.split('::')) c_prefixes = [ # demangle prefixes for c++ symbols '_ZN', # namespace '_ZTIN', # typeinfo for '_ZTSN', # typeinfo name for '_ZTTN', # VTT for '_ZTVN', # vtable for '_ZNK', # const methods ] c_sym = '' while symbols: s = symbols.popleft() if s == '*': c_sym += '*' break if '*' in s and len(s) > 1: raise Exception('Unsupported format, cannot guess length of symbol: ' + s) c_sym += str(len(s)) + s if symbols: raise Exception('Unsupported format: ' + sym) if c_sym[-1] != '*': c_sym += 'E*' return ['{prefix}{sym}'.format(prefix=prefix, sym=c_sym) for prefix in c_prefixes] def fix_darwin_param(ex): for item in ex: if item.get('linux_version'): continue if item['lang'] == 'C': yield '-Wl,-exported_symbol,_' + item['sym'] elif item['lang'] == 'C++': for sym in to_c(item['sym']): yield '-Wl,-exported_symbol,_' + sym else: raise Exception('unsupported lang: ' + item['lang']) def fix_gnu_param(arch, ex): d = collections.defaultdict(list) version = None for item in ex: if item.get('linux_version'): if not version: version = item.get('linux_version') else: raise Exception('More than one linux_version defined') elif item['lang'] == 'C++': d['C'].extend(to_c(item['sym'])) else: d[item['lang']].append(item['sym']) with tempfile.NamedTemporaryFile(mode='wt', delete=False) as f: if version: f.write('{} {{\nglobal:\n'.format(version)) else: f.write('{\nglobal:\n') for k, v in d.items(): f.write(' extern "' + k + '" {\n') for x in v: f.write(' ' + x + ';\n') f.write(' };\n') f.write('local: *;\n};\n') ret = ['-Wl,--version-script=' + f.name] if arch == 'ANDROID': ret += ['-Wl,--export-dynamic'] return ret def fix_windows_param(ex): with tempfile.NamedTemporaryFile(delete=False) as def_file: exports = [] for item in ex: if item.get('lang') == 'C': exports.append(item.get('sym')) def_file.write('EXPORTS\n') for export in exports: def_file.write(' {}\n'.format(export)) return ['/DEF:{}'.format(def_file.name)] MUSL_LIBS = '-lc', '-lcrypt', '-ldl', '-lm', '-lpthread', '-lrt', '-lutil' CUDA_LIBRARIES = { '-lcublas_static': '-lcublas', '-lcublasLt_static': '-lcublasLt', '-lcudart_static': '-lcudart', '-lcudnn_static': '-lcudnn', '-lcufft_static_nocallback': '-lcufft', '-lcurand_static': '-lcurand', '-lcusolver_static': '-lcusolver', '-lcusparse_static': '-lcusparse', '-lmyelin_compiler_static': '-lmyelin', '-lmyelin_executor_static': '-lnvcaffe_parser', '-lmyelin_pattern_library_static': '', '-lmyelin_pattern_runtime_static': '', '-lnvinfer_static': '-lnvinfer', '-lnvinfer_plugin_static': '-lnvinfer_plugin', '-lnvonnxparser_static': '-lnvonnxparser', '-lnvparsers_static': '-lnvparsers', } def fix_cmd(arch, c): if arch == 'WINDOWS': prefix = '/DEF:' f = fix_windows_param else: prefix = '-Wl,--version-script=' if arch in ('DARWIN', 'IOS', 'IOSSIM'): f = fix_darwin_param else: f = lambda x: fix_gnu_param(arch, x) def do_fix(p): if p.startswith(prefix) and p.endswith('.exports'): fname = p[len(prefix) :] return list(f(list(parse_export_file(fname)))) if p.endswith('.supp'): return [] if p.endswith('.pkg.fake'): return [] return [p] return sum((do_fix(x) for x in c), []) def fix_cmd_for_musl(cmd): flags = [] for flag in cmd: if flag not in MUSL_LIBS: flags.append(flag) return flags def fix_cmd_for_dynamic_cuda(cmd): flags = [] for flag in cmd: if flag in CUDA_LIBRARIES: flags.append(CUDA_LIBRARIES[flag]) else: flags.append(flag) return flags def fix_blas_resolving(cmd): # Intel mkl comes as a precompiled static library and thus can not be recompiled with sanitizer runtime instrumentation. # That's why we prefer to use cblas instead of Intel mkl as a drop-in replacement under sanitizers. # But if the library has dependencies on mkl and cblas simultaneously, it will get a linking error. # Hence we assume that it's probably compiling without sanitizers and we can easily remove cblas to prevent multiple definitions of the same symbol at link time. for arg in cmd: if arg.startswith('contrib/libs') and arg.endswith('mkl-lp64.a'): return [arg for arg in cmd if not arg.endswith('libcontrib-libs-cblas.a')] return cmd def parse_args(): parser = optparse.OptionParser() parser.disable_interspersed_args() parser.add_option('--arch') parser.add_option('--target') parser.add_option('--soname') parser.add_option('--source-root') parser.add_option('--build-root') parser.add_option('--fix-elf') parser.add_option('--linker-output') parser.add_option('--musl', action='store_true') parser.add_option('--dynamic-cuda', action='store_true') parser.add_option('--cuda-architectures', help='List of supported CUDA architectures, separated by ":" (e.g. "sm_52:compute_70:lto_90a"') parser.add_option('--nvprune-exe') parser.add_option('--objcopy-exe') parser.add_option('--whole-archive-peers', action='append') parser.add_option('--whole-archive-libs', action='append') parser.add_option('--custom-step') parser.add_option('--python') thinlto_cache.add_options(parser) return parser.parse_args() if __name__ == '__main__': opts, args = parse_args() assert opts.arch assert opts.target cmd = fix_blas_resolving(args) cmd = fix_cmd(opts.arch, cmd) cmd = fix_py2(cmd) if opts.musl: cmd = fix_cmd_for_musl(cmd) if opts.dynamic_cuda: cmd = fix_cmd_for_dynamic_cuda(cmd) else: cuda_manager = link_exe.CUDAManager(opts.cuda_architectures, opts.nvprune_exe) cmd = link_exe.process_cuda_libraries_by_nvprune(cmd, cuda_manager, opts.build_root) cmd = link_exe.process_cuda_libraries_by_objcopy(cmd, opts.build_root, opts.objcopy_exe) cmd = ProcessWholeArchiveOption(opts.arch, opts.whole_archive_peers, opts.whole_archive_libs).construct_cmd(cmd) thinlto_cache.preprocess(opts, cmd) if opts.custom_step: assert opts.python subprocess.check_call([opts.python] + [opts.custom_step] + cmd) if opts.linker_output: stdout = open(opts.linker_output, 'w') else: stdout = sys.stdout proc = subprocess.Popen(cmd, shell=False, stderr=sys.stderr, stdout=stdout) proc.communicate() thinlto_cache.postprocess(opts) if proc.returncode: print('linker has failed with retcode:', proc.returncode, file=sys.stderr) print('linker command:', shlex_join(cmd), file=sys.stderr) sys.exit(proc.returncode) if opts.fix_elf: cmd = [opts.fix_elf, opts.target] proc = subprocess.Popen(cmd, shell=False, stderr=sys.stderr, stdout=sys.stdout) proc.communicate() if proc.returncode: print('fix_elf has failed with retcode:', proc.returncode, file=sys.stderr) print('fix_elf command:', shlex_join(cmd), file=sys.stderr) sys.exit(proc.returncode) if opts.soname and opts.soname != opts.target: if os.path.exists(opts.soname): os.unlink(opts.soname) os.link(opts.target, opts.soname) # -----------------Test---------------- # def write_temp_file(content): import yatest.common as yc filename = yc.output_path('test.exports') with open(filename, 'w') as f: f.write(content) return filename def test_fix_cmd_darwin(): export_file_content = """ C++ geobase5::details::lookup_impl::* C++ geobase5::hardcoded_service """ filename = write_temp_file(export_file_content) args = ['-Wl,--version-script={}'.format(filename)] assert fix_cmd('DARWIN', args) == [ '-Wl,-exported_symbol,__ZN8geobase57details11lookup_impl*', '-Wl,-exported_symbol,__ZTIN8geobase57details11lookup_impl*', '-Wl,-exported_symbol,__ZTSN8geobase57details11lookup_impl*', '-Wl,-exported_symbol,__ZTTN8geobase57details11lookup_impl*', '-Wl,-exported_symbol,__ZTVN8geobase57details11lookup_impl*', '-Wl,-exported_symbol,__ZNK8geobase57details11lookup_impl*', '-Wl,-exported_symbol,__ZN8geobase517hardcoded_serviceE*', '-Wl,-exported_symbol,__ZTIN8geobase517hardcoded_serviceE*', '-Wl,-exported_symbol,__ZTSN8geobase517hardcoded_serviceE*', '-Wl,-exported_symbol,__ZTTN8geobase517hardcoded_serviceE*', '-Wl,-exported_symbol,__ZTVN8geobase517hardcoded_serviceE*', '-Wl,-exported_symbol,__ZNK8geobase517hardcoded_serviceE*', ] def run_fix_gnu_param(export_file_content): filename = write_temp_file(export_file_content) result = fix_gnu_param('LINUX', list(parse_export_file(filename)))[0] version_script_path = result[len('-Wl,--version-script=') :] with open(version_script_path) as f: content = f.read() return content def test_fix_gnu_param(): export_file_content = """ C++ geobase5::details::lookup_impl::* C getFactoryMap """ assert ( run_fix_gnu_param(export_file_content) == """{ global: extern "C" { _ZN8geobase57details11lookup_impl*; _ZTIN8geobase57details11lookup_impl*; _ZTSN8geobase57details11lookup_impl*; _ZTTN8geobase57details11lookup_impl*; _ZTVN8geobase57details11lookup_impl*; _ZNK8geobase57details11lookup_impl*; getFactoryMap; }; local: *; }; """ ) def test_fix_gnu_param_with_linux_version(): export_file_content = """ C++ geobase5::details::lookup_impl::* linux_version ver1.0 C getFactoryMap """ assert ( run_fix_gnu_param(export_file_content) == """ver1.0 { global: extern "C" { _ZN8geobase57details11lookup_impl*; _ZTIN8geobase57details11lookup_impl*; _ZTSN8geobase57details11lookup_impl*; _ZTTN8geobase57details11lookup_impl*; _ZTVN8geobase57details11lookup_impl*; _ZNK8geobase57details11lookup_impl*; getFactoryMap; }; local: *; }; """ )