go_tool.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897
  1. import argparse
  2. import copy
  3. import json
  4. import os
  5. import re
  6. import shutil
  7. import subprocess
  8. import sys
  9. import tarfile
  10. import tempfile
  11. import threading
  12. import traceback
  13. from contextlib import contextmanager
  14. from functools import reduce
  15. import process_command_files as pcf
  16. import process_whole_archive_option as pwa
  17. arc_project_prefix = 'a.yandex-team.ru/'
  18. # FIXME: make version-independent
  19. std_lib_prefix = 'contrib/go/_std_1.19/src/'
  20. vendor_prefix = 'vendor/'
  21. vet_info_ext = '.vet.out'
  22. vet_report_ext = '.vet.txt'
  23. FIXED_CGO1_SUFFIX='.fixed.cgo1.go'
  24. COMPILE_OPTIMIZATION_FLAGS=('-N',)
  25. def get_trimpath_args(args):
  26. return ['-trimpath', args.trimpath] if args.trimpath else []
  27. def preprocess_cgo1(src_path, dst_path, source_root):
  28. with open(src_path, 'r') as f:
  29. content = f.read()
  30. content = content.replace('__ARCADIA_SOURCE_ROOT_PREFIX__', source_root)
  31. with open(dst_path, 'w') as f:
  32. f.write(content)
  33. def preprocess_args(args):
  34. # Temporary work around for noauto
  35. if args.cgo_srcs and len(args.cgo_srcs) > 0:
  36. cgo_srcs_set = set(args.cgo_srcs)
  37. args.srcs = [x for x in args.srcs if x not in cgo_srcs_set]
  38. args.pkg_root = os.path.join(args.toolchain_root, 'pkg')
  39. toolchain_tool_root = os.path.join(args.pkg_root, 'tool', '{}_{}'.format(args.host_os, args.host_arch))
  40. args.go_compile = os.path.join(toolchain_tool_root, 'compile')
  41. args.go_cgo = os.path.join(toolchain_tool_root, 'cgo')
  42. args.go_link = os.path.join(toolchain_tool_root, 'link')
  43. args.go_asm = os.path.join(toolchain_tool_root, 'asm')
  44. args.go_pack = os.path.join(toolchain_tool_root, 'pack')
  45. args.go_vet = os.path.join(toolchain_tool_root, 'vet') if args.vet is True else args.vet
  46. args.output = os.path.normpath(args.output)
  47. args.vet_report_output = vet_report_output_name(args.output, args.vet_report_ext)
  48. args.trimpath = None
  49. if args.debug_root_map:
  50. roots = {'build': args.build_root, 'source': args.source_root, 'tools': args.tools_root}
  51. replaces = []
  52. for root in args.debug_root_map.split(';'):
  53. src, dst = root.split('=', 1)
  54. assert src in roots
  55. replaces.append('{}=>{}'.format(roots[src], dst))
  56. del roots[src]
  57. assert len(replaces) > 0
  58. args.trimpath = ';'.join(replaces)
  59. args.build_root = os.path.normpath(args.build_root)
  60. args.build_root_dir = args.build_root + os.path.sep
  61. args.source_root = os.path.normpath(args.source_root)
  62. args.source_root_dir = args.source_root + os.path.sep
  63. args.output_root = os.path.normpath(args.output_root)
  64. args.import_map = {}
  65. args.module_map = {}
  66. if args.cgo_peers:
  67. args.cgo_peers = [x for x in args.cgo_peers if not x.endswith('.fake.pkg')]
  68. srcs = []
  69. for f in args.srcs:
  70. if f.endswith('.gosrc'):
  71. with tarfile.open(f, 'r') as tar:
  72. srcs.extend(os.path.join(args.output_root, src) for src in tar.getnames())
  73. tar.extractall(path=args.output_root)
  74. else:
  75. srcs.append(f)
  76. args.srcs = srcs
  77. assert args.mode == 'test' or args.test_srcs is None and args.xtest_srcs is None
  78. # add lexical oreder by basename for go sources
  79. args.srcs.sort(key=lambda x: os.path.basename(x))
  80. if args.test_srcs:
  81. args.srcs += sorted(args.test_srcs, key=lambda x: os.path.basename(x))
  82. del args.test_srcs
  83. if args.xtest_srcs:
  84. args.xtest_srcs.sort(key=lambda x: os.path.basename(x))
  85. # compute root relative module dir path
  86. assert args.output is None or args.output_root == os.path.dirname(args.output)
  87. assert args.output_root.startswith(args.build_root_dir)
  88. args.module_path = args.output_root[len(args.build_root_dir):]
  89. args.source_module_dir = os.path.join(args.source_root, args.test_import_path or args.module_path) + os.path.sep
  90. assert len(args.module_path) > 0
  91. args.import_path, args.is_std = get_import_path(args.module_path)
  92. assert args.asmhdr is None or args.word == 'go'
  93. srcs = []
  94. for f in args.srcs:
  95. if f.endswith(FIXED_CGO1_SUFFIX) and f.startswith(args.build_root_dir):
  96. path = os.path.join(args.output_root, '{}.cgo1.go'.format(os.path.basename(f[:-len(FIXED_CGO1_SUFFIX)])))
  97. srcs.append(path)
  98. preprocess_cgo1(f, path, args.source_root)
  99. else:
  100. srcs.append(f)
  101. args.srcs = srcs
  102. if args.extldflags:
  103. args.extldflags = pwa.ProcessWholeArchiveOption(args.targ_os).construct_cmd(args.extldflags)
  104. classify_srcs(args.srcs, args)
  105. def compare_versions(version1, version2):
  106. def last_index(version):
  107. index = version.find('beta')
  108. return len(version) if index < 0 else index
  109. v1 = tuple(x.zfill(8) for x in version1[:last_index(version1)].split('.'))
  110. v2 = tuple(x.zfill(8) for x in version2[:last_index(version2)].split('.'))
  111. if v1 == v2:
  112. return 0
  113. return 1 if v1 < v2 else -1
  114. def get_symlink_or_copyfile():
  115. os_symlink = getattr(os, 'symlink', None)
  116. if os_symlink is None or os.name == 'nt':
  117. os_symlink = shutil.copyfile
  118. return os_symlink
  119. def copy_args(args):
  120. return copy.copy(args)
  121. def get_vendor_index(import_path):
  122. index = import_path.rfind('/' + vendor_prefix)
  123. if index < 0:
  124. index = 0 if import_path.startswith(vendor_prefix) else index
  125. else:
  126. index = index + 1
  127. return index
  128. def get_import_path(module_path):
  129. assert len(module_path) > 0
  130. import_path = module_path.replace('\\', '/')
  131. is_std_module = import_path.startswith(std_lib_prefix)
  132. if is_std_module:
  133. import_path = import_path[len(std_lib_prefix):]
  134. elif import_path.startswith(vendor_prefix):
  135. import_path = import_path[len(vendor_prefix):]
  136. else:
  137. import_path = arc_project_prefix + import_path
  138. assert len(import_path) > 0
  139. return import_path, is_std_module
  140. def call(cmd, cwd, env=None):
  141. # sys.stderr.write('{}\n'.format(' '.join(cmd)))
  142. return subprocess.check_output(cmd, stdin=None, stderr=subprocess.STDOUT, cwd=cwd, env=env, text=True)
  143. def classify_srcs(srcs, args):
  144. args.go_srcs = [x for x in srcs if x.endswith('.go')]
  145. args.asm_srcs = [x for x in srcs if x.endswith('.s')]
  146. args.objects = [x for x in srcs if x.endswith('.o') or x.endswith('.obj')]
  147. args.symabis = [x for x in srcs if x.endswith('.symabis')]
  148. args.sysos = [x for x in srcs if x.endswith('.syso')]
  149. def get_import_config_info(peers, gen_importmap, import_map={}, module_map={}):
  150. info = {'importmap': [], 'packagefile': [], 'standard': {}}
  151. if gen_importmap:
  152. for key, value in import_map.items():
  153. info['importmap'].append((key, value))
  154. for peer in peers:
  155. peer_import_path, is_std = get_import_path(os.path.dirname(peer))
  156. if gen_importmap:
  157. index = get_vendor_index(peer_import_path)
  158. if index >= 0:
  159. index += len(vendor_prefix)
  160. info['importmap'].append((peer_import_path[index:], peer_import_path))
  161. info['packagefile'].append((peer_import_path, os.path.join(args.build_root, peer)))
  162. if is_std:
  163. info['standard'][peer_import_path] = True
  164. for key, value in module_map.items():
  165. info['packagefile'].append((key, value))
  166. return info
  167. def create_import_config(peers, gen_importmap, import_map={}, module_map={}):
  168. lines = []
  169. info = get_import_config_info(peers, gen_importmap, import_map, module_map)
  170. for key in ('importmap', 'packagefile'):
  171. for item in info[key]:
  172. lines.append('{} {}={}'.format(key, *item))
  173. if len(lines) > 0:
  174. lines.append('')
  175. content = '\n'.join(lines)
  176. # sys.stderr.writelines('{}\n'.format(l) for l in lines)
  177. with tempfile.NamedTemporaryFile(delete=False) as f:
  178. f.write(content.encode('UTF-8'))
  179. return f.name
  180. return None
  181. def create_embed_config(args):
  182. data = {
  183. 'Patterns': {},
  184. 'Files': {},
  185. }
  186. for info in args.embed:
  187. embed_dir = os.path.normpath(info[0])
  188. assert embed_dir == args.source_module_dir[:-1] or embed_dir.startswith((args.source_module_dir, args.build_root))
  189. pattern = info[1]
  190. if pattern.endswith('/**/*'):
  191. pattern = pattern[:-3]
  192. files = {os.path.relpath(f, embed_dir).replace('\\', '/'): f for f in info[2:]}
  193. data['Patterns'][pattern] = list(files.keys())
  194. data['Files'].update(files)
  195. # sys.stderr.write('{}\n'.format(json.dumps(data, indent=4)))
  196. with tempfile.NamedTemporaryFile(delete=False, suffix='.embedcfg') as f:
  197. f.write(json.dumps(data).encode('UTF-8'))
  198. return f.name
  199. def vet_info_output_name(path, ext=None):
  200. return '{}{}'.format(path, ext or vet_info_ext)
  201. def vet_report_output_name(path, ext=None):
  202. return '{}{}'.format(path, ext or vet_report_ext)
  203. def get_source_path(args):
  204. return args.test_import_path or args.module_path
  205. def gen_vet_info(args):
  206. import_path = args.real_import_path if hasattr(args, 'real_import_path') else args.import_path
  207. info = get_import_config_info(args.peers, True, args.import_map, args.module_map)
  208. import_map = dict(info['importmap'])
  209. # FIXME(snermolaev): it seems that adding import map for 'fake' package
  210. # does't make any harm (it needs to be revised later)
  211. import_map['unsafe'] = 'unsafe'
  212. for (key, _) in info['packagefile']:
  213. if key not in import_map:
  214. import_map[key] = key
  215. data = {
  216. 'ID': import_path,
  217. 'Compiler': 'gc',
  218. 'Dir': os.path.join(args.source_root, get_source_path(args)),
  219. 'ImportPath': import_path,
  220. 'GoFiles': [x for x in args.go_srcs if x.endswith('.go')],
  221. 'NonGoFiles': [x for x in args.go_srcs if not x.endswith('.go')],
  222. 'ImportMap': import_map,
  223. 'PackageFile': dict(info['packagefile']),
  224. 'Standard': dict(info['standard']),
  225. 'PackageVetx': dict((key, vet_info_output_name(value)) for key, value in info['packagefile']),
  226. 'VetxOnly': False,
  227. 'VetxOutput': vet_info_output_name(args.output),
  228. 'SucceedOnTypecheckFailure': False
  229. }
  230. # sys.stderr.write('{}\n'.format(json.dumps(data, indent=4)))
  231. return data
  232. def create_vet_config(args, info):
  233. with tempfile.NamedTemporaryFile(delete=False, suffix='.cfg') as f:
  234. f.write(json.dumps(info).encode('UTF-8'))
  235. return f.name
  236. def decode_vet_report(json_report):
  237. report = ''
  238. if json_report:
  239. try:
  240. full_diags = json.JSONDecoder().decode(json_report.decode('UTF-8'))
  241. except ValueError:
  242. report = json_report
  243. else:
  244. messages = []
  245. for _, module_diags in full_diags.items():
  246. for _, type_diags in module_diags.items():
  247. for diag in type_diags:
  248. messages.append('{}: {}'.format(diag['posn'], json.dumps(diag['message'])))
  249. report = '\n'.join(messages)
  250. return report
  251. def dump_vet_report(args, report):
  252. if report:
  253. report = report.replace(args.build_root, '$B')
  254. report = report.replace(args.source_root, '$S')
  255. with open(args.vet_report_output, 'w') as f:
  256. f.write(report)
  257. def read_vet_report(args):
  258. assert args
  259. report = ''
  260. if os.path.exists(args.vet_report_output):
  261. with open(args.vet_report_output, 'r') as f:
  262. report += f.read()
  263. return report
  264. def dump_vet_report_for_tests(args, *test_args_list):
  265. dump_vet_report(args, reduce(lambda x, y: x + read_vet_report(y), [_f for _f in test_args_list if _f], ''))
  266. def do_vet(args):
  267. assert args.vet
  268. info = gen_vet_info(args)
  269. vet_config = create_vet_config(args, info)
  270. cmd = [args.go_vet, '-json']
  271. if args.vet_flags:
  272. cmd.extend(args.vet_flags)
  273. cmd.append(vet_config)
  274. # sys.stderr.write('>>>> [{}]\n'.format(' '.join(cmd)))
  275. p_vet = subprocess.Popen(cmd, stdin=None, stderr=subprocess.PIPE, stdout=subprocess.PIPE, cwd=args.source_root)
  276. vet_out, vet_err = p_vet.communicate()
  277. report = decode_vet_report(vet_out) if vet_out else ''
  278. dump_vet_report(args, report)
  279. if p_vet.returncode:
  280. raise subprocess.CalledProcessError(returncode=p_vet.returncode, cmd=cmd, output=vet_err)
  281. def _do_compile_go(args):
  282. import_path, is_std_module = args.import_path, args.is_std
  283. cmd = [
  284. args.go_compile,
  285. '-o',
  286. args.output,
  287. '-p',
  288. import_path if import_path != "unsafe" else "",
  289. '-D',
  290. '""',
  291. ]
  292. if args.lang:
  293. cmd.append('-lang=go{}'.format(args.lang))
  294. cmd.extend(get_trimpath_args(args))
  295. compiling_runtime = False
  296. if is_std_module:
  297. cmd.append('-std')
  298. if import_path in ('runtime', 'internal/abi', 'internal/bytealg', 'internal/cpu') or import_path.startswith('runtime/internal/'):
  299. cmd.append('-+')
  300. compiling_runtime = True
  301. import_config_name = create_import_config(args.peers, True, args.import_map, args.module_map)
  302. if import_config_name:
  303. cmd += ['-importcfg', import_config_name]
  304. else:
  305. if import_path == 'unsafe' or len(args.objects) > 0 or args.asmhdr:
  306. pass
  307. else:
  308. cmd.append('-complete')
  309. # if compare_versions('1.16', args.goversion) >= 0:
  310. if args.embed:
  311. embed_config_name = create_embed_config(args)
  312. cmd.extend(['-embedcfg', embed_config_name])
  313. if args.asmhdr:
  314. cmd += ['-asmhdr', args.asmhdr]
  315. # Use .symabis (starting from 1.12 version)
  316. if args.symabis:
  317. cmd += ['-symabis'] + args.symabis
  318. # If 1.12 <= version < 1.13 we have to pass -allabis for 'runtime' and 'runtime/internal/atomic'
  319. # if compare_versions('1.13', args.goversion) >= 0:
  320. # pass
  321. # elif import_path in ('runtime', 'runtime/internal/atomic'):
  322. # cmd.append('-allabis')
  323. compile_workers = '4'
  324. if args.compile_flags:
  325. if compiling_runtime:
  326. cmd.extend(x for x in args.compile_flags if x not in COMPILE_OPTIMIZATION_FLAGS)
  327. else:
  328. cmd.extend(args.compile_flags)
  329. if any([x in ('-race', '-shared') for x in args.compile_flags]):
  330. compile_workers = '1'
  331. cmd += ['-pack', '-c={}'.format(compile_workers)]
  332. cmd += args.go_srcs
  333. call(cmd, args.build_root)
  334. class VetThread(threading.Thread):
  335. def __init__(self, target, args):
  336. super(VetThread, self).__init__(target=target, args=args)
  337. self.exc_info = None
  338. def run(self):
  339. try:
  340. super(VetThread, self).run()
  341. except:
  342. self.exc_info = sys.exc_info()
  343. def join_with_exception(self, reraise_exception):
  344. self.join()
  345. if reraise_exception and self.exc_info:
  346. raise self.exc_info[0].with_traceback(self.exc_info[1], self.exc_info[2])
  347. def do_compile_go(args):
  348. raise_exception_from_vet = False
  349. if args.vet:
  350. run_vet = VetThread(target=do_vet, args=(args,))
  351. run_vet.start()
  352. try:
  353. _do_compile_go(args)
  354. raise_exception_from_vet = True
  355. finally:
  356. if args.vet:
  357. run_vet.join_with_exception(raise_exception_from_vet)
  358. def do_compile_asm(args):
  359. def need_compiling_runtime(import_path):
  360. return import_path in ('runtime', 'reflect', 'syscall') or \
  361. import_path.startswith('runtime/internal/') or \
  362. compare_versions('1.17', args.goversion) >= 0 and import_path == 'internal/bytealg'
  363. assert(len(args.srcs) == 1 and len(args.asm_srcs) == 1)
  364. cmd = [args.go_asm]
  365. cmd += get_trimpath_args(args)
  366. cmd += ['-I', args.output_root, '-I', os.path.join(args.pkg_root, 'include')]
  367. cmd += ['-D', 'GOOS_' + args.targ_os, '-D', 'GOARCH_' + args.targ_arch, '-o', args.output]
  368. # if compare_versions('1.16', args.goversion) >= 0:
  369. cmd += ['-p', args.import_path]
  370. if need_compiling_runtime(args.import_path):
  371. cmd += ['-compiling-runtime']
  372. if args.asm_flags:
  373. cmd += args.asm_flags
  374. cmd += args.asm_srcs
  375. call(cmd, args.build_root)
  376. def do_link_lib(args):
  377. if len(args.asm_srcs) > 0:
  378. asmargs = copy_args(args)
  379. asmargs.asmhdr = os.path.join(asmargs.output_root, 'go_asm.h')
  380. do_compile_go(asmargs)
  381. for src in asmargs.asm_srcs:
  382. asmargs.srcs = [src]
  383. asmargs.asm_srcs = [src]
  384. asmargs.output = os.path.join(asmargs.output_root, os.path.basename(src) + '.o')
  385. do_compile_asm(asmargs)
  386. args.objects.append(asmargs.output)
  387. else:
  388. do_compile_go(args)
  389. if args.objects or args.sysos:
  390. cmd = [args.go_pack, 'r', args.output] + args.objects + args.sysos
  391. call(cmd, args.build_root)
  392. def do_link_exe(args):
  393. assert args.extld is not None
  394. assert args.non_local_peers is not None
  395. compile_args = copy_args(args)
  396. compile_args.output = os.path.join(args.output_root, 'main.a')
  397. compile_args.real_import_path = compile_args.import_path
  398. compile_args.import_path = 'main'
  399. if args.vcs and os.path.isfile(compile_args.vcs):
  400. build_info = os.path.join('library', 'go', 'core', 'buildinfo')
  401. if any([x.startswith(build_info) for x in compile_args.peers]):
  402. compile_args.go_srcs.append(compile_args.vcs)
  403. do_link_lib(compile_args)
  404. cmd = [args.go_link, '-o', args.output]
  405. import_config_name = create_import_config(args.peers + args.non_local_peers, False, args.import_map, args.module_map)
  406. if import_config_name:
  407. cmd += ['-importcfg', import_config_name]
  408. if args.link_flags:
  409. cmd += args.link_flags
  410. extldflags = []
  411. if args.buildmode:
  412. cmd.append('-buildmode={}'.format(args.buildmode))
  413. elif args.mode in ('exe', 'test'):
  414. cmd.append('-buildmode=exe')
  415. if 'ld.lld' in str(args):
  416. extldflags.append('-Wl,-no-pie')
  417. elif args.mode == 'dll':
  418. cmd.append('-buildmode=c-shared')
  419. else:
  420. assert False, 'Unexpected mode: {}'.format(args.mode)
  421. cmd.append('-extld={}'.format(args.extld))
  422. if args.extldflags is not None:
  423. filter_musl = bool
  424. if args.musl:
  425. cmd.append('-linkmode=external')
  426. extldflags.append('-static')
  427. filter_musl = lambda x: x not in ('-lc', '-ldl', '-lm', '-lpthread', '-lrt')
  428. extldflags += [x for x in args.extldflags if filter_musl(x)]
  429. cgo_peers = []
  430. if args.cgo_peers is not None and len(args.cgo_peers) > 0:
  431. is_group = args.targ_os == 'linux'
  432. if is_group:
  433. cgo_peers.append('-Wl,--start-group')
  434. cgo_peers.extend(args.cgo_peers)
  435. if is_group:
  436. cgo_peers.append('-Wl,--end-group')
  437. try:
  438. index = extldflags.index('--cgo-peers')
  439. extldflags = extldflags[:index] + cgo_peers + extldflags[index+1:]
  440. except ValueError:
  441. extldflags.extend(cgo_peers)
  442. if len(extldflags) > 0:
  443. cmd.append('-extldflags={}'.format(' '.join(extldflags)))
  444. cmd.append(compile_args.output)
  445. call(cmd, args.build_root)
  446. def gen_cover_info(args):
  447. lines = []
  448. lines.extend([
  449. """
  450. var (
  451. coverCounters = make(map[string][]uint32)
  452. coverBlocks = make(map[string][]testing.CoverBlock)
  453. )
  454. """,
  455. 'func init() {',
  456. ])
  457. for var, file in (x.split(':') for x in args.cover_info):
  458. lines.append(' coverRegisterFile("{file}", _cover0.{var}.Count[:], _cover0.{var}.Pos[:], _cover0.{var}.NumStmt[:])'.format(file=file, var=var))
  459. lines.extend([
  460. '}',
  461. """
  462. func coverRegisterFile(fileName string, counter []uint32, pos []uint32, numStmts []uint16) {
  463. if 3*len(counter) != len(pos) || len(counter) != len(numStmts) {
  464. panic("coverage: mismatched sizes")
  465. }
  466. if coverCounters[fileName] != nil {
  467. // Already registered.
  468. return
  469. }
  470. coverCounters[fileName] = counter
  471. block := make([]testing.CoverBlock, len(counter))
  472. for i := range counter {
  473. block[i] = testing.CoverBlock{
  474. Line0: pos[3*i+0],
  475. Col0: uint16(pos[3*i+2]),
  476. Line1: pos[3*i+1],
  477. Col1: uint16(pos[3*i+2]>>16),
  478. Stmts: numStmts[i],
  479. }
  480. }
  481. coverBlocks[fileName] = block
  482. }
  483. """,
  484. ])
  485. return lines
  486. def filter_out_skip_tests(tests, skip_tests):
  487. skip_set = set()
  488. star_skip_set = set()
  489. for t in skip_tests:
  490. work_set = star_skip_set if '*' in t else skip_set
  491. work_set.add(t)
  492. re_star_tests = None
  493. if len(star_skip_set) > 0:
  494. re_star_tests = re.compile(re.sub(r'(\*)+', r'.\1', '^({})$'.format('|'.join(star_skip_set))))
  495. return [x for x in tests if not (x in skip_tests or re_star_tests and re_star_tests.match(x))]
  496. @contextmanager
  497. def create_strip_symlink():
  498. # This function creates symlink of llvm-strip as strip for golink needs.
  499. # We believe that cc-binaries path is a first element in PATH enviroment variable.
  500. tmpdir = None
  501. if os.getenv("CC") == "clang":
  502. tmpdir = tempfile.mkdtemp()
  503. cc_path = os.getenv("PATH").split(os.pathsep)[0]
  504. os.environ["PATH"] += os.pathsep + tmpdir
  505. src_strip_path = os.path.join(cc_path, 'llvm-strip')
  506. dst_strip_path = os.path.join(tmpdir, 'strip')
  507. os.symlink(src_strip_path, dst_strip_path)
  508. try:
  509. yield
  510. finally:
  511. if tmpdir:
  512. shutil.rmtree(tmpdir)
  513. def gen_test_main(args, test_lib_args, xtest_lib_args):
  514. assert args and (test_lib_args or xtest_lib_args)
  515. test_miner = args.test_miner
  516. test_module_path = test_lib_args.import_path if test_lib_args else xtest_lib_args.import_path
  517. is_cover = args.cover_info and len(args.cover_info) > 0
  518. # Prepare GOPATH
  519. # $BINDIR
  520. # |- __go__
  521. # |- src
  522. # |- pkg
  523. # |- ${TARGET_OS}_${TARGET_ARCH}
  524. go_path_root = os.path.join(args.output_root, '__go__')
  525. test_src_dir = os.path.join(go_path_root, 'src')
  526. target_os_arch = '_'.join([args.targ_os, args.targ_arch])
  527. test_pkg_dir = os.path.join(go_path_root, 'pkg', target_os_arch, os.path.dirname(test_module_path))
  528. os.makedirs(test_pkg_dir)
  529. my_env = os.environ.copy()
  530. my_env['GOROOT'] = ''
  531. my_env['GOPATH'] = go_path_root
  532. my_env['GOARCH'] = args.targ_arch
  533. my_env['GOOS'] = args.targ_os
  534. tests = []
  535. xtests = []
  536. os_symlink = get_symlink_or_copyfile()
  537. # Get the list of "internal" tests
  538. if test_lib_args:
  539. os.makedirs(os.path.join(test_src_dir, test_module_path))
  540. os_symlink(test_lib_args.output, os.path.join(test_pkg_dir, os.path.basename(test_module_path) + '.a'))
  541. cmd = [test_miner, '-benchmarks', '-tests', test_module_path]
  542. tests = [x for x in (call(cmd, test_lib_args.output_root, my_env) or '').strip().split('\n') if len(x) > 0]
  543. if args.skip_tests:
  544. tests = filter_out_skip_tests(tests, args.skip_tests)
  545. test_main_found = '#TestMain' in tests
  546. # Get the list of "external" tests
  547. if xtest_lib_args:
  548. xtest_module_path = xtest_lib_args.import_path
  549. os.makedirs(os.path.join(test_src_dir, xtest_module_path))
  550. os_symlink(xtest_lib_args.output, os.path.join(test_pkg_dir, os.path.basename(xtest_module_path) + '.a'))
  551. cmd = [test_miner, '-benchmarks', '-tests', xtest_module_path]
  552. xtests = [x for x in (call(cmd, xtest_lib_args.output_root, my_env) or '').strip().split('\n') if len(x) > 0]
  553. if args.skip_tests:
  554. xtests = filter_out_skip_tests(xtests, args.skip_tests)
  555. xtest_main_found = '#TestMain' in xtests
  556. test_main_package = None
  557. if test_main_found and xtest_main_found:
  558. assert False, 'multiple definition of TestMain'
  559. elif test_main_found:
  560. test_main_package = '_test'
  561. elif xtest_main_found:
  562. test_main_package = '_xtest'
  563. shutil.rmtree(go_path_root)
  564. lines = ['package main', '', 'import (']
  565. if test_main_package is None:
  566. lines.append(' "os"')
  567. lines.extend([' "testing"', ' "testing/internal/testdeps"'])
  568. lines.extend([' _ "a.yandex-team.ru/library/go/test/yatest"'])
  569. if len(tests) > 0:
  570. lines.append(' _test "{}"'.format(test_module_path))
  571. elif test_lib_args:
  572. lines.append(' _ "{}"'.format(test_module_path))
  573. if len(xtests) > 0:
  574. lines.append(' _xtest "{}"'.format(xtest_module_path))
  575. elif xtest_lib_args:
  576. lines.append(' _ "{}"'.format(xtest_module_path))
  577. if is_cover:
  578. lines.append(' _cover0 "{}"'.format(test_module_path))
  579. lines.extend([')', ''])
  580. if compare_versions('1.18', args.goversion) < 0:
  581. kinds = ['Test', 'Benchmark', 'Example']
  582. else:
  583. kinds = ['Test', 'Benchmark', 'FuzzTarget', 'Example']
  584. var_names = []
  585. for kind in kinds:
  586. var_name = '{}s'.format(kind.lower())
  587. var_names.append(var_name)
  588. lines.append('var {} = []testing.Internal{}{{'.format(var_name, kind))
  589. for test in [x for x in tests if x.startswith(kind)]:
  590. lines.append(' {{"{test}", _test.{test}}},'.format(test=test))
  591. for test in [x for x in xtests if x.startswith(kind)]:
  592. lines.append(' {{"{test}", _xtest.{test}}},'.format(test=test))
  593. lines.extend(['}', ''])
  594. if is_cover:
  595. lines.extend(gen_cover_info(args))
  596. lines.append('func main() {')
  597. if is_cover:
  598. lines.extend([
  599. ' testing.RegisterCover(testing.Cover{',
  600. ' Mode: "set",',
  601. ' Counters: coverCounters,',
  602. ' Blocks: coverBlocks,',
  603. ' CoveredPackages: "",',
  604. ' })',
  605. ])
  606. lines.extend([
  607. ' m := testing.MainStart(testdeps.TestDeps{{}}, {})'.format(', '.join(var_names)),
  608. '',
  609. ])
  610. if test_main_package:
  611. lines.append(' {}.TestMain(m)'.format(test_main_package))
  612. else:
  613. lines.append(' os.Exit(m.Run())')
  614. lines.extend(['}', ''])
  615. content = '\n'.join(lines)
  616. # sys.stderr.write('{}\n'.format(content))
  617. return content
  618. def do_link_test(args):
  619. assert args.srcs or args.xtest_srcs
  620. assert args.test_miner is not None
  621. test_module_path = get_source_path(args)
  622. test_import_path, _ = get_import_path(test_module_path)
  623. test_lib_args = copy_args(args) if args.srcs else None
  624. xtest_lib_args = copy_args(args) if args.xtest_srcs else None
  625. if xtest_lib_args is not None:
  626. xtest_lib_args.embed = args.embed_xtest if args.embed_xtest else None
  627. ydx_file_name = None
  628. xtest_ydx_file_name = None
  629. need_append_ydx = test_lib_args and xtest_lib_args and args.ydx_file and args.vet_flags
  630. if need_append_ydx:
  631. def find_ydx_file_name(name, flags):
  632. for i, elem in enumerate(flags):
  633. if elem.endswith(name):
  634. return (i, elem)
  635. assert False, 'Unreachable code'
  636. idx, ydx_file_name = find_ydx_file_name(xtest_lib_args.ydx_file, xtest_lib_args.vet_flags)
  637. xtest_ydx_file_name = '{}_xtest'.format(ydx_file_name)
  638. xtest_lib_args.vet_flags = copy.copy(xtest_lib_args.vet_flags)
  639. xtest_lib_args.vet_flags[idx] = xtest_ydx_file_name
  640. if test_lib_args:
  641. test_lib_args.output = os.path.join(args.output_root, 'test.a')
  642. test_lib_args.vet_report_output = vet_report_output_name(test_lib_args.output)
  643. test_lib_args.module_path = test_module_path
  644. test_lib_args.import_path = test_import_path
  645. do_link_lib(test_lib_args)
  646. if xtest_lib_args:
  647. xtest_lib_args.srcs = xtest_lib_args.xtest_srcs
  648. classify_srcs(xtest_lib_args.srcs, xtest_lib_args)
  649. xtest_lib_args.output = os.path.join(args.output_root, 'xtest.a')
  650. xtest_lib_args.vet_report_output = vet_report_output_name(xtest_lib_args.output)
  651. xtest_lib_args.module_path = test_module_path + '_test'
  652. xtest_lib_args.import_path = test_import_path + '_test'
  653. if test_lib_args:
  654. xtest_lib_args.module_map[test_import_path] = test_lib_args.output
  655. need_append_ydx = args.ydx_file and args.srcs and args.vet_flags
  656. do_link_lib(xtest_lib_args)
  657. if need_append_ydx:
  658. with open(os.path.join(args.build_root, ydx_file_name), 'ab') as dst_file:
  659. with open(os.path.join(args.build_root, xtest_ydx_file_name), 'rb') as src_file:
  660. dst_file.write(src_file.read())
  661. test_main_content = gen_test_main(args, test_lib_args, xtest_lib_args)
  662. test_main_name = os.path.join(args.output_root, '_test_main.go')
  663. with open(test_main_name, "w") as f:
  664. f.write(test_main_content)
  665. test_args = copy_args(args)
  666. test_args.embed = None
  667. test_args.srcs = [test_main_name]
  668. if test_args.test_import_path is None:
  669. # it seems that we can do it unconditionally, but this kind
  670. # of mangling doesn't really looks good to me and we leave it
  671. # for pure GO_TEST module
  672. test_args.module_path = test_args.module_path + '___test_main__'
  673. test_args.import_path = test_args.import_path + '___test_main__'
  674. classify_srcs(test_args.srcs, test_args)
  675. if test_lib_args:
  676. test_args.module_map[test_lib_args.import_path] = test_lib_args.output
  677. if xtest_lib_args:
  678. test_args.module_map[xtest_lib_args.import_path] = xtest_lib_args.output
  679. if args.vet:
  680. dump_vet_report_for_tests(test_args, test_lib_args, xtest_lib_args)
  681. test_args.vet = False
  682. do_link_exe(test_args)
  683. if __name__ == '__main__':
  684. args = pcf.get_args(sys.argv[1:])
  685. parser = argparse.ArgumentParser(prefix_chars='+')
  686. parser.add_argument('++mode', choices=['dll', 'exe', 'lib', 'test'], required=True)
  687. parser.add_argument('++buildmode', choices=['c-shared', 'exe', 'pie'])
  688. parser.add_argument('++srcs', nargs='*', required=True)
  689. parser.add_argument('++cgo-srcs', nargs='*')
  690. parser.add_argument('++test_srcs', nargs='*')
  691. parser.add_argument('++xtest_srcs', nargs='*')
  692. parser.add_argument('++cover_info', nargs='*')
  693. parser.add_argument('++output', nargs='?', default=None)
  694. parser.add_argument('++source-root', default=None)
  695. parser.add_argument('++build-root', required=True)
  696. parser.add_argument('++tools-root', default=None)
  697. parser.add_argument('++output-root', required=True)
  698. parser.add_argument('++toolchain-root', required=True)
  699. parser.add_argument('++host-os', choices=['linux', 'darwin', 'windows'], required=True)
  700. parser.add_argument('++host-arch', choices=['amd64', 'arm64'], required=True)
  701. parser.add_argument('++targ-os', choices=['linux', 'darwin', 'windows'], required=True)
  702. parser.add_argument('++targ-arch', choices=['amd64', 'x86', 'arm64'], required=True)
  703. parser.add_argument('++peers', nargs='*')
  704. parser.add_argument('++non-local-peers', nargs='*')
  705. parser.add_argument('++cgo-peers', nargs='*')
  706. parser.add_argument('++asmhdr', nargs='?', default=None)
  707. parser.add_argument('++test-import-path', nargs='?')
  708. parser.add_argument('++test-miner', nargs='?')
  709. parser.add_argument('++arc-project-prefix', nargs='?', default=arc_project_prefix)
  710. parser.add_argument('++std-lib-prefix', nargs='?', default=std_lib_prefix)
  711. parser.add_argument('++vendor-prefix', nargs='?', default=vendor_prefix)
  712. parser.add_argument('++extld', nargs='?', default=None)
  713. parser.add_argument('++extldflags', nargs='+', default=None)
  714. parser.add_argument('++goversion', required=True)
  715. parser.add_argument('++lang', nargs='?', default=None)
  716. parser.add_argument('++asm-flags', nargs='*')
  717. parser.add_argument('++compile-flags', nargs='*')
  718. parser.add_argument('++link-flags', nargs='*')
  719. parser.add_argument('++vcs', nargs='?', default=None)
  720. parser.add_argument('++vet', nargs='?', const=True, default=False)
  721. parser.add_argument('++vet-flags', nargs='*', default=None)
  722. parser.add_argument('++vet-info-ext', default=vet_info_ext)
  723. parser.add_argument('++vet-report-ext', default=vet_report_ext)
  724. parser.add_argument('++musl', action='store_true')
  725. parser.add_argument('++skip-tests', nargs='*', default=None)
  726. parser.add_argument('++ydx-file', default='')
  727. parser.add_argument('++debug-root-map', default=None)
  728. parser.add_argument('++embed', action='append', nargs='*')
  729. parser.add_argument('++embed_xtest', action='append', nargs='*')
  730. args = parser.parse_args(args)
  731. arc_project_prefix = args.arc_project_prefix
  732. std_lib_prefix = args.std_lib_prefix
  733. vendor_prefix = args.vendor_prefix
  734. vet_info_ext = args.vet_info_ext
  735. vet_report_ext = args.vet_report_ext
  736. preprocess_args(args)
  737. try:
  738. os.unlink(args.output)
  739. except OSError:
  740. pass
  741. # We are going to support only 'lib', 'exe' and 'cgo' build modes currently
  742. # and as a result we are going to generate only one build node per module
  743. # (or program)
  744. dispatch = {
  745. 'exe': do_link_exe,
  746. 'dll': do_link_exe,
  747. 'lib': do_link_lib,
  748. 'test': do_link_test
  749. }
  750. exit_code = 1
  751. try:
  752. with create_strip_symlink():
  753. dispatch[args.mode](args)
  754. exit_code = 0
  755. except KeyError:
  756. sys.stderr.write('Unknown build mode [{}]...\n'.format(args.mode))
  757. except subprocess.CalledProcessError as e:
  758. sys.stderr.write('{} returned non-zero exit code {}.\n{}\n'.format(' '.join(e.cmd), e.returncode, e.output))
  759. exit_code = e.returncode
  760. except AssertionError as e:
  761. traceback.print_exc(file=sys.stderr)
  762. except Exception as e:
  763. sys.stderr.write('Unhandled exception [{}]...\n'.format(str(e)))
  764. sys.exit(exit_code)