_dart_fields.py 45 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414
  1. import base64
  2. import functools
  3. import json
  4. import operator
  5. import os
  6. import re
  7. import shlex
  8. import sys
  9. from functools import reduce
  10. import six
  11. import ymake
  12. import _common
  13. import lib.test_const as consts
  14. CANON_RESULT_FILE_NAME = 'result.json'
  15. CANON_DATA_DIR_NAME = 'canondata'
  16. CANON_OUTPUT_STORAGE = 'canondata_storage'
  17. KTLINT_CURRENT_EDITOR_CONFIG = "arcadia/build/platform/java/ktlint/.editorconfig"
  18. KTLINT_OLD_EDITOR_CONFIG = "arcadia/build/platform/java/ktlint_old/.editorconfig"
  19. ARCADIA_ROOT = '${ARCADIA_ROOT}/'
  20. SOURCE_ROOT_SHORT = '$S/'
  21. class DartValueError(ValueError):
  22. pass
  23. def create_dart_record(fields, *args):
  24. return reduce(operator.or_, (value for field in fields if (value := field(*args))), {})
  25. def with_fields(fields):
  26. def inner(func):
  27. @functools.wraps(func)
  28. def innermost(*args, **kwargs):
  29. func(fields, *args, **kwargs)
  30. return innermost
  31. return inner
  32. def serialize_list(lst):
  33. lst = list(filter(None, lst))
  34. return '\"' + ';'.join(lst) + '\"' if lst else ''
  35. def deserialize_list(val):
  36. return list(filter(None, val.replace('"', "").split(";")))
  37. def get_unit_list_variable(unit, name):
  38. items = unit.get(name)
  39. if items:
  40. items = items.split(' ')
  41. assert items[0] == "${}".format(name), (items, name)
  42. return items[1:]
  43. return []
  44. def get_values_list(unit, key):
  45. res = map(str.strip, (unit.get(key) or '').replace('$' + key, '').strip().split())
  46. return [r for r in res if r and r not in ['""', "''"]]
  47. def _get_test_tags(unit, spec_args=None):
  48. if spec_args is None:
  49. spec_args = {}
  50. tags = spec_args.get('TAG', []) + get_values_list(unit, 'TEST_TAGS_VALUE')
  51. tags = set(tags)
  52. if unit.get('EXPORT_SEM') == 'yes':
  53. filter_only_tags = sorted(t for t in tags if ':' not in t)
  54. unit.set(['FILTER_ONLY_TEST_TAGS', ' '.join(filter_only_tags)])
  55. # DEVTOOLS-7571
  56. if unit.get('SKIP_TEST_VALUE') and consts.YaTestTags.Fat in tags:
  57. tags.add(consts.YaTestTags.NotAutocheck)
  58. return tags
  59. def format_recipes(data: str | None) -> str:
  60. if not data:
  61. return ""
  62. data = data.replace('"USE_RECIPE_DELIM"', "\n")
  63. data = data.replace("$TEST_RECIPES_VALUE", "")
  64. return data
  65. def prepare_recipes(data: str | None) -> bytes:
  66. formatted = format_recipes(data)
  67. return base64.b64encode(six.ensure_binary(formatted))
  68. def prepare_env(data):
  69. data = data.replace("$TEST_ENV_VALUE", "")
  70. return serialize_list(shlex.split(data))
  71. def get_norm_paths(unit, key):
  72. # return paths without trailing (back)slash
  73. return [x.rstrip('\\/').replace('${ARCADIA_ROOT}/', '') for x in get_values_list(unit, key)]
  74. def _load_canonical_file(filename, unit_path):
  75. try:
  76. with open(filename, 'rb') as results_file:
  77. return json.load(results_file)
  78. except Exception as e:
  79. print("malformed canonical data in {}: {} ({})".format(unit_path, e, filename), file=sys.stderr)
  80. return {}
  81. def _get_resource_from_uri(uri):
  82. m = consts.CANON_MDS_RESOURCE_REGEX.match(uri)
  83. if m:
  84. key = m.group(1)
  85. return "{}:{}".format(consts.MDS_SCHEME, key)
  86. m = consts.CANON_BACKEND_RESOURCE_REGEX.match(uri)
  87. if m:
  88. key = m.group(1)
  89. return "{}:{}".format(consts.MDS_SCHEME, key)
  90. m = consts.CANON_SBR_RESOURCE_REGEX.match(uri)
  91. if m:
  92. # There might be conflict between resources, because all resources in sandbox have 'resource.tar.gz' name
  93. # That's why we use notation with '=' to specify specific path for resource
  94. uri = m.group(1)
  95. res_id = m.group(2)
  96. return "{}={}".format(uri, '/'.join([CANON_OUTPUT_STORAGE, res_id]))
  97. def _get_external_resources_from_canon_data(data):
  98. # Method should work with both canonization versions:
  99. # result.json: {'uri':X 'checksum':Y}
  100. # result.json: {'testname': {'uri':X 'checksum':Y}}
  101. # result.json: {'testname': [{'uri':X 'checksum':Y}]}
  102. # Also there is a bug - if user returns {'uri': 1} from test - machinery will fail
  103. # That's why we check 'uri' and 'checksum' fields presence
  104. # (it's still a bug - user can return {'uri':X, 'checksum': Y}, we need to unify canonization format)
  105. res = set()
  106. if isinstance(data, dict):
  107. if 'uri' in data and 'checksum' in data:
  108. resource = _get_resource_from_uri(data['uri'])
  109. if resource:
  110. res.add(resource)
  111. else:
  112. for k, v in six.iteritems(data):
  113. res.update(_get_external_resources_from_canon_data(v))
  114. elif isinstance(data, list):
  115. for e in data:
  116. res.update(_get_external_resources_from_canon_data(e))
  117. return res
  118. def _get_canonical_data_resources_v2(filename, unit_path):
  119. return (_get_external_resources_from_canon_data(_load_canonical_file(filename, unit_path)), [filename])
  120. def get_canonical_test_resources(unit):
  121. unit_path = unit.path()
  122. if unit.get("CUSTOM_CANONDATA_PATH"):
  123. path_to_canondata = unit_path.replace("$S", unit.get("CUSTOM_CANONDATA_PATH"))
  124. else:
  125. path_to_canondata = unit.resolve(unit_path)
  126. canon_data_dir = os.path.join(path_to_canondata, CANON_DATA_DIR_NAME, unit.get('CANONIZE_SUB_PATH') or '')
  127. try:
  128. _, dirs, files = next(os.walk(canon_data_dir))
  129. except StopIteration:
  130. # path doesn't exist
  131. return [], []
  132. if CANON_RESULT_FILE_NAME in files:
  133. return _get_canonical_data_resources_v2(os.path.join(canon_data_dir, CANON_RESULT_FILE_NAME), unit_path)
  134. return [], []
  135. def java_srcdirs_to_data(unit, var, serialize_result=True):
  136. extra_data = []
  137. for srcdir in (unit.get(var) or '').replace('$' + var, '').split():
  138. if srcdir == '.':
  139. srcdir = unit.get('MODDIR')
  140. if srcdir.startswith('${ARCADIA_ROOT}/') or srcdir.startswith('$ARCADIA_ROOT/'):
  141. srcdir = srcdir.replace('${ARCADIA_ROOT}/', '$S/')
  142. srcdir = srcdir.replace('$ARCADIA_ROOT/', '$S/')
  143. if srcdir.startswith('${CURDIR}') or srcdir.startswith('$CURDIR'):
  144. srcdir = srcdir.replace('${CURDIR}', os.path.join('$S', unit.get('MODDIR')))
  145. srcdir = srcdir.replace('$CURDIR', os.path.join('$S', unit.get('MODDIR')))
  146. srcdir = unit.resolve_arc_path(srcdir)
  147. if not srcdir.startswith('$'):
  148. srcdir = os.path.join('$S', unit.get('MODDIR'), srcdir)
  149. if srcdir.startswith('$S'):
  150. extra_data.append(srcdir.replace('$S', 'arcadia'))
  151. return serialize_list(extra_data) if serialize_result else extra_data
  152. def extract_java_system_properties(unit, args):
  153. if len(args) % 2:
  154. return [], 'Wrong use of SYSTEM_PROPERTIES in {}: odd number of arguments'.format(unit.path())
  155. props = []
  156. for x, y in zip(args[::2], args[1::2]):
  157. if x == 'FILE':
  158. if y.startswith('${BINDIR}') or y.startswith('${ARCADIA_BUILD_ROOT}') or y.startswith('/'):
  159. return [], 'Wrong use of SYSTEM_PROPERTIES in {}: absolute/build file path {}'.format(unit.path(), y)
  160. y = _common.rootrel_arc_src(y, unit)
  161. if not os.path.exists(unit.resolve('$S/' + y)):
  162. return [], 'Wrong use of SYSTEM_PROPERTIES in {}: can\'t resolve {}'.format(unit.path(), y)
  163. y = '${ARCADIA_ROOT}/' + y
  164. props.append({'type': 'file', 'path': y})
  165. else:
  166. props.append({'type': 'inline', 'key': x, 'value': y})
  167. return props, None
  168. def _resolve_module_files(unit, mod_dir, file_paths):
  169. mod_dir_with_sep_len = len(mod_dir) + 1
  170. resolved_files = []
  171. for path in file_paths:
  172. resolved = _common.rootrel_arc_src(path, unit)
  173. if resolved.startswith(mod_dir):
  174. resolved = resolved[mod_dir_with_sep_len:]
  175. resolved_files.append(resolved)
  176. return resolved_files
  177. def _resolve_config_path(unit, test_runner, rel_to):
  178. config_path = unit.get("ESLINT_CONFIG_PATH") if test_runner == "eslint" else unit.get("TS_TEST_CONFIG_PATH")
  179. arc_config_path = unit.resolve_arc_path(config_path)
  180. abs_config_path = unit.resolve(arc_config_path)
  181. if not abs_config_path:
  182. raise Exception("{} config not found: {}".format(test_runner, config_path))
  183. unit.onsrcs([arc_config_path])
  184. abs_rel_to = unit.resolve(unit.resolve_arc_path(unit.get(rel_to)))
  185. return os.path.relpath(abs_config_path, start=abs_rel_to)
  186. def _get_ts_test_data_dirs(unit):
  187. return sorted(
  188. set(
  189. [
  190. os.path.dirname(_common.rootrel_arc_src(p, unit))
  191. for p in (get_values_list(unit, "_TS_TEST_DATA_VALUE") or [])
  192. ]
  193. )
  194. )
  195. @_common.cache_by_second_arg
  196. def get_linter_configs(unit, config_paths):
  197. rel_config_path = _common.rootrel_arc_src(config_paths, unit)
  198. arc_config_path = unit.resolve_arc_path(rel_config_path)
  199. abs_config_path = unit.resolve(arc_config_path)
  200. with open(abs_config_path, 'r') as fd:
  201. return json.load(fd)
  202. def _reference_group_var(varname: str, extensions: list[str] | None = None) -> str:
  203. if extensions is None:
  204. return f'"${{join=\\;:{varname}}}"'
  205. return serialize_list(f'${{ext={ext};join=\\;:{varname}}}' for ext in extensions)
  206. def assert_file_exists(unit, path):
  207. path = unit.resolve(SOURCE_ROOT_SHORT + path)
  208. if not os.path.exists(path):
  209. message = 'File {} is not found'.format(path)
  210. ymake.report_configure_error(message)
  211. raise DartValueError()
  212. class AndroidApkTestActivity:
  213. KEY = 'ANDROID_APK_TEST_ACTIVITY'
  214. @classmethod
  215. def value(cls, unit, flat_args, spec_args):
  216. return {cls.KEY: unit.get('ANDROID_APK_TEST_ACTIVITY_VALUE')}
  217. class BenchmarkOpts:
  218. KEY = 'BENCHMARK-OPTS'
  219. @classmethod
  220. def value(cls, unit, flat_args, spec_args):
  221. return {cls.KEY: serialize_list(get_unit_list_variable(unit, 'BENCHMARK_OPTS_VALUE'))}
  222. class BinaryPath:
  223. KEY = 'BINARY-PATH'
  224. @classmethod
  225. def normalized(cls, unit, flat_args, spec_args):
  226. unit_path = _common.get_norm_unit_path(unit)
  227. return {cls.KEY: os.path.join(unit_path, unit.filename())}
  228. @classmethod
  229. def stripped(cls, unit, flat_args, spec_args):
  230. unit_path = unit.path()
  231. binary_path = os.path.join(unit_path, unit.filename())
  232. if binary_path:
  233. return {cls.KEY: _common.strip_roots(binary_path)}
  234. @classmethod
  235. def stripped_without_pkg_ext(cls, unit, flat_args, spec_args):
  236. value = _common.strip_roots(os.path.join(unit.path(), unit.filename()).replace(".pkg", ""))
  237. return {cls.KEY: value}
  238. class Blob:
  239. KEY = 'BLOB'
  240. @classmethod
  241. def value(cls, unit, flat_args, spec_args):
  242. return {cls.KEY: unit.get('TEST_BLOB_DATA')}
  243. class BuildFolderPath:
  244. KEY = 'BUILD-FOLDER-PATH'
  245. @classmethod
  246. def normalized(cls, unit, flat_args, spec_args):
  247. return {cls.KEY: _common.get_norm_unit_path(unit)}
  248. @classmethod
  249. def stripped(cls, unit, flat_args, spec_args):
  250. return {cls.KEY: _common.strip_roots(unit.path())}
  251. class CanonizeSubPath:
  252. KEY = 'CANONIZE_SUB_PATH'
  253. @classmethod
  254. def value(cls, unit, flat_args, spec_args):
  255. return {cls.KEY: unit.get('CANONIZE_SUB_PATH')}
  256. class Classpath:
  257. KEY = 'CLASSPATH'
  258. @classmethod
  259. def value(cls, unit, flat_args, spec_args):
  260. ymake_java_test = unit.get('YMAKE_JAVA_TEST') == 'yes'
  261. if ymake_java_test:
  262. value = '$B/{}/{}.jar ${{DART_CLASSPATH}}'.format(unit.get('MODDIR'), unit.get('REALPRJNAME'))
  263. return {cls.KEY: value}
  264. class ConfigPath:
  265. KEY = 'CONFIG-PATH'
  266. @classmethod
  267. def value(cls, unit, flat_args, spec_args):
  268. test_runner, rel_to = flat_args
  269. return {cls.KEY: _resolve_config_path(unit, test_runner, rel_to=rel_to)}
  270. class CustomDependencies:
  271. KEY = 'CUSTOM-DEPENDENCIES'
  272. @classmethod
  273. def all_standard(cls, unit, flat_args, spec_args):
  274. custom_deps = ' '.join(spec_args.get('DEPENDS', []) + get_values_list(unit, 'TEST_DEPENDS_VALUE'))
  275. return {cls.KEY: custom_deps}
  276. @classmethod
  277. def depends_only(cls, unit, flat_args, spec_args):
  278. return {cls.KEY: " ".join(spec_args.get('DEPENDS', []))}
  279. @classmethod
  280. def test_depends_only(cls, unit, flat_args, spec_args):
  281. custom_deps = get_values_list(unit, 'TEST_DEPENDS_VALUE')
  282. return {cls.KEY: " ".join(custom_deps)}
  283. @classmethod
  284. def depends_with_linter(cls, unit, flat_args, spec_args):
  285. linter = Linter.value(unit, flat_args, spec_args)[Linter.KEY]
  286. deps = spec_args.get('DEPENDS', []) + [os.path.dirname(linter)]
  287. for dep in deps:
  288. unit.ondepends(dep)
  289. return {cls.KEY: " ".join(deps)}
  290. @classmethod
  291. def nots_with_recipies(cls, unit, flat_args, spec_args):
  292. deps = flat_args[0]
  293. recipes_lines = format_recipes(unit.get("TEST_RECIPES_VALUE")).strip().splitlines()
  294. if recipes_lines:
  295. deps = deps or []
  296. deps.extend([os.path.dirname(r.strip().split(" ")[0]) for r in recipes_lines])
  297. return {cls.KEY: " ".join(deps)}
  298. class EslintConfigPath:
  299. KEY = 'ESLINT_CONFIG_PATH'
  300. @classmethod
  301. def value(cls, unit, flat_args, spec_args):
  302. test_runner, rel_to = flat_args
  303. return {cls.KEY: _resolve_config_path(unit, test_runner, rel_to=rel_to)}
  304. class ForkMode:
  305. KEY = 'FORK-MODE'
  306. @classmethod
  307. def from_macro_and_unit(cls, unit, flat_args, spec_args):
  308. fork_mode = []
  309. if 'FORK_SUBTESTS' in spec_args:
  310. fork_mode.append('subtests')
  311. if 'FORK_TESTS' in spec_args:
  312. fork_mode.append('tests')
  313. fork_mode = fork_mode or spec_args.get('FORK_MODE', []) or unit.get('TEST_FORK_MODE').split()
  314. fork_mode = ' '.join(fork_mode) if fork_mode else ''
  315. return {cls.KEY: fork_mode}
  316. @classmethod
  317. def test_fork_mode(cls, unit, flat_args, spec_args):
  318. return {cls.KEY: unit.get('TEST_FORK_MODE')}
  319. class ForkTestFiles:
  320. KEY = 'FORK-TEST-FILES'
  321. @classmethod
  322. def value(cls, unit, flat_args, spec_args):
  323. return {cls.KEY: unit.get('FORK_TEST_FILES_MODE')}
  324. class FuzzDicts:
  325. KEY = 'FUZZ-DICTS'
  326. @classmethod
  327. def value(cls, unit, flat_args, spec_args):
  328. value = serialize_list(spec_args.get('FUZZ_DICTS', []) + get_unit_list_variable(unit, 'FUZZ_DICTS_VALUE'))
  329. return {cls.KEY: value}
  330. class FuzzOpts:
  331. KEY = 'FUZZ-OPTS'
  332. @classmethod
  333. def value(cls, unit, flat_args, spec_args):
  334. value = serialize_list(spec_args.get('FUZZ_OPTS', []) + get_unit_list_variable(unit, 'FUZZ_OPTS_VALUE'))
  335. return {cls.KEY: value}
  336. class Fuzzing:
  337. KEY = 'FUZZING'
  338. @classmethod
  339. def value(cls, unit, flat_args, spec_args):
  340. if unit.get('FUZZING') == 'yes':
  341. return {cls.KEY: '1'}
  342. class GlobalLibraryPath:
  343. KEY = 'GLOBAL-LIBRARY-PATH'
  344. @classmethod
  345. def value(cls, unit, flat_args, spec_args):
  346. return {cls.KEY: unit.global_filename()}
  347. class GoBenchTimeout:
  348. KEY = 'GO_BENCH_TIMEOUT'
  349. @classmethod
  350. def value(cls, unit, flat_args, spec_args):
  351. return {cls.KEY: unit.get('GO_BENCH_TIMEOUT')}
  352. class IgnoreClasspathClash:
  353. KEY = 'IGNORE_CLASSPATH_CLASH'
  354. @classmethod
  355. def value(cls, unit, flat_args, spec_args):
  356. value = ' '.join(get_values_list(unit, 'JAVA_IGNORE_CLASSPATH_CLASH_VALUE'))
  357. return {cls.KEY: value}
  358. class JavaClasspathCmdType:
  359. KEY = 'JAVA_CLASSPATH_CMD_TYPE'
  360. @classmethod
  361. def value(cls, unit, flat_args, spec_args):
  362. java_cp_arg_type = unit.get('JAVA_CLASSPATH_CMD_TYPE_VALUE') or 'MANIFEST'
  363. if java_cp_arg_type not in ('MANIFEST', 'COMMAND_FILE', 'LIST'):
  364. # TODO move error reporting out of field classes
  365. ymake.report_configure_error(
  366. '{}: TEST_JAVA_CLASSPATH_CMD_TYPE({}) are invalid. Choose argument from MANIFEST, COMMAND_FILE or LIST)'.format(
  367. unit.path(), java_cp_arg_type
  368. )
  369. )
  370. raise DartValueError()
  371. return {cls.KEY: java_cp_arg_type}
  372. class JdkForTests:
  373. KEY = 'JDK_FOR_TESTS'
  374. @classmethod
  375. def value(cls, unit, flat_args, spec_args):
  376. value = 'JDK' + (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION') or '_DEFAULT') + '_FOR_TESTS'
  377. return {cls.KEY: value}
  378. class JdkLatestVersion:
  379. KEY = 'JDK_LATEST_VERSION'
  380. @classmethod
  381. def value(cls, unit, flat_args, spec_args):
  382. return {cls.KEY: unit.get('JDK_LATEST_VERSION')}
  383. class JdkResource:
  384. KEY = 'JDK_RESOURCE'
  385. @classmethod
  386. def value(cls, unit, flat_args, spec_args):
  387. value = 'JDK' + (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION') or '_DEFAULT')
  388. return {cls.KEY: value}
  389. class KtlintBaselineFile:
  390. KEY = 'KTLINT_BASELINE_FILE'
  391. @classmethod
  392. def value(cls, unit, flat_args, spec_args):
  393. if unit.get('_USE_KTLINT_OLD') != 'yes':
  394. baseline_path_relative = unit.get('_KTLINT_BASELINE_FILE')
  395. if baseline_path_relative:
  396. return {cls.KEY: baseline_path_relative}
  397. class KtlintBinary:
  398. KEY = 'KTLINT_BINARY'
  399. @classmethod
  400. def value(cls, unit, flat_args, spec_args):
  401. value = '$(KTLINT_OLD)/run.bat' if unit.get('_USE_KTLINT_OLD') == 'yes' else '$(KTLINT)/run.bat'
  402. return {cls.KEY: value}
  403. class Linter:
  404. KEY = 'LINTER'
  405. @classmethod
  406. def value(cls, unit, flat_args, spec_args):
  407. return {cls.KEY: spec_args['LINTER'][0]}
  408. class LintConfigs:
  409. KEY = 'LINT-CONFIGS'
  410. @classmethod
  411. def python_configs(cls, unit, flat_args, spec_args):
  412. resolved_configs = []
  413. project_to_config_map = spec_args.get('PROJECT_TO_CONFIG_MAP', [])
  414. if project_to_config_map:
  415. # ruff, TODO rewrite once custom configs migrated to autoincludes scheme
  416. project_to_config_map = project_to_config_map[0]
  417. assert_file_exists(unit, project_to_config_map)
  418. resolved_configs.append(project_to_config_map)
  419. cfgs = get_linter_configs(unit, project_to_config_map).values()
  420. for c in cfgs:
  421. assert_file_exists(unit, c)
  422. resolved_configs.append(c)
  423. return {cls.KEY: serialize_list(resolved_configs)}
  424. custom_config = spec_args.get('CUSTOM_CONFIG', [])
  425. if custom_config:
  426. # black if custom config is passed
  427. # TODO rewrite once custom configs migrated to autoincludes scheme
  428. custom_config = custom_config[0]
  429. assert_file_exists(unit, custom_config)
  430. resolved_configs.append(custom_config)
  431. return {cls.KEY: serialize_list(resolved_configs)}
  432. config = spec_args['CONFIGS'][0]
  433. # black without custom config or flake8, using default configs file
  434. assert_file_exists(unit, config)
  435. name = spec_args['NAME'][0]
  436. cfg = get_linter_configs(unit, config)[name]
  437. assert_file_exists(unit, cfg)
  438. resolved_configs.append(cfg)
  439. if name in ('flake8', 'py2_flake8'):
  440. resolved_configs.extend(spec_args.get('FLAKE_MIGRATIONS_CONFIG', []))
  441. return {cls.KEY: serialize_list(resolved_configs)}
  442. @classmethod
  443. def cpp_configs(cls, unit, flat_args, spec_args):
  444. custom_config = spec_args.get('CUSTOM_CONFIG')
  445. if custom_config:
  446. config = custom_config[0]
  447. assert_file_exists(unit, config)
  448. else:
  449. # file with default configs
  450. config = spec_args.get('CONFIGS')[0]
  451. assert_file_exists(unit, config)
  452. name = spec_args['NAME'][0]
  453. config = get_linter_configs(unit, config)[name]
  454. assert_file_exists(unit, config)
  455. return {cls.KEY: serialize_list([config])}
  456. class LintExtraParams:
  457. KEY = 'LINT-EXTRA-PARAMS'
  458. @classmethod
  459. def from_macro_args(cls, unit, flat_args, spec_args):
  460. extra_params = spec_args.get('EXTRA_PARAMS', [])
  461. for arg in extra_params:
  462. if '=' not in arg:
  463. message = 'Wrong EXTRA_PARAMS value: "{}". Values must have format "name=value".'.format(arg)
  464. ymake.report_configure_error(message)
  465. raise DartValueError()
  466. return {cls.KEY: serialize_list(extra_params)}
  467. class LintFileProcessingTime:
  468. KEY = 'LINT-FILE-PROCESSING-TIME'
  469. @classmethod
  470. def from_macro_args(cls, unit, flat_args, spec_args):
  471. return {cls.KEY: spec_args.get('FILE_PROCESSING_TIME', [''])[0]}
  472. class LintName:
  473. KEY = 'LINT-NAME'
  474. @classmethod
  475. def value(cls, unit, flat_args, spec_args):
  476. lint_name = spec_args['NAME'][0]
  477. if lint_name in ('flake8', 'py2_flake8') and (unit.get('DISABLE_FLAKE8') or 'no') == 'yes':
  478. unit.message(['INFO', 'Flake8 linting is disabled by `DISABLE_FLAKE8`'])
  479. raise DartValueError()
  480. return {cls.KEY: lint_name}
  481. class ModuleLang:
  482. KEY = 'MODULE_LANG'
  483. @classmethod
  484. def value(cls, unit, flat_args, spec_args):
  485. return {cls.KEY: unit.get("MODULE_LANG").lower() or consts.ModuleLang.UNKNOWN}
  486. class ModuleType:
  487. KEY = 'MODULE_TYPE'
  488. @classmethod
  489. def value(cls, unit, flat_args, spec_args):
  490. return {cls.KEY: unit.get('MODULE_TYPE')}
  491. class NoCheck:
  492. KEY = 'NO-CHECK'
  493. @classmethod
  494. def value(cls, unit, flat_args, spec_args):
  495. if unit.get('NO_CHECK_IMPORTS_FOR_VALUE') != "None":
  496. value = serialize_list(get_values_list(unit, 'NO_CHECK_IMPORTS_FOR_VALUE') or ["*"])
  497. return {cls.KEY: value}
  498. class NodejsRootVarName:
  499. KEY = 'NODEJS-ROOT-VAR-NAME'
  500. @classmethod
  501. def value(cls, unit, flat_args, spec_args):
  502. return {cls.KEY: unit.get("NODEJS-ROOT-VAR-NAME")}
  503. class NodeModulesBundleFilename:
  504. KEY = 'NODE-MODULES-BUNDLE-FILENAME'
  505. @classmethod
  506. def value(cls, unit, flat_args, spec_args):
  507. return {cls.KEY: spec_args.get('nm_bundle')}
  508. class PythonPaths:
  509. KEY = 'PYTHON-PATHS'
  510. @classmethod
  511. def value(cls, unit, flat_args, spec_args):
  512. python_paths = get_values_list(unit, 'TEST_PYTHON_PATH_VALUE')
  513. return {cls.KEY: serialize_list(python_paths)}
  514. class Requirements:
  515. KEY = 'REQUIREMENTS'
  516. @classmethod
  517. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  518. test_requirements = spec_args.get('REQUIREMENTS', []) + get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')
  519. return {cls.KEY: serialize_list(test_requirements)}
  520. @classmethod
  521. def with_maybe_fuzzing(cls, unit, flat_args, spec_args):
  522. test_requirements = serialize_list(
  523. spec_args.get('REQUIREMENTS', []) + get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')
  524. )
  525. if unit.get('FUZZING') == 'yes':
  526. value = serialize_list(filter(None, deserialize_list(test_requirements) + ["cpu:all", "ram:all"]))
  527. return {cls.KEY: value}
  528. else:
  529. return {cls.KEY: test_requirements}
  530. @classmethod
  531. def from_macro_args(cls, unit, flat_args, spec_args):
  532. value = " ".join(spec_args.get('REQUIREMENTS', []))
  533. return {cls.KEY: value}
  534. @classmethod
  535. def from_unit(cls, unit, flat_args, spec_args):
  536. requirements = get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')
  537. return {cls.KEY: serialize_list(requirements)}
  538. @classmethod
  539. def from_unit_with_full_network(cls, unit, flat_args, spec_args):
  540. requirements = sorted(set(["network:full"] + get_values_list(unit, "TEST_REQUIREMENTS_VALUE")))
  541. return {cls.KEY: serialize_list(requirements)}
  542. class SbrUidExt:
  543. KEY = 'SBR-UID-EXT'
  544. @classmethod
  545. def value(cls, unit, flat_args, spec_args):
  546. uid_ext = unit.get("SBR_UID_EXT").split(" ", 1)[-1] # strip variable name
  547. return {cls.KEY: uid_ext}
  548. class ScriptRelPath:
  549. KEY = 'SCRIPT-REL-PATH'
  550. @classmethod
  551. def second_flat(cls, unit, flat_args, spec_args):
  552. return {cls.KEY: flat_args[1]}
  553. @classmethod
  554. def first_flat(cls, unit, flat_args, spec_args):
  555. return {cls.KEY: flat_args[0]}
  556. @classmethod
  557. def pytest(cls, unit, flat_args, spec_args):
  558. return {cls.KEY: 'py3test.bin' if (unit.get("PYTHON3") == 'yes') else "pytest.bin"}
  559. @classmethod
  560. def junit(cls, unit, flat_args, spec_args):
  561. return {cls.KEY: 'junit5.test' if unit.get('MODULE_TYPE') == 'JUNIT5' else 'junit.test'}
  562. class Size:
  563. KEY = 'SIZE'
  564. @classmethod
  565. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  566. return {cls.KEY: ''.join(spec_args.get('SIZE', [])) or unit.get('TEST_SIZE_NAME')}
  567. @classmethod
  568. def from_unit(cls, unit, flat_args, spec_args):
  569. return {cls.KEY: unit.get('TEST_SIZE_NAME')}
  570. class SkipTest:
  571. KEY = 'SKIP_TEST'
  572. @classmethod
  573. def value(cls, unit, flat_args, spec_args):
  574. return {cls.KEY: unit.get('SKIP_TEST_VALUE')}
  575. class SourceFolderPath:
  576. KEY = 'SOURCE-FOLDER-PATH'
  577. @classmethod
  578. def normalized(cls, unit, flat_args, spec_args):
  579. return {cls.KEY: _common.get_norm_unit_path(unit)}
  580. @classmethod
  581. def test_dir(cls, unit, flat_args, spec_args):
  582. test_dir = _common.get_norm_unit_path(unit)
  583. test_files = flat_args[1:]
  584. if test_files:
  585. test_dir = os.path.dirname(test_files[0]).lstrip("$S/")
  586. return {cls.KEY: test_dir}
  587. class SplitFactor:
  588. KEY = 'SPLIT-FACTOR'
  589. @classmethod
  590. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  591. value = ''.join(spec_args.get('SPLIT_FACTOR', [])) or unit.get('TEST_SPLIT_FACTOR')
  592. return {cls.KEY: value}
  593. @classmethod
  594. def from_unit(cls, unit, flat_args, spec_args):
  595. return {cls.KEY: unit.get('TEST_SPLIT_FACTOR')}
  596. class Tag:
  597. KEY = 'TAG'
  598. @classmethod
  599. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  600. tags = serialize_list(sorted(_get_test_tags(unit, spec_args)))
  601. return {cls.KEY: tags}
  602. @classmethod
  603. def from_unit(cls, unit, flat_args, spec_args):
  604. tags = serialize_list(get_values_list(unit, "TEST_TAGS_VALUE"))
  605. return {cls.KEY: tags}
  606. @classmethod
  607. def from_unit_fat_external_no_retries(cls, unit, flat_args, spec_args):
  608. tags = sorted(set(["ya:fat", "ya:external", "ya:noretries"] + get_values_list(unit, "TEST_TAGS_VALUE")))
  609. return {cls.KEY: serialize_list(tags)}
  610. class TestClasspath:
  611. KEY = 'TEST_CLASSPATH'
  612. @classmethod
  613. def value(cls, unit, flat_args, spec_args):
  614. test_classpath_origins = unit.get('TEST_CLASSPATH_VALUE')
  615. ymake_java_test = unit.get('YMAKE_JAVA_TEST') == 'yes'
  616. if test_classpath_origins:
  617. value = '${TEST_CLASSPATH_MANAGED}'
  618. return {cls.KEY: value}
  619. elif ymake_java_test:
  620. value = '${DART_CLASSPATH}'
  621. return {cls.KEY: value}
  622. class TestClasspathDeps:
  623. KEY = 'TEST_CLASSPATH_DEPS'
  624. @classmethod
  625. def value(cls, unit, flat_args, spec_args):
  626. test_classpath_origins = unit.get('TEST_CLASSPATH_VALUE')
  627. ymake_java_test = unit.get('YMAKE_JAVA_TEST') == 'yes'
  628. if not test_classpath_origins and ymake_java_test:
  629. return {cls.KEY: '${DART_CLASSPATH_DEPS}'}
  630. class TestClasspathOrigins:
  631. KEY = 'TEST_CLASSPATH_ORIGINS'
  632. @classmethod
  633. def value(cls, unit, flat_args, spec_args):
  634. test_classpath_origins = unit.get('TEST_CLASSPATH_VALUE')
  635. if test_classpath_origins:
  636. return {cls.KEY: test_classpath_origins}
  637. class TestCwd:
  638. KEY = 'TEST-CWD'
  639. @classmethod
  640. def from_unit(cls, unit, flat_args, spec_args):
  641. test_cwd = unit.get('TEST_CWD_VALUE') # TODO: validate test_cwd value
  642. return {cls.KEY: test_cwd}
  643. @classmethod
  644. def keywords_replaced(cls, unit, flat_args, spec_args):
  645. test_cwd = unit.get('TEST_CWD_VALUE') or ''
  646. if test_cwd:
  647. test_cwd = test_cwd.replace("$TEST_CWD_VALUE", "").replace('"MACRO_CALLS_DELIM"', "").strip()
  648. return {cls.KEY: test_cwd}
  649. @classmethod
  650. def moddir(cls, unit, flat_args, spec_args):
  651. return {cls.KEY: unit.get("MODDIR")}
  652. class TestData:
  653. KEY = 'TEST-DATA'
  654. @classmethod
  655. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  656. test_data = sorted(
  657. _common.filter_out_by_keyword(
  658. spec_args.get('DATA', []) + get_norm_paths(unit, 'TEST_DATA_VALUE'), 'AUTOUPDATED'
  659. )
  660. )
  661. return {cls.KEY: serialize_list(test_data)}
  662. @classmethod
  663. def from_macro_args_and_unit_with_canonical(cls, unit, flat_args, spec_args):
  664. test_data = sorted(
  665. _common.filter_out_by_keyword(
  666. spec_args.get('DATA', []) + get_norm_paths(unit, 'TEST_DATA_VALUE'), 'AUTOUPDATED'
  667. )
  668. )
  669. data, _ = get_canonical_test_resources(unit)
  670. test_data += data
  671. value = serialize_list(sorted(test_data))
  672. return {cls.KEY: value}
  673. @classmethod
  674. def ktlint(cls, unit, flat_args, spec_args):
  675. if unit.get('_USE_KTLINT_OLD') == 'yes':
  676. extra_test_data = [KTLINT_OLD_EDITOR_CONFIG]
  677. else:
  678. data_list = [KTLINT_CURRENT_EDITOR_CONFIG]
  679. baseline_path_relative = unit.get('_KTLINT_BASELINE_FILE')
  680. if baseline_path_relative:
  681. baseline_path = unit.resolve_arc_path(baseline_path_relative).replace('$S', 'arcadia')
  682. data_list += [baseline_path]
  683. extra_test_data = data_list
  684. # XXX
  685. if unit.get('_WITH_YA_1931') != 'yes':
  686. extra_test_data += java_srcdirs_to_data(unit, 'ALL_SRCDIRS', serialize_result=False)
  687. extra_test_data = serialize_list(extra_test_data)
  688. return {cls.KEY: extra_test_data}
  689. @classmethod
  690. def java_style(cls, unit, flat_args, spec_args):
  691. ymake_java_test = unit.get('YMAKE_JAVA_TEST') == 'yes'
  692. if ymake_java_test:
  693. return {cls.KEY: java_srcdirs_to_data(unit, 'ALL_SRCDIRS')}
  694. @classmethod
  695. def from_unit_with_canonical(cls, unit, flat_args, spec_args):
  696. test_data = get_norm_paths(unit, 'TEST_DATA_VALUE')
  697. data, _ = get_canonical_test_resources(unit)
  698. test_data += data
  699. value = serialize_list(sorted(_common.filter_out_by_keyword(test_data, 'AUTOUPDATED')))
  700. return {cls.KEY: value}
  701. @classmethod
  702. def java_test(cls, unit, flat_args, spec_args):
  703. test_data = get_norm_paths(unit, 'TEST_DATA_VALUE')
  704. test_data.append('arcadia/build/scripts/run_junit.py')
  705. test_data.append('arcadia/build/scripts/unpacking_jtest_runner.py')
  706. data, _ = get_canonical_test_resources(unit)
  707. test_data += data
  708. props, error_mgs = extract_java_system_properties(unit, get_values_list(unit, 'SYSTEM_PROPERTIES_VALUE'))
  709. if error_mgs:
  710. ymake.report_configure_error(error_mgs)
  711. raise DartValueError()
  712. for prop in props:
  713. if prop['type'] == 'file':
  714. test_data.append(prop['path'].replace('${ARCADIA_ROOT}', 'arcadia'))
  715. value = serialize_list(sorted(_common.filter_out_by_keyword(test_data, 'AUTOUPDATED')))
  716. return {cls.KEY: value}
  717. @classmethod
  718. def from_unit(cls, unit, flat_args, spec_args):
  719. return {cls.KEY: serialize_list(get_values_list(unit, "TEST_DATA_VALUE"))}
  720. class DockerImage:
  721. KEY = 'DOCKER-IMAGES'
  722. @staticmethod
  723. def _validate(images):
  724. docker_image_re = consts.DOCKER_LINK_RE
  725. for img in images:
  726. msg = None
  727. if "=" in img:
  728. link, _ = img.rsplit('=', 1)
  729. if docker_image_re.match(link) is None:
  730. msg = 'Invalid docker url format: {}. Link should be provided in format docker://<repo>@sha256:<digest>'.format(
  731. link
  732. )
  733. else:
  734. msg = 'Invalid docker image: {}. Image should be provided in format <link>=<tag>'.format(img)
  735. if msg:
  736. ymake.report_configure_error(msg)
  737. raise DartValueError(msg)
  738. @classmethod
  739. def value(cls, unit, flat_args, spec_args):
  740. raw_value = get_values_list(unit, 'DOCKER_IMAGES_VALUE')
  741. images = sorted(raw_value)
  742. if images:
  743. cls._validate(images)
  744. return {cls.KEY: serialize_list(images)}
  745. class TsConfigPath:
  746. KEY = 'TS_CONFIG_PATH'
  747. class TsStylelintConfig:
  748. KEY = 'TS_STYLELINT_CONFIG'
  749. @classmethod
  750. def value(cls, unit, flat_args, spec_args):
  751. test_config = unit.get('_TS_STYLELINT_CONFIG')
  752. abs_test_config = unit.resolve(unit.resolve_arc_path(test_config))
  753. if not abs_test_config:
  754. ymake.report_configure_error(
  755. f"Config for stylelint not found: {test_config}.\n"
  756. "Set the correct value in `TS_STYLELINT(<config_filename>)` macro in the `ya.make` file."
  757. )
  758. return {cls.KEY: test_config}
  759. class TsTestDataDirs:
  760. KEY = 'TS-TEST-DATA-DIRS'
  761. @classmethod
  762. def value(cls, unit, flat_args, spec_args):
  763. value = serialize_list(_get_ts_test_data_dirs(unit))
  764. return {cls.KEY: value}
  765. class TsTestDataDirsRename:
  766. KEY = 'TS-TEST-DATA-DIRS-RENAME'
  767. @classmethod
  768. def value(cls, unit, flat_args, spec_args):
  769. return {cls.KEY: unit.get("_TS_TEST_DATA_DIRS_RENAME_VALUE")}
  770. class TsTestForPath:
  771. KEY = 'TS-TEST-FOR-PATH'
  772. @classmethod
  773. def value(cls, unit, flat_args, spec_args):
  774. return {cls.KEY: unit.get("TS_TEST_FOR_PATH")}
  775. class TestedProjectFilename:
  776. KEY = 'TESTED-PROJECT-FILENAME'
  777. @classmethod
  778. def value(cls, unit, flat_args, spec_args):
  779. return {cls.KEY: unit.filename()}
  780. class TestedProjectName:
  781. KEY = 'TESTED-PROJECT-NAME'
  782. @classmethod
  783. def unit_name(cls, unit, flat_args, spec_args):
  784. return {cls.KEY: unit.name()}
  785. @classmethod
  786. def normalized_basename(cls, unit, flat_args, spec_args):
  787. test_dir = _common.get_norm_unit_path(unit)
  788. return {cls.KEY: os.path.basename(test_dir)}
  789. @classmethod
  790. def test_dir(cls, unit, flat_args, spec_args):
  791. test_dir = _common.get_norm_unit_path(unit)
  792. test_files = flat_args[1:]
  793. if test_files:
  794. test_dir = os.path.dirname(test_files[0]).lstrip("$S/")
  795. return {cls.KEY: os.path.basename(test_dir)}
  796. @classmethod
  797. def path_filename_basename(cls, unit, flat_args, spec_args):
  798. binary_path = os.path.join(unit.path(), unit.filename())
  799. return {cls.KEY: os.path.basename(binary_path)}
  800. @classmethod
  801. def normalized(cls, unit, flat_args, spec_args):
  802. return {cls.KEY: _common.get_norm_unit_path(unit)}
  803. @classmethod
  804. def path_filename_basename_without_pkg_ext(cls, unit, flat_args, spec_args):
  805. value = os.path.basename(os.path.join(unit.path(), unit.filename()).replace(".pkg", ""))
  806. return {cls.KEY: value}
  807. @classmethod
  808. def filename_without_ext(cls, unit, flat_args, spec_args):
  809. return {cls.KEY: os.path.splitext(unit.filename())[0]}
  810. class TestFiles:
  811. KEY = 'TEST-FILES'
  812. # TODO remove FILES, see DEVTOOLS-7052, currently it's required
  813. # https://a.yandex-team.ru/arcadia/devtools/ya/test/dartfile/__init__.py?rev=r14292146#L10
  814. KEY2 = 'FILES'
  815. @classmethod
  816. def value(cls, unit, flat_args, spec_args):
  817. data_re = re.compile(r"sbr:/?/?(\d+)=?.*")
  818. data = flat_args[1:]
  819. resources = []
  820. for f in data:
  821. matched = re.match(data_re, f)
  822. if matched:
  823. resources.append(matched.group(1))
  824. value = serialize_list(resources)
  825. return {cls.KEY: value, cls.KEY2: value}
  826. @classmethod
  827. def flat_args_wo_first(cls, unit, flat_args, spec_args):
  828. value = serialize_list(flat_args[1:])
  829. return {cls.KEY: value, cls.KEY2: value}
  830. @classmethod
  831. def java_style(cls, unit, flat_args, spec_args):
  832. test_files = flat_args[1:]
  833. check_level = flat_args[1]
  834. allowed_levels = {
  835. 'base': '/yandex_checks.xml',
  836. 'strict': '/yandex_checks_strict.xml',
  837. 'extended': '/yandex_checks_extended.xml',
  838. 'library': '/yandex_checks_library.xml',
  839. }
  840. if check_level not in allowed_levels:
  841. raise Exception("'{}' is not allowed in LINT(), use one of {}".format(check_level, allowed_levels.keys()))
  842. test_files[0] = allowed_levels[check_level]
  843. value = serialize_list(test_files)
  844. return {cls.KEY: value, cls.KEY2: value}
  845. @classmethod
  846. def normalized(cls, unit, flat_args, spec_args):
  847. value = serialize_list([_common.get_norm_unit_path(unit, unit.filename())])
  848. return {cls.KEY: value, cls.KEY2: value}
  849. @classmethod
  850. def test_srcs(cls, unit, flat_args, spec_args):
  851. test_files = get_values_list(unit, 'TEST_SRCS_VALUE')
  852. value = serialize_list(test_files)
  853. return {cls.KEY: value, cls.KEY2: value}
  854. @classmethod
  855. def ts_test_srcs(cls, unit, flat_args, spec_args):
  856. test_files = get_values_list(unit, "_TS_TEST_SRCS_VALUE")
  857. test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files)
  858. value = serialize_list(test_files)
  859. return {cls.KEY: value, cls.KEY2: value}
  860. @classmethod
  861. def ts_input_files(cls, unit, flat_args, spec_args):
  862. typecheck_files = get_values_list(unit, "TS_INPUT_FILES")
  863. test_files = [_common.resolve_common_const(f) for f in typecheck_files]
  864. value = serialize_list(test_files)
  865. return {cls.KEY: value, cls.KEY2: value}
  866. @classmethod
  867. def ts_lint_srcs(cls, unit, flat_args, spec_args):
  868. test_files = get_values_list(unit, "_TS_LINT_SRCS_VALUE")
  869. test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files)
  870. value = serialize_list(test_files)
  871. return {cls.KEY: value, cls.KEY2: value}
  872. @classmethod
  873. def stylesheets(cls, unit, flat_args, spec_args):
  874. test_files = get_values_list(unit, "_TS_STYLELINT_FILES")
  875. test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files)
  876. value = serialize_list(test_files)
  877. return {cls.KEY: value, cls.KEY2: value}
  878. @classmethod
  879. def py_linter_files(cls, unit, flat_args, spec_args):
  880. files = unit.get('PY_LINTER_FILES')
  881. if not files:
  882. raise DartValueError()
  883. files = json.loads(files)
  884. test_files = []
  885. for path in files:
  886. if path.startswith(ARCADIA_ROOT):
  887. test_files.append(path.replace(ARCADIA_ROOT, SOURCE_ROOT_SHORT, 1))
  888. elif path.startswith(SOURCE_ROOT_SHORT):
  889. test_files.append(path)
  890. if not test_files:
  891. lint_name = LintName.value(unit, flat_args, spec_args)[LintName.KEY]
  892. message = 'No files to lint for {}'.format(lint_name)
  893. raise DartValueError(message)
  894. test_files = serialize_list(test_files)
  895. return {cls.KEY: test_files, cls.KEY2: test_files}
  896. @classmethod
  897. def cpp_linter_files(cls, unit, flat_args, spec_args):
  898. files_dart = _reference_group_var("ALL_SRCS", consts.STYLE_CPP_ALL_EXTS)
  899. return {cls.KEY: files_dart, cls.KEY2: files_dart}
  900. class TestEnv:
  901. KEY = 'TEST-ENV'
  902. @classmethod
  903. def value(cls, unit, flat_args, spec_args):
  904. return {cls.KEY: prepare_env(unit.get("TEST_ENV_VALUE"))}
  905. class TestIosDeviceType:
  906. KEY = 'TEST_IOS_DEVICE_TYPE'
  907. @classmethod
  908. def value(cls, unit, flat_args, spec_args):
  909. return {cls.KEY: unit.get('TEST_IOS_DEVICE_TYPE_VALUE')}
  910. class TestIosRuntimeType:
  911. KEY = 'TEST_IOS_RUNTIME_TYPE'
  912. @classmethod
  913. def value(cls, unit, flat_args, spec_args):
  914. return {cls.KEY: unit.get('TEST_IOS_RUNTIME_TYPE_VALUE')}
  915. class TestJar:
  916. KEY = 'TEST_JAR'
  917. @classmethod
  918. def value(cls, unit, flat_args, spec_args):
  919. test_classpath_origins = unit.get('TEST_CLASSPATH_VALUE')
  920. ymake_java_test = unit.get('YMAKE_JAVA_TEST') == 'yes'
  921. if not test_classpath_origins and ymake_java_test:
  922. if unit.get('UNITTEST_DIR'):
  923. value = '${UNITTEST_MOD}'
  924. else:
  925. value = '{}/{}.jar'.format(unit.get('MODDIR'), unit.get('REALPRJNAME'))
  926. return {cls.KEY: value}
  927. class TestName:
  928. KEY = 'TEST-NAME'
  929. @classmethod
  930. def value(cls, unit, flat_args, spec_args):
  931. return {cls.KEY: flat_args[0]}
  932. @classmethod
  933. def first_flat_with_bench(cls, unit, flat_args, spec_args):
  934. return {cls.KEY: flat_args[0] + '_bench'}
  935. @classmethod
  936. def first_flat(cls, unit, flat_args, spec_args):
  937. return {cls.KEY: flat_args[0].lower()}
  938. @classmethod
  939. def filename_without_ext(cls, unit, flat_args, spec_args):
  940. test_name = os.path.basename(os.path.join(unit.path(), unit.filename()))
  941. return {cls.KEY: os.path.splitext(test_name)[0]}
  942. @classmethod
  943. def normalized_joined_dir_basename(cls, unit, flat_args, spec_args):
  944. path = _common.get_norm_unit_path(unit)
  945. value = '-'.join([os.path.basename(os.path.dirname(path)), os.path.basename(path)])
  946. return {cls.KEY: value}
  947. @classmethod
  948. def normalized_joined_dir_basename_deps(cls, unit, flat_args, spec_args):
  949. path = _common.get_norm_unit_path(unit)
  950. value = '-'.join([os.path.basename(os.path.dirname(path)), os.path.basename(path), 'dependencies']).strip('-')
  951. return {cls.KEY: value}
  952. @classmethod
  953. def filename_without_pkg_ext(cls, unit, flat_args, spec_args):
  954. test_name = os.path.basename(os.path.join(unit.path(), unit.filename()).replace(".pkg", ""))
  955. return {cls.KEY: os.path.splitext(test_name)[0]}
  956. @classmethod
  957. def name_from_macro_args(cls, unit, flat_args, spec_args):
  958. return {cls.KEY: spec_args['NAME'][0]}
  959. class TestPartition:
  960. KEY = 'TEST_PARTITION'
  961. @classmethod
  962. def value(cls, unit, flat_args, spec_args):
  963. return {cls.KEY: unit.get("TEST_PARTITION")}
  964. class TestRecipes:
  965. KEY = 'TEST-RECIPES'
  966. @classmethod
  967. def value(cls, unit, flat_args, spec_args):
  968. return {cls.KEY: prepare_recipes(unit.get("TEST_RECIPES_VALUE"))}
  969. class TestRunnerBin:
  970. KEY = 'TEST-RUNNER-BIN'
  971. @classmethod
  972. def value(cls, unit, flat_args, spec_args):
  973. runner_bin = spec_args.get('RUNNER_BIN', [None])[0]
  974. if runner_bin:
  975. return {cls.KEY: runner_bin}
  976. class TestTimeout:
  977. KEY = 'TEST-TIMEOUT'
  978. @classmethod
  979. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  980. test_timeout = ''.join(spec_args.get('TIMEOUT', [])) or unit.get('TEST_TIMEOUT') or ''
  981. return {cls.KEY: test_timeout}
  982. @classmethod
  983. def from_unit_with_default(cls, unit, flat_args, spec_args):
  984. timeout = list(filter(None, [unit.get(["TEST_TIMEOUT"])]))
  985. if timeout:
  986. timeout = timeout[0]
  987. else:
  988. timeout = '0'
  989. return {cls.KEY: timeout}
  990. @classmethod
  991. def from_unit(cls, unit, flat_args, spec_args):
  992. return {cls.KEY: unit.get('TEST_TIMEOUT')}
  993. class TsResources:
  994. KEY = "{}-ROOT-VAR-NAME"
  995. @classmethod
  996. def value(cls, unit, flat_args, spec_args):
  997. erm_json = spec_args['erm_json']
  998. ret = {}
  999. for tool in erm_json.list_npm_packages():
  1000. tool_resource_label = cls.KEY.format(tool.upper())
  1001. tool_resource_value = unit.get(tool_resource_label)
  1002. if tool_resource_value:
  1003. ret[tool_resource_label] = tool_resource_value
  1004. return ret
  1005. class JvmArgs:
  1006. KEY = 'JVM_ARGS'
  1007. @classmethod
  1008. def value(cls, unit, flat_args, spec_args):
  1009. value = serialize_list(get_values_list(unit, 'JVM_ARGS_VALUE'))
  1010. return {cls.KEY: value}
  1011. class StrictClasspathClash:
  1012. KEY = 'STRICT_CLASSPATH_CLASH'
  1013. class SystemProperties:
  1014. KEY = 'SYSTEM_PROPERTIES'
  1015. @classmethod
  1016. def value(cls, unit, flat_args, spec_args):
  1017. props, error_mgs = extract_java_system_properties(unit, get_values_list(unit, 'SYSTEM_PROPERTIES_VALUE'))
  1018. if error_mgs:
  1019. ymake.report_configure_error(error_mgs)
  1020. raise DartValueError()
  1021. props = base64.b64encode(six.ensure_binary(json.dumps(props)))
  1022. return {cls.KEY: props}
  1023. class UnittestDir:
  1024. KEY = 'UNITTEST_DIR'
  1025. @classmethod
  1026. def value(cls, unit, flat_args, spec_args):
  1027. return {cls.KEY: unit.get('UNITTEST_DIR')}
  1028. class UseArcadiaPython:
  1029. KEY = 'USE_ARCADIA_PYTHON'
  1030. @classmethod
  1031. def value(cls, unit, flat_args, spec_args):
  1032. return {cls.KEY: unit.get('USE_ARCADIA_PYTHON')}
  1033. class UseKtlintOld:
  1034. KEY = 'USE_KTLINT_OLD'
  1035. @classmethod
  1036. def value(cls, unit, flat_args, spec_args):
  1037. if unit.get('_USE_KTLINT_OLD') == 'yes':
  1038. return {cls.KEY: 'yes'}
  1039. class YtSpec:
  1040. KEY = 'YT-SPEC'
  1041. @classmethod
  1042. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  1043. value = serialize_list(spec_args.get('YT_SPEC', []) + get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE'))
  1044. return {cls.KEY: value}
  1045. @classmethod
  1046. def from_unit(cls, unit, flat_args, spec_args):
  1047. yt_spec = get_values_list(unit, 'TEST_YT_SPEC_VALUE')
  1048. if yt_spec:
  1049. return {cls.KEY: serialize_list(yt_spec)}
  1050. @classmethod
  1051. def from_unit_list_var(cls, unit, flat_args, spec_args):
  1052. yt_spec_values = get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')
  1053. return {cls.KEY: serialize_list(yt_spec_values)}