_dart_fields.py 45 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431
  1. import base64
  2. import functools
  3. import json
  4. import operator
  5. import os
  6. import re
  7. import shlex
  8. import sys
  9. from functools import reduce
  10. import six
  11. import ymake
  12. import _common
  13. import lib.test_const as consts
  14. CANON_RESULT_FILE_NAME = 'result.json'
  15. CANON_DATA_DIR_NAME = 'canondata'
  16. CANON_OUTPUT_STORAGE = 'canondata_storage'
  17. KTLINT_CURRENT_EDITOR_CONFIG = "arcadia/build/platform/java/ktlint/.editorconfig"
  18. KTLINT_OLD_EDITOR_CONFIG = "arcadia/build/platform/java/ktlint_old/.editorconfig"
  19. ARCADIA_ROOT = '${ARCADIA_ROOT}/'
  20. SOURCE_ROOT_SHORT = '$S/'
  21. class DartValueError(ValueError):
  22. pass
  23. def create_dart_record(fields, *args):
  24. try:
  25. return reduce(operator.or_, (value for field in fields if (value := field(*args))), {})
  26. except Exception as e:
  27. if str(e) != "":
  28. ymake.report_configure_error("Exception: {}".format(e))
  29. else:
  30. raise (e)
  31. return None
  32. def with_fields(fields):
  33. def inner(func):
  34. @functools.wraps(func)
  35. def innermost(*args, **kwargs):
  36. func(fields, *args, **kwargs)
  37. return innermost
  38. return inner
  39. def serialize_list(lst):
  40. lst = list(filter(None, lst))
  41. return '\"' + ';'.join(lst) + '\"' if lst else ''
  42. def deserialize_list(val):
  43. return list(filter(None, val.replace('"', "").split(";")))
  44. def get_unit_list_variable(unit, name):
  45. items = unit.get(name)
  46. if items:
  47. items = items.split(' ')
  48. assert items[0] == "${}".format(name), (items, name)
  49. return items[1:]
  50. return []
  51. def get_values_list(unit, key):
  52. res = map(str.strip, (unit.get(key) or '').replace('$' + key, '').strip().split())
  53. return [r for r in res if r and r not in ['""', "''"]]
  54. def _get_test_tags(unit, spec_args=None):
  55. if spec_args is None:
  56. spec_args = {}
  57. tags = spec_args.get('TAG', []) + get_values_list(unit, 'TEST_TAGS_VALUE')
  58. tags = set(tags)
  59. if unit.get('EXPORT_SEM') == 'yes':
  60. filter_only_tags = sorted(t for t in tags if ':' not in t)
  61. unit.set(['FILTER_ONLY_TEST_TAGS', ' '.join(filter_only_tags)])
  62. # DEVTOOLS-7571
  63. if unit.get('SKIP_TEST_VALUE') and consts.YaTestTags.Fat in tags:
  64. tags.add(consts.YaTestTags.NotAutocheck)
  65. return tags
  66. def format_recipes(data: str | None) -> str:
  67. if not data:
  68. return ""
  69. data = data.replace('"USE_RECIPE_DELIM"', "\n")
  70. data = data.replace("$TEST_RECIPES_VALUE", "")
  71. return data
  72. def prepare_recipes(data: str | None) -> bytes:
  73. formatted = format_recipes(data)
  74. return base64.b64encode(six.ensure_binary(formatted))
  75. def prepare_env(data):
  76. data = data.replace("$TEST_ENV_VALUE", "")
  77. return serialize_list(shlex.split(data))
  78. def get_norm_paths(unit, key):
  79. # return paths without trailing (back)slash
  80. return [x.rstrip('\\/').replace('${ARCADIA_ROOT}/', '') for x in get_values_list(unit, key)]
  81. def _load_canonical_file(filename, unit_path):
  82. try:
  83. with open(filename, 'rb') as results_file:
  84. return json.load(results_file)
  85. except Exception as e:
  86. print("malformed canonical data in {}: {} ({})".format(unit_path, e, filename), file=sys.stderr)
  87. return {}
  88. def _get_resource_from_uri(uri):
  89. m = consts.CANON_MDS_RESOURCE_REGEX.match(uri)
  90. if m:
  91. key = m.group(1)
  92. return "{}:{}".format(consts.MDS_SCHEME, key)
  93. m = consts.CANON_BACKEND_RESOURCE_REGEX.match(uri)
  94. if m:
  95. key = m.group(1)
  96. return "{}:{}".format(consts.MDS_SCHEME, key)
  97. m = consts.CANON_SBR_RESOURCE_REGEX.match(uri)
  98. if m:
  99. # There might be conflict between resources, because all resources in sandbox have 'resource.tar.gz' name
  100. # That's why we use notation with '=' to specify specific path for resource
  101. uri = m.group(1)
  102. res_id = m.group(2)
  103. return "{}={}".format(uri, '/'.join([CANON_OUTPUT_STORAGE, res_id]))
  104. def _get_external_resources_from_canon_data(data):
  105. # Method should work with both canonization versions:
  106. # result.json: {'uri':X 'checksum':Y}
  107. # result.json: {'testname': {'uri':X 'checksum':Y}}
  108. # result.json: {'testname': [{'uri':X 'checksum':Y}]}
  109. # Also there is a bug - if user returns {'uri': 1} from test - machinery will fail
  110. # That's why we check 'uri' and 'checksum' fields presence
  111. # (it's still a bug - user can return {'uri':X, 'checksum': Y}, we need to unify canonization format)
  112. res = set()
  113. if isinstance(data, dict):
  114. if 'uri' in data and 'checksum' in data:
  115. resource = _get_resource_from_uri(data['uri'])
  116. if resource:
  117. res.add(resource)
  118. else:
  119. for k, v in six.iteritems(data):
  120. res.update(_get_external_resources_from_canon_data(v))
  121. elif isinstance(data, list):
  122. for e in data:
  123. res.update(_get_external_resources_from_canon_data(e))
  124. return res
  125. def _get_canonical_data_resources_v2(filename, unit_path):
  126. return (_get_external_resources_from_canon_data(_load_canonical_file(filename, unit_path)), [filename])
  127. def get_canonical_test_resources(unit):
  128. unit_path = unit.path()
  129. if unit.get("CUSTOM_CANONDATA_PATH"):
  130. path_to_canondata = unit_path.replace("$S", unit.get("CUSTOM_CANONDATA_PATH"))
  131. else:
  132. path_to_canondata = unit.resolve(unit_path)
  133. canon_data_dir = os.path.join(path_to_canondata, CANON_DATA_DIR_NAME, unit.get('CANONIZE_SUB_PATH') or '')
  134. try:
  135. _, dirs, files = next(os.walk(canon_data_dir))
  136. except StopIteration:
  137. # path doesn't exist
  138. return [], []
  139. if CANON_RESULT_FILE_NAME in files:
  140. return _get_canonical_data_resources_v2(os.path.join(canon_data_dir, CANON_RESULT_FILE_NAME), unit_path)
  141. return [], []
  142. def java_srcdirs_to_data(unit, var, serialize_result=True):
  143. extra_data = []
  144. for srcdir in (unit.get(var) or '').replace('$' + var, '').split():
  145. if srcdir == '.':
  146. srcdir = unit.get('MODDIR')
  147. if srcdir.startswith('${ARCADIA_ROOT}/') or srcdir.startswith('$ARCADIA_ROOT/'):
  148. srcdir = srcdir.replace('${ARCADIA_ROOT}/', '$S/')
  149. srcdir = srcdir.replace('$ARCADIA_ROOT/', '$S/')
  150. if srcdir.startswith('${CURDIR}') or srcdir.startswith('$CURDIR'):
  151. srcdir = srcdir.replace('${CURDIR}', os.path.join('$S', unit.get('MODDIR')))
  152. srcdir = srcdir.replace('$CURDIR', os.path.join('$S', unit.get('MODDIR')))
  153. srcdir = unit.resolve_arc_path(srcdir)
  154. if not srcdir.startswith('$'):
  155. srcdir = os.path.join('$S', unit.get('MODDIR'), srcdir)
  156. if srcdir.startswith('$S'):
  157. extra_data.append(srcdir.replace('$S', 'arcadia'))
  158. return serialize_list(extra_data) if serialize_result else extra_data
  159. def extract_java_system_properties(unit, args):
  160. if len(args) % 2:
  161. return [], 'Wrong use of SYSTEM_PROPERTIES in {}: odd number of arguments'.format(unit.path())
  162. props = []
  163. for x, y in zip(args[::2], args[1::2]):
  164. if x == 'FILE':
  165. if y.startswith('${BINDIR}') or y.startswith('${ARCADIA_BUILD_ROOT}') or y.startswith('/'):
  166. return [], 'Wrong use of SYSTEM_PROPERTIES in {}: absolute/build file path {}'.format(unit.path(), y)
  167. y = _common.rootrel_arc_src(y, unit)
  168. if not os.path.exists(unit.resolve('$S/' + y)):
  169. return [], 'Wrong use of SYSTEM_PROPERTIES in {}: can\'t resolve {}'.format(unit.path(), y)
  170. y = '${ARCADIA_ROOT}/' + y
  171. props.append({'type': 'file', 'path': y})
  172. else:
  173. props.append({'type': 'inline', 'key': x, 'value': y})
  174. return props, None
  175. def _resolve_module_files(unit, mod_dir, file_paths):
  176. mod_dir_with_sep_len = len(mod_dir) + 1
  177. resolved_files = []
  178. for path in file_paths:
  179. resolved = _common.rootrel_arc_src(path, unit)
  180. if resolved.startswith(mod_dir):
  181. resolved = resolved[mod_dir_with_sep_len:]
  182. resolved_files.append(resolved)
  183. return resolved_files
  184. def _resolve_config_path(unit, test_runner, rel_to):
  185. config_path = unit.get("ESLINT_CONFIG_PATH") if test_runner == "eslint" else unit.get("TS_TEST_CONFIG_PATH")
  186. arc_config_path = unit.resolve_arc_path(config_path)
  187. abs_config_path = unit.resolve(arc_config_path)
  188. if not abs_config_path:
  189. raise Exception("{} config not found: {}".format(test_runner, config_path))
  190. unit.onsrcs([arc_config_path])
  191. abs_rel_to = unit.resolve(unit.resolve_arc_path(unit.get(rel_to)))
  192. return os.path.relpath(abs_config_path, start=abs_rel_to)
  193. def _get_ts_test_data_dirs(unit):
  194. return sorted(
  195. set(
  196. [
  197. os.path.dirname(_common.rootrel_arc_src(p, unit))
  198. for p in (get_values_list(unit, "_TS_TEST_DATA_VALUE") or [])
  199. ]
  200. )
  201. )
  202. @_common.cache_by_second_arg
  203. def get_linter_configs(unit, config_paths):
  204. rel_config_path = _common.rootrel_arc_src(config_paths, unit)
  205. arc_config_path = unit.resolve_arc_path(rel_config_path)
  206. abs_config_path = unit.resolve(arc_config_path)
  207. with open(abs_config_path, 'r') as fd:
  208. return json.load(fd)
  209. def _reference_group_var(varname: str, extensions: list[str] | None = None) -> str:
  210. if extensions is None:
  211. return f'"${{join=\\;:{varname}}}"'
  212. return serialize_list(f'${{ext={ext};join=\\;:{varname}}}' for ext in extensions)
  213. def assert_file_exists(unit, path):
  214. path = unit.resolve(SOURCE_ROOT_SHORT + path)
  215. if not os.path.exists(path):
  216. message = 'File {} is not found'.format(path)
  217. ymake.report_configure_error(message)
  218. raise DartValueError()
  219. class AndroidApkTestActivity:
  220. KEY = 'ANDROID_APK_TEST_ACTIVITY'
  221. @classmethod
  222. def value(cls, unit, flat_args, spec_args):
  223. return {cls.KEY: unit.get('ANDROID_APK_TEST_ACTIVITY_VALUE')}
  224. class BenchmarkOpts:
  225. KEY = 'BENCHMARK-OPTS'
  226. @classmethod
  227. def value(cls, unit, flat_args, spec_args):
  228. return {cls.KEY: serialize_list(get_unit_list_variable(unit, 'BENCHMARK_OPTS_VALUE'))}
  229. class BinaryPath:
  230. KEY = 'BINARY-PATH'
  231. @classmethod
  232. def normalized(cls, unit, flat_args, spec_args):
  233. unit_path = _common.get_norm_unit_path(unit)
  234. return {cls.KEY: os.path.join(unit_path, unit.filename())}
  235. @classmethod
  236. def stripped(cls, unit, flat_args, spec_args):
  237. unit_path = unit.path()
  238. binary_path = os.path.join(unit_path, unit.filename())
  239. if binary_path:
  240. return {cls.KEY: _common.strip_roots(binary_path)}
  241. @classmethod
  242. def stripped_without_pkg_ext(cls, unit, flat_args, spec_args):
  243. value = _common.strip_roots(os.path.join(unit.path(), unit.filename()).replace(".pkg", ""))
  244. return {cls.KEY: value}
  245. class Blob:
  246. KEY = 'BLOB'
  247. @classmethod
  248. def value(cls, unit, flat_args, spec_args):
  249. return {cls.KEY: unit.get('TEST_BLOB_DATA')}
  250. class BuildFolderPath:
  251. KEY = 'BUILD-FOLDER-PATH'
  252. @classmethod
  253. def normalized(cls, unit, flat_args, spec_args):
  254. return {cls.KEY: _common.get_norm_unit_path(unit)}
  255. @classmethod
  256. def stripped(cls, unit, flat_args, spec_args):
  257. return {cls.KEY: _common.strip_roots(unit.path())}
  258. class CanonizeSubPath:
  259. KEY = 'CANONIZE_SUB_PATH'
  260. @classmethod
  261. def value(cls, unit, flat_args, spec_args):
  262. return {cls.KEY: unit.get('CANONIZE_SUB_PATH')}
  263. class Classpath:
  264. KEY = 'CLASSPATH'
  265. @classmethod
  266. def value(cls, unit, flat_args, spec_args):
  267. value = '$B/{}/{}.jar ${{DART_CLASSPATH}}'.format(unit.get('MODDIR'), unit.get('REALPRJNAME'))
  268. return {cls.KEY: value}
  269. class ConfigPath:
  270. KEY = 'CONFIG-PATH'
  271. @classmethod
  272. def value(cls, unit, flat_args, spec_args):
  273. test_runner, rel_to = flat_args
  274. return {cls.KEY: _resolve_config_path(unit, test_runner, rel_to=rel_to)}
  275. class CustomDependencies:
  276. KEY = 'CUSTOM-DEPENDENCIES'
  277. @classmethod
  278. def all_standard(cls, unit, flat_args, spec_args):
  279. custom_deps = ' '.join(spec_args.get('DEPENDS', []) + get_values_list(unit, 'TEST_DEPENDS_VALUE'))
  280. return {cls.KEY: custom_deps}
  281. @classmethod
  282. def depends_only(cls, unit, flat_args, spec_args):
  283. return {cls.KEY: " ".join(spec_args.get('DEPENDS', []))}
  284. @classmethod
  285. def test_depends_only(cls, unit, flat_args, spec_args):
  286. custom_deps = get_values_list(unit, 'TEST_DEPENDS_VALUE')
  287. return {cls.KEY: " ".join(custom_deps)}
  288. @classmethod
  289. def depends_with_linter(cls, unit, flat_args, spec_args):
  290. linter = Linter.value(unit, flat_args, spec_args)[Linter.KEY]
  291. deps = spec_args.get('DEPENDS', []) + [os.path.dirname(linter)]
  292. for dep in deps:
  293. unit.ondepends(dep)
  294. return {cls.KEY: " ".join(deps)}
  295. @classmethod
  296. def nots_with_recipies(cls, unit, flat_args, spec_args):
  297. deps = flat_args[0]
  298. recipes_lines = format_recipes(unit.get("TEST_RECIPES_VALUE")).strip().splitlines()
  299. if recipes_lines:
  300. deps = deps or []
  301. deps.extend([os.path.dirname(r.strip().split(" ")[0]) for r in recipes_lines])
  302. return {cls.KEY: " ".join(deps)}
  303. class EslintConfigPath:
  304. KEY = 'ESLINT_CONFIG_PATH'
  305. @classmethod
  306. def value(cls, unit, flat_args, spec_args):
  307. test_runner, rel_to = flat_args
  308. return {cls.KEY: _resolve_config_path(unit, test_runner, rel_to=rel_to)}
  309. class ForkMode:
  310. KEY = 'FORK-MODE'
  311. @classmethod
  312. def from_macro_and_unit(cls, unit, flat_args, spec_args):
  313. fork_mode = []
  314. if 'FORK_SUBTESTS' in spec_args:
  315. fork_mode.append('subtests')
  316. if 'FORK_TESTS' in spec_args:
  317. fork_mode.append('tests')
  318. fork_mode = fork_mode or spec_args.get('FORK_MODE', []) or unit.get('TEST_FORK_MODE').split()
  319. fork_mode = ' '.join(fork_mode) if fork_mode else ''
  320. return {cls.KEY: fork_mode}
  321. @classmethod
  322. def test_fork_mode(cls, unit, flat_args, spec_args):
  323. return {cls.KEY: unit.get('TEST_FORK_MODE')}
  324. class ForkTestFiles:
  325. KEY = 'FORK-TEST-FILES'
  326. @classmethod
  327. def value(cls, unit, flat_args, spec_args):
  328. return {cls.KEY: unit.get('FORK_TEST_FILES_MODE')}
  329. class FuzzDicts:
  330. KEY = 'FUZZ-DICTS'
  331. @classmethod
  332. def value(cls, unit, flat_args, spec_args):
  333. value = serialize_list(spec_args.get('FUZZ_DICTS', []) + get_unit_list_variable(unit, 'FUZZ_DICTS_VALUE'))
  334. return {cls.KEY: value}
  335. class FuzzOpts:
  336. KEY = 'FUZZ-OPTS'
  337. @classmethod
  338. def value(cls, unit, flat_args, spec_args):
  339. value = serialize_list(spec_args.get('FUZZ_OPTS', []) + get_unit_list_variable(unit, 'FUZZ_OPTS_VALUE'))
  340. return {cls.KEY: value}
  341. class Fuzzing:
  342. KEY = 'FUZZING'
  343. @classmethod
  344. def value(cls, unit, flat_args, spec_args):
  345. if unit.get('FUZZING') == 'yes':
  346. return {cls.KEY: '1'}
  347. class GlobalLibraryPath:
  348. KEY = 'GLOBAL-LIBRARY-PATH'
  349. @classmethod
  350. def value(cls, unit, flat_args, spec_args):
  351. return {cls.KEY: unit.global_filename()}
  352. class GoBenchTimeout:
  353. KEY = 'GO_BENCH_TIMEOUT'
  354. @classmethod
  355. def value(cls, unit, flat_args, spec_args):
  356. return {cls.KEY: unit.get('GO_BENCH_TIMEOUT')}
  357. class IgnoreClasspathClash:
  358. KEY = 'IGNORE_CLASSPATH_CLASH'
  359. @classmethod
  360. def value(cls, unit, flat_args, spec_args):
  361. value = ' '.join(get_values_list(unit, 'JAVA_IGNORE_CLASSPATH_CLASH_VALUE'))
  362. return {cls.KEY: value}
  363. class JavaClasspathCmdType:
  364. KEY = 'JAVA_CLASSPATH_CMD_TYPE'
  365. @classmethod
  366. def value(cls, unit, flat_args, spec_args):
  367. java_cp_arg_type = unit.get('JAVA_CLASSPATH_CMD_TYPE_VALUE') or 'MANIFEST'
  368. if java_cp_arg_type not in ('MANIFEST', 'COMMAND_FILE', 'LIST'):
  369. # TODO move error reporting out of field classes
  370. ymake.report_configure_error(
  371. '{}: TEST_JAVA_CLASSPATH_CMD_TYPE({}) are invalid. Choose argument from MANIFEST, COMMAND_FILE or LIST)'.format(
  372. unit.path(), java_cp_arg_type
  373. )
  374. )
  375. raise DartValueError()
  376. return {cls.KEY: java_cp_arg_type}
  377. class JdkForTests:
  378. KEY = 'JDK_FOR_TESTS'
  379. @classmethod
  380. def value(cls, unit, flat_args, spec_args):
  381. value = 'JDK' + (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION') or '_DEFAULT') + '_FOR_TESTS'
  382. return {cls.KEY: value}
  383. class JdkLatestVersion:
  384. KEY = 'JDK_LATEST_VERSION'
  385. @classmethod
  386. def value(cls, unit, flat_args, spec_args):
  387. return {cls.KEY: unit.get('JDK_LATEST_VERSION')}
  388. class JdkResource:
  389. KEY = 'JDK_RESOURCE'
  390. @classmethod
  391. def value(cls, unit, flat_args, spec_args):
  392. value = 'JDK' + (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION') or '_DEFAULT')
  393. return {cls.KEY: value}
  394. class KtlintBaselineFile:
  395. KEY = 'KTLINT_BASELINE_FILE'
  396. @classmethod
  397. def value(cls, unit, flat_args, spec_args):
  398. if unit.get('_USE_KTLINT_OLD') != 'yes':
  399. baseline_path_relative = unit.get('_KTLINT_BASELINE_FILE')
  400. if baseline_path_relative:
  401. return {cls.KEY: baseline_path_relative}
  402. class KtlintBinary:
  403. KEY = 'KTLINT_BINARY'
  404. @classmethod
  405. def value(cls, unit, flat_args, spec_args):
  406. value = '$(KTLINT_OLD)/run.bat' if unit.get('_USE_KTLINT_OLD') == 'yes' else '$(KTLINT)/run.bat'
  407. return {cls.KEY: value}
  408. class Linter:
  409. KEY = 'LINTER'
  410. @classmethod
  411. def value(cls, unit, flat_args, spec_args):
  412. return {cls.KEY: spec_args['LINTER'][0]}
  413. class LintConfigs:
  414. KEY = 'LINT-CONFIGS'
  415. @staticmethod
  416. def _from_config_type(unit, spec_args):
  417. if not spec_args.get('CONFIG_TYPE') or not spec_args.get('CONFIG_TYPE')[0]:
  418. return
  419. linter_name = spec_args['NAME'][0]
  420. config_type = spec_args.get('CONFIG_TYPE')[0]
  421. if config_type not in consts.LINTER_CONFIG_TYPES[linter_name]:
  422. message = "Unknown {} linter config type: {}. Allowed types: {}".format(
  423. linter_name, config_type, ', '.join(consts.LINTER_CONFIG_TYPES[linter_name])
  424. )
  425. ymake.report_configure_error(message)
  426. raise DartValueError()
  427. if common_configs_dir := unit.get('MODULE_COMMON_CONFIGS_DIR'):
  428. config = os.path.join(common_configs_dir, config_type)
  429. path = unit.resolve(config)
  430. if os.path.exists(path):
  431. return _common.strip_roots(config)
  432. message = "File not found: {}".format(path)
  433. ymake.report_configure_error(message)
  434. raise DartValueError()
  435. else:
  436. message = "Config type specifier is only allowed with autoincludes"
  437. ymake.report_configure_error(message)
  438. raise DartValueError()
  439. @classmethod
  440. def python_configs(cls, unit, flat_args, spec_args):
  441. resolved_configs = []
  442. if (custom_config := spec_args.get('CUSTOM_CONFIG')) and '/' in custom_config[0]:
  443. # black if custom config is passed.
  444. # XXX During migration we want to use the same macro parameter
  445. # for path to linter config and config type
  446. # thus, we check if '/' is present, if it is then it's a path
  447. # TODO delete once custom configs migrated to autoincludes scheme
  448. custom_config = custom_config[0]
  449. assert_file_exists(unit, custom_config)
  450. resolved_configs.append(custom_config)
  451. return {cls.KEY: serialize_list(resolved_configs)}
  452. if config := cls._from_config_type(unit, spec_args):
  453. # specified by config type, autoincludes scheme
  454. return {cls.KEY: serialize_list([config])}
  455. if project_to_config_map := spec_args.get('PROJECT_TO_CONFIG_MAP'):
  456. # ruff, TODO delete once custom configs migrated to autoincludes scheme
  457. project_to_config_map = project_to_config_map[0]
  458. assert_file_exists(unit, project_to_config_map)
  459. resolved_configs.append(project_to_config_map)
  460. cfgs = get_linter_configs(unit, project_to_config_map).values()
  461. for c in cfgs:
  462. assert_file_exists(unit, c)
  463. resolved_configs.append(c)
  464. return {cls.KEY: serialize_list(resolved_configs)}
  465. # default config
  466. linter_name = spec_args['NAME'][0]
  467. config = spec_args['CONFIGS'][0]
  468. assert_file_exists(unit, config)
  469. cfg = get_linter_configs(unit, config)[linter_name]
  470. assert_file_exists(unit, cfg)
  471. resolved_configs.append(cfg)
  472. if linter_name in ('flake8', 'py2_flake8'):
  473. resolved_configs.extend(spec_args.get('FLAKE_MIGRATIONS_CONFIG', []))
  474. return {cls.KEY: serialize_list(resolved_configs)}
  475. @classmethod
  476. def cpp_configs(cls, unit, flat_args, spec_args):
  477. custom_config = spec_args.get('CUSTOM_CONFIG')
  478. if custom_config:
  479. # TODO delete CUSTOM_CONFIG, it's used only by arc
  480. config = custom_config[0]
  481. assert_file_exists(unit, config)
  482. return {cls.KEY: serialize_list([config])}
  483. if config := cls._from_config_type(unit, spec_args):
  484. # specified by config type, autoincludes scheme
  485. return {cls.KEY: serialize_list([config])}
  486. # default config
  487. linter_name = spec_args['NAME'][0]
  488. config = spec_args.get('CONFIGS')[0]
  489. assert_file_exists(unit, config)
  490. config = get_linter_configs(unit, config)[linter_name]
  491. assert_file_exists(unit, config)
  492. return {cls.KEY: serialize_list([config])}
  493. class LintExtraParams:
  494. KEY = 'LINT-EXTRA-PARAMS'
  495. @classmethod
  496. def from_macro_args(cls, unit, flat_args, spec_args):
  497. extra_params = spec_args.get('EXTRA_PARAMS', [])
  498. for arg in extra_params:
  499. if '=' not in arg:
  500. message = 'Wrong EXTRA_PARAMS value: "{}". Values must have format "name=value".'.format(arg)
  501. ymake.report_configure_error(message)
  502. raise DartValueError()
  503. return {cls.KEY: serialize_list(extra_params)}
  504. class LintFileProcessingTime:
  505. KEY = 'LINT-FILE-PROCESSING-TIME'
  506. @classmethod
  507. def from_macro_args(cls, unit, flat_args, spec_args):
  508. return {cls.KEY: spec_args.get('FILE_PROCESSING_TIME', [''])[0]}
  509. class LintName:
  510. KEY = 'LINT-NAME'
  511. @classmethod
  512. def value(cls, unit, flat_args, spec_args):
  513. lint_name = spec_args['NAME'][0]
  514. if lint_name in ('flake8', 'py2_flake8') and (unit.get('DISABLE_FLAKE8') or 'no') == 'yes':
  515. unit.message(['INFO', 'Flake8 linting is disabled by `DISABLE_FLAKE8`'])
  516. raise DartValueError()
  517. return {cls.KEY: lint_name}
  518. class ModuleLang:
  519. KEY = 'MODULE_LANG'
  520. @classmethod
  521. def value(cls, unit, flat_args, spec_args):
  522. return {cls.KEY: unit.get("MODULE_LANG").lower() or consts.ModuleLang.UNKNOWN}
  523. class ModuleType:
  524. KEY = 'MODULE_TYPE'
  525. @classmethod
  526. def value(cls, unit, flat_args, spec_args):
  527. return {cls.KEY: unit.get('MODULE_TYPE')}
  528. class NoCheck:
  529. KEY = 'NO-CHECK'
  530. @classmethod
  531. def value(cls, unit, flat_args, spec_args):
  532. if unit.get('NO_CHECK_IMPORTS_FOR_VALUE') != "None":
  533. value = serialize_list(get_values_list(unit, 'NO_CHECK_IMPORTS_FOR_VALUE') or ["*"])
  534. return {cls.KEY: value}
  535. class NodejsRootVarName:
  536. KEY = 'NODEJS-ROOT-VAR-NAME'
  537. @classmethod
  538. def value(cls, unit, flat_args, spec_args):
  539. return {cls.KEY: unit.get("NODEJS-ROOT-VAR-NAME")}
  540. class NodeModulesBundleFilename:
  541. KEY = 'NODE-MODULES-BUNDLE-FILENAME'
  542. @classmethod
  543. def value(cls, unit, flat_args, spec_args):
  544. return {cls.KEY: spec_args.get('nm_bundle')}
  545. class PythonPaths:
  546. KEY = 'PYTHON-PATHS'
  547. @classmethod
  548. def value(cls, unit, flat_args, spec_args):
  549. python_paths = get_values_list(unit, 'TEST_PYTHON_PATH_VALUE')
  550. return {cls.KEY: serialize_list(python_paths)}
  551. class Requirements:
  552. KEY = 'REQUIREMENTS'
  553. @classmethod
  554. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  555. test_requirements = spec_args.get('REQUIREMENTS', []) + get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')
  556. return {cls.KEY: serialize_list(test_requirements)}
  557. @classmethod
  558. def with_maybe_fuzzing(cls, unit, flat_args, spec_args):
  559. test_requirements = serialize_list(
  560. spec_args.get('REQUIREMENTS', []) + get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')
  561. )
  562. if unit.get('FUZZING') == 'yes':
  563. value = serialize_list(filter(None, deserialize_list(test_requirements) + ["cpu:all", "ram:all"]))
  564. return {cls.KEY: value}
  565. else:
  566. return {cls.KEY: test_requirements}
  567. @classmethod
  568. def from_macro_args(cls, unit, flat_args, spec_args):
  569. value = " ".join(spec_args.get('REQUIREMENTS', []))
  570. return {cls.KEY: value}
  571. @classmethod
  572. def from_unit(cls, unit, flat_args, spec_args):
  573. requirements = get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')
  574. return {cls.KEY: serialize_list(requirements)}
  575. @classmethod
  576. def from_unit_with_full_network(cls, unit, flat_args, spec_args):
  577. requirements = sorted(set(["network:full"] + get_values_list(unit, "TEST_REQUIREMENTS_VALUE")))
  578. return {cls.KEY: serialize_list(requirements)}
  579. class SbrUidExt:
  580. KEY = 'SBR-UID-EXT'
  581. @classmethod
  582. def value(cls, unit, flat_args, spec_args):
  583. uid_ext = unit.get("SBR_UID_EXT").split(" ", 1)[-1] # strip variable name
  584. return {cls.KEY: uid_ext}
  585. class ScriptRelPath:
  586. KEY = 'SCRIPT-REL-PATH'
  587. @classmethod
  588. def second_flat(cls, unit, flat_args, spec_args):
  589. return {cls.KEY: flat_args[1]}
  590. @classmethod
  591. def first_flat(cls, unit, flat_args, spec_args):
  592. return {cls.KEY: flat_args[0]}
  593. @classmethod
  594. def pytest(cls, unit, flat_args, spec_args):
  595. return {cls.KEY: 'py3test.bin' if (unit.get("PYTHON3") == 'yes') else "pytest.bin"}
  596. @classmethod
  597. def junit(cls, unit, flat_args, spec_args):
  598. return {cls.KEY: 'junit5.test' if unit.get('MODULE_TYPE') == 'JUNIT5' else 'junit.test'}
  599. class Size:
  600. KEY = 'SIZE'
  601. @classmethod
  602. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  603. return {cls.KEY: ''.join(spec_args.get('SIZE', [])) or unit.get('TEST_SIZE_NAME')}
  604. @classmethod
  605. def from_unit(cls, unit, flat_args, spec_args):
  606. return {cls.KEY: unit.get('TEST_SIZE_NAME')}
  607. class SkipTest:
  608. KEY = 'SKIP_TEST'
  609. @classmethod
  610. def value(cls, unit, flat_args, spec_args):
  611. return {cls.KEY: unit.get('SKIP_TEST_VALUE')}
  612. class SourceFolderPath:
  613. KEY = 'SOURCE-FOLDER-PATH'
  614. @classmethod
  615. def normalized(cls, unit, flat_args, spec_args):
  616. return {cls.KEY: _common.get_norm_unit_path(unit)}
  617. @classmethod
  618. def test_dir(cls, unit, flat_args, spec_args):
  619. test_dir = _common.get_norm_unit_path(unit)
  620. test_files = flat_args[1:]
  621. if test_files:
  622. test_dir = os.path.dirname(test_files[0]).lstrip("$S/")
  623. return {cls.KEY: test_dir}
  624. class SplitFactor:
  625. KEY = 'SPLIT-FACTOR'
  626. @classmethod
  627. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  628. value = ''.join(spec_args.get('SPLIT_FACTOR', [])) or unit.get('TEST_SPLIT_FACTOR')
  629. return {cls.KEY: value}
  630. @classmethod
  631. def from_unit(cls, unit, flat_args, spec_args):
  632. return {cls.KEY: unit.get('TEST_SPLIT_FACTOR')}
  633. class Tag:
  634. KEY = 'TAG'
  635. @classmethod
  636. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  637. tags = serialize_list(sorted(_get_test_tags(unit, spec_args)))
  638. return {cls.KEY: tags}
  639. @classmethod
  640. def from_unit(cls, unit, flat_args, spec_args):
  641. tags = serialize_list(get_values_list(unit, "TEST_TAGS_VALUE"))
  642. return {cls.KEY: tags}
  643. @classmethod
  644. def from_unit_fat_external_no_retries(cls, unit, flat_args, spec_args):
  645. tags = sorted(set(["ya:fat", "ya:external", "ya:noretries"] + get_values_list(unit, "TEST_TAGS_VALUE")))
  646. return {cls.KEY: serialize_list(tags)}
  647. class TestClasspath:
  648. KEY = 'TEST_CLASSPATH'
  649. @classmethod
  650. def value(cls, unit, flat_args, spec_args):
  651. value = '${DART_CLASSPATH}'
  652. return {cls.KEY: value}
  653. class TestClasspathDeps:
  654. KEY = 'TEST_CLASSPATH_DEPS'
  655. @classmethod
  656. def value(cls, unit, flat_args, spec_args):
  657. return {cls.KEY: '${DART_CLASSPATH_DEPS}'}
  658. class TestCwd:
  659. KEY = 'TEST-CWD'
  660. @classmethod
  661. def from_unit(cls, unit, flat_args, spec_args):
  662. test_cwd = unit.get('TEST_CWD_VALUE') # TODO: validate test_cwd value
  663. return {cls.KEY: test_cwd}
  664. @classmethod
  665. def keywords_replaced(cls, unit, flat_args, spec_args):
  666. test_cwd = unit.get('TEST_CWD_VALUE') or ''
  667. if test_cwd:
  668. test_cwd = test_cwd.replace("$TEST_CWD_VALUE", "").replace('"MACRO_CALLS_DELIM"', "").strip()
  669. return {cls.KEY: test_cwd}
  670. @classmethod
  671. def moddir(cls, unit, flat_args, spec_args):
  672. return {cls.KEY: unit.get("MODDIR")}
  673. class TestData:
  674. KEY = 'TEST-DATA'
  675. @classmethod
  676. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  677. test_data = sorted(
  678. _common.filter_out_by_keyword(
  679. spec_args.get('DATA', []) + get_norm_paths(unit, 'TEST_DATA_VALUE'), 'AUTOUPDATED'
  680. )
  681. )
  682. return {cls.KEY: serialize_list(test_data)}
  683. @classmethod
  684. def from_macro_args_and_unit_with_canonical(cls, unit, flat_args, spec_args):
  685. test_data = sorted(
  686. _common.filter_out_by_keyword(
  687. spec_args.get('DATA', []) + get_norm_paths(unit, 'TEST_DATA_VALUE'), 'AUTOUPDATED'
  688. )
  689. )
  690. data, _ = get_canonical_test_resources(unit)
  691. test_data += data
  692. value = serialize_list(sorted(test_data))
  693. return {cls.KEY: value}
  694. @classmethod
  695. def ktlint(cls, unit, flat_args, spec_args):
  696. if unit.get('_USE_KTLINT_OLD') == 'yes':
  697. extra_test_data = [KTLINT_OLD_EDITOR_CONFIG]
  698. else:
  699. data_list = [KTLINT_CURRENT_EDITOR_CONFIG]
  700. baseline_path_relative = unit.get('_KTLINT_BASELINE_FILE')
  701. if baseline_path_relative:
  702. baseline_path = unit.resolve_arc_path(baseline_path_relative).replace('$S', 'arcadia')
  703. data_list += [baseline_path]
  704. extra_test_data = data_list
  705. # XXX
  706. if unit.get('_WITH_YA_1931') != 'yes':
  707. extra_test_data += java_srcdirs_to_data(unit, 'ALL_SRCDIRS', serialize_result=False)
  708. extra_test_data = serialize_list(extra_test_data)
  709. return {cls.KEY: extra_test_data}
  710. @classmethod
  711. def java_style(cls, unit, flat_args, spec_args):
  712. return {cls.KEY: java_srcdirs_to_data(unit, 'ALL_SRCDIRS')}
  713. @classmethod
  714. def from_unit_with_canonical(cls, unit, flat_args, spec_args):
  715. test_data = get_norm_paths(unit, 'TEST_DATA_VALUE')
  716. data, _ = get_canonical_test_resources(unit)
  717. test_data += data
  718. value = serialize_list(sorted(_common.filter_out_by_keyword(test_data, 'AUTOUPDATED')))
  719. return {cls.KEY: value}
  720. @classmethod
  721. def java_test(cls, unit, flat_args, spec_args):
  722. test_data = get_norm_paths(unit, 'TEST_DATA_VALUE')
  723. test_data.append('arcadia/build/scripts/run_junit.py')
  724. test_data.append('arcadia/build/scripts/unpacking_jtest_runner.py')
  725. data, _ = get_canonical_test_resources(unit)
  726. test_data += data
  727. props, error_mgs = extract_java_system_properties(unit, get_values_list(unit, 'SYSTEM_PROPERTIES_VALUE'))
  728. if error_mgs:
  729. ymake.report_configure_error(error_mgs)
  730. raise DartValueError()
  731. for prop in props:
  732. if prop['type'] == 'file':
  733. test_data.append(prop['path'].replace('${ARCADIA_ROOT}', 'arcadia'))
  734. value = serialize_list(sorted(_common.filter_out_by_keyword(test_data, 'AUTOUPDATED')))
  735. return {cls.KEY: value}
  736. @classmethod
  737. def from_unit(cls, unit, flat_args, spec_args):
  738. return {cls.KEY: serialize_list(get_values_list(unit, "TEST_DATA_VALUE"))}
  739. class DockerImage:
  740. KEY = 'DOCKER-IMAGES'
  741. @staticmethod
  742. def _validate(images):
  743. docker_image_re = consts.DOCKER_LINK_RE
  744. for img in images:
  745. msg = None
  746. if "=" in img:
  747. link, _ = img.rsplit('=', 1)
  748. if docker_image_re.match(link) is None:
  749. msg = 'Invalid docker url format: {}. Link should be provided in format docker://<repo>@sha256:<digest>'.format(
  750. link
  751. )
  752. else:
  753. msg = 'Invalid docker image: {}. Image should be provided in format <link>=<tag>'.format(img)
  754. if msg:
  755. ymake.report_configure_error(msg)
  756. raise DartValueError(msg)
  757. @classmethod
  758. def value(cls, unit, flat_args, spec_args):
  759. raw_value = get_values_list(unit, 'DOCKER_IMAGES_VALUE')
  760. images = sorted(raw_value)
  761. if images:
  762. cls._validate(images)
  763. return {cls.KEY: serialize_list(images)}
  764. class TsConfigPath:
  765. KEY = 'TS_CONFIG_PATH'
  766. class TsStylelintConfig:
  767. KEY = 'TS_STYLELINT_CONFIG'
  768. @classmethod
  769. def value(cls, unit, flat_args, spec_args):
  770. test_config = unit.get('_TS_STYLELINT_CONFIG')
  771. abs_test_config = unit.resolve(unit.resolve_arc_path(test_config))
  772. if not abs_test_config:
  773. ymake.report_configure_error(
  774. f"Config for stylelint not found: {test_config}.\n"
  775. "Set the correct value in `TS_STYLELINT(<config_filename>)` macro in the `ya.make` file."
  776. )
  777. return {cls.KEY: test_config}
  778. class TsTestDataDirs:
  779. KEY = 'TS-TEST-DATA-DIRS'
  780. @classmethod
  781. def value(cls, unit, flat_args, spec_args):
  782. value = serialize_list(_get_ts_test_data_dirs(unit))
  783. return {cls.KEY: value}
  784. class TsTestDataDirsRename:
  785. KEY = 'TS-TEST-DATA-DIRS-RENAME'
  786. @classmethod
  787. def value(cls, unit, flat_args, spec_args):
  788. return {cls.KEY: unit.get("_TS_TEST_DATA_DIRS_RENAME_VALUE")}
  789. class TsTestForPath:
  790. KEY = 'TS-TEST-FOR-PATH'
  791. @classmethod
  792. def value(cls, unit, flat_args, spec_args):
  793. return {cls.KEY: unit.get("TS_TEST_FOR_PATH")}
  794. class TestedProjectFilename:
  795. KEY = 'TESTED-PROJECT-FILENAME'
  796. @classmethod
  797. def value(cls, unit, flat_args, spec_args):
  798. return {cls.KEY: unit.filename()}
  799. class TestedProjectName:
  800. KEY = 'TESTED-PROJECT-NAME'
  801. @classmethod
  802. def unit_name(cls, unit, flat_args, spec_args):
  803. return {cls.KEY: unit.name()}
  804. @classmethod
  805. def normalized_basename(cls, unit, flat_args, spec_args):
  806. test_dir = _common.get_norm_unit_path(unit)
  807. return {cls.KEY: os.path.basename(test_dir)}
  808. @classmethod
  809. def test_dir(cls, unit, flat_args, spec_args):
  810. test_dir = _common.get_norm_unit_path(unit)
  811. test_files = flat_args[1:]
  812. if test_files:
  813. test_dir = os.path.dirname(test_files[0]).lstrip("$S/")
  814. return {cls.KEY: os.path.basename(test_dir)}
  815. @classmethod
  816. def path_filename_basename(cls, unit, flat_args, spec_args):
  817. binary_path = os.path.join(unit.path(), unit.filename())
  818. return {cls.KEY: os.path.basename(binary_path)}
  819. @classmethod
  820. def normalized(cls, unit, flat_args, spec_args):
  821. return {cls.KEY: _common.get_norm_unit_path(unit)}
  822. @classmethod
  823. def path_filename_basename_without_pkg_ext(cls, unit, flat_args, spec_args):
  824. value = os.path.basename(os.path.join(unit.path(), unit.filename()).replace(".pkg", ""))
  825. return {cls.KEY: value}
  826. @classmethod
  827. def filename_without_ext(cls, unit, flat_args, spec_args):
  828. return {cls.KEY: os.path.splitext(unit.filename())[0]}
  829. class TestFiles:
  830. KEY = 'TEST-FILES'
  831. # TODO remove FILES, see DEVTOOLS-7052, currently it's required
  832. # https://a.yandex-team.ru/arcadia/devtools/ya/test/dartfile/__init__.py?rev=r14292146#L10
  833. KEY2 = 'FILES'
  834. @classmethod
  835. def value(cls, unit, flat_args, spec_args):
  836. data_re = re.compile(r"sbr:/?/?(\d+)=?.*")
  837. data = flat_args[1:]
  838. resources = []
  839. for f in data:
  840. matched = re.match(data_re, f)
  841. if matched:
  842. resources.append(matched.group(1))
  843. value = serialize_list(resources)
  844. return {cls.KEY: value, cls.KEY2: value}
  845. @classmethod
  846. def flat_args_wo_first(cls, unit, flat_args, spec_args):
  847. value = serialize_list(flat_args[1:])
  848. return {cls.KEY: value, cls.KEY2: value}
  849. @classmethod
  850. def java_style(cls, unit, flat_args, spec_args):
  851. test_files = flat_args[1:]
  852. check_level = flat_args[1]
  853. allowed_levels = {
  854. 'base': '/yandex_checks.xml',
  855. 'strict': '/yandex_checks_strict.xml',
  856. 'extended': '/yandex_checks_extended.xml',
  857. 'library': '/yandex_checks_library.xml',
  858. }
  859. if check_level not in allowed_levels:
  860. raise Exception("'{}' is not allowed in LINT(), use one of {}".format(check_level, allowed_levels.keys()))
  861. test_files[0] = allowed_levels[check_level]
  862. value = serialize_list(test_files)
  863. return {cls.KEY: value, cls.KEY2: value}
  864. @classmethod
  865. def normalized(cls, unit, flat_args, spec_args):
  866. value = serialize_list([_common.get_norm_unit_path(unit, unit.filename())])
  867. return {cls.KEY: value, cls.KEY2: value}
  868. @classmethod
  869. def test_srcs(cls, unit, flat_args, spec_args):
  870. test_files = get_values_list(unit, 'TEST_SRCS_VALUE')
  871. value = serialize_list(test_files)
  872. return {cls.KEY: value, cls.KEY2: value}
  873. @classmethod
  874. def ts_test_srcs(cls, unit, flat_args, spec_args):
  875. test_files = get_values_list(unit, "_TS_TEST_SRCS_VALUE")
  876. test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files)
  877. value = serialize_list(test_files)
  878. return {cls.KEY: value, cls.KEY2: value}
  879. @classmethod
  880. def ts_input_files(cls, unit, flat_args, spec_args):
  881. typecheck_files = get_values_list(unit, "TS_INPUT_FILES")
  882. test_files = [_common.resolve_common_const(f) for f in typecheck_files]
  883. value = serialize_list(test_files)
  884. return {cls.KEY: value, cls.KEY2: value}
  885. @classmethod
  886. def ts_lint_srcs(cls, unit, flat_args, spec_args):
  887. test_files = get_values_list(unit, "_TS_LINT_SRCS_VALUE")
  888. test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files)
  889. value = serialize_list(test_files)
  890. return {cls.KEY: value, cls.KEY2: value}
  891. @classmethod
  892. def stylesheets(cls, unit, flat_args, spec_args):
  893. test_files = get_values_list(unit, "_TS_STYLELINT_FILES")
  894. test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files)
  895. value = serialize_list(test_files)
  896. return {cls.KEY: value, cls.KEY2: value}
  897. @classmethod
  898. def py_linter_files(cls, unit, flat_args, spec_args):
  899. files = unit.get('PY_LINTER_FILES')
  900. if not files:
  901. raise DartValueError()
  902. files = json.loads(files)
  903. test_files = []
  904. for path in files:
  905. if path.startswith(ARCADIA_ROOT):
  906. test_files.append(path.replace(ARCADIA_ROOT, SOURCE_ROOT_SHORT, 1))
  907. elif path.startswith(SOURCE_ROOT_SHORT):
  908. test_files.append(path)
  909. if not test_files:
  910. lint_name = LintName.value(unit, flat_args, spec_args)[LintName.KEY]
  911. message = 'No files to lint for {}'.format(lint_name)
  912. raise DartValueError(message)
  913. test_files = serialize_list(test_files)
  914. return {cls.KEY: test_files, cls.KEY2: test_files}
  915. @classmethod
  916. def cpp_linter_files(cls, unit, flat_args, spec_args):
  917. files_dart = _reference_group_var("ALL_SRCS", consts.STYLE_CPP_ALL_EXTS)
  918. return {cls.KEY: files_dart, cls.KEY2: files_dart}
  919. class TestEnv:
  920. KEY = 'TEST-ENV'
  921. @classmethod
  922. def value(cls, unit, flat_args, spec_args):
  923. return {cls.KEY: prepare_env(unit.get("TEST_ENV_VALUE"))}
  924. class TestIosDeviceType:
  925. KEY = 'TEST_IOS_DEVICE_TYPE'
  926. @classmethod
  927. def value(cls, unit, flat_args, spec_args):
  928. return {cls.KEY: unit.get('TEST_IOS_DEVICE_TYPE_VALUE')}
  929. class TestIosRuntimeType:
  930. KEY = 'TEST_IOS_RUNTIME_TYPE'
  931. @classmethod
  932. def value(cls, unit, flat_args, spec_args):
  933. return {cls.KEY: unit.get('TEST_IOS_RUNTIME_TYPE_VALUE')}
  934. class TestJar:
  935. KEY = 'TEST_JAR'
  936. @classmethod
  937. def value(cls, unit, flat_args, spec_args):
  938. if unit.get('UNITTEST_DIR'):
  939. value = '${UNITTEST_MOD}'
  940. else:
  941. value = '{}/{}.jar'.format(unit.get('MODDIR'), unit.get('REALPRJNAME'))
  942. return {cls.KEY: value}
  943. class TestName:
  944. KEY = 'TEST-NAME'
  945. @classmethod
  946. def value(cls, unit, flat_args, spec_args):
  947. return {cls.KEY: flat_args[0]}
  948. @classmethod
  949. def first_flat_with_bench(cls, unit, flat_args, spec_args):
  950. return {cls.KEY: flat_args[0] + '_bench'}
  951. @classmethod
  952. def first_flat(cls, unit, flat_args, spec_args):
  953. return {cls.KEY: flat_args[0].lower()}
  954. @classmethod
  955. def filename_without_ext(cls, unit, flat_args, spec_args):
  956. test_name = os.path.basename(os.path.join(unit.path(), unit.filename()))
  957. return {cls.KEY: os.path.splitext(test_name)[0]}
  958. @classmethod
  959. def normalized_joined_dir_basename(cls, unit, flat_args, spec_args):
  960. path = _common.get_norm_unit_path(unit)
  961. value = '-'.join([os.path.basename(os.path.dirname(path)), os.path.basename(path)])
  962. return {cls.KEY: value}
  963. @classmethod
  964. def normalized_joined_dir_basename_deps(cls, unit, flat_args, spec_args):
  965. path = _common.get_norm_unit_path(unit)
  966. value = '-'.join([os.path.basename(os.path.dirname(path)), os.path.basename(path), 'dependencies']).strip('-')
  967. return {cls.KEY: value}
  968. @classmethod
  969. def filename_without_pkg_ext(cls, unit, flat_args, spec_args):
  970. test_name = os.path.basename(os.path.join(unit.path(), unit.filename()).replace(".pkg", ""))
  971. return {cls.KEY: os.path.splitext(test_name)[0]}
  972. @classmethod
  973. def name_from_macro_args(cls, unit, flat_args, spec_args):
  974. return {cls.KEY: spec_args['NAME'][0]}
  975. class TestPartition:
  976. KEY = 'TEST_PARTITION'
  977. @classmethod
  978. def value(cls, unit, flat_args, spec_args):
  979. return {cls.KEY: unit.get("TEST_PARTITION")}
  980. class TestRecipes:
  981. KEY = 'TEST-RECIPES'
  982. @classmethod
  983. def value(cls, unit, flat_args, spec_args):
  984. return {cls.KEY: prepare_recipes(unit.get("TEST_RECIPES_VALUE"))}
  985. class TestRunnerBin:
  986. KEY = 'TEST-RUNNER-BIN'
  987. @classmethod
  988. def value(cls, unit, flat_args, spec_args):
  989. runner_bin = spec_args.get('RUNNER_BIN', [None])[0]
  990. if runner_bin:
  991. return {cls.KEY: runner_bin}
  992. class TestTimeout:
  993. KEY = 'TEST-TIMEOUT'
  994. @classmethod
  995. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  996. test_timeout = ''.join(spec_args.get('TIMEOUT', [])) or unit.get('TEST_TIMEOUT') or ''
  997. return {cls.KEY: test_timeout}
  998. @classmethod
  999. def from_unit_with_default(cls, unit, flat_args, spec_args):
  1000. timeout = list(filter(None, [unit.get(["TEST_TIMEOUT"])]))
  1001. if timeout:
  1002. timeout = timeout[0]
  1003. else:
  1004. timeout = '0'
  1005. return {cls.KEY: timeout}
  1006. @classmethod
  1007. def from_unit(cls, unit, flat_args, spec_args):
  1008. return {cls.KEY: unit.get('TEST_TIMEOUT')}
  1009. class TsResources:
  1010. KEY = "{}-ROOT-VAR-NAME"
  1011. @classmethod
  1012. def value(cls, unit, flat_args, spec_args):
  1013. erm_json = spec_args['erm_json']
  1014. ret = {}
  1015. for tool in erm_json.list_npm_packages():
  1016. tool_resource_label = cls.KEY.format(tool.upper())
  1017. tool_resource_value = unit.get(tool_resource_label)
  1018. if tool_resource_value:
  1019. ret[tool_resource_label] = tool_resource_value
  1020. return ret
  1021. class JvmArgs:
  1022. KEY = 'JVM_ARGS'
  1023. @classmethod
  1024. def value(cls, unit, flat_args, spec_args):
  1025. value = serialize_list(get_values_list(unit, 'JVM_ARGS_VALUE'))
  1026. return {cls.KEY: value}
  1027. class StrictClasspathClash:
  1028. KEY = 'STRICT_CLASSPATH_CLASH'
  1029. class SystemProperties:
  1030. KEY = 'SYSTEM_PROPERTIES'
  1031. @classmethod
  1032. def value(cls, unit, flat_args, spec_args):
  1033. props, error_mgs = extract_java_system_properties(unit, get_values_list(unit, 'SYSTEM_PROPERTIES_VALUE'))
  1034. if error_mgs:
  1035. ymake.report_configure_error(error_mgs)
  1036. raise DartValueError()
  1037. props = base64.b64encode(six.ensure_binary(json.dumps(props)))
  1038. return {cls.KEY: props}
  1039. class UnittestDir:
  1040. KEY = 'UNITTEST_DIR'
  1041. @classmethod
  1042. def value(cls, unit, flat_args, spec_args):
  1043. return {cls.KEY: unit.get('UNITTEST_DIR')}
  1044. class UseArcadiaPython:
  1045. KEY = 'USE_ARCADIA_PYTHON'
  1046. @classmethod
  1047. def value(cls, unit, flat_args, spec_args):
  1048. return {cls.KEY: unit.get('USE_ARCADIA_PYTHON')}
  1049. class UseKtlintOld:
  1050. KEY = 'USE_KTLINT_OLD'
  1051. @classmethod
  1052. def value(cls, unit, flat_args, spec_args):
  1053. if unit.get('_USE_KTLINT_OLD') == 'yes':
  1054. return {cls.KEY: 'yes'}
  1055. class YtSpec:
  1056. KEY = 'YT-SPEC'
  1057. @classmethod
  1058. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  1059. value = serialize_list(spec_args.get('YT_SPEC', []) + get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE'))
  1060. return {cls.KEY: value}
  1061. @classmethod
  1062. def from_unit(cls, unit, flat_args, spec_args):
  1063. yt_spec = get_values_list(unit, 'TEST_YT_SPEC_VALUE')
  1064. if yt_spec:
  1065. return {cls.KEY: serialize_list(yt_spec)}
  1066. @classmethod
  1067. def from_unit_list_var(cls, unit, flat_args, spec_args):
  1068. yt_spec_values = get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')
  1069. return {cls.KEY: serialize_list(yt_spec_values)}