_dart_fields.py 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439
  1. import base64
  2. import functools
  3. import json
  4. import operator
  5. import os
  6. import re
  7. import shlex
  8. import sys
  9. from functools import reduce
  10. import six
  11. import ymake
  12. import _common
  13. import lib.test_const as consts
  14. CANON_RESULT_FILE_NAME = 'result.json'
  15. CANON_DATA_DIR_NAME = 'canondata'
  16. CANON_OUTPUT_STORAGE = 'canondata_storage'
  17. KTLINT_CURRENT_EDITOR_CONFIG = "arcadia/build/platform/java/ktlint/.editorconfig"
  18. KTLINT_OLD_EDITOR_CONFIG = "arcadia/build/platform/java/ktlint_old/.editorconfig"
  19. ARCADIA_ROOT = '${ARCADIA_ROOT}/'
  20. SOURCE_ROOT_SHORT = '$S/'
  21. class DartValueError(ValueError):
  22. pass
  23. def create_dart_record(fields, *args):
  24. try:
  25. return reduce(operator.or_, (value for field in fields if (value := field(*args))), {})
  26. except Exception as e:
  27. if str(e) != "":
  28. ymake.report_configure_error("Exception: {}".format(e))
  29. else:
  30. raise (e)
  31. return None
  32. def with_fields(fields):
  33. def inner(func):
  34. @functools.wraps(func)
  35. def innermost(*args, **kwargs):
  36. func(fields, *args, **kwargs)
  37. return innermost
  38. return inner
  39. def serialize_list(lst):
  40. lst = list(filter(None, lst))
  41. return '\"' + ';'.join(lst) + '\"' if lst else ''
  42. def deserialize_list(val):
  43. return list(filter(None, val.replace('"', "").split(";")))
  44. def get_unit_list_variable(unit, name):
  45. items = unit.get(name)
  46. if items:
  47. items = items.split(' ')
  48. assert items[0] == "${}".format(name), (items, name)
  49. return items[1:]
  50. return []
  51. def get_values_list(unit, key):
  52. res = map(str.strip, (unit.get(key) or '').replace('$' + key, '').strip().split())
  53. return [r for r in res if r and r not in ['""', "''"]]
  54. def _get_test_tags(unit, spec_args=None):
  55. if spec_args is None:
  56. spec_args = {}
  57. tags = spec_args.get('TAG', []) + get_values_list(unit, 'TEST_TAGS_VALUE')
  58. tags = set(tags)
  59. if unit.get('EXPORT_SEM') == 'yes':
  60. filter_only_tags = sorted(t for t in tags if ':' not in t)
  61. unit.set(['FILTER_ONLY_TEST_TAGS', ' '.join(filter_only_tags)])
  62. # DEVTOOLS-7571
  63. if unit.get('SKIP_TEST_VALUE') and consts.YaTestTags.Fat in tags:
  64. tags.add(consts.YaTestTags.NotAutocheck)
  65. return tags
  66. def format_recipes(data: str | None) -> str:
  67. if not data:
  68. return ""
  69. data = data.replace('"USE_RECIPE_DELIM"', "\n")
  70. data = data.replace("$TEST_RECIPES_VALUE", "")
  71. return data
  72. def prepare_recipes(data: str | None) -> bytes:
  73. formatted = format_recipes(data)
  74. return base64.b64encode(six.ensure_binary(formatted))
  75. def prepare_env(data):
  76. data = data.replace("$TEST_ENV_VALUE", "")
  77. return serialize_list(shlex.split(data))
  78. def get_norm_paths(unit, key):
  79. # return paths without trailing (back)slash
  80. return [x.rstrip('\\/').replace('${ARCADIA_ROOT}/', '') for x in get_values_list(unit, key)]
  81. def _load_canonical_file(filename, unit_path):
  82. try:
  83. with open(filename, 'rb') as results_file:
  84. return json.load(results_file)
  85. except Exception as e:
  86. print("malformed canonical data in {}: {} ({})".format(unit_path, e, filename), file=sys.stderr)
  87. return {}
  88. def _get_resource_from_uri(uri):
  89. m = consts.CANON_MDS_RESOURCE_REGEX.match(uri)
  90. if m:
  91. key = m.group(1)
  92. return "{}:{}".format(consts.MDS_SCHEME, key)
  93. m = consts.CANON_BACKEND_RESOURCE_REGEX.match(uri)
  94. if m:
  95. key = m.group(1)
  96. return "{}:{}".format(consts.MDS_SCHEME, key)
  97. m = consts.CANON_SBR_RESOURCE_REGEX.match(uri)
  98. if m:
  99. # There might be conflict between resources, because all resources in sandbox have 'resource.tar.gz' name
  100. # That's why we use notation with '=' to specify specific path for resource
  101. uri = m.group(1)
  102. res_id = m.group(2)
  103. return "{}={}".format(uri, '/'.join([CANON_OUTPUT_STORAGE, res_id]))
  104. def _get_external_resources_from_canon_data(data):
  105. # Method should work with both canonization versions:
  106. # result.json: {'uri':X 'checksum':Y}
  107. # result.json: {'testname': {'uri':X 'checksum':Y}}
  108. # result.json: {'testname': [{'uri':X 'checksum':Y}]}
  109. # Also there is a bug - if user returns {'uri': 1} from test - machinery will fail
  110. # That's why we check 'uri' and 'checksum' fields presence
  111. # (it's still a bug - user can return {'uri':X, 'checksum': Y}, we need to unify canonization format)
  112. res = set()
  113. if isinstance(data, dict):
  114. if 'uri' in data and 'checksum' in data:
  115. resource = _get_resource_from_uri(data['uri'])
  116. if resource:
  117. res.add(resource)
  118. else:
  119. for k, v in six.iteritems(data):
  120. res.update(_get_external_resources_from_canon_data(v))
  121. elif isinstance(data, list):
  122. for e in data:
  123. res.update(_get_external_resources_from_canon_data(e))
  124. return res
  125. def _get_canonical_data_resources_v2(filename, unit_path):
  126. return (_get_external_resources_from_canon_data(_load_canonical_file(filename, unit_path)), [filename])
  127. def get_canonical_test_resources(unit):
  128. unit_path = unit.path()
  129. if unit.get("CUSTOM_CANONDATA_PATH"):
  130. path_to_canondata = unit_path.replace("$S", unit.get("CUSTOM_CANONDATA_PATH"))
  131. else:
  132. path_to_canondata = unit.resolve(unit_path)
  133. canon_data_dir = os.path.join(path_to_canondata, CANON_DATA_DIR_NAME, unit.get('CANONIZE_SUB_PATH') or '')
  134. try:
  135. _, dirs, files = next(os.walk(canon_data_dir))
  136. except StopIteration:
  137. # path doesn't exist
  138. return [], []
  139. if CANON_RESULT_FILE_NAME in files:
  140. return _get_canonical_data_resources_v2(os.path.join(canon_data_dir, CANON_RESULT_FILE_NAME), unit_path)
  141. return [], []
  142. def java_srcdirs_to_data(unit, var, serialize_result=True):
  143. extra_data = []
  144. for srcdir in (unit.get(var) or '').replace('$' + var, '').split():
  145. if srcdir == '.':
  146. srcdir = unit.get('MODDIR')
  147. if srcdir.startswith('${ARCADIA_ROOT}/') or srcdir.startswith('$ARCADIA_ROOT/'):
  148. srcdir = srcdir.replace('${ARCADIA_ROOT}/', '$S/')
  149. srcdir = srcdir.replace('$ARCADIA_ROOT/', '$S/')
  150. if srcdir.startswith('${CURDIR}') or srcdir.startswith('$CURDIR'):
  151. srcdir = srcdir.replace('${CURDIR}', os.path.join('$S', unit.get('MODDIR')))
  152. srcdir = srcdir.replace('$CURDIR', os.path.join('$S', unit.get('MODDIR')))
  153. srcdir = unit.resolve_arc_path(srcdir)
  154. if not srcdir.startswith('$'):
  155. srcdir = os.path.join('$S', unit.get('MODDIR'), srcdir)
  156. if srcdir.startswith('$S'):
  157. extra_data.append(srcdir.replace('$S', 'arcadia'))
  158. return serialize_list(extra_data) if serialize_result else extra_data
  159. def extract_java_system_properties(unit, args):
  160. if len(args) % 2:
  161. return [], 'Wrong use of SYSTEM_PROPERTIES in {}: odd number of arguments'.format(unit.path())
  162. props = []
  163. for x, y in zip(args[::2], args[1::2]):
  164. if x == 'FILE':
  165. if y.startswith('${BINDIR}') or y.startswith('${ARCADIA_BUILD_ROOT}') or y.startswith('/'):
  166. return [], 'Wrong use of SYSTEM_PROPERTIES in {}: absolute/build file path {}'.format(unit.path(), y)
  167. y = _common.rootrel_arc_src(y, unit)
  168. if not os.path.exists(unit.resolve('$S/' + y)):
  169. return [], 'Wrong use of SYSTEM_PROPERTIES in {}: can\'t resolve {}'.format(unit.path(), y)
  170. y = '${ARCADIA_ROOT}/' + y
  171. props.append({'type': 'file', 'path': y})
  172. else:
  173. props.append({'type': 'inline', 'key': x, 'value': y})
  174. return props, None
  175. def _resolve_module_files(unit, mod_dir, file_paths):
  176. mod_dir_with_sep_len = len(mod_dir) + 1
  177. resolved_files = []
  178. for path in file_paths:
  179. resolved = _common.rootrel_arc_src(path, unit)
  180. if resolved.startswith(mod_dir):
  181. resolved = resolved[mod_dir_with_sep_len:]
  182. resolved_files.append(resolved)
  183. return resolved_files
  184. def _resolve_config_path(unit, test_runner, rel_to):
  185. config_path = unit.get("ESLINT_CONFIG_PATH") if test_runner == "eslint" else unit.get("TS_TEST_CONFIG_PATH")
  186. arc_config_path = unit.resolve_arc_path(config_path)
  187. abs_config_path = unit.resolve(arc_config_path)
  188. if not abs_config_path:
  189. raise Exception("{} config not found: {}".format(test_runner, config_path))
  190. unit.onsrcs([arc_config_path])
  191. abs_rel_to = unit.resolve(unit.resolve_arc_path(unit.get(rel_to)))
  192. return os.path.relpath(abs_config_path, start=abs_rel_to)
  193. def _get_ts_test_data_dirs(unit):
  194. return sorted(
  195. set(
  196. [
  197. os.path.dirname(_common.rootrel_arc_src(p, unit))
  198. for p in (get_values_list(unit, "_TS_TEST_DATA_VALUE") or [])
  199. ]
  200. )
  201. )
  202. @_common.cache_by_second_arg
  203. def get_linter_configs(unit, config_paths):
  204. rel_config_path = _common.rootrel_arc_src(config_paths, unit)
  205. arc_config_path = unit.resolve_arc_path(rel_config_path)
  206. abs_config_path = unit.resolve(arc_config_path)
  207. with open(abs_config_path, 'r') as fd:
  208. return json.load(fd)
  209. def _reference_group_var(varname: str, extensions: list[str] | None = None) -> str:
  210. if extensions is None:
  211. return f'"${{join=\\;:{varname}}}"'
  212. return serialize_list(f'${{ext={ext};join=\\;:{varname}}}' for ext in extensions)
  213. def assert_file_exists(unit, path):
  214. path = unit.resolve(SOURCE_ROOT_SHORT + path)
  215. if not os.path.exists(path):
  216. message = 'File {} is not found'.format(path)
  217. ymake.report_configure_error(message)
  218. raise DartValueError()
  219. class AndroidApkTestActivity:
  220. KEY = 'ANDROID_APK_TEST_ACTIVITY'
  221. @classmethod
  222. def value(cls, unit, flat_args, spec_args):
  223. return {cls.KEY: unit.get('ANDROID_APK_TEST_ACTIVITY_VALUE')}
  224. class BenchmarkOpts:
  225. KEY = 'BENCHMARK-OPTS'
  226. @classmethod
  227. def value(cls, unit, flat_args, spec_args):
  228. return {cls.KEY: serialize_list(get_unit_list_variable(unit, 'BENCHMARK_OPTS_VALUE'))}
  229. class BinaryPath:
  230. KEY = 'BINARY-PATH'
  231. @classmethod
  232. def normalized(cls, unit, flat_args, spec_args):
  233. unit_path = _common.get_norm_unit_path(unit)
  234. return {cls.KEY: os.path.join(unit_path, unit.filename())}
  235. @classmethod
  236. def stripped(cls, unit, flat_args, spec_args):
  237. unit_path = unit.path()
  238. binary_path = os.path.join(unit_path, unit.filename())
  239. if binary_path:
  240. return {cls.KEY: _common.strip_roots(binary_path)}
  241. @classmethod
  242. def stripped_without_pkg_ext(cls, unit, flat_args, spec_args):
  243. value = _common.strip_roots(os.path.join(unit.path(), unit.filename()).replace(".pkg", ""))
  244. return {cls.KEY: value}
  245. class Blob:
  246. KEY = 'BLOB'
  247. @classmethod
  248. def value(cls, unit, flat_args, spec_args):
  249. return {cls.KEY: unit.get('TEST_BLOB_DATA')}
  250. class BuildFolderPath:
  251. KEY = 'BUILD-FOLDER-PATH'
  252. @classmethod
  253. def normalized(cls, unit, flat_args, spec_args):
  254. return {cls.KEY: _common.get_norm_unit_path(unit)}
  255. @classmethod
  256. def stripped(cls, unit, flat_args, spec_args):
  257. return {cls.KEY: _common.strip_roots(unit.path())}
  258. class CanonizeSubPath:
  259. KEY = 'CANONIZE_SUB_PATH'
  260. @classmethod
  261. def value(cls, unit, flat_args, spec_args):
  262. return {cls.KEY: unit.get('CANONIZE_SUB_PATH')}
  263. class Classpath:
  264. KEY = 'CLASSPATH'
  265. @classmethod
  266. def value(cls, unit, flat_args, spec_args):
  267. value = '$B/{}/{}.jar ${{DART_CLASSPATH}}'.format(unit.get('MODDIR'), unit.get('REALPRJNAME'))
  268. return {cls.KEY: value}
  269. class ConfigPath:
  270. KEY = 'CONFIG-PATH'
  271. @classmethod
  272. def value(cls, unit, flat_args, spec_args):
  273. test_runner, rel_to = flat_args
  274. return {cls.KEY: _resolve_config_path(unit, test_runner, rel_to=rel_to)}
  275. class CustomDependencies:
  276. KEY = 'CUSTOM-DEPENDENCIES'
  277. @classmethod
  278. def all_standard(cls, unit, flat_args, spec_args):
  279. custom_deps = ' '.join(spec_args.get('DEPENDS', []) + get_values_list(unit, 'TEST_DEPENDS_VALUE'))
  280. return {cls.KEY: custom_deps}
  281. @classmethod
  282. def depends_only(cls, unit, flat_args, spec_args):
  283. return {cls.KEY: " ".join(spec_args.get('DEPENDS', []))}
  284. @classmethod
  285. def test_depends_only(cls, unit, flat_args, spec_args):
  286. custom_deps = get_values_list(unit, 'TEST_DEPENDS_VALUE')
  287. return {cls.KEY: " ".join(custom_deps)}
  288. @classmethod
  289. def depends_with_linter(cls, unit, flat_args, spec_args):
  290. linter = Linter.value(unit, flat_args, spec_args)[Linter.KEY]
  291. deps = spec_args.get('DEPENDS', []) + [os.path.dirname(linter)]
  292. for dep in deps:
  293. unit.ondepends(dep)
  294. return {cls.KEY: " ".join(deps)}
  295. @classmethod
  296. def nots_with_recipies(cls, unit, flat_args, spec_args):
  297. deps = flat_args[0]
  298. recipes_lines = format_recipes(unit.get("TEST_RECIPES_VALUE")).strip().splitlines()
  299. if recipes_lines:
  300. deps = deps or []
  301. deps.extend([os.path.dirname(r.strip().split(" ")[0]) for r in recipes_lines])
  302. return {cls.KEY: " ".join(deps)}
  303. class EslintConfigPath:
  304. KEY = 'ESLINT_CONFIG_PATH'
  305. @classmethod
  306. def value(cls, unit, flat_args, spec_args):
  307. test_runner, rel_to = flat_args
  308. return {cls.KEY: _resolve_config_path(unit, test_runner, rel_to=rel_to)}
  309. class ForkMode:
  310. KEY = 'FORK-MODE'
  311. @classmethod
  312. def from_macro_and_unit(cls, unit, flat_args, spec_args):
  313. fork_mode = []
  314. if 'FORK_SUBTESTS' in spec_args:
  315. fork_mode.append('subtests')
  316. if 'FORK_TESTS' in spec_args:
  317. fork_mode.append('tests')
  318. fork_mode = fork_mode or spec_args.get('FORK_MODE', []) or unit.get('TEST_FORK_MODE').split()
  319. fork_mode = ' '.join(fork_mode) if fork_mode else ''
  320. return {cls.KEY: fork_mode}
  321. @classmethod
  322. def test_fork_mode(cls, unit, flat_args, spec_args):
  323. return {cls.KEY: unit.get('TEST_FORK_MODE')}
  324. class ForkTestFiles:
  325. KEY = 'FORK-TEST-FILES'
  326. @classmethod
  327. def value(cls, unit, flat_args, spec_args):
  328. return {cls.KEY: unit.get('FORK_TEST_FILES_MODE')}
  329. class FuzzDicts:
  330. KEY = 'FUZZ-DICTS'
  331. @classmethod
  332. def value(cls, unit, flat_args, spec_args):
  333. value = serialize_list(spec_args.get('FUZZ_DICTS', []) + get_unit_list_variable(unit, 'FUZZ_DICTS_VALUE'))
  334. return {cls.KEY: value}
  335. class FuzzOpts:
  336. KEY = 'FUZZ-OPTS'
  337. @classmethod
  338. def value(cls, unit, flat_args, spec_args):
  339. value = serialize_list(spec_args.get('FUZZ_OPTS', []) + get_unit_list_variable(unit, 'FUZZ_OPTS_VALUE'))
  340. return {cls.KEY: value}
  341. class Fuzzing:
  342. KEY = 'FUZZING'
  343. @classmethod
  344. def value(cls, unit, flat_args, spec_args):
  345. if unit.get('FUZZING') == 'yes':
  346. return {cls.KEY: '1'}
  347. class GlobalLibraryPath:
  348. KEY = 'GLOBAL-LIBRARY-PATH'
  349. @classmethod
  350. def value(cls, unit, flat_args, spec_args):
  351. return {cls.KEY: unit.global_filename()}
  352. class GoBenchTimeout:
  353. KEY = 'GO_BENCH_TIMEOUT'
  354. @classmethod
  355. def value(cls, unit, flat_args, spec_args):
  356. return {cls.KEY: unit.get('GO_BENCH_TIMEOUT')}
  357. class IgnoreClasspathClash:
  358. KEY = 'IGNORE_CLASSPATH_CLASH'
  359. @classmethod
  360. def value(cls, unit, flat_args, spec_args):
  361. value = ' '.join(get_values_list(unit, 'JAVA_IGNORE_CLASSPATH_CLASH_VALUE'))
  362. return {cls.KEY: value}
  363. class JavaClasspathCmdType:
  364. KEY = 'JAVA_CLASSPATH_CMD_TYPE'
  365. @classmethod
  366. def value(cls, unit, flat_args, spec_args):
  367. java_cp_arg_type = unit.get('JAVA_CLASSPATH_CMD_TYPE_VALUE') or 'MANIFEST'
  368. if java_cp_arg_type not in ('MANIFEST', 'COMMAND_FILE', 'LIST'):
  369. # TODO move error reporting out of field classes
  370. ymake.report_configure_error(
  371. '{}: TEST_JAVA_CLASSPATH_CMD_TYPE({}) are invalid. Choose argument from MANIFEST, COMMAND_FILE or LIST)'.format(
  372. unit.path(), java_cp_arg_type
  373. )
  374. )
  375. raise DartValueError()
  376. return {cls.KEY: java_cp_arg_type}
  377. class JdkForTests:
  378. KEY = 'JDK_FOR_TESTS'
  379. @classmethod
  380. def value(cls, unit, flat_args, spec_args):
  381. value = 'JDK' + (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION') or '_DEFAULT') + '_FOR_TESTS'
  382. return {cls.KEY: value}
  383. class JdkLatestVersion:
  384. KEY = 'JDK_LATEST_VERSION'
  385. @classmethod
  386. def value(cls, unit, flat_args, spec_args):
  387. return {cls.KEY: unit.get('JDK_LATEST_VERSION')}
  388. class JdkResource:
  389. KEY = 'JDK_RESOURCE'
  390. @classmethod
  391. def value(cls, unit, flat_args, spec_args):
  392. value = 'JDK' + (unit.get('JDK_VERSION') or unit.get('JDK_REAL_VERSION') or '_DEFAULT')
  393. return {cls.KEY: value}
  394. class KtlintBaselineFile:
  395. KEY = 'KTLINT_BASELINE_FILE'
  396. @classmethod
  397. def value(cls, unit, flat_args, spec_args):
  398. if unit.get('_USE_KTLINT_OLD') != 'yes':
  399. baseline_path_relative = unit.get('_KTLINT_BASELINE_FILE')
  400. if baseline_path_relative:
  401. return {cls.KEY: baseline_path_relative}
  402. class KtlintBinary:
  403. KEY = 'KTLINT_BINARY'
  404. @classmethod
  405. def value(cls, unit, flat_args, spec_args):
  406. value = '$(KTLINT_OLD)/run.bat' if unit.get('_USE_KTLINT_OLD') == 'yes' else '$(KTLINT)/run.bat'
  407. return {cls.KEY: value}
  408. class Linter:
  409. KEY = 'LINTER'
  410. @classmethod
  411. def value(cls, unit, flat_args, spec_args):
  412. return {cls.KEY: spec_args['LINTER'][0]}
  413. class LintConfigs:
  414. KEY = 'LINT-CONFIGS'
  415. @staticmethod
  416. def _from_config_type(unit, spec_args):
  417. if not spec_args.get('CONFIG_TYPE') or not spec_args.get('CONFIG_TYPE')[0]:
  418. return
  419. linter_name = spec_args['NAME'][0]
  420. config_type = spec_args.get('CONFIG_TYPE')[0]
  421. if config_type not in consts.LINTER_CONFIG_TYPES[linter_name]:
  422. message = "Unknown {} linter config type: {}. Allowed types: {}".format(
  423. linter_name, config_type, ', '.join(consts.LINTER_CONFIG_TYPES[linter_name])
  424. )
  425. ymake.report_configure_error(message)
  426. raise DartValueError()
  427. if common_configs_dir := unit.get('MODULE_COMMON_CONFIGS_DIR'):
  428. config = os.path.join(common_configs_dir, config_type)
  429. path = unit.resolve(config)
  430. if os.path.exists(path):
  431. return _common.strip_roots(config)
  432. message = "File not found: {}".format(path)
  433. ymake.report_configure_error(message)
  434. raise DartValueError()
  435. else:
  436. message = "Config type specifier is only allowed with autoincludes"
  437. ymake.report_configure_error(message)
  438. raise DartValueError()
  439. @classmethod
  440. def python_configs(cls, unit, flat_args, spec_args):
  441. resolved_configs = []
  442. if (custom_config := spec_args.get('CUSTOM_CONFIG')) and '/' in custom_config[0]:
  443. # black if custom config is passed.
  444. # XXX During migration we want to use the same macro parameter
  445. # for path to linter config and config type
  446. # thus, we check if '/' is present, if it is then it's a path
  447. # TODO delete once custom configs migrated to autoincludes scheme
  448. custom_config = custom_config[0]
  449. assert_file_exists(unit, custom_config)
  450. resolved_configs.append(custom_config)
  451. return {cls.KEY: serialize_list(resolved_configs)}
  452. if config := cls._from_config_type(unit, spec_args):
  453. # specified by config type, autoincludes scheme
  454. return {cls.KEY: serialize_list([config])}
  455. if project_to_config_map := spec_args.get('PROJECT_TO_CONFIG_MAP'):
  456. # ruff, TODO delete once custom configs migrated to autoincludes scheme
  457. project_to_config_map = project_to_config_map[0]
  458. assert_file_exists(unit, project_to_config_map)
  459. resolved_configs.append(project_to_config_map)
  460. cfgs = get_linter_configs(unit, project_to_config_map).values()
  461. for c in cfgs:
  462. assert_file_exists(unit, c)
  463. resolved_configs.append(c)
  464. return {cls.KEY: serialize_list(resolved_configs)}
  465. # default config
  466. linter_name = spec_args['NAME'][0]
  467. config = spec_args['CONFIGS'][0]
  468. assert_file_exists(unit, config)
  469. cfg = get_linter_configs(unit, config)[linter_name]
  470. assert_file_exists(unit, cfg)
  471. resolved_configs.append(cfg)
  472. if linter_name in ('flake8', 'py2_flake8'):
  473. resolved_configs.extend(spec_args.get('FLAKE_MIGRATIONS_CONFIG', []))
  474. return {cls.KEY: serialize_list(resolved_configs)}
  475. @classmethod
  476. def cpp_configs(cls, unit, flat_args, spec_args):
  477. custom_config = spec_args.get('CUSTOM_CONFIG')
  478. if custom_config:
  479. # TODO delete CUSTOM_CONFIG, it's used only by arc
  480. config = custom_config[0]
  481. assert_file_exists(unit, config)
  482. return {cls.KEY: serialize_list([config])}
  483. if config := cls._from_config_type(unit, spec_args):
  484. # specified by config type, autoincludes scheme
  485. return {cls.KEY: serialize_list([config])}
  486. # default config
  487. linter_name = spec_args['NAME'][0]
  488. config = spec_args.get('CONFIGS')[0]
  489. assert_file_exists(unit, config)
  490. config = get_linter_configs(unit, config)[linter_name]
  491. assert_file_exists(unit, config)
  492. return {cls.KEY: serialize_list([config])}
  493. class LintExtraParams:
  494. KEY = 'LINT-EXTRA-PARAMS'
  495. _CUSTOM_CLANG_FORMAT_BIN_ALLOWED_PATHS = ('ads', 'bigrt', 'grut')
  496. @classmethod
  497. def from_macro_args(cls, unit, flat_args, spec_args):
  498. extra_params = spec_args.get('EXTRA_PARAMS', [])
  499. for arg in extra_params:
  500. if '=' not in arg:
  501. message = 'Wrong EXTRA_PARAMS value: "{}". Values must have format "name=value".'.format(arg)
  502. ymake.report_configure_error(message)
  503. raise DartValueError()
  504. if 'clang_format_bin' in arg:
  505. upath = unit.path()[3:]
  506. if not upath.startswith(cls._CUSTOM_CLANG_FORMAT_BIN_ALLOWED_PATHS):
  507. message = f'Custom clang-format is not allowed in upath: {upath}'
  508. ymake.report_configure_error(message)
  509. raise DartValueError()
  510. return {cls.KEY: serialize_list(extra_params)}
  511. class LintFileProcessingTime:
  512. KEY = 'LINT-FILE-PROCESSING-TIME'
  513. @classmethod
  514. def from_macro_args(cls, unit, flat_args, spec_args):
  515. return {cls.KEY: spec_args.get('FILE_PROCESSING_TIME', [''])[0]}
  516. class LintName:
  517. KEY = 'LINT-NAME'
  518. @classmethod
  519. def value(cls, unit, flat_args, spec_args):
  520. lint_name = spec_args['NAME'][0]
  521. if lint_name in ('flake8', 'py2_flake8') and (unit.get('DISABLE_FLAKE8') or 'no') == 'yes':
  522. unit.message(['INFO', 'Flake8 linting is disabled by `DISABLE_FLAKE8`'])
  523. raise DartValueError()
  524. return {cls.KEY: lint_name}
  525. class ModuleLang:
  526. KEY = 'MODULE_LANG'
  527. @classmethod
  528. def value(cls, unit, flat_args, spec_args):
  529. return {cls.KEY: unit.get("MODULE_LANG").lower() or consts.ModuleLang.UNKNOWN}
  530. class ModuleType:
  531. KEY = 'MODULE_TYPE'
  532. @classmethod
  533. def value(cls, unit, flat_args, spec_args):
  534. return {cls.KEY: unit.get('MODULE_TYPE')}
  535. class NoCheck:
  536. KEY = 'NO-CHECK'
  537. @classmethod
  538. def value(cls, unit, flat_args, spec_args):
  539. if unit.get('NO_CHECK_IMPORTS_FOR_VALUE') != "None":
  540. value = serialize_list(get_values_list(unit, 'NO_CHECK_IMPORTS_FOR_VALUE') or ["*"])
  541. return {cls.KEY: value}
  542. class NodejsRootVarName:
  543. KEY = 'NODEJS-ROOT-VAR-NAME'
  544. @classmethod
  545. def value(cls, unit, flat_args, spec_args):
  546. return {cls.KEY: unit.get("NODEJS-ROOT-VAR-NAME")}
  547. class NodeModulesBundleFilename:
  548. KEY = 'NODE-MODULES-BUNDLE-FILENAME'
  549. @classmethod
  550. def value(cls, unit, flat_args, spec_args):
  551. return {cls.KEY: spec_args.get('nm_bundle')}
  552. class PythonPaths:
  553. KEY = 'PYTHON-PATHS'
  554. @classmethod
  555. def value(cls, unit, flat_args, spec_args):
  556. python_paths = get_values_list(unit, 'TEST_PYTHON_PATH_VALUE')
  557. return {cls.KEY: serialize_list(python_paths)}
  558. class Requirements:
  559. KEY = 'REQUIREMENTS'
  560. @classmethod
  561. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  562. test_requirements = spec_args.get('REQUIREMENTS', []) + get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')
  563. return {cls.KEY: serialize_list(test_requirements)}
  564. @classmethod
  565. def with_maybe_fuzzing(cls, unit, flat_args, spec_args):
  566. test_requirements = serialize_list(
  567. spec_args.get('REQUIREMENTS', []) + get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')
  568. )
  569. if unit.get('FUZZING') == 'yes':
  570. value = serialize_list(filter(None, deserialize_list(test_requirements) + ["cpu:all", "ram:all"]))
  571. return {cls.KEY: value}
  572. else:
  573. return {cls.KEY: test_requirements}
  574. @classmethod
  575. def from_macro_args(cls, unit, flat_args, spec_args):
  576. value = " ".join(spec_args.get('REQUIREMENTS', []))
  577. return {cls.KEY: value}
  578. @classmethod
  579. def from_unit(cls, unit, flat_args, spec_args):
  580. requirements = get_values_list(unit, 'TEST_REQUIREMENTS_VALUE')
  581. return {cls.KEY: serialize_list(requirements)}
  582. @classmethod
  583. def from_unit_with_full_network(cls, unit, flat_args, spec_args):
  584. requirements = sorted(set(["network:full"] + get_values_list(unit, "TEST_REQUIREMENTS_VALUE")))
  585. return {cls.KEY: serialize_list(requirements)}
  586. class SbrUidExt:
  587. KEY = 'SBR-UID-EXT'
  588. @classmethod
  589. def value(cls, unit, flat_args, spec_args):
  590. uid_ext = unit.get("SBR_UID_EXT").split(" ", 1)[-1] # strip variable name
  591. return {cls.KEY: uid_ext}
  592. class ScriptRelPath:
  593. KEY = 'SCRIPT-REL-PATH'
  594. @classmethod
  595. def second_flat(cls, unit, flat_args, spec_args):
  596. return {cls.KEY: flat_args[1]}
  597. @classmethod
  598. def first_flat(cls, unit, flat_args, spec_args):
  599. return {cls.KEY: flat_args[0]}
  600. @classmethod
  601. def pytest(cls, unit, flat_args, spec_args):
  602. return {cls.KEY: 'py3test.bin' if (unit.get("PYTHON3") == 'yes') else "pytest.bin"}
  603. @classmethod
  604. def junit(cls, unit, flat_args, spec_args):
  605. return {cls.KEY: 'junit5.test' if unit.get('MODULE_TYPE') == 'JUNIT5' else 'junit.test'}
  606. class Size:
  607. KEY = 'SIZE'
  608. @classmethod
  609. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  610. return {cls.KEY: ''.join(spec_args.get('SIZE', [])) or unit.get('TEST_SIZE_NAME')}
  611. @classmethod
  612. def from_unit(cls, unit, flat_args, spec_args):
  613. return {cls.KEY: unit.get('TEST_SIZE_NAME')}
  614. class SkipTest:
  615. KEY = 'SKIP_TEST'
  616. @classmethod
  617. def value(cls, unit, flat_args, spec_args):
  618. return {cls.KEY: unit.get('SKIP_TEST_VALUE')}
  619. class SourceFolderPath:
  620. KEY = 'SOURCE-FOLDER-PATH'
  621. @classmethod
  622. def normalized(cls, unit, flat_args, spec_args):
  623. return {cls.KEY: _common.get_norm_unit_path(unit)}
  624. @classmethod
  625. def test_dir(cls, unit, flat_args, spec_args):
  626. test_dir = _common.get_norm_unit_path(unit)
  627. test_files = flat_args[1:]
  628. if test_files:
  629. test_dir = os.path.dirname(test_files[0]).lstrip("$S/")
  630. return {cls.KEY: test_dir}
  631. class SplitFactor:
  632. KEY = 'SPLIT-FACTOR'
  633. @classmethod
  634. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  635. value = ''.join(spec_args.get('SPLIT_FACTOR', [])) or unit.get('TEST_SPLIT_FACTOR')
  636. return {cls.KEY: value}
  637. @classmethod
  638. def from_unit(cls, unit, flat_args, spec_args):
  639. return {cls.KEY: unit.get('TEST_SPLIT_FACTOR')}
  640. class Tag:
  641. KEY = 'TAG'
  642. @classmethod
  643. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  644. tags = serialize_list(sorted(_get_test_tags(unit, spec_args)))
  645. return {cls.KEY: tags}
  646. @classmethod
  647. def from_unit(cls, unit, flat_args, spec_args):
  648. tags = serialize_list(get_values_list(unit, "TEST_TAGS_VALUE"))
  649. return {cls.KEY: tags}
  650. @classmethod
  651. def from_unit_fat_external_no_retries(cls, unit, flat_args, spec_args):
  652. tags = sorted(set(["ya:fat", "ya:external", "ya:noretries"] + get_values_list(unit, "TEST_TAGS_VALUE")))
  653. return {cls.KEY: serialize_list(tags)}
  654. class TestClasspath:
  655. KEY = 'TEST_CLASSPATH'
  656. @classmethod
  657. def value(cls, unit, flat_args, spec_args):
  658. value = '${DART_CLASSPATH}'
  659. return {cls.KEY: value}
  660. class TestClasspathDeps:
  661. KEY = 'TEST_CLASSPATH_DEPS'
  662. @classmethod
  663. def value(cls, unit, flat_args, spec_args):
  664. return {cls.KEY: '${DART_CLASSPATH_DEPS}'}
  665. class TestCwd:
  666. KEY = 'TEST-CWD'
  667. @classmethod
  668. def from_unit(cls, unit, flat_args, spec_args):
  669. test_cwd = unit.get('TEST_CWD_VALUE') # TODO: validate test_cwd value
  670. return {cls.KEY: test_cwd}
  671. @classmethod
  672. def keywords_replaced(cls, unit, flat_args, spec_args):
  673. test_cwd = unit.get('TEST_CWD_VALUE') or ''
  674. if test_cwd:
  675. test_cwd = test_cwd.replace("$TEST_CWD_VALUE", "").replace('"MACRO_CALLS_DELIM"', "").strip()
  676. return {cls.KEY: test_cwd}
  677. @classmethod
  678. def moddir(cls, unit, flat_args, spec_args):
  679. return {cls.KEY: unit.get("MODDIR")}
  680. class TestData:
  681. KEY = 'TEST-DATA'
  682. @classmethod
  683. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  684. test_data = sorted(
  685. _common.filter_out_by_keyword(
  686. spec_args.get('DATA', []) + get_norm_paths(unit, 'TEST_DATA_VALUE'), 'AUTOUPDATED'
  687. )
  688. )
  689. return {cls.KEY: serialize_list(test_data)}
  690. @classmethod
  691. def from_macro_args_and_unit_with_canonical(cls, unit, flat_args, spec_args):
  692. test_data = sorted(
  693. _common.filter_out_by_keyword(
  694. spec_args.get('DATA', []) + get_norm_paths(unit, 'TEST_DATA_VALUE'), 'AUTOUPDATED'
  695. )
  696. )
  697. data, _ = get_canonical_test_resources(unit)
  698. test_data += data
  699. value = serialize_list(sorted(test_data))
  700. return {cls.KEY: value}
  701. @classmethod
  702. def ktlint(cls, unit, flat_args, spec_args):
  703. if unit.get('_USE_KTLINT_OLD') == 'yes':
  704. extra_test_data = [KTLINT_OLD_EDITOR_CONFIG]
  705. else:
  706. data_list = [KTLINT_CURRENT_EDITOR_CONFIG]
  707. baseline_path_relative = unit.get('_KTLINT_BASELINE_FILE')
  708. if baseline_path_relative:
  709. baseline_path = unit.resolve_arc_path(baseline_path_relative).replace('$S', 'arcadia')
  710. data_list += [baseline_path]
  711. extra_test_data = data_list
  712. # XXX
  713. if unit.get('_WITH_YA_1931') != 'yes':
  714. extra_test_data += java_srcdirs_to_data(unit, 'ALL_SRCDIRS', serialize_result=False)
  715. extra_test_data = serialize_list(extra_test_data)
  716. return {cls.KEY: extra_test_data}
  717. @classmethod
  718. def java_style(cls, unit, flat_args, spec_args):
  719. return {cls.KEY: java_srcdirs_to_data(unit, 'ALL_SRCDIRS')}
  720. @classmethod
  721. def from_unit_with_canonical(cls, unit, flat_args, spec_args):
  722. test_data = get_norm_paths(unit, 'TEST_DATA_VALUE')
  723. data, _ = get_canonical_test_resources(unit)
  724. test_data += data
  725. value = serialize_list(sorted(_common.filter_out_by_keyword(test_data, 'AUTOUPDATED')))
  726. return {cls.KEY: value}
  727. @classmethod
  728. def java_test(cls, unit, flat_args, spec_args):
  729. test_data = get_norm_paths(unit, 'TEST_DATA_VALUE')
  730. test_data.append('arcadia/build/scripts/run_junit.py')
  731. test_data.append('arcadia/build/scripts/unpacking_jtest_runner.py')
  732. data, _ = get_canonical_test_resources(unit)
  733. test_data += data
  734. props, error_mgs = extract_java_system_properties(unit, get_values_list(unit, 'SYSTEM_PROPERTIES_VALUE'))
  735. if error_mgs:
  736. ymake.report_configure_error(error_mgs)
  737. raise DartValueError()
  738. for prop in props:
  739. if prop['type'] == 'file':
  740. test_data.append(prop['path'].replace('${ARCADIA_ROOT}', 'arcadia'))
  741. value = serialize_list(sorted(_common.filter_out_by_keyword(test_data, 'AUTOUPDATED')))
  742. return {cls.KEY: value}
  743. @classmethod
  744. def from_unit(cls, unit, flat_args, spec_args):
  745. return {cls.KEY: serialize_list(get_values_list(unit, "TEST_DATA_VALUE"))}
  746. class DockerImage:
  747. KEY = 'DOCKER-IMAGES'
  748. @staticmethod
  749. def _validate(images):
  750. docker_image_re = consts.DOCKER_LINK_RE
  751. for img in images:
  752. msg = None
  753. if "=" in img:
  754. link, _ = img.rsplit('=', 1)
  755. if docker_image_re.match(link) is None:
  756. msg = 'Invalid docker url format: {}. Link should be provided in format docker://<repo>@sha256:<digest>'.format(
  757. link
  758. )
  759. else:
  760. msg = 'Invalid docker image: {}. Image should be provided in format <link>=<tag>'.format(img)
  761. if msg:
  762. ymake.report_configure_error(msg)
  763. raise DartValueError(msg)
  764. @classmethod
  765. def value(cls, unit, flat_args, spec_args):
  766. raw_value = get_values_list(unit, 'DOCKER_IMAGES_VALUE')
  767. images = sorted(raw_value)
  768. if images:
  769. cls._validate(images)
  770. return {cls.KEY: serialize_list(images)}
  771. class TsConfigPath:
  772. KEY = 'TS_CONFIG_PATH'
  773. class TsStylelintConfig:
  774. KEY = 'TS_STYLELINT_CONFIG'
  775. @classmethod
  776. def value(cls, unit, flat_args, spec_args):
  777. test_config = unit.get('_TS_STYLELINT_CONFIG')
  778. abs_test_config = unit.resolve(unit.resolve_arc_path(test_config))
  779. if not abs_test_config:
  780. ymake.report_configure_error(
  781. f"Config for stylelint not found: {test_config}.\n"
  782. "Set the correct value in `TS_STYLELINT(<config_filename>)` macro in the `ya.make` file."
  783. )
  784. return {cls.KEY: test_config}
  785. class TsTestDataDirs:
  786. KEY = 'TS-TEST-DATA-DIRS'
  787. @classmethod
  788. def value(cls, unit, flat_args, spec_args):
  789. value = serialize_list(_get_ts_test_data_dirs(unit))
  790. return {cls.KEY: value}
  791. class TsTestDataDirsRename:
  792. KEY = 'TS-TEST-DATA-DIRS-RENAME'
  793. @classmethod
  794. def value(cls, unit, flat_args, spec_args):
  795. return {cls.KEY: unit.get("_TS_TEST_DATA_DIRS_RENAME_VALUE")}
  796. class TsTestForPath:
  797. KEY = 'TS-TEST-FOR-PATH'
  798. @classmethod
  799. def value(cls, unit, flat_args, spec_args):
  800. return {cls.KEY: unit.get("TS_TEST_FOR_PATH")}
  801. class TestedProjectFilename:
  802. KEY = 'TESTED-PROJECT-FILENAME'
  803. @classmethod
  804. def value(cls, unit, flat_args, spec_args):
  805. return {cls.KEY: unit.filename()}
  806. class TestedProjectName:
  807. KEY = 'TESTED-PROJECT-NAME'
  808. @classmethod
  809. def unit_name(cls, unit, flat_args, spec_args):
  810. return {cls.KEY: unit.name()}
  811. @classmethod
  812. def normalized_basename(cls, unit, flat_args, spec_args):
  813. test_dir = _common.get_norm_unit_path(unit)
  814. return {cls.KEY: os.path.basename(test_dir)}
  815. @classmethod
  816. def test_dir(cls, unit, flat_args, spec_args):
  817. test_dir = _common.get_norm_unit_path(unit)
  818. test_files = flat_args[1:]
  819. if test_files:
  820. test_dir = os.path.dirname(test_files[0]).lstrip("$S/")
  821. return {cls.KEY: os.path.basename(test_dir)}
  822. @classmethod
  823. def path_filename_basename(cls, unit, flat_args, spec_args):
  824. binary_path = os.path.join(unit.path(), unit.filename())
  825. return {cls.KEY: os.path.basename(binary_path)}
  826. @classmethod
  827. def normalized(cls, unit, flat_args, spec_args):
  828. return {cls.KEY: _common.get_norm_unit_path(unit)}
  829. @classmethod
  830. def path_filename_basename_without_pkg_ext(cls, unit, flat_args, spec_args):
  831. value = os.path.basename(os.path.join(unit.path(), unit.filename()).replace(".pkg", ""))
  832. return {cls.KEY: value}
  833. @classmethod
  834. def filename_without_ext(cls, unit, flat_args, spec_args):
  835. return {cls.KEY: os.path.splitext(unit.filename())[0]}
  836. class TestFiles:
  837. KEY = 'TEST-FILES'
  838. # TODO remove FILES, see DEVTOOLS-7052, currently it's required
  839. # https://a.yandex-team.ru/arcadia/devtools/ya/test/dartfile/__init__.py?rev=r14292146#L10
  840. KEY2 = 'FILES'
  841. @classmethod
  842. def value(cls, unit, flat_args, spec_args):
  843. data_re = re.compile(r"sbr:/?/?(\d+)=?.*")
  844. data = flat_args[1:]
  845. resources = []
  846. for f in data:
  847. matched = re.match(data_re, f)
  848. if matched:
  849. resources.append(matched.group(1))
  850. value = serialize_list(resources)
  851. return {cls.KEY: value, cls.KEY2: value}
  852. @classmethod
  853. def flat_args_wo_first(cls, unit, flat_args, spec_args):
  854. value = serialize_list(flat_args[1:])
  855. return {cls.KEY: value, cls.KEY2: value}
  856. @classmethod
  857. def java_style(cls, unit, flat_args, spec_args):
  858. test_files = flat_args[1:]
  859. check_level = flat_args[1]
  860. allowed_levels = {
  861. 'base': '/yandex_checks.xml',
  862. 'strict': '/yandex_checks_strict.xml',
  863. 'extended': '/yandex_checks_extended.xml',
  864. 'library': '/yandex_checks_library.xml',
  865. }
  866. if check_level not in allowed_levels:
  867. raise Exception("'{}' is not allowed in LINT(), use one of {}".format(check_level, allowed_levels.keys()))
  868. test_files[0] = allowed_levels[check_level]
  869. value = serialize_list(test_files)
  870. return {cls.KEY: value, cls.KEY2: value}
  871. @classmethod
  872. def normalized(cls, unit, flat_args, spec_args):
  873. value = serialize_list([_common.get_norm_unit_path(unit, unit.filename())])
  874. return {cls.KEY: value, cls.KEY2: value}
  875. @classmethod
  876. def test_srcs(cls, unit, flat_args, spec_args):
  877. test_files = get_values_list(unit, 'TEST_SRCS_VALUE')
  878. value = serialize_list(test_files)
  879. return {cls.KEY: value, cls.KEY2: value}
  880. @classmethod
  881. def ts_test_srcs(cls, unit, flat_args, spec_args):
  882. test_files = get_values_list(unit, "_TS_TEST_SRCS_VALUE")
  883. test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files)
  884. value = serialize_list(test_files)
  885. return {cls.KEY: value, cls.KEY2: value}
  886. @classmethod
  887. def ts_input_files(cls, unit, flat_args, spec_args):
  888. typecheck_files = get_values_list(unit, "TS_INPUT_FILES")
  889. test_files = [_common.resolve_common_const(f) for f in typecheck_files]
  890. value = serialize_list(test_files)
  891. return {cls.KEY: value, cls.KEY2: value}
  892. @classmethod
  893. def ts_lint_srcs(cls, unit, flat_args, spec_args):
  894. test_files = get_values_list(unit, "_TS_LINT_SRCS_VALUE")
  895. test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files)
  896. value = serialize_list(test_files)
  897. return {cls.KEY: value, cls.KEY2: value}
  898. @classmethod
  899. def stylesheets(cls, unit, flat_args, spec_args):
  900. test_files = get_values_list(unit, "_TS_STYLELINT_FILES")
  901. test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files)
  902. value = serialize_list(test_files)
  903. return {cls.KEY: value, cls.KEY2: value}
  904. @classmethod
  905. def py_linter_files(cls, unit, flat_args, spec_args):
  906. files = unit.get('PY_LINTER_FILES')
  907. if not files:
  908. raise DartValueError()
  909. files = json.loads(files)
  910. test_files = []
  911. for path in files:
  912. if path.startswith(ARCADIA_ROOT):
  913. test_files.append(path.replace(ARCADIA_ROOT, SOURCE_ROOT_SHORT, 1))
  914. elif path.startswith(SOURCE_ROOT_SHORT):
  915. test_files.append(path)
  916. if not test_files:
  917. lint_name = LintName.value(unit, flat_args, spec_args)[LintName.KEY]
  918. message = 'No files to lint for {}'.format(lint_name)
  919. raise DartValueError(message)
  920. test_files = serialize_list(test_files)
  921. return {cls.KEY: test_files, cls.KEY2: test_files}
  922. @classmethod
  923. def cpp_linter_files(cls, unit, flat_args, spec_args):
  924. files_dart = _reference_group_var("ALL_SRCS", consts.STYLE_CPP_ALL_EXTS)
  925. return {cls.KEY: files_dart, cls.KEY2: files_dart}
  926. class TestEnv:
  927. KEY = 'TEST-ENV'
  928. @classmethod
  929. def value(cls, unit, flat_args, spec_args):
  930. return {cls.KEY: prepare_env(unit.get("TEST_ENV_VALUE"))}
  931. class TestIosDeviceType:
  932. KEY = 'TEST_IOS_DEVICE_TYPE'
  933. @classmethod
  934. def value(cls, unit, flat_args, spec_args):
  935. return {cls.KEY: unit.get('TEST_IOS_DEVICE_TYPE_VALUE')}
  936. class TestIosRuntimeType:
  937. KEY = 'TEST_IOS_RUNTIME_TYPE'
  938. @classmethod
  939. def value(cls, unit, flat_args, spec_args):
  940. return {cls.KEY: unit.get('TEST_IOS_RUNTIME_TYPE_VALUE')}
  941. class TestJar:
  942. KEY = 'TEST_JAR'
  943. @classmethod
  944. def value(cls, unit, flat_args, spec_args):
  945. if unit.get('UNITTEST_DIR'):
  946. value = '${UNITTEST_MOD}'
  947. else:
  948. value = '{}/{}.jar'.format(unit.get('MODDIR'), unit.get('REALPRJNAME'))
  949. return {cls.KEY: value}
  950. class TestName:
  951. KEY = 'TEST-NAME'
  952. @classmethod
  953. def value(cls, unit, flat_args, spec_args):
  954. return {cls.KEY: flat_args[0]}
  955. @classmethod
  956. def first_flat_with_bench(cls, unit, flat_args, spec_args):
  957. return {cls.KEY: flat_args[0] + '_bench'}
  958. @classmethod
  959. def first_flat(cls, unit, flat_args, spec_args):
  960. return {cls.KEY: flat_args[0].lower()}
  961. @classmethod
  962. def filename_without_ext(cls, unit, flat_args, spec_args):
  963. test_name = os.path.basename(os.path.join(unit.path(), unit.filename()))
  964. return {cls.KEY: os.path.splitext(test_name)[0]}
  965. @classmethod
  966. def normalized_joined_dir_basename(cls, unit, flat_args, spec_args):
  967. path = _common.get_norm_unit_path(unit)
  968. value = '-'.join([os.path.basename(os.path.dirname(path)), os.path.basename(path)])
  969. return {cls.KEY: value}
  970. @classmethod
  971. def normalized_joined_dir_basename_deps(cls, unit, flat_args, spec_args):
  972. path = _common.get_norm_unit_path(unit)
  973. value = '-'.join([os.path.basename(os.path.dirname(path)), os.path.basename(path), 'dependencies']).strip('-')
  974. return {cls.KEY: value}
  975. @classmethod
  976. def filename_without_pkg_ext(cls, unit, flat_args, spec_args):
  977. test_name = os.path.basename(os.path.join(unit.path(), unit.filename()).replace(".pkg", ""))
  978. return {cls.KEY: os.path.splitext(test_name)[0]}
  979. @classmethod
  980. def name_from_macro_args(cls, unit, flat_args, spec_args):
  981. return {cls.KEY: spec_args['NAME'][0]}
  982. class TestPartition:
  983. KEY = 'TEST_PARTITION'
  984. @classmethod
  985. def value(cls, unit, flat_args, spec_args):
  986. return {cls.KEY: unit.get("TEST_PARTITION")}
  987. class TestRecipes:
  988. KEY = 'TEST-RECIPES'
  989. @classmethod
  990. def value(cls, unit, flat_args, spec_args):
  991. return {cls.KEY: prepare_recipes(unit.get("TEST_RECIPES_VALUE"))}
  992. class TestRunnerBin:
  993. KEY = 'TEST-RUNNER-BIN'
  994. @classmethod
  995. def value(cls, unit, flat_args, spec_args):
  996. runner_bin = spec_args.get('RUNNER_BIN', [None])[0]
  997. if runner_bin:
  998. return {cls.KEY: runner_bin}
  999. class TestTimeout:
  1000. KEY = 'TEST-TIMEOUT'
  1001. @classmethod
  1002. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  1003. test_timeout = ''.join(spec_args.get('TIMEOUT', [])) or unit.get('TEST_TIMEOUT') or ''
  1004. return {cls.KEY: test_timeout}
  1005. @classmethod
  1006. def from_unit_with_default(cls, unit, flat_args, spec_args):
  1007. timeout = list(filter(None, [unit.get(["TEST_TIMEOUT"])]))
  1008. if timeout:
  1009. timeout = timeout[0]
  1010. else:
  1011. timeout = '0'
  1012. return {cls.KEY: timeout}
  1013. @classmethod
  1014. def from_unit(cls, unit, flat_args, spec_args):
  1015. return {cls.KEY: unit.get('TEST_TIMEOUT')}
  1016. class TsResources:
  1017. KEY = "{}-ROOT-VAR-NAME"
  1018. @classmethod
  1019. def value(cls, unit, flat_args, spec_args):
  1020. erm_json = spec_args['erm_json']
  1021. ret = {}
  1022. for tool in erm_json.list_npm_packages():
  1023. tool_resource_label = cls.KEY.format(tool.upper())
  1024. tool_resource_value = unit.get(tool_resource_label)
  1025. if tool_resource_value:
  1026. ret[tool_resource_label] = tool_resource_value
  1027. return ret
  1028. class JvmArgs:
  1029. KEY = 'JVM_ARGS'
  1030. @classmethod
  1031. def value(cls, unit, flat_args, spec_args):
  1032. value = serialize_list(get_values_list(unit, 'JVM_ARGS_VALUE'))
  1033. return {cls.KEY: value}
  1034. class StrictClasspathClash:
  1035. KEY = 'STRICT_CLASSPATH_CLASH'
  1036. class SystemProperties:
  1037. KEY = 'SYSTEM_PROPERTIES'
  1038. @classmethod
  1039. def value(cls, unit, flat_args, spec_args):
  1040. props, error_mgs = extract_java_system_properties(unit, get_values_list(unit, 'SYSTEM_PROPERTIES_VALUE'))
  1041. if error_mgs:
  1042. ymake.report_configure_error(error_mgs)
  1043. raise DartValueError()
  1044. props = base64.b64encode(six.ensure_binary(json.dumps(props)))
  1045. return {cls.KEY: props}
  1046. class UnittestDir:
  1047. KEY = 'UNITTEST_DIR'
  1048. @classmethod
  1049. def value(cls, unit, flat_args, spec_args):
  1050. return {cls.KEY: unit.get('UNITTEST_DIR')}
  1051. class UseArcadiaPython:
  1052. KEY = 'USE_ARCADIA_PYTHON'
  1053. @classmethod
  1054. def value(cls, unit, flat_args, spec_args):
  1055. return {cls.KEY: unit.get('USE_ARCADIA_PYTHON')}
  1056. class UseKtlintOld:
  1057. KEY = 'USE_KTLINT_OLD'
  1058. @classmethod
  1059. def value(cls, unit, flat_args, spec_args):
  1060. if unit.get('_USE_KTLINT_OLD') == 'yes':
  1061. return {cls.KEY: 'yes'}
  1062. class YtSpec:
  1063. KEY = 'YT-SPEC'
  1064. @classmethod
  1065. def from_macro_args_and_unit(cls, unit, flat_args, spec_args):
  1066. value = serialize_list(spec_args.get('YT_SPEC', []) + get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE'))
  1067. return {cls.KEY: value}
  1068. @classmethod
  1069. def from_unit(cls, unit, flat_args, spec_args):
  1070. yt_spec = get_values_list(unit, 'TEST_YT_SPEC_VALUE')
  1071. if yt_spec:
  1072. return {cls.KEY: serialize_list(yt_spec)}
  1073. @classmethod
  1074. def from_unit_list_var(cls, unit, flat_args, spec_args):
  1075. yt_spec_values = get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')
  1076. return {cls.KEY: serialize_list(yt_spec_values)}