nots.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684
  1. import os
  2. import ymake
  3. import ytest
  4. from _common import resolve_common_const, get_norm_unit_path, rootrel_arc_src, to_yesno
  5. # 1 is 60 files per chunk for TIMEOUT(60) - default timeout for SIZE(SMALL)
  6. # 0.5 is 120 files per chunk for TIMEOUT(60) - default timeout for SIZE(SMALL)
  7. # 0.2 is 300 files per chunk for TIMEOUT(60) - default timeout for SIZE(SMALL)
  8. ESLINT_FILE_PROCESSING_TIME_DEFAULT = 0.2 # seconds per file
  9. class PluginLogger(object):
  10. def __init__(self):
  11. self.unit = None
  12. self.prefix = ""
  13. def reset(self, unit, prefix=""):
  14. self.unit = unit
  15. self.prefix = prefix
  16. def get_state(self):
  17. return (self.unit, self.prefix)
  18. def _stringify_messages(self, messages):
  19. parts = []
  20. for m in messages:
  21. if m is None:
  22. parts.append("None")
  23. else:
  24. parts.append(m if isinstance(m, str) else repr(m))
  25. # cyan color (code 36) for messages
  26. return "\033[0;32m{}\033[0;49m\n\033[0;36m{}\033[0;49m".format(self.prefix, " ".join(parts))
  27. def info(self, *messages):
  28. if self.unit:
  29. self.unit.message(["INFO", self._stringify_messages(messages)])
  30. def warn(self, *messages):
  31. if self.unit:
  32. self.unit.message(["WARN", self._stringify_messages(messages)])
  33. def error(self, *messages):
  34. if self.unit:
  35. self.unit.message(["ERROR", self._stringify_messages(messages)])
  36. def print_vars(self, *variables):
  37. if self.unit:
  38. values = ["{}={}".format(v, self.unit.get(v)) for v in variables]
  39. self.info("\n".join(values))
  40. logger = PluginLogger()
  41. def _with_report_configure_error(fn):
  42. def _wrapper(*args, **kwargs):
  43. last_state = logger.get_state()
  44. unit = args[0]
  45. logger.reset(unit if unit.get("TS_LOG") == "yes" else None, fn.__name__)
  46. try:
  47. fn(*args, **kwargs)
  48. except Exception as exc:
  49. ymake.report_configure_error(str(exc))
  50. if unit.get("TS_RAISE") == "yes":
  51. raise
  52. else:
  53. unit.message(["WARN", "Configure error is reported. Add -DTS_RAISE to see actual exception"])
  54. finally:
  55. logger.reset(*last_state)
  56. return _wrapper
  57. def _build_directives(name, flags, paths):
  58. # type: (str, list[str]|tuple[str], list[str]) -> str
  59. parts = [p for p in [name] + (flags or []) if p]
  60. parts_str = ";".join(parts)
  61. expressions = ['${{{parts}:"{path}"}}'.format(parts=parts_str, path=path) for path in paths]
  62. return " ".join(expressions)
  63. def _build_cmd_input_paths(paths, hide=False, disable_include_processor=False):
  64. # type: (list[str]|tuple[str], bool, bool) -> str
  65. hide_part = "hide" if hide else ""
  66. disable_ip_part = "context=TEXT" if disable_include_processor else ""
  67. return _build_directives("input", [hide_part, disable_ip_part], paths)
  68. def _create_pm(unit):
  69. from lib.nots.package_manager import manager
  70. sources_path = unit.path()
  71. module_path = unit.get("MODDIR")
  72. if unit.get("TS_TEST_FOR"):
  73. sources_path = unit.get("TS_TEST_FOR_DIR")
  74. module_path = unit.get("TS_TEST_FOR_PATH")
  75. return manager(
  76. sources_path=unit.resolve(sources_path),
  77. build_root="$B",
  78. build_path=unit.path().replace("$S", "$B", 1),
  79. contribs_path=unit.get("NPM_CONTRIBS_PATH"),
  80. nodejs_bin_path=None,
  81. script_path=None,
  82. module_path=module_path,
  83. )
  84. def _create_erm_json(unit):
  85. from lib.nots.erm_json_lite import ErmJsonLite
  86. erm_packages_path = unit.get("ERM_PACKAGES_PATH")
  87. path = unit.resolve(unit.resolve_arc_path(erm_packages_path))
  88. return ErmJsonLite.load(path)
  89. @_with_report_configure_error
  90. def on_set_append_with_directive(unit, var_name, dir, *values):
  91. wrapped = ['${{{dir}:"{v}"}}'.format(dir=dir, v=v) for v in values]
  92. __set_append(unit, var_name, " ".join(wrapped))
  93. @_with_report_configure_error
  94. def on_from_npm_lockfiles(unit, *args):
  95. from lib.nots.package_manager.base import PackageManagerError
  96. pm = _create_pm(unit)
  97. lf_paths = []
  98. for lf_path in args:
  99. abs_lf_path = unit.resolve(unit.resolve_arc_path(lf_path))
  100. if abs_lf_path:
  101. lf_paths.append(abs_lf_path)
  102. elif unit.get("TS_STRICT_FROM_NPM_LOCKFILES") == "yes":
  103. ymake.report_configure_error("lockfile not found: {}".format(lf_path))
  104. try:
  105. for pkg in pm.extract_packages_meta_from_lockfiles(lf_paths):
  106. unit.on_from_npm([pkg.tarball_url, pkg.sky_id, pkg.integrity, pkg.integrity_algorithm, pkg.tarball_path])
  107. except PackageManagerError as e:
  108. logger.warn(str(e))
  109. pass
  110. def _check_nodejs_version(unit, major):
  111. if major < 14:
  112. raise Exception(
  113. "Node.js {} is unsupported. Update Node.js please. See https://nda.ya.ru/t/joB9Mivm6h4znu".format(major)
  114. )
  115. if major < 18:
  116. unit.message(
  117. [
  118. "WARN",
  119. "Node.js {} is deprecated. Update Node.js please. See https://nda.ya.ru/t/joB9Mivm6h4znu".format(major),
  120. ]
  121. )
  122. @_with_report_configure_error
  123. def on_peerdir_ts_resource(unit, *resources):
  124. pm = _create_pm(unit)
  125. pj = pm.load_package_json_from_dir(pm.sources_path)
  126. erm_json = _create_erm_json(unit)
  127. dirs = []
  128. nodejs_version = _select_matching_version(erm_json, "nodejs", pj.get_nodejs_version())
  129. _check_nodejs_version(unit, nodejs_version.major)
  130. for tool in resources:
  131. dir_name = erm_json.canonize_name(tool)
  132. if erm_json.use_resource_directly(tool):
  133. # raises the configuration error when the version is unsupported
  134. _select_matching_version(erm_json, tool, pj.get_dep_specifier(tool), dep_is_required=True)
  135. elif tool == "nodejs":
  136. dirs.append(os.path.join("build", "platform", dir_name, str(nodejs_version)))
  137. _set_resource_vars(unit, erm_json, tool, nodejs_version)
  138. elif erm_json.is_resource_multiplatform(tool):
  139. v = _select_matching_version(erm_json, tool, pj.get_dep_specifier(tool))
  140. sb_resources = [
  141. sbr for sbr in erm_json.get_sb_resources(tool, v) if sbr.get("nodejs") == nodejs_version.major
  142. ]
  143. nodejs_dir = "NODEJS_{}".format(nodejs_version.major)
  144. if len(sb_resources) > 0:
  145. dirs.append(os.path.join("build", "external_resources", dir_name, str(v), nodejs_dir))
  146. _set_resource_vars(unit, erm_json, tool, v, nodejs_version.major)
  147. else:
  148. unit.message(["WARN", "Missing {}@{} for {}".format(tool, str(v), nodejs_dir)])
  149. else:
  150. v = _select_matching_version(erm_json, tool, pj.get_dep_specifier(tool))
  151. dirs.append(os.path.join("build", "external_resources", dir_name, str(v)))
  152. _set_resource_vars(unit, erm_json, tool, v, nodejs_version.major)
  153. if dirs:
  154. unit.onpeerdir(dirs)
  155. @_with_report_configure_error
  156. def on_ts_configure(unit, *tsconfig_paths):
  157. # type: (Unit, *str) -> None
  158. from lib.nots.package_manager.base import PackageJson
  159. from lib.nots.package_manager.base.utils import build_pj_path
  160. from lib.nots.typescript import TsConfig
  161. # for use in CMD as inputs
  162. __set_append(
  163. unit, "TS_CONFIG_FILES", _build_cmd_input_paths(tsconfig_paths, hide=True, disable_include_processor=True)
  164. )
  165. mod_dir = unit.get("MODDIR")
  166. cur_dir = unit.get("TS_TEST_FOR_PATH") if unit.get("TS_TEST_FOR") else mod_dir
  167. pj_path = build_pj_path(unit.resolve(unit.resolve_arc_path(cur_dir)))
  168. dep_paths = PackageJson.load(pj_path).get_dep_paths_by_names()
  169. # reversed for using the first tsconfig as the config for include processor (legacy)
  170. for tsconfig_path in reversed(tsconfig_paths):
  171. abs_tsconfig_path = unit.resolve(unit.resolve_arc_path(tsconfig_path))
  172. if not abs_tsconfig_path:
  173. raise Exception("tsconfig not found: {}".format(tsconfig_path))
  174. tsconfig = TsConfig.load(abs_tsconfig_path)
  175. config_files = tsconfig.inline_extend(dep_paths)
  176. config_files = _resolve_module_files(unit, mod_dir, config_files)
  177. use_tsconfig_outdir = unit.get("TS_CONFIG_USE_OUTDIR") == "yes"
  178. tsconfig.validate(use_tsconfig_outdir)
  179. # add tsconfig files from which root tsconfig files were extended
  180. __set_append(
  181. unit, "TS_CONFIG_FILES", _build_cmd_input_paths(config_files, hide=True, disable_include_processor=True)
  182. )
  183. # region include processor
  184. unit.set(["TS_CONFIG_ROOT_DIR", tsconfig.compiler_option("rootDir")]) # also for hermione
  185. if use_tsconfig_outdir:
  186. unit.set(["TS_CONFIG_OUT_DIR", tsconfig.compiler_option("outDir")]) # also for hermione
  187. unit.set(["TS_CONFIG_SOURCE_MAP", to_yesno(tsconfig.compiler_option("sourceMap"))])
  188. unit.set(["TS_CONFIG_DECLARATION", to_yesno(tsconfig.compiler_option("declaration"))])
  189. unit.set(["TS_CONFIG_DECLARATION_MAP", to_yesno(tsconfig.compiler_option("declarationMap"))])
  190. unit.set(["TS_CONFIG_PRESERVE_JSX", to_yesno(tsconfig.compiler_option("jsx") == "preserve")])
  191. # endregion
  192. _filter_inputs_by_rules_from_tsconfig(unit, tsconfig)
  193. _setup_eslint(unit)
  194. _setup_tsc_typecheck(unit, tsconfig_paths)
  195. def __set_append(unit, var_name, value):
  196. # type: (Unit, str, str|list[str]|tuple[str]) -> None
  197. """
  198. SET_APPEND() python naive implementation - append value/values to the list of values
  199. """
  200. previous_value = unit.get(var_name) or ""
  201. value_in_str = " ".join(value) if isinstance(value, list) or isinstance(value, tuple) else value
  202. new_value = previous_value + " " + value_in_str
  203. unit.set([var_name, new_value])
  204. def __strip_prefix(prefix, line):
  205. # type: (str, str) -> str
  206. if line.startswith(prefix):
  207. prefix_len = len(prefix)
  208. return line[prefix_len:]
  209. return line
  210. def _filter_inputs_by_rules_from_tsconfig(unit, tsconfig):
  211. """
  212. Reduce file list from the TS_GLOB_FILES variable following tsconfig.json rules
  213. """
  214. mod_dir = unit.get("MODDIR")
  215. target_path = os.path.join("${ARCADIA_ROOT}", mod_dir, "") # To have "/" in the end
  216. all_files = [__strip_prefix(target_path, f) for f in unit.get("TS_GLOB_FILES").split(" ")]
  217. filtered_files = tsconfig.filter_files(all_files)
  218. __set_append(unit, "TS_INPUT_FILES", [os.path.join(target_path, f) for f in filtered_files])
  219. def _get_ts_test_data_dirs(unit):
  220. return sorted(
  221. set(
  222. [
  223. os.path.dirname(rootrel_arc_src(p, unit))
  224. for p in (ytest.get_values_list(unit, "_TS_TEST_DATA_VALUE") or [])
  225. ]
  226. )
  227. )
  228. def _resolve_config_path(unit, test_runner, rel_to):
  229. config_path = unit.get("ESLINT_CONFIG_PATH") if test_runner == "eslint" else unit.get("TS_TEST_CONFIG_PATH")
  230. arc_config_path = unit.resolve_arc_path(config_path)
  231. abs_config_path = unit.resolve(arc_config_path)
  232. if not abs_config_path:
  233. raise Exception("{} config not found: {}".format(test_runner, config_path))
  234. unit.onsrcs([arc_config_path])
  235. abs_rel_to = unit.resolve(unit.resolve_arc_path(unit.get(rel_to)))
  236. return os.path.relpath(abs_config_path, start=abs_rel_to)
  237. def _is_tests_enabled(unit):
  238. if unit.get("TIDY") == "yes":
  239. return False
  240. return True
  241. def _get_test_runner_handlers():
  242. return {
  243. "jest": _add_jest_ts_test,
  244. "hermione": _add_hermione_ts_test,
  245. "playwright": _add_playwright_ts_test,
  246. }
  247. def _add_jest_ts_test(unit, test_runner, test_files, deps, test_record):
  248. test_record.update(
  249. {
  250. "CONFIG-PATH": _resolve_config_path(unit, test_runner, rel_to="TS_TEST_FOR_PATH"),
  251. }
  252. )
  253. _add_test(unit, test_runner, test_files, deps, test_record)
  254. def _add_hermione_ts_test(unit, test_runner, test_files, deps, test_record):
  255. test_tags = sorted(set(["ya:fat", "ya:external", "ya:noretries"] + ytest.get_values_list(unit, "TEST_TAGS_VALUE")))
  256. test_requirements = sorted(set(["network:full"] + ytest.get_values_list(unit, "TEST_REQUIREMENTS_VALUE")))
  257. test_record.update(
  258. {
  259. "SIZE": "LARGE",
  260. "TAG": ytest.serialize_list(test_tags),
  261. "REQUIREMENTS": ytest.serialize_list(test_requirements),
  262. "CONFIG-PATH": _resolve_config_path(unit, test_runner, rel_to="TS_TEST_FOR_PATH"),
  263. }
  264. )
  265. _add_test(unit, test_runner, test_files, deps, test_record)
  266. def _add_playwright_ts_test(unit, test_runner, test_files, deps, test_record):
  267. test_record.update(
  268. {
  269. "CONFIG-PATH": _resolve_config_path(unit, test_runner, rel_to="TS_TEST_FOR_PATH"),
  270. }
  271. )
  272. _add_test(unit, test_runner, test_files, deps, test_record)
  273. def _setup_eslint(unit):
  274. if not _is_tests_enabled(unit):
  275. return
  276. if unit.get("_NO_LINT_VALUE") == "none":
  277. return
  278. lint_files = ytest.get_values_list(unit, "_TS_LINT_SRCS_VALUE")
  279. if not lint_files:
  280. return
  281. unit.on_peerdir_ts_resource("eslint")
  282. user_recipes = unit.get("TEST_RECIPES_VALUE")
  283. unit.on_setup_extract_node_modules_recipe(unit.get("MODDIR"))
  284. mod_dir = unit.get("MODDIR")
  285. lint_files = _resolve_module_files(unit, mod_dir, lint_files)
  286. deps = _create_pm(unit).get_peers_from_package_json()
  287. test_record = {
  288. "ESLINT_CONFIG_PATH": _resolve_config_path(unit, "eslint", rel_to="MODDIR"),
  289. "LINT-FILE-PROCESSING-TIME": str(ESLINT_FILE_PROCESSING_TIME_DEFAULT),
  290. }
  291. _add_test(unit, "eslint", lint_files, deps, test_record, mod_dir)
  292. unit.set(["TEST_RECIPES_VALUE", user_recipes])
  293. def _setup_tsc_typecheck(unit, tsconfig_paths: list[str]):
  294. if not _is_tests_enabled(unit):
  295. return
  296. if unit.get("_TS_TYPECHECK_VALUE") == "none":
  297. return
  298. typecheck_files = ytest.get_values_list(unit, "TS_INPUT_FILES")
  299. if not typecheck_files:
  300. return
  301. tsconfig_path = tsconfig_paths[0]
  302. if len(tsconfig_paths) > 1:
  303. tsconfig_path = unit.get("_TS_TYPECHECK_TSCONFIG")
  304. if not tsconfig_path:
  305. macros = " or ".join([f"TS_TYPECHECK({p})" for p in tsconfig_paths])
  306. raise Exception(f"Module uses several tsconfig files, specify which one to use for typecheck: {macros}")
  307. abs_tsconfig_path = unit.resolve(unit.resolve_arc_path(tsconfig_path))
  308. if not abs_tsconfig_path:
  309. raise Exception(f"tsconfig for typecheck not found: {tsconfig_path}")
  310. unit.on_peerdir_ts_resource("typescript")
  311. user_recipes = unit.get("TEST_RECIPES_VALUE")
  312. unit.on_setup_install_node_modules_recipe()
  313. unit.on_setup_extract_output_tars_recipe([unit.get("MODDIR")])
  314. _add_test(
  315. unit,
  316. test_type="tsc_typecheck",
  317. test_files=[resolve_common_const(f) for f in typecheck_files],
  318. deps=_create_pm(unit).get_peers_from_package_json(),
  319. test_record={"TS_CONFIG_PATH": tsconfig_path},
  320. test_cwd=unit.get("MODDIR"),
  321. )
  322. unit.set(["TEST_RECIPES_VALUE", user_recipes])
  323. def _resolve_module_files(unit, mod_dir, file_paths):
  324. mod_dir_with_sep_len = len(mod_dir) + 1
  325. resolved_files = []
  326. for path in file_paths:
  327. resolved = rootrel_arc_src(path, unit)
  328. if resolved.startswith(mod_dir):
  329. resolved = resolved[mod_dir_with_sep_len:]
  330. resolved_files.append(resolved)
  331. return resolved_files
  332. def _add_test(unit, test_type, test_files, deps=None, test_record=None, test_cwd=None):
  333. from lib.nots.package_manager import constants
  334. def sort_uniq(text):
  335. return sorted(set(text))
  336. recipes_lines = ytest.format_recipes(unit.get("TEST_RECIPES_VALUE")).strip().splitlines()
  337. if recipes_lines:
  338. deps = deps or []
  339. deps.extend([os.path.dirname(r.strip().split(" ")[0]) for r in recipes_lines])
  340. if deps:
  341. joined_deps = "\n".join(deps)
  342. logger.info(f"{test_type} deps: \n{joined_deps}")
  343. unit.ondepends(deps)
  344. test_dir = get_norm_unit_path(unit)
  345. full_test_record = {
  346. # Key to discover suite (see devtools/ya/test/explore/__init__.py#gen_suite)
  347. "SCRIPT-REL-PATH": test_type,
  348. # Test name as shown in PR check, should be unique inside one module
  349. "TEST-NAME": test_type.lower(),
  350. "TEST-TIMEOUT": unit.get("TEST_TIMEOUT") or "",
  351. "TEST-ENV": ytest.prepare_env(unit.get("TEST_ENV_VALUE")),
  352. "TESTED-PROJECT-NAME": os.path.splitext(unit.filename())[0],
  353. "TEST-RECIPES": ytest.prepare_recipes(unit.get("TEST_RECIPES_VALUE")),
  354. "SOURCE-FOLDER-PATH": test_dir,
  355. "BUILD-FOLDER-PATH": test_dir,
  356. "BINARY-PATH": os.path.join(test_dir, unit.filename()),
  357. "SPLIT-FACTOR": unit.get("TEST_SPLIT_FACTOR") or "",
  358. "FORK-MODE": unit.get("TEST_FORK_MODE") or "",
  359. "SIZE": unit.get("TEST_SIZE_NAME") or "",
  360. "TEST-DATA": ytest.serialize_list(ytest.get_values_list(unit, "TEST_DATA_VALUE")),
  361. "TEST-FILES": ytest.serialize_list(test_files),
  362. "TEST-CWD": test_cwd or "",
  363. "TAG": ytest.serialize_list(ytest.get_values_list(unit, "TEST_TAGS_VALUE")),
  364. "REQUIREMENTS": ytest.serialize_list(ytest.get_values_list(unit, "TEST_REQUIREMENTS_VALUE")),
  365. "NODEJS-ROOT-VAR-NAME": unit.get("NODEJS-ROOT-VAR-NAME"),
  366. "NODE-MODULES-BUNDLE-FILENAME": constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME,
  367. "CUSTOM-DEPENDENCIES": " ".join(sort_uniq((deps or []) + ytest.get_values_list(unit, "TEST_DEPENDS_VALUE"))),
  368. }
  369. if test_record:
  370. full_test_record.update(test_record)
  371. data = ytest.dump_test(unit, full_test_record)
  372. if data:
  373. unit.set_property(["DART_DATA", data])
  374. def _set_resource_vars(unit, erm_json, tool, version, nodejs_major=None):
  375. # type: (any, ErmJsonLite, Version, str|None, int|None) -> None
  376. resource_name = erm_json.canonize_name(tool).upper()
  377. # example: NODEJS_12_18_4 | HERMIONE_7_0_4_NODEJS_18
  378. version_str = str(version).replace(".", "_")
  379. yamake_resource_name = "{}_{}".format(resource_name, version_str)
  380. if erm_json.is_resource_multiplatform(tool):
  381. yamake_resource_name += "_NODEJS_{}".format(nodejs_major)
  382. yamake_resource_var = "{}_RESOURCE_GLOBAL".format(yamake_resource_name)
  383. unit.set(["{}_ROOT".format(resource_name), "${}".format(yamake_resource_var)])
  384. unit.set(["{}-ROOT-VAR-NAME".format(resource_name), yamake_resource_var])
  385. def _select_matching_version(erm_json, resource_name, range_str, dep_is_required=False):
  386. # type: (ErmJsonLite, str, str, bool) -> Version
  387. if dep_is_required and range_str is None:
  388. raise Exception(
  389. "Please install the '{tool}' package to the project. Run the command:\n"
  390. " ya tool nots add -D {tool}".format(tool=resource_name)
  391. )
  392. try:
  393. version = erm_json.select_version_of(resource_name, range_str)
  394. if version:
  395. return version
  396. raise ValueError("There is no allowed version to satisfy this range: '{}'".format(range_str))
  397. except Exception as error:
  398. toolchain_versions = erm_json.get_versions_of(erm_json.get_resource(resource_name))
  399. raise Exception(
  400. "Requested {} version range '{}' could not be satisfied. \n"
  401. "Please use a range that would include one of the following: {}. \n"
  402. "For further details please visit the link: {} \nOriginal error: {} \n".format(
  403. resource_name,
  404. range_str,
  405. ", ".join(map(str, toolchain_versions)),
  406. "https://docs.yandex-team.ru/ya-make/manual/typescript/toolchain",
  407. str(error),
  408. )
  409. )
  410. @_with_report_configure_error
  411. def on_prepare_deps_configure(unit):
  412. # Originally this peerdir was in .conf file
  413. # but it kept taking default value of NPM_CONTRIBS_PATH
  414. # before it was updated by CUSTOM_CONTRIB_TYPESCRIPT()
  415. # so I moved it here.
  416. unit.onpeerdir(unit.get("NPM_CONTRIBS_PATH"))
  417. pm = _create_pm(unit)
  418. pj = pm.load_package_json_from_dir(pm.sources_path)
  419. has_deps = pj.has_dependencies()
  420. ins, outs = pm.calc_prepare_deps_inouts(unit.get("_TARBALLS_STORE"), has_deps)
  421. if pj.has_dependencies():
  422. unit.onpeerdir(pm.get_local_peers_from_package_json())
  423. __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("input", ["hide"], sorted(ins)))
  424. __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("output", ["hide"], sorted(outs)))
  425. else:
  426. __set_append(unit, "_PREPARE_DEPS_INOUTS", _build_directives("output", [], sorted(outs)))
  427. unit.set(["_PREPARE_DEPS_CMD", "$_PREPARE_NO_DEPS_CMD"])
  428. @_with_report_configure_error
  429. def on_node_modules_configure(unit):
  430. pm = _create_pm(unit)
  431. pj = pm.load_package_json_from_dir(pm.sources_path)
  432. if pj.has_dependencies():
  433. unit.onpeerdir(pm.get_local_peers_from_package_json())
  434. local_cli = unit.get("TS_LOCAL_CLI") == "yes"
  435. ins, outs = pm.calc_node_modules_inouts(local_cli)
  436. __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("input", ["hide"], sorted(ins)))
  437. if not unit.get("TS_TEST_FOR"):
  438. __set_append(unit, "_NODE_MODULES_INOUTS", _build_directives("output", ["hide"], sorted(outs)))
  439. if pj.get_use_prebuilder():
  440. lf = pm.load_lockfile_from_dir(pm.sources_path)
  441. is_valid, invalid_keys = lf.validate_has_addons_flags()
  442. if not is_valid:
  443. ymake.report_configure_error(
  444. "Project is configured to use @yatool/prebuilder. \n"
  445. + "Some packages in the pnpm-lock.yaml are misconfigured.\n"
  446. + "Run `ya tool nots update-lockfile` to fix lockfile.\n"
  447. + "All packages with `requiresBuild:true` have to be marked with `hasAddons:true/false`.\n"
  448. + "Misconfigured keys: \n"
  449. + " - "
  450. + "\n - ".join(invalid_keys)
  451. )
  452. unit.on_peerdir_ts_resource("@yatool/prebuilder")
  453. unit.set(
  454. [
  455. "_YATOOL_PREBUILDER_ARG",
  456. "--yatool-prebuilder-path $YATOOL_PREBUILDER_ROOT/node_modules/@yatool/prebuilder",
  457. ]
  458. )
  459. @_with_report_configure_error
  460. def on_ts_test_for_configure(unit, test_runner, default_config, node_modules_filename):
  461. if not _is_tests_enabled(unit):
  462. return
  463. if unit.enabled('TS_COVERAGE'):
  464. unit.on_peerdir_ts_resource("nyc")
  465. for_mod_path = unit.get("TS_TEST_FOR_PATH")
  466. unit.onpeerdir([for_mod_path])
  467. unit.on_setup_extract_node_modules_recipe([for_mod_path])
  468. unit.on_setup_extract_output_tars_recipe([for_mod_path])
  469. root = "$B" if test_runner == "hermione" else "$(BUILD_ROOT)"
  470. unit.set(["TS_TEST_NM", os.path.join(root, for_mod_path, node_modules_filename)])
  471. config_path = unit.get("TS_TEST_CONFIG_PATH")
  472. if not config_path:
  473. config_path = os.path.join(for_mod_path, default_config)
  474. unit.set(["TS_TEST_CONFIG_PATH", config_path])
  475. test_record = _add_ts_resources_to_test_record(
  476. unit,
  477. {
  478. "TS-TEST-FOR-PATH": for_mod_path,
  479. "TS-TEST-DATA-DIRS": ytest.serialize_list(_get_ts_test_data_dirs(unit)),
  480. "TS-TEST-DATA-DIRS-RENAME": unit.get("_TS_TEST_DATA_DIRS_RENAME_VALUE"),
  481. },
  482. )
  483. test_files = ytest.get_values_list(unit, "_TS_TEST_SRCS_VALUE")
  484. test_files = _resolve_module_files(unit, unit.get("MODDIR"), test_files)
  485. if not test_files:
  486. ymake.report_configure_error("No tests found")
  487. return
  488. deps = _create_pm(unit).get_peers_from_package_json()
  489. add_ts_test = _get_test_runner_handlers()[test_runner]
  490. add_ts_test(unit, test_runner, test_files, deps, test_record)
  491. @_with_report_configure_error
  492. def on_validate_ts_test_for_args(unit, for_mod, root):
  493. # FBP-1085
  494. is_arc_root = root == "${ARCADIA_ROOT}"
  495. is_rel_for_mod = for_mod.startswith(".")
  496. if is_arc_root and is_rel_for_mod:
  497. ymake.report_configure_error(
  498. "You are using a relative path for a module. "
  499. + "You have to add RELATIVE key, like (RELATIVE {})".format(for_mod)
  500. )
  501. @_with_report_configure_error
  502. def on_set_ts_test_for_vars(unit, for_mod):
  503. unit.set(["TS_TEST_FOR", "yes"])
  504. unit.set(["TS_TEST_FOR_DIR", unit.resolve_arc_path(for_mod)])
  505. unit.set(["TS_TEST_FOR_PATH", rootrel_arc_src(for_mod, unit)])
  506. def _add_ts_resources_to_test_record(unit, test_record):
  507. erm_json = _create_erm_json(unit)
  508. for tool in erm_json.list_npm_packages():
  509. tool_resource_label = "{}-ROOT-VAR-NAME".format(tool.upper())
  510. tool_resource_value = unit.get(tool_resource_label)
  511. if tool_resource_value:
  512. test_record[tool_resource_label] = tool_resource_value
  513. return test_record
  514. @_with_report_configure_error
  515. def on_ts_files(unit, *files):
  516. new_cmds = ['$COPY_CMD ${{input;context=TEXT:"{0}"}} ${{output;noauto:"{0}"}}'.format(f) for f in files]
  517. all_cmds = unit.get("_TS_FILES_COPY_CMD")
  518. if all_cmds:
  519. new_cmds.insert(0, all_cmds)
  520. unit.set(["_TS_FILES_COPY_CMD", " && ".join(new_cmds)])
  521. @_with_report_configure_error
  522. def on_depends_on_mod(unit):
  523. if unit.get("_TS_TEST_DEPENDS_ON_BUILD"):
  524. for_mod_path = unit.get("TS_TEST_FOR_PATH")
  525. unit.ondepends([for_mod_path])