from __future__ import print_function import base64 import collections import copy import json import os import re import six import subprocess try: from StringIO import StringIO except ImportError: from io import StringIO import _common import _dart_fields as df import _requirements as reqs import lib.test_const as consts import ymake from _dart_fields import ( serialize_list, get_unit_list_variable, deserialize_list, create_dart_record, ) BLOCK_SEPARATOR = '=============================================================' SPLIT_FACTOR_MAX_VALUE = 1000 SPLIT_FACTOR_TEST_FILES_MAX_VALUE = 4250 PARTITION_MODS = ('SEQUENTIAL', 'MODULO') DEFAULT_TIDY_CONFIG = "build/config/tests/clang_tidy/config.yaml" DEFAULT_TIDY_CONFIG_MAP_PATH = "build/yandex_specific/config/clang_tidy/tidy_default_map.json" PROJECT_TIDY_CONFIG_MAP_PATH = "build/yandex_specific/config/clang_tidy/tidy_project_map.json" KTLINT_CURRENT_EDITOR_CONFIG = "arcadia/build/platform/java/ktlint/.editorconfig" KTLINT_OLD_EDITOR_CONFIG = "arcadia/build/platform/java/ktlint_old/.editorconfig" YTEST_FIELDS_BASE = ( df.AndroidApkTestActivity.value, df.BinaryPath.normalized, df.BuildFolderPath.normalized, df.CustomDependencies.all_standard, df.GlobalLibraryPath.value, df.ScriptRelPath.second_flat, df.SkipTest.value, df.SourceFolderPath.normalized, df.SplitFactor.from_macro_args_and_unit, df.TestCwd.from_unit, df.TestedProjectFilename.value, df.TestedProjectName.unit_name, df.TestEnv.value, df.TestIosDeviceType.value, df.TestIosRuntimeType.value, df.TestRecipes.value, ) YTEST_FIELDS_EXTRA = ( df.Blob.value, df.ForkMode.from_macro_and_unit, df.Size.from_macro_args_and_unit, df.Tag.from_macro_args_and_unit, df.TestTimeout.from_macro_args_and_unit, df.YtSpec.from_macro_args_and_unit, ) PY_EXEC_FIELDS_BASE = ( df.Blob.value, df.BuildFolderPath.stripped, df.CanonizeSubPath.value, df.CustomDependencies.test_depends_only, df.ForkMode.test_fork_mode, df.ForkTestFiles.value, df.PythonPaths.value, df.Requirements.from_unit, df.Size.from_unit, df.SkipTest.value, df.SourceFolderPath.normalized, df.SplitFactor.from_unit, df.Tag.from_macro_args_and_unit, df.TestCwd.keywords_replaced, df.TestData.from_unit_with_canonical, df.TestEnv.value, df.TestFiles.test_srcs, df.TestPartition.value, df.TestRecipes.value, df.TestTimeout.from_unit_with_default, df.UseArcadiaPython.value, ) CHECK_FIELDS_BASE = ( df.CustomDependencies.depends_only, df.Requirements.from_macro_args, df.ScriptRelPath.first_flat, df.TestEnv.value, df.TestName.first_flat, df.UseArcadiaPython.value, ) LINTER_FIELDS_BASE = ( df.LintName.value, df.LintExtraParams.from_macro_args, df.TestName.name_from_macro_args, df.TestedProjectName.unit_name, df.SourceFolderPath.normalized, df.TestEnv.value, df.UseArcadiaPython.value, df.LintFileProcessingTime.from_macro_args, df.Linter.value, df.CustomDependencies.depends_with_linter, ) tidy_config_map = None def ontest_data(unit, *args): ymake.report_configure_error("TEST_DATA is removed in favour of DATA") def is_yt_spec_contain_pool_info(filename): # XXX switch to yson in ymake + perf test for configure pool_re = re.compile(r"""['"]*pool['"]*\s*?=""") cypress_root_re = re.compile(r"""['"]*cypress_root['"]*\s*=""") with open(filename, 'r') as afile: yt_spec = afile.read() return pool_re.search(yt_spec) and cypress_root_re.search(yt_spec) def validate_test(unit, kw): def get_list(key): return deserialize_list(kw.get(key, "")) valid_kw = copy.deepcopy(kw) errors = [] warnings = [] mandatory_fields = {"SCRIPT-REL-PATH", "SOURCE-FOLDER-PATH", "TEST-NAME"} for field in mandatory_fields - valid_kw.keys(): errors.append(f"Mandatory field {field!r} is not set in DART") if valid_kw.get('SCRIPT-REL-PATH') == 'boost.test': project_path = valid_kw.get('BUILD-FOLDER-PATH', "") if not project_path.startswith( ("contrib", "mail", "maps", "tools/idl", "metrika", "devtools", "mds", "yandex_io", "smart_devices") ): errors.append("BOOSTTEST is not allowed here") size_timeout = collections.OrderedDict(sorted(consts.TestSize.DefaultTimeouts.items(), key=lambda t: t[1])) size = valid_kw.get('SIZE', consts.TestSize.Small).lower() tags = set(get_list("TAG")) requirements_orig = get_list("REQUIREMENTS") in_autocheck = consts.YaTestTags.NotAutocheck not in tags and consts.YaTestTags.Manual not in tags is_fat = consts.YaTestTags.Fat in tags is_force_sandbox = consts.YaTestTags.ForceDistbuild not in tags and is_fat is_ytexec_run = consts.YaTestTags.YtRunner in tags is_fuzzing = valid_kw.get("FUZZING", False) is_kvm = 'kvm' in requirements_orig requirements = {} secret_requirements = ('sb_vault', 'yav') list_requirements = secret_requirements for req in requirements_orig: if req in ('kvm',): requirements[req] = str(True) continue if ":" in req: req_name, req_value = req.split(":", 1) if req_name in list_requirements: requirements[req_name] = ",".join(filter(None, [requirements.get(req_name), req_value])) else: if req_name in requirements: if req_value in ["0"]: warnings.append( "Requirement [[imp]]{}[[rst]] is dropped [[imp]]{}[[rst]] -> [[imp]]{}[[rst]]".format( req_name, requirements[req_name], req_value ) ) del requirements[req_name] elif requirements[req_name] != req_value: warnings.append( "Requirement [[imp]]{}[[rst]] is redefined [[imp]]{}[[rst]] -> [[imp]]{}[[rst]]".format( req_name, requirements[req_name], req_value ) ) requirements[req_name] = req_value else: requirements[req_name] = req_value else: errors.append("Invalid requirement syntax [[imp]]{}[[rst]]: expect :".format(req)) if not errors: for req_name, req_value in requirements.items(): try: error_msg = reqs.validate_requirement( req_name, req_value, size, is_force_sandbox, in_autocheck, is_fuzzing, is_kvm, is_ytexec_run, requirements, ) except Exception as e: error_msg = str(e) if error_msg: errors += [error_msg] invalid_requirements_for_distbuild = [ requirement for requirement in requirements.keys() if requirement not in ('ram', 'ram_disk', 'cpu', 'network') ] sb_tags = [] # XXX Unfortunately, some users have already started using colons # in their tag names. Use skip set to avoid treating their tag as system ones. # Remove this check when all such user tags are removed. skip_set = ('ynmt_benchmark', 'bert_models', 'zeliboba_map') # Verify the prefixes of the system tags to avoid pointless use of the REQUIREMENTS macro parameters in the TAG macro. for tag in tags: if tag.startswith('sb:'): sb_tags.append(tag) elif ':' in tag and not tag.startswith('ya:') and tag.split(':')[0] not in skip_set: errors.append( "Only [[imp]]sb:[[rst]] and [[imp]]ya:[[rst]] prefixes are allowed in system tags: {}".format(tag) ) if is_fat: if size != consts.TestSize.Large: errors.append("Only LARGE test may have ya:fat tag") if in_autocheck and not is_force_sandbox: if invalid_requirements_for_distbuild: errors.append( "'{}' REQUIREMENTS options can be used only for FAT tests without ya:force_distbuild tag. Remove TAG(ya:force_distbuild) or an option.".format( invalid_requirements_for_distbuild ) ) if sb_tags: errors.append( "You can set sandbox tags '{}' only for FAT tests without ya:force_distbuild. Remove TAG(ya:force_sandbox) or sandbox tags.".format( sb_tags ) ) if consts.YaTestTags.SandboxCoverage in tags: errors.append("You can set 'ya:sandbox_coverage' tag only for FAT tests without ya:force_distbuild.") if is_ytexec_run: errors.append( "Running LARGE tests over YT (ya:yt) on Distbuild (ya:force_distbuild) is forbidden. Consider removing TAG(ya:force_distbuild)." ) else: if is_force_sandbox: errors.append('ya:force_sandbox can be used with LARGE tests only') if consts.YaTestTags.Privileged in tags: errors.append("ya:privileged can be used with LARGE tests only") if in_autocheck and size == consts.TestSize.Large: errors.append("LARGE test must have ya:fat tag") if consts.YaTestTags.Privileged in tags and 'container' not in requirements: errors.append("Only tests with 'container' requirement can have 'ya:privileged' tag") if size not in size_timeout: errors.append( "Unknown test size: [[imp]]{}[[rst]], choose from [[imp]]{}[[rst]]".format( size.upper(), ", ".join([sz.upper() for sz in size_timeout.keys()]) ) ) else: try: timeout = int(valid_kw.get('TEST-TIMEOUT', size_timeout[size]) or size_timeout[size]) script_rel_path = valid_kw.get('SCRIPT-REL-PATH') if timeout < 0: raise Exception("Timeout must be > 0") skip_timeout_verification = script_rel_path in ('java.style', 'ktlint') if size_timeout[size] < timeout and in_autocheck and not skip_timeout_verification: suggested_size = None for s, t in size_timeout.items(): if timeout <= t: suggested_size = s break if suggested_size: suggested_size = ", suggested size: [[imp]]{}[[rst]]".format(suggested_size.upper()) else: suggested_size = "" errors.append( "Max allowed timeout for test size [[imp]]{}[[rst]] is [[imp]]{} sec[[rst]]{}".format( size.upper(), size_timeout[size], suggested_size ) ) except Exception as e: errors.append("Error when parsing test timeout: [[bad]]{}[[rst]]".format(e)) requirements_list = [] for req_name, req_value in six.iteritems(requirements): requirements_list.append(req_name + ":" + req_value) valid_kw['REQUIREMENTS'] = serialize_list(sorted(requirements_list)) # Mark test with ya:external tag if it requests any secret from external storages # It's not stable and nonreproducible by definition for x in secret_requirements: if x in requirements: tags.add(consts.YaTestTags.External) if valid_kw.get("FUZZ-OPTS"): for option in get_list("FUZZ-OPTS"): if not option.startswith("-"): errors.append( "Unrecognized fuzzer option '[[imp]]{}[[rst]]'. All fuzzer options should start with '-'".format( option ) ) break eqpos = option.find("=") if eqpos == -1 or len(option) == eqpos + 1: errors.append( "Unrecognized fuzzer option '[[imp]]{}[[rst]]'. All fuzzer options should obtain value specified after '='".format( option ) ) break if option[eqpos - 1] == " " or option[eqpos + 1] == " ": errors.append("Spaces are not allowed: '[[imp]]{}[[rst]]'".format(option)) break if option[:eqpos] in ("-runs", "-dict", "-jobs", "-workers", "-artifact_prefix", "-print_final_stats"): errors.append( "You can't use '[[imp]]{}[[rst]]' - it will be automatically calculated or configured during run".format( option ) ) break if valid_kw.get("YT-SPEC"): if not is_ytexec_run: errors.append("You can use YT_SPEC macro only tests marked with ya:yt tag") else: for filename in get_list("YT-SPEC"): filename = unit.resolve('$S/' + filename) if not os.path.exists(filename): errors.append("File '{}' specified in the YT_SPEC macro doesn't exist".format(filename)) continue if not is_yt_spec_contain_pool_info(filename): tags.add(consts.YaTestTags.External) tags.add("ya:yt_research_pool") partition = valid_kw.get('TEST_PARTITION', 'SEQUENTIAL') if partition not in PARTITION_MODS: raise ValueError('partition mode should be one of {}, detected: {}'.format(PARTITION_MODS, partition)) if valid_kw.get('SPLIT-FACTOR'): if valid_kw.get('FORK-MODE') == 'none': errors.append('SPLIT_FACTOR must be use with FORK_TESTS() or FORK_SUBTESTS() macro') value = 1 try: value = int(valid_kw.get('SPLIT-FACTOR')) if value <= 0: raise ValueError("must be > 0") if value > SPLIT_FACTOR_MAX_VALUE: raise ValueError("the maximum allowed value is {}".format(SPLIT_FACTOR_MAX_VALUE)) except ValueError as e: errors.append('Incorrect SPLIT_FACTOR value: {}'.format(e)) if valid_kw.get('FORK-TEST-FILES') and size != consts.TestSize.Large: nfiles = count_entries(valid_kw.get('TEST-FILES')) if nfiles * value > SPLIT_FACTOR_TEST_FILES_MAX_VALUE: errors.append( 'Too much chunks generated:{} (limit: {}). Remove FORK_TEST_FILES() macro or reduce SPLIT_FACTOR({}).'.format( nfiles * value, SPLIT_FACTOR_TEST_FILES_MAX_VALUE, value ) ) if tags: valid_kw['TAG'] = serialize_list(sorted(tags)) unit_path = _common.get_norm_unit_path(unit) if ( not is_fat and consts.YaTestTags.Noretries in tags and not is_ytexec_run and not unit_path.startswith("devtools/dummy_arcadia/test/noretries") ): errors.append("Only LARGE tests can have 'ya:noretries' tag") if errors: return None, warnings, errors return valid_kw, warnings, errors def dump_test(unit, kw): kw = {k: v for k, v in kw.items() if v and (not isinstance(v, str | bytes) or v.strip())} valid_kw, warnings, errors = validate_test(unit, kw) for w in warnings: unit.message(['warn', w]) for e in errors: ymake.report_configure_error(e) if valid_kw is None: return None string_handler = StringIO() for k, v in six.iteritems(valid_kw): print(k + ': ' + six.ensure_str(v), file=string_handler) print(BLOCK_SEPARATOR, file=string_handler) data = string_handler.getvalue() string_handler.close() return data def count_entries(x): # see (de)serialize_list assert x is None or isinstance(x, str), type(x) if not x: return 0 return x.count(";") + 1 def implies(a, b): return bool((not a) or b) def match_coverage_extractor_requirements(unit): # we add test if return all( ( # tests are requested unit.get("TESTS_REQUESTED") == "yes", # build implies clang coverage, which supports segment extraction from the binaries unit.get("CLANG_COVERAGE") == "yes", # contrib was requested implies( _common.get_norm_unit_path(unit).startswith("contrib/"), unit.get("ENABLE_CONTRIB_COVERAGE") == "yes" ), ) ) def get_tidy_config_map(unit, map_path): config_map_path = unit.resolve(os.path.join("$S", map_path)) config_map = {} try: with open(config_map_path, 'r') as afile: config_map = json.load(afile) except ValueError: ymake.report_configure_error("{} is invalid json".format(map_path)) except Exception as e: ymake.report_configure_error(str(e)) return config_map def prepare_config_map(config_map): return list(reversed(sorted(config_map.items()))) def get_default_tidy_config(unit): unit_path = _common.get_norm_unit_path(unit) tidy_default_config_map = prepare_config_map(get_tidy_config_map(unit, DEFAULT_TIDY_CONFIG_MAP_PATH)) for project_prefix, config_path in tidy_default_config_map: if unit_path.startswith(project_prefix): return config_path return DEFAULT_TIDY_CONFIG ordered_tidy_map = None def get_project_tidy_config(unit): global ordered_tidy_map if ordered_tidy_map is None: ordered_tidy_map = prepare_config_map(get_tidy_config_map(unit, PROJECT_TIDY_CONFIG_MAP_PATH)) unit_path = _common.get_norm_unit_path(unit) for project_prefix, config_path in ordered_tidy_map: if unit_path.startswith(project_prefix): return config_path else: return get_default_tidy_config(unit) @df.with_fields( CHECK_FIELDS_BASE + ( df.TestedProjectName.normalized_basename, df.SourceFolderPath.normalized, df.SbrUidExt.value, df.TestFiles.value, ) ) def check_data(fields, unit, *args): flat_args, spec_args = _common.sort_by_keywords( { "DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1, }, args, ) dart_record = create_dart_record(fields, unit, flat_args, spec_args) if not dart_record[df.TestFiles.KEY]: return data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( CHECK_FIELDS_BASE + ( df.TestedProjectName.normalized_basename, df.SourceFolderPath.normalized, df.SbrUidExt.value, df.TestFiles.flat_args_wo_first, ) ) def check_resource(fields, unit, *args): flat_args, spec_args = _common.sort_by_keywords( { "DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1, }, args, ) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( CHECK_FIELDS_BASE + ( df.TestedProjectName.normalized_basename, df.SourceFolderPath.normalized, df.TestData.ktlint, df.TestFiles.flat_args_wo_first, df.ModuleLang.value, df.KtlintBinary.value, df.UseKtlintOld.value, df.KtlintBaselineFile.value, ) ) def ktlint(fields, unit, *args): flat_args, spec_args = _common.sort_by_keywords( { "DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1, }, args, ) dart_record = create_dart_record(fields, unit, flat_args, spec_args) dart_record[df.TestTimeout.KEY] = '120' data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( CHECK_FIELDS_BASE + ( df.TestedProjectName.normalized_basename, df.SourceFolderPath.normalized, df.TestData.java_style, df.ForkMode.test_fork_mode, df.TestFiles.java_style, df.JdkLatestVersion.value, df.JdkResource.value, df.ModuleLang.value, ) ) def java_style(fields, unit, *args): flat_args, spec_args = _common.sort_by_keywords( { "DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1, }, args, ) if len(flat_args) < 2: raise Exception("Not enough arguments for JAVA_STYLE check") # jstyle should use the latest jdk unit.onpeerdir([unit.get('JDK_LATEST_PEERDIR')]) dart_record = create_dart_record(fields, unit, flat_args, spec_args) dart_record[df.TestTimeout.KEY] = '240' dart_record[df.ScriptRelPath.KEY] = 'java.style' data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( CHECK_FIELDS_BASE + ( df.TestedProjectName.test_dir, df.SourceFolderPath.test_dir, df.ForkMode.test_fork_mode, df.TestFiles.flat_args_wo_first, df.ModuleLang.value, ) ) def gofmt(fields, unit, *args): flat_args, spec_args = _common.sort_by_keywords( { "DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1, }, args, ) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( CHECK_FIELDS_BASE + ( df.TestedProjectName.normalized_basename, df.SourceFolderPath.normalized, df.ForkMode.test_fork_mode, df.TestFiles.flat_args_wo_first, df.ModuleLang.value, ) ) def govet(fields, unit, *args): flat_args, spec_args = _common.sort_by_keywords( { "DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1, }, args, ) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( CHECK_FIELDS_BASE + ( df.TestedProjectName.normalized_basename, df.SourceFolderPath.normalized, df.TestFiles.flat_args_wo_first, df.ModuleLang.value, ) ) def detekt_report(fields, unit, *args): flat_args, spec_args = _common.sort_by_keywords( { "DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1, }, args, ) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) def onadd_check(unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests return flat_args, *_ = _common.sort_by_keywords( { "DEPENDS": -1, "TIMEOUT": 1, "DATA": -1, "TAG": -1, "REQUIREMENTS": -1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, "SIZE": 1, }, args, ) check_type = flat_args[0] if check_type == "check.data" and unit.get('VALIDATE_DATA') != "no": check_data(unit, *args) elif check_type == "check.resource" and unit.get('VALIDATE_DATA') != "no": check_resource(unit, *args) elif check_type == "ktlint": ktlint(unit, *args) elif check_type == "JAVA_STYLE" and (unit.get('YMAKE_JAVA_TEST') != 'yes' or unit.get('ALL_SRCDIRS')): java_style(unit, *args) elif check_type == "gofmt": gofmt(unit, *args) elif check_type == "govet": govet(unit, *args) elif check_type == "detekt.report": detekt_report(unit, *args) def on_register_no_check_imports(unit): s = unit.get('NO_CHECK_IMPORTS_FOR_VALUE') if s not in ('', 'None'): unit.onresource(['-', 'py/no_check_imports/{}="{}"'.format(_common.pathid(s), s)]) @df.with_fields( ( df.TestedProjectName.normalized_basename, df.SourceFolderPath.normalized, df.TestEnv.value, df.UseArcadiaPython.value, df.TestFiles.normalized, df.ModuleLang.value, df.NoCheck.value, ) ) def onadd_check_py_imports(fields, unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests return if unit.get('NO_CHECK_IMPORTS_FOR_VALUE').strip() == "": return unit.onpeerdir(['library/python/testing/import_test']) dart_record = create_dart_record(fields, unit, (), {}) dart_record[df.TestName.KEY] = 'pyimports' dart_record[df.ScriptRelPath.KEY] = 'py.imports' data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( PY_EXEC_FIELDS_BASE + ( df.TestName.filename_without_ext, df.ScriptRelPath.pytest, df.TestedProjectName.path_filename_basename, df.ModuleLang.value, df.BinaryPath.stripped, df.TestRunnerBin.value, df.DockerImage.value, ) ) def onadd_pytest_bin(fields, unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests return flat_args, spec_args = _common.sort_by_keywords({'RUNNER_BIN': 1}, args) if flat_args: ymake.report_configure_error( 'Unknown arguments found while processing add_pytest_bin macro: {!r}'.format(flat_args) ) if unit.get('ADD_SRCDIR_TO_TEST_DATA') == "yes": unit.ondata_files(_common.get_norm_unit_path(unit)) yt_spec = df.YtSpec.from_unit(unit, flat_args, spec_args) if yt_spec and yt_spec[df.YtSpec.KEY]: unit.ondata_files(deserialize_list(yt_spec[df.YtSpec.KEY])) dart_record = create_dart_record(fields, unit, flat_args, spec_args) if yt_spec: dart_record |= yt_spec data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( ( df.SourceFolderPath.normalized, df.TestName.normalized_joined_dir_basename, df.ScriptRelPath.junit, df.TestTimeout.from_unit, df.TestedProjectName.normalized, df.TestEnv.value, df.TestData.java_test, df.ForkMode.test_fork_mode, df.SplitFactor.from_unit, df.CustomDependencies.test_depends_only, df.Tag.from_macro_args_and_unit, df.Size.from_unit, df.Requirements.with_maybe_fuzzing, df.TestRecipes.value, df.ModuleType.value, df.UnittestDir.value, df.JvmArgs.value, # TODO optimize, SystemProperties is used in TestData df.SystemProperties.value, df.TestCwd.from_unit, df.SkipTest.value, df.JavaClasspathCmdType.value, df.JdkResource.value, df.JdkForTests.value, df.ModuleLang.value, df.TestClasspath.value, df.TestClasspathOrigins.value, df.TestClasspathDeps.value, df.TestJar.value, df.DockerImage.value, ) ) def onjava_test(fields, unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests return assert unit.get('MODULE_TYPE') is not None if unit.get('MODULE_TYPE') == 'JTEST_FOR': if not unit.get('UNITTEST_DIR'): ymake.report_configure_error('skip JTEST_FOR in {}: no args provided'.format(unit.path())) return if unit.get('ADD_SRCDIR_TO_TEST_DATA') == "yes": unit.ondata_files(_common.get_norm_unit_path(unit)) yt_spec = df.YtSpec.from_unit_list_var(unit, (), {}) unit.ondata_files(deserialize_list(yt_spec[df.YtSpec.KEY])) try: dart_record = create_dart_record(fields, unit, (), {}) except df.DartValueError: return dart_record |= yt_spec data = dump_test(unit, dart_record) if data: unit.set_property(['DART_DATA', data]) @df.with_fields( ( df.SourceFolderPath.normalized, df.TestName.normalized_joined_dir_basename_deps, df.TestedProjectName.normalized, df.CustomDependencies.test_depends_only, df.IgnoreClasspathClash.value, df.ModuleType.value, df.ModuleLang.value, df.Classpath.value, ) ) def onjava_test_deps(fields, unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests return assert unit.get('MODULE_TYPE') is not None assert len(args) == 1 mode = args[0] dart_record = create_dart_record(fields, unit, (args[0],), {}) dart_record[df.ScriptRelPath.KEY] = 'java.dependency.test' if mode == 'strict': dart_record[df.StrictClasspathClash.KEY] = 'yes' data = dump_test(unit, dart_record) unit.set_property(['DART_DATA', data]) def onsetup_pytest_bin(unit, *args): use_arcadia_python = unit.get('USE_ARCADIA_PYTHON') == "yes" if use_arcadia_python: unit.onresource(['-', 'PY_MAIN={}'.format("library.python.pytest.main:main")]) # XXX unit.onadd_pytest_bin(list(args)) def onrun(unit, *args): exectest_cmd = unit.get(["EXECTEST_COMMAND_VALUE"]) or '' exectest_cmd += "\n" + subprocess.list2cmdline(args) unit.set(["EXECTEST_COMMAND_VALUE", exectest_cmd]) @df.with_fields( PY_EXEC_FIELDS_BASE + ( df.TestName.filename_without_pkg_ext, df.TestedProjectName.path_filename_basename_without_pkg_ext, df.BinaryPath.stripped_without_pkg_ext, df.DockerImage.value, ) ) def onsetup_exectest(fields, unit, *args): if unit.get("TIDY") == "yes": # graph changed for clang_tidy tests return command = unit.get(["EXECTEST_COMMAND_VALUE"]) if command is None: ymake.report_configure_error("EXECTEST must have at least one RUN macro") return command = command.replace("$EXECTEST_COMMAND_VALUE", "") if "PYTHON_BIN" in command: unit.ondepends('contrib/tools/python') unit.set(["TEST_BLOB_DATA", base64.b64encode(six.ensure_binary(command))]) if unit.get('ADD_SRCDIR_TO_TEST_DATA') == "yes": unit.ondata_files(_common.get_norm_unit_path(unit)) yt_spec = df.YtSpec.from_unit(unit, (), {}) if yt_spec and yt_spec[df.YtSpec.KEY]: unit.ondata_files(deserialize_list(yt_spec[df.YtSpec.KEY])) dart_record = create_dart_record(fields, unit, (), {}) dart_record[df.ScriptRelPath.KEY] = 'exectest' if yt_spec: dart_record |= yt_spec data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) def onsetup_run_python(unit): if unit.get("USE_ARCADIA_PYTHON") == "yes": unit.ondepends('contrib/tools/python') @df.with_fields( ( df.TestFiles.cpp_linter_files, df.LintConfigs.cpp_configs, ) + LINTER_FIELDS_BASE ) def on_add_cpp_linter_check(fields, unit, *args): if unit.get("TIDY") == "yes": return no_lint_value = _common.get_no_lint_value(unit) if no_lint_value in ("none", "none_internal"): return unlimited = -1 keywords = { "NAME": 1, "LINTER": 1, "DEPENDS": unlimited, "CONFIGS": 1, "CUSTOM_CONFIG": 1, "GLOBAL_RESOURCES": unlimited, "FILE_PROCESSING_TIME": 1, "EXTRA_PARAMS": unlimited, } _, spec_args = _common.sort_by_keywords(keywords, args) global_resources = spec_args.get('GLOBAL_RESOURCES', []) for resource in global_resources: unit.onpeerdir(resource) try: dart_record = create_dart_record(fields, unit, (), spec_args) except df.DartValueError as e: if msg := str(e): unit.message(['WARN', msg]) return dart_record[df.ScriptRelPath.KEY] = 'custom_lint' data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( ( df.TestFiles.py_linter_files, df.LintConfigs.python_configs, ) + LINTER_FIELDS_BASE ) def on_add_py_linter_check(fields, unit, *args): if unit.get("TIDY") == "yes": return no_lint_value = _common.get_no_lint_value(unit) if no_lint_value in ("none", "none_internal"): return unlimited = -1 keywords = { "NAME": 1, "LINTER": 1, "DEPENDS": unlimited, "CONFIGS": 1, "GLOBAL_RESOURCES": unlimited, "FILE_PROCESSING_TIME": 1, "EXTRA_PARAMS": unlimited, "PROJECT_TO_CONFIG_MAP": 1, "FLAKE_MIGRATIONS_CONFIG": 1, "CUSTOM_CONFIG": 1, } _, spec_args = _common.sort_by_keywords(keywords, args) global_resources = spec_args.get('GLOBAL_RESOURCES', []) for resource in global_resources: unit.onpeerdir(resource) try: dart_record = create_dart_record(fields, unit, (), spec_args) except df.DartValueError as e: if msg := str(e): unit.message(['WARN', msg]) return dart_record[df.ScriptRelPath.KEY] = 'custom_lint' data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( YTEST_FIELDS_BASE + ( df.TestName.value, df.TestPartition.value, df.ModuleLang.value, ) ) def clang_tidy(fields, unit, *args): keywords = { "DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, } flat_args, spec_args = _common.sort_by_keywords(keywords, args) if unit.get("TIDY_CONFIG"): default_config_path = unit.get("TIDY_CONFIG") project_config_path = unit.get("TIDY_CONFIG") else: default_config_path = get_default_tidy_config(unit) project_config_path = get_project_tidy_config(unit) unit.set(["DEFAULT_TIDY_CONFIG", default_config_path]) unit.set(["PROJECT_TIDY_CONFIG", project_config_path]) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( YTEST_FIELDS_BASE + YTEST_FIELDS_EXTRA + ( df.TestName.value, df.TestData.from_macro_args_and_unit, df.Requirements.from_macro_args_and_unit, df.TestPartition.value, df.ModuleLang.value, df.DockerImage.value, ) ) def unittest_py(fields, unit, *args): keywords = { "DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, } flat_args, spec_args = _common.sort_by_keywords(keywords, args) if unit.get('ADD_SRCDIR_TO_TEST_DATA') == "yes": unit.ondata_files(_common.get_norm_unit_path(unit)) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( YTEST_FIELDS_BASE + YTEST_FIELDS_EXTRA + ( df.TestName.value, df.TestData.from_macro_args_and_unit, df.Requirements.from_macro_args_and_unit, df.TestPartition.value, df.ModuleLang.value, df.DockerImage.value, ) ) def gunittest(fields, unit, *args): keywords = { "DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, } flat_args, spec_args = _common.sort_by_keywords(keywords, args) if unit.get('ADD_SRCDIR_TO_TEST_DATA') == "yes": unit.ondata_files(_common.get_norm_unit_path(unit)) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( YTEST_FIELDS_BASE + YTEST_FIELDS_EXTRA + ( df.TestName.value, df.TestData.from_macro_args_and_unit, df.Requirements.from_macro_args_and_unit, df.TestPartition.value, df.ModuleLang.value, df.BenchmarkOpts.value, df.DockerImage.value, ) ) def g_benchmark(fields, unit, *args): keywords = { "DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, } flat_args, spec_args = _common.sort_by_keywords(keywords, args) if unit.get('ADD_SRCDIR_TO_TEST_DATA') == "yes": unit.ondata_files(_common.get_norm_unit_path(unit)) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( YTEST_FIELDS_BASE + YTEST_FIELDS_EXTRA + ( df.TestName.value, df.TestData.from_macro_args_and_unit_with_canonical, df.Requirements.from_macro_args_and_unit, df.TestPartition.value, df.ModuleLang.value, df.DockerImage.value, ) ) def go_test(fields, unit, *args): keywords = { "DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, } flat_args, spec_args = _common.sort_by_keywords(keywords, args) if unit.get('ADD_SRCDIR_TO_TEST_DATA') == "yes": unit.ondata_files(_common.get_norm_unit_path(unit)) unit.ondata_files(get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( YTEST_FIELDS_BASE + YTEST_FIELDS_EXTRA + ( df.TestName.value, df.TestData.from_macro_args_and_unit, df.Requirements.from_macro_args_and_unit, df.TestPartition.value, df.DockerImage.value, ) ) def boost_test(fields, unit, *args): keywords = { "DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, } flat_args, spec_args = _common.sort_by_keywords(keywords, args) if unit.get('ADD_SRCDIR_TO_TEST_DATA') == "yes": unit.ondata_files(_common.get_norm_unit_path(unit)) unit.ondata_files(get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( YTEST_FIELDS_BASE + YTEST_FIELDS_EXTRA + ( df.TestName.value, df.TestData.from_macro_args_and_unit, df.Requirements.with_maybe_fuzzing, df.FuzzDicts.value, df.FuzzOpts.value, df.Fuzzing.value, df.DockerImage.value, ) ) def fuzz_test(fields, unit, *args): keywords = { "DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, } flat_args, spec_args = _common.sort_by_keywords(keywords, args) if unit.get('ADD_SRCDIR_TO_TEST_DATA') == "yes": unit.ondata_files(_common.get_norm_unit_path(unit)) unit.ondata_files("fuzzing/{}/corpus.json".format(_common.get_norm_unit_path(unit))) unit.ondata_files(get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( YTEST_FIELDS_BASE + YTEST_FIELDS_EXTRA + ( df.TestName.value, df.TestData.from_macro_args_and_unit, df.Requirements.from_macro_args_and_unit, df.TestPartition.value, df.ModuleLang.value, df.BenchmarkOpts.value, df.DockerImage.value, ) ) def y_benchmark(fields, unit, *args): keywords = { "DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, } flat_args, spec_args = _common.sort_by_keywords(keywords, args) unit.ondata_files(get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( YTEST_FIELDS_BASE + YTEST_FIELDS_EXTRA + ( df.TestName.value, df.TestData.from_macro_args_and_unit, df.Requirements.from_macro_args_and_unit, df.TestPartition.value, ) ) def coverage_extractor(fields, unit, *args): keywords = { "DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, } flat_args, spec_args = _common.sort_by_keywords(keywords, args) unit.ondata_files(get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) @df.with_fields( YTEST_FIELDS_BASE + YTEST_FIELDS_EXTRA + ( df.TestName.first_flat_with_bench, df.TestData.from_macro_args_and_unit, df.Requirements.from_macro_args_and_unit, df.TestPartition.value, df.GoBenchTimeout.value, df.ModuleLang.value, df.DockerImage.value, ) ) def go_bench(fields, unit, *args): keywords = { "DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, } flat_args, spec_args = _common.sort_by_keywords(keywords, args) tags = df.Tag.from_macro_args_and_unit(unit, flat_args, spec_args)[df.Tag.KEY] if "ya:run_go_benchmark" not in tags: return unit.ondata_files(get_unit_list_variable(unit, 'TEST_YT_SPEC_VALUE')) dart_record = create_dart_record(fields, unit, flat_args, spec_args) data = dump_test(unit, dart_record) if data: unit.set_property(["DART_DATA", data]) def onadd_ytest(unit, *args): keywords = { "DEPENDS": -1, "DATA": -1, "TIMEOUT": 1, "FORK_MODE": 1, "SPLIT_FACTOR": 1, "FORK_SUBTESTS": 0, "FORK_TESTS": 0, } flat_args, *_ = _common.sort_by_keywords(keywords, args) test_type = flat_args[1] # TIDY not supported for module if unit.get("TIDY_ENABLED") == "yes" and test_type != "clang_tidy": return # TIDY explicitly disabled for module in ymake.core.conf elif test_type == "clang_tidy" and unit.get("TIDY_ENABLED") != "yes": return # TIDY disabled for module in ya.make elif unit.get("TIDY") == "yes" and unit.get("TIDY_ENABLED") != "yes": return elif test_type == "no.test": return elif test_type == "clang_tidy" and unit.get("TIDY_ENABLED") == "yes": clang_tidy(unit, *args) elif test_type == "unittest.py": unittest_py(unit, *args) elif test_type == "gunittest": gunittest(unit, *args) elif test_type == "g_benchmark": g_benchmark(unit, *args) elif test_type == "go.test": go_test(unit, *args) elif test_type == "boost.test": boost_test(unit, *args) elif test_type == "fuzz.test": fuzz_test(unit, *args) elif test_type == "y_benchmark": y_benchmark(unit, *args) elif test_type == "coverage.extractor" and match_coverage_extractor_requirements(unit): coverage_extractor(unit, *args) elif test_type == "go.bench": go_bench(unit, *args)