import base64 import itertools from hashlib import md5 import os import six from _common import rootrel_arc_src, tobuilddir import ymake runtime_cgo_path = os.path.join('runtime', 'cgo') runtime_msan_path = os.path.join('runtime', 'msan') runtime_race_path = os.path.join('runtime', 'race') arc_project_prefix = 'a.yandex-team.ru/' import_runtime_cgo_false = { 'norace': (runtime_cgo_path, runtime_msan_path, runtime_race_path), 'race': (runtime_cgo_path, runtime_msan_path), } import_syscall_false = { 'norace': (runtime_cgo_path), 'race': (runtime_cgo_path, runtime_race_path), } def get_import_path(unit): # std_lib_prefix = unit.get('GO_STD_LIB_PREFIX') # unit.get() doesn't evalutate the value of variable, so the line above doesn't really work std_lib_prefix = unit.get('GOSTD') + '/' arc_project_prefix = unit.get('GO_ARCADIA_PROJECT_PREFIX') vendor_prefix = unit.get('GO_CONTRIB_PROJECT_PREFIX') module_path = rootrel_arc_src(unit.path(), unit) assert len(module_path) > 0 if go_package_name(unit) == "main": return "main" import_path = module_path.replace('\\', '/') if import_path.startswith(std_lib_prefix): import_path = import_path[len(std_lib_prefix) :] elif import_path.startswith(vendor_prefix): import_path = import_path[len(vendor_prefix) :] else: import_path = arc_project_prefix + import_path assert len(import_path) > 0 if import_path.endswith("/gotest"): return import_path[:-7] return import_path def get_appended_values(unit, key): values = [] raw_value = unit.get(key) if raw_value: values = [x for x in raw_value.split(' ') if x] if len(values) > 0 and values[0] == '$' + key: values.pop(0) return values def compare_versions(version1, version2): def last_index(version): index = version.find('beta') return len(version) if index < 0 else index v1 = tuple(x.zfill(8) for x in version1[: last_index(version1)].split('.')) v2 = tuple(x.zfill(8) for x in version2[: last_index(version2)].split('.')) if v1 == v2: return 0 return 1 if v1 < v2 else -1 def go_package_name(unit): name = unit.get('_GO_PACKAGE_VALUE') if not name and unit.enabled('GO_TEST_MODULE'): name = unit.get('GO_PACKAGE_VALUE') if not name: name = unit.get('GO_TEST_IMPORT_PATH') if name: name = os.path.basename(os.path.normpath(name)) elif unit.get('MODULE_TYPE') == 'PROGRAM': name = 'main' else: name = unit.get('REALPRJNAME') return name def need_lint(path): return not path.startswith('$S/vendor/') and not path.startswith('$S/contrib/') def on_go_process_srcs(unit): """ _GO_PROCESS_SRCS() macro processes only 'CGO' files. All remaining *.go files and other input files are currently processed by a link command of the GO module (GO_LIBRARY, GO_PROGRAM) """ srcs_files = get_appended_values(unit, '_GO_SRCS_VALUE') asm_files = [] c_files = [] cxx_files = [] ev_files = [] fbs_files = [] go_files = [] in_files = [] proto_files = [] s_files = [] syso_files = [] classified_files = { '.c': c_files, '.cc': cxx_files, '.cpp': cxx_files, '.cxx': cxx_files, '.ev': ev_files, '.fbs': fbs_files, '.go': go_files, '.in': in_files, '.proto': proto_files, '.s': asm_files, '.syso': syso_files, '.C': cxx_files, '.S': s_files, } # Classify files specifed in _GO_SRCS() macro by extension and process CGO_EXPORT keyword # which can preceed C/C++ files only is_cgo_export = False for f in srcs_files: _, ext = os.path.splitext(f) ext_files = classified_files.get(ext) if ext_files is not None: if is_cgo_export: is_cgo_export = False if ext in ('.c', '.cc', '.cpp', '.cxx', '.C'): unit.oncopy_file_with_context([f, f, 'OUTPUT_INCLUDES', '${BINDIR}/_cgo_export.h']) f = '${BINDIR}/' + f else: ymake.report_configure_error('Unmatched CGO_EXPORT keyword in SRCS() macro') ext_files.append(f) elif f == 'CGO_EXPORT': is_cgo_export = True else: # FIXME(snermolaev): We can report an unsupported files for _GO_SRCS here pass if is_cgo_export: ymake.report_configure_error('Unmatched CGO_EXPORT keyword in SRCS() macro') for f in go_files: if f.endswith('_test.go'): ymake.report_configure_error('file {} must be listed in GO_TEST_SRCS() or GO_XTEST_SRCS() macros'.format(f)) go_test_files = get_appended_values(unit, '_GO_TEST_SRCS_VALUE') go_xtest_files = get_appended_values(unit, '_GO_XTEST_SRCS_VALUE') for f in go_test_files + go_xtest_files: if not f.endswith('_test.go'): ymake.report_configure_error( 'file {} should not be listed in GO_TEST_SRCS() or GO_XTEST_SRCS() macros'.format(f) ) is_test_module = unit.enabled('GO_TEST_MODULE') unit_path = unit.path() # Add gofmt style checks add_fmt = True if not unit.enabled('_GO_FMT_ADD_CHECK'): allow_skip_fmt = unit.get('_GO_FMT_ALLOW_SKIP') allow_skip_fmt_list = None if allow_skip_fmt: allow_skip_fmt_list = unit.get('_GO_FMT_ALLOW_SKIP').split(' ') if allow_skip_fmt_list: unit_rel_path = rootrel_arc_src(unit_path, unit) if unit_rel_path in allow_skip_fmt_list: add_fmt = False else: for item in allow_skip_fmt_list: if item: prefix = item if item[-1] == '/' else item + '/' if unit_rel_path.startswith(prefix): add_fmt = False break if add_fmt: ymake.report_configure_error('Disabling gofmt is prohibited, please contact devtools') if add_fmt: resolved_go_files = [] go_source_files = [] if is_test_module and unit.get(['GO_TEST_FOR_DIR']) else go_files for path in itertools.chain(go_source_files, go_test_files, go_xtest_files): if path.endswith('.go'): resolved = unit.resolve_arc_path([path]) if resolved != path and need_lint(resolved): resolved_go_files.append(resolved) if resolved_go_files: basedirs = {} for f in resolved_go_files: basedir = os.path.dirname(f) if basedir not in basedirs: basedirs[basedir] = [] basedirs[basedir].append(f) for basedir in basedirs: unit.onadd_check(['gofmt'] + basedirs[basedir]) # Go coverage instrumentation (NOTE! go_files list is modified here) if is_test_module and unit.enabled('GO_TEST_COVER'): cover_info = [] for f in go_files: if f.endswith('_test.go'): continue cover_var = 'GoCover' + six.ensure_str(base64.b32encode(six.ensure_binary(f))).rstrip('=') cover_file = unit.resolve_arc_path(f) cover_file_output = '{}/{}'.format(unit_path, os.path.basename(f)) unit.on_go_gen_cover_go([cover_file, cover_file_output, cover_var]) if cover_file.startswith('$S/'): cover_file = arc_project_prefix + cover_file[3:] cover_info.append('{}:{}'.format(cover_var, cover_file)) # go_files should be empty now since the initial list shouldn't contain # any non-go or go test file. The value of go_files list will be used later # to update the value of _GO_SRCS_VALUE go_files = [] unit.set(['GO_COVER_INFO_VALUE', ' '.join(cover_info)]) # We have cleaned up the list of files from _GO_SRCS_VALUE var and we have to update # the value since it is used in module command line unit.set(['_GO_SRCS_VALUE', ' '.join(itertools.chain(go_files, asm_files, syso_files))]) # Add go vet check if unit.enabled('_GO_VET_ADD_CHECK') and need_lint(unit_path): vet_report_file_name = os.path.join(unit_path, '{}{}'.format(unit.filename(), unit.get('GO_VET_REPORT_EXT'))) unit.onadd_check(["govet", '$(BUILD_ROOT)/' + tobuilddir(vet_report_file_name)[3:]]) for f in ev_files: ev_proto_file = '{}.proto'.format(f) unit.oncopy_file_with_context([f, ev_proto_file]) proto_files.append(ev_proto_file) # Process .proto files for f in proto_files: unit.on_go_proto_cmd(f) # Process .fbs files for f in fbs_files: unit.on_go_flatc_cmd([f, go_package_name(unit)]) # Process .in files for f in in_files: unit.onsrc(f) # Generate .symabis for .s files (starting from 1.12 version) if len(asm_files) > 0: symabis_flags = [] gostd_version = unit.get('GOSTD_VERSION') if compare_versions('1.16', gostd_version) >= 0: import_path = get_import_path(unit) symabis_flags.extend(['FLAGS', '-p', import_path]) unit.on_go_compile_symabis(asm_files + symabis_flags) # Process cgo files cgo_files = get_appended_values(unit, '_CGO_SRCS_VALUE') cgo_cflags = [] if len(c_files) + len(cxx_files) + len(s_files) + len(cgo_files) > 0: if is_test_module: go_test_for_dir = unit.get('GO_TEST_FOR_DIR') if go_test_for_dir and go_test_for_dir.startswith('$S/'): unit.onaddincl(['FOR', 'c', go_test_for_dir[3:]]) unit.onaddincl(['FOR', 'c', unit.get('MODDIR')]) cgo_cflags = get_appended_values(unit, 'CGO_CFLAGS_VALUE') for f in itertools.chain(c_files, cxx_files, s_files): unit.onsrc([f] + cgo_cflags) if len(cgo_files) > 0: if not unit.enabled('CGO_ENABLED'): ymake.report_configure_error('trying to build with CGO (CGO_SRCS is non-empty) when CGO is disabled') import_path = get_import_path(unit) if import_path != runtime_cgo_path: go_std_root = unit.get('GOSTD') unit.onpeerdir(os.path.join(go_std_root, runtime_cgo_path)) race_mode = 'race' if unit.enabled('RACE') else 'norace' import_runtime_cgo = 'false' if import_path in import_runtime_cgo_false[race_mode] else 'true' import_syscall = 'false' if import_path in import_syscall_false[race_mode] else 'true' args = ( [import_path] + cgo_files + ['FLAGS', '-import_runtime_cgo=' + import_runtime_cgo, '-import_syscall=' + import_syscall] ) unit.on_go_compile_cgo1(args) cgo2_cflags = get_appended_values(unit, 'CGO2_CFLAGS_VALUE') for f in cgo_files: if f.endswith('.go'): unit.onsrc([f[:-2] + 'cgo2.c'] + cgo_cflags + cgo2_cflags) else: ymake.report_configure_error('file {} should not be listed in CGO_SRCS() macros'.format(f)) args = [go_package_name(unit)] + cgo_files if len(c_files) > 0: args += ['C_FILES'] + c_files if len(s_files) > 0: args += ['S_FILES'] + s_files if len(syso_files) > 0: args += ['OBJ_FILES'] + syso_files unit.on_go_compile_cgo2(args) def on_go_resource(unit, *args): args = list(args) files = args[::2] keys = args[1::2] suffix_md5 = md5(six.ensure_binary('@'.join(args))).hexdigest() resource_go = os.path.join("resource.{}.res.go".format(suffix_md5)) unit.onpeerdir(["library/go/core/resource"]) if len(files) != len(keys): ymake.report_configure_error("last file {} is missing resource key".format(files[-1])) for i, (key, filename) in enumerate(zip(keys, files)): if not key: ymake.report_configure_error("file key must be non empty") return if filename == "-" and "=" not in key: ymake.report_configure_error("key \"{}\" must contain = sign".format(key)) return # quote key, to avoid automatic substitution of filename by absolute # path in RUN_PROGRAM args[2 * i + 1] = "notafile" + args[2 * i + 1] files = [file for file in files if file != "-"] unit.onrun_program( ["library/go/core/resource/cc", "-package", go_package_name(unit), "-o", resource_go] + list(args) + ["IN"] + files + ["OUT", resource_go] )