intermediate changes

ref:128a8f4d9d8b1f87d5a730a31e6abaecd194bfe9

build/conf/docs.conf

@@ -2,6 +2,14 @@
 TOUCH_DOCS=$YMAKE_PYTHON ${input:"build/scripts/touch.py"} ${kv;hide:"p DC"} ${kv;hide:"pc light-cyan"} $TARGET
 TOUCH_DOCS_MF=$TOUCH_DOCS && $GENERATE_MF
 
+# tag:docs
+### @usage: DOCS_COPY_FILE(SRCDIR src_dir [NAMESPCE dst_dir] files...)
+###
+### Copy files from src_dir to $BINDIR/dst_dir
+macro DOCS_COPY_FILES(SRCDIR="${CURDIR}", NAMESPACE=".", FILES...) {
+    .CMD=$YMAKE_PYTHON ${input:"build/scripts/copy_docs_files.py"} --source-root $ARCADIA_ROOT --build-root $ARCADIA_BUILD_ROOT --src-dir $SRCDIR --dst-dir $BINDIR/$NAMESPACE $FILES ${input;hide;context=TEXT;pre=${SRCDIR}/:FILES} ${output;hide;pre=${NAMESPACE}/:FILES}
+}
+
 # tag:docs
 _DOCS_USE_PLANTUML=no
 _DOCS_EXTRA_TOOLS=

build/rules/go/vendor.policy

@@ -1068,9 +1068,9 @@ ALLOW trust/psp/core/handler -> vendor/github.com/gorilla/mux
 ALLOW cloud/compute/go-common/pkg/validation -> vendor/github.com/go-playground/validator/v10
 
 # CONTRIB-2412
-ALLOW library/go/protoc_gen_crd -> vendor/github.com/google/gnostic/openapiv3
-ALLOW library/go/protoc_gen_crd -> vendor/github.com/google/gnostic/compiler
-ALLOW library/go/protoc_gen_crd -> vendor/gopkg.in/yaml.v3
+ALLOW library/go/k8s/protoc_gen_crd -> vendor/github.com/google/gnostic/openapiv3
+ALLOW library/go/k8s/protoc_gen_crd -> vendor/github.com/google/gnostic/compiler
+ALLOW library/go/k8s/protoc_gen_crd -> vendor/gopkg.in/yaml.v3
 ALLOW infra/infractl/.* -> vendor/github.com/google/gnostic/openapiv3
 ALLOW infra/infractl/.* -> vendor/github.com/google/gnostic/compiler
 ALLOW infra/infractl/.* -> vendor/gopkg.in/yaml.v3

build/scripts/copy_docs_files.py

@@ -0,0 +1,76 @@
+import argparse
+import errno
+import os
+import process_command_files as pcf
+import shutil
+import sys
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--build-root', required=True)
+    parser.add_argument('--dst-dir', required=True)
+    parser.add_argument('--existing', choices=('skip', 'overwrite'), default='overwrite')
+    parser.add_argument('--source-root', required=True)
+    parser.add_argument('--src-dir', required=None)
+    parser.add_argument('files', nargs='*')
+    return parser.parse_args(pcf.get_args(sys.argv[1:]))
+
+
+def makedirs(dirname):
+    try:
+        os.makedirs(dirname)
+    except OSError as e:
+        if e.errno == errno.EEXIST and os.path.isdir(dirname):
+            pass
+        else:
+            raise
+
+
+def copy_file(src, dst, overwrite=False, orig_path=None, generated=False):
+    if os.path.exists(dst) and not overwrite:
+        return
+
+    makedirs(os.path.dirname(dst))
+
+    with open(src, 'r') as fsrc, open(dst, 'w') as fdst:
+        if (orig_path or generated) and src.endswith('.md'):
+            fdst.write('---\n{}\n\n---\n'.format('generated: true' if generated else 'vcsPath: {}'.format(orig_path)))
+        shutil.copyfileobj(fsrc, fdst)
+
+
+def main():
+    args = parse_args()
+
+    source_root = os.path.normpath(args.source_root) + os.path.sep
+    build_root = os.path.normpath(args.build_root) + os.path.sep
+
+    dst_dir = os.path.normpath(args.dst_dir)
+    assert dst_dir.startswith(build_root)
+    makedirs(dst_dir)
+
+    src_dir = os.path.normpath(args.src_dir) + os.path.sep
+    assert src_dir.startswith(source_root)
+
+    if src_dir.startswith(source_root):
+        root = source_root
+        is_from_source_root = True
+    elif src_dir.startswith(build_root):
+        root = build_root
+        is_from_source_root = False
+    else:
+        assert False, 'src_dir [{}] should start with [{}] or [{}]'.format(src_dir, source_root, build_root)
+
+    is_overwrite_existing = args.existing == 'overwrite'
+
+    for f in [os.path.normpath(f) for f in args.files]:
+        src_file = os.path.join(src_dir, f)
+        dst_file = os.path.join(dst_dir, f)
+        if src_file == dst_file:
+            continue
+        rel_path = src_file[len(root):] if is_from_source_root else None
+        copy_file(src_file, dst_file, overwrite=is_overwrite_existing, orig_path=rel_path)
+
+
+if __name__ == '__main__':
+    main()

build/scripts/copy_docs_files_to_dir.py

@@ -14,6 +14,7 @@ def parse_args():
     parser.add_argument('--docs-dir', action='append', nargs=2, dest='docs_dirs', default=None)
     parser.add_argument('--existing', choices=('skip', 'overwrite'), default='overwrite')
     parser.add_argument('--source-root', required=True)
+    parser.add_argument('--src-dir', action='append', nargs='*', dest='src_dirs', default=None)
     parser.add_argument('files', nargs='*')
     return parser.parse_args(pcf.get_args(sys.argv[1:]))
 
@@ -28,16 +29,15 @@ def makedirs(dirname):
             raise
 
 
-def copy_file(src, dst, overwrite, orig_path, generated=False):
+def copy_file(src, dst, overwrite=False, orig_path=None, generated=False):
     if os.path.exists(dst) and not overwrite:
         return
 
     makedirs(os.path.dirname(dst))
 
     with open(src, 'r') as fsrc, open(dst, 'w') as fdst:
-        # FIXME(snermolaev): uncomment lines below when yfm is ready
-        # if src.endswith('.md'):
-        #     fdst.write('---\n{}\n\n---\n'.format('generated: true' if generated else 'vcsPath: {}'.format(orig_path)))
+        # if (orig_path or generated) and src.endswith('.md'):
+        #    fdst.write('---\n{}\n\n---\n'.format('generated: true' if generated else 'vcsPath: {}'.format(orig_path)))
         shutil.copyfileobj(fsrc, fdst)
 
 
@@ -55,11 +55,11 @@ def main():
     if args.docs_dirs:
         for item in args.docs_dirs:
             assert len(item) == 2
-            docs_dir, docs_dir_namespace = item[0], item[1]
+            docs_dir, nm = item[0], item[1]
             assert not os.path.isabs(docs_dir)
-            if docs_dir_namespace and docs_dir_namespace != '.':
-                assert not os.path.isabs(docs_dir_namespace)
-                dst = os.path.join(dest_dir, docs_dir_namespace)
+            if nm and nm != '.':
+                assert not os.path.isabs(nm)
+                dst = os.path.join(dest_dir, nm)
             else:
                 dst = dest_dir
 
@@ -72,7 +72,33 @@ def main():
                     file_src = os.path.join(root, f)
                     assert file_src.startswith(source_root)
                     file_dst = os.path.join(dst, os.path.relpath(root, abs_docs_dir), f)
-                    copy_file(file_src, file_dst, is_overwrite_existing, file_src[len(source_root):])
+                    copy_file(file_src, file_dst, overwrite=is_overwrite_existing, orig_path=file_src[len(source_root):])
+
+    if args.src_dirs:
+        for item in args.src_dirs:
+            assert len(item) > 1
+            src_dir, nm = os.path.normpath(item[0]), item[1]
+            assert os.path.isabs(src_dir)
+            if nm and nm != '.':
+                assert not os.path.isabs(nm)
+                dst = os.path.join(dest_dir, nm)
+            else:
+                dst = dest_dir
+
+            if src_dir.startswith(source_root):
+                root = source_root
+                is_from_source_root = True
+            else:
+                assert src_dir.startswith(build_root)
+                root = build_root
+                is_from_source_root = False
+
+            for f in item[2:]:
+                file_src = os.path.normpath(f)
+                assert file_src.startswith(root)
+                rel_path = file_src[len(root):] if is_from_source_root else None
+                file_dst = os.path.join(dst, file_src[len(src_dir):])
+                copy_file(file_src, file_dst, overwrite=is_overwrite_existing, orig_path=rel_path)
 
     if args.bin_dir:
         assert len(args.bin_dir) > 1
@@ -88,7 +114,7 @@ def main():
             assert os.path.isfile(file_src)
             assert file_src.startswith(bin_dir)
             file_dst = os.path.join(dst, file_src[len(bin_dir):])
-            copy_file(file_src, file_dst, is_overwrite_existing, None, generated=True)
+            copy_file(file_src, file_dst, overwrite=is_overwrite_existing, orig_path=None)
 
     for src in args.files:
         generated = False
@@ -98,7 +124,7 @@ def main():
         if file_src.startswith(source_root):
             rel_path = file_src[len(source_root):]
         elif file_src.startswith(build_root):
-            generated = True
+            # generated = True
             rel_path = file_src[len(build_root):]
         else:
             raise Exception('Unexpected file path [{}].'.format(file_src))

build/ya.conf.json

@@ -8225,7 +8225,7 @@
         },
         "sedem": {
             "formula": {
-                "sandbox_id": 1248571726,
+                "sandbox_id": 1253638317,
                 "match": "SEDEM archive"
             },
             "executable": {
@@ -8559,7 +8559,7 @@
         },
         "lama": {
             "formula": {
-                "sandbox_id": 1247043408,
+                "sandbox_id": 1254576600,
                 "match": "lama"
             },
             "executable": {

build/ymake.core.conf

@@ -6279,6 +6279,10 @@ macro _PY3_COMPILE_BYTECODE(SrcX, Src, Dst) {
     .CMD=${env:"PYTHONHASHSEED=0"} ${tool:"contrib/tools/python3/pycc"} $SrcX ${input:Src} ${output;noauto:Dst.yapyc3} ${kv;hide:"p PY"} ${kv;hide:"pc yellow"}
 }
 
+macro _ARCHIVE_SEM_HELPER(OUT, DONTCOMPRESS, Files...) {
+    .SEM=add_custom_command OUTPUT ${OUT} DEPENDS ${Files} COMMAND $ARCH_TOOL -q $DONTCOMPRESS ${join=\: :Files}: -o ${OUT}
+}
+
 ### @usage: ARCHIVE_ASM(NAME archive_name files...)
 ###
 ### Similar to the macro ARCHIVE, but:
@@ -6286,6 +6290,7 @@ macro _PY3_COMPILE_BYTECODE(SrcX, Src, Dst) {
 ### 2. Different syntax (see examples in codesearch or users/pg/tests/archive_test)
 macro ARCHIVE_ASM(NAME="", DONTCOMPRESS?"-p":"", Files...) {
     .CMD=$ARCH_TOOL -q $DONTCOMPRESS ${input;join=\: :Files}: -o ${output;chksum;suf=$OBJ_SUF.rodata:NAME} ${kv;hide:"p AR"} ${kv;hide:"pc light-cyan"}
+    .SEM=$_ARCHIVE_SEM_HELPER(${output;suf=$OBJ_SUF.rodata:NAME}, $DONTCOMPRESS, ${input:Files})
 }
 
 # tag:yweb-specific
@@ -6307,6 +6312,7 @@ macro PIRE_INLINE(FILES...) {
 ### Example: https://wiki.yandex-team.ru/yatool/howtowriteyamakefiles/#a1ispolzujjtekomanduarchive
 macro ARCHIVE(NAME="", DONTCOMPRESS?"-p":"", Files...) {
     .CMD=$ARCH_TOOL -q -x $DONTCOMPRESS ${input;join=\: :Files}: -o ${output;chksum;addincl;noauto:NAME} ${kv;hide:"p AR"} ${kv;hide:"pc light-red"}
+    .SEM=$_ARCHIVE_SEM_HELPER(${output;addincl;noauto:NAME}, $DONTCOMPRESS, ${input:Files})
 }
 
 ### @usage: ARCHIVE_BY_KEYS(archive_name key [DONT_COMPRESS] files...)
@@ -7068,13 +7074,18 @@ macro CTEMPLATE_VARNAMES(File) {
 
 LLVM_OPTS=
 CLANG_ROOT=$CLANG_RESOURCE_GLOBAL
-CLANG_CPP_COMPILER_BIN=$CLANG_ROOT/bin/clang++
-CLANG_C_COMPILER_BIN=$CLANG_ROOT/bin/clang
+CLANG_BINARIES_ROOT=$CLANG_ROOT/bin
+CLANG_CPP_COMPILER_BIN=$CLANG_BINARIES_ROOT/clang++
+CLANG_C_COMPILER_BIN=$CLANG_BINARIES_ROOT/clang
 
 when ($OS_ANDROID == "yes") {
     CLANG_ROOT=$CLANG_ANDROID_RESOURCE_GLOBAL
-    CLANG_CPP_COMPILER_BIN=$CLANG_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin/clang++
-    CLANG_C_COMPILER_BIN=$CLANG_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin/clang
+    when ($HOST_OS_DARWIN) {
+        CLANG_BINARIES_ROOT=$CLANG_ROOT/toolchains/llvm/prebuilt/darwin-x86_64/bin
+    }
+    otherwise {
+        CLANG_BINARIES_ROOT=$CLANG_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin
+    }
 }
 
 ### @usage: GENERATED_SRCS(srcs... PARSE_META_FROM cpp_srcs... [OUTPUT_INCLUDES output_includes...] [OPTIONS])
@@ -7155,7 +7166,7 @@ macro BPF_STATIC(Input, Output, Opts...) {
 ### Compile LLVM bytecode to object representation.
 ### Note: Output name is used as is, no extension added.
 macro LLVM_COMPILE_LL(Input, Output, Opts...) {
-    .CMD=$CLANG_ROOT/bin/llvm-as ${input:Input} -o ${output;noauto:Output} ${kv;hide:"p BC"} ${kv;hide:"pc light-green"}
+    .CMD=$CLANG_BINARIES_ROOT/llvm-as ${input:Input} -o ${output;noauto:Output} ${kv;hide:"p BC"} ${kv;hide:"pc light-green"}
     PEERDIR(build/platform/clang)
 }
 
@@ -7164,7 +7175,7 @@ macro LLVM_COMPILE_LL(Input, Output, Opts...) {
 ### Call llvm-link on set of Inputs to produce Output.
 ### Note: Unlike many other macros output argument goes first. Output name is used as is, no extension added.
 macro LLVM_LINK(Output, Inputs...) {
-    .CMD=$CLANG_ROOT/bin/llvm-link ${input:Inputs} -o ${output;noauto:Output} ${kv;hide:"p LD"} ${kv;hide:"pc light-red"}
+    .CMD=$CLANG_BINARIES_ROOT/llvm-link ${input:Inputs} -o ${output;noauto:Output} ${kv;hide:"p LD"} ${kv;hide:"pc light-red"}
     .SEM=add_custom_command OUTPUT ${output;noauto:Output} DEPENDS ${input:Inputs} COMMAND ${LLVMLINK} ${input:Inputs} -o ${output;noauto:Output}
     PEERDIR(build/platform/clang)
 }
@@ -7174,7 +7185,7 @@ macro LLVM_LINK(Output, Inputs...) {
 ### Call llvm-opt with set of Opts on Input to produce Output.
 ### Note: Output name is used as is, no extension added.
 macro LLVM_OPT(Input, Output, Opts...) {
-    .CMD=$YMAKE_PYTHON ${input:"build/scripts/llvm_opt_wrapper.py"} $CLANG_ROOT/bin/opt ${input:Input} -o ${output;noauto:Output} $Opts ${kv;hide:"p OP"} ${kv;hide:"pc yellow"}
+    .CMD=$YMAKE_PYTHON ${input:"build/scripts/llvm_opt_wrapper.py"} $CLANG_BINARIES_ROOT/opt ${input:Input} -o ${output;noauto:Output} $Opts ${kv;hide:"p OP"} ${kv;hide:"pc yellow"}
     .SEM=add_custom_command OUTPUT ${output;noauto:Output} DEPENDS ${input:Input} COMMAND ${LLVMOPT} ${input:Input} -o ${output;noauto:Output} $Opts
     PEERDIR(build/platform/clang)
 }