Revert commit rXXXXXX, collect .proto files by PROTO_DESCRIPTIONS

b9f5656ae46daedf9083c3721d8cb31a02b1c4c7

build/conf/proto.conf

@@ -639,12 +639,12 @@ macro LIST_PROTO(TO="files.proto", Files...) {
 }
 
 # tag:proto
-macro _PROTO_DESC_RAWPROTO_CMD(File) {
-    .CMD=${cwd;rootdir;input:File} $YMAKE_PYTHON3 ${input:"build/scripts/desc_rawproto_wrapper.py"} --desc-output ${output;suf=.desc:File} --rawproto-output ${output;norel;suf=.rawproto:File} --proto-file ${input;rootrel:File} -- $PROTOC -I=./$PROTO_NAMESPACE -I=$ARCADIA_ROOT/$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_BUILD_ROOT -I=$PROTOBUF_INCLUDE_PATH --include_source_info $_PROTOC_FLAGS ${hide:PROTO_FAKEID}
+macro _PROTO_DESC_CMD(File) {
+    .CMD=${cwd;rootdir;input:File} $PROTOC -I=./$PROTO_NAMESPACE -I=$ARCADIA_ROOT/$PROTO_NAMESPACE ${pre=-I=:_PROTO__INCLUDE} -I=$ARCADIA_BUILD_ROOT -I=$PROTOBUF_INCLUDE_PATH --descriptor_set_out=${output;suf=.desc:File} --include_source_info $_PROTOC_FLAGS ${input;rootrel:File} ${hide:PROTO_FAKEID}
 }
 
-_PROTO_DESC_MERGE_CMD=$YMAKE_PYTHON3 ${input:"build/scripts/merge_files.py"} $TARGET ${ext=.desc:AUTO_INPUT} ${kv;hide:"p PD"} ${kv;hide:"pc light-cyan"} && ${cwd:ARCADIA_BUILD_ROOT} $YMAKE_PYTHON3 ${input:"build/scripts/collect_rawproto.py"} --output ${output;suf=.protosrc:REALPRJNAME} ${rootrel;ext=.rawproto:AUTO_INPUT}
-_PROTO_DESC_MERGE_PEERS_CMD=$YMAKE_PYTHON3 ${input:"build/scripts/merge_files.py"} $TARGET $PEERS ${ext=.protodesc:SRCS_GLOBAL}  ${kv;hide:"p PD"} ${kv;hide:"pc light-cyan"} && ${cwd:ARCADIA_BUILD_ROOT} $YMAKE_PYTHON3 ${input:"build/scripts/merge_protosrc.py"} --output ${output;suf=.tar:REALPRJNAME} ${rootrel:PEERS}
+_PROTO_DESC_MERGE_CMD=$YMAKE_PYTHON ${input:"build/scripts/merge_files.py"} $TARGET $AUTO_INPUT ${kv;hide:"p PD"} ${kv;hide:"pc light-cyan"}
+_PROTO_DESC_MERGE_PEERS_CMD=$YMAKE_PYTHON ${input:"build/scripts/merge_files.py"} $TARGET $PEERS ${ext=.protodesc:SRCS_GLOBAL}  ${kv;hide:"p PD"} ${kv;hide:"pc light-cyan"}
 
 NEED_GOOGLE_PROTO_PEERDIRS=yes
 
@@ -807,7 +807,7 @@ multimodule PROTO_LIBRARY {
     module DESC_PROTO: _BARE_UNIT {
         .CMD=_PROTO_DESC_MERGE_CMD
         .SEM=IGNORED
-        .EXTS=.desc .rawproto
+        .EXTS=.desc
         .NODE_TYPE=Library
         .IGNORED=GENERATE_ENUM_SERIALIZATION GENERATE_ENUM_SERIALIZATION_WITH_HEADER YMAPS_SPROTO RESOURCE GO_PROTO_PLUGIN GRPC
 
@@ -869,6 +869,6 @@ _CPP_PROTO_EVLOG_CMDLINE=$_CPP_PROTO_EVLOG_CMD($SRC)
 _CPP_VANILLA_PROTO_CMDLINE=$_CPP_VANILLA_PROTO_CMD($SRC)
 _JAVA_EVLOG_CMDLINE=$_JAVA_EVLOG_CMD($SRC)
 _JAVA_PROTO_CMDLINE=$_JAVA_PROTO_CMD($SRC)
-_PROTO_DESC_CMDLINE=$_PROTO_DESC_RAWPROTO_CMD($SRC)
+_PROTO_DESC_CMDLINE=$_PROTO_DESC_CMD($SRC)
 _PY_EVLOG_CMDLINE=$_PY_EVLOG_CMD($SRC)
 _PY_PROTO_CMDLINE=$_PY_PROTO_CMD($SRC)

build/scripts/collect_rawproto.py

@@ -1,37 +0,0 @@
-import argparse
-import os
-import tarfile
-import stat
-import sys
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--output', required=True)
-    parser.add_argument('args', nargs='*')
-    return parser.parse_args()
-
-
-def main(args):
-    rawprotos = args.args
-    with tarfile.open(args.output, 'w') as fout:
-        for rawproto in sorted(os.path.normpath(r).replace('\\', '/') for r in rawprotos):
-            assert rawproto.endswith('.rawproto')
-            arcname = rawproto[:-len('.rawproto')]
-            with open(rawproto, 'rb') as fin:
-                tarinfo = fout.gettarinfo(rawproto, arcname)
-                tarinfo.mode = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH if tarinfo.mode | stat.S_IXUSR else 0
-                tarinfo.mode = (
-                    tarinfo.mode | stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH
-                )
-                tarinfo.mtime = 0
-                tarinfo.uid = 0
-                tarinfo.gid = 0
-                tarinfo.uname = 'dummy'
-                tarinfo.gname = 'dummy'
-                fout.addfile(tarinfo, fin)
-    return 0
-
-
-if __name__ == '__main__':
-    sys.exit(main(parse_args()))

build/scripts/desc_rawproto_wrapper.py

@@ -1,34 +0,0 @@
-import argparse
-import shutil
-import subprocess
-import sys
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--desc-output', required=True)
-    parser.add_argument('--rawproto-output', required=True)
-    parser.add_argument('--proto-file', required=True)
-    parser.add_argument('args', nargs='+')
-
-    return parser.parse_args()
-
-
-def main(args):
-    cmd = list(args.args)
-    cmd.append(f'--descriptor_set_out={args.desc_output}')
-    cmd.append(args.proto_file)
-
-    try:
-        subprocess.run(cmd, stdin=None, stderr=subprocess.STDOUT, text=True, check=True)
-    except subrocess.CalledProcessError as e:
-        sys.stderr.write(f'{e.cmd} returned non-zero exit code {e.returncode}.\n{e.output}\n')
-        return e.returncode
-
-    shutil.copyfile(args.proto_file, args.rawproto_output)
-
-    return 0
-
-
-if __name__ == '__main__':
-    sys.exit(main(parse_args()))

build/scripts/merge_files.py

@@ -2,7 +2,7 @@ import sys
 
 
 if __name__ == "__main__":
-    with open(sys.argv[1], "wb") as f:
+    with open(sys.argv[1], "w") as f:
         for appended in sys.argv[2:]:
-            with open(appended, "rb") as a:
+            with open(appended) as a:
                 f.write(a.read())

build/scripts/merge_protosrc.py

@@ -1,39 +0,0 @@
-import argparse
-import os
-import tarfile
-import sys
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--output', required=True)
-    parser.add_argument('args', nargs='+')
-    return parser.parse_args()
-
-
-def main(args):
-    peers = args.args
-
-    compression_mode = ''
-    if args.output.endswith(('.tar.gz', '.tgz')):
-        compression_mode = 'gz'
-    elif args.output.endswith('.bzip2'):
-        compression_mode = 'bz2'
-
-    files = set()
-    with tarfile.open(args.output, f'w:{compression_mode}') as dest:
-        for psrc in [p[:-len('.self.protodesc')]+'.protosrc' for p in peers if p.endswith('.self.protodesc')]:
-            with tarfile.open(psrc, 'r') as src:
-                for tarinfo in [m for m in src.getmembers() if m.name != '']:
-                    if tarinfo.name in files:
-                        continue
-                    files.add(tarinfo.name)
-                    if tarinfo.isdir():
-                        dest.addfile(tarinfo)
-                    else:
-                        dest.addfile(tarinfo, src.extractfile(tarinfo))
-    return 0
-
-
-if __name__ == '__main__':
-    sys.exit(main(parse_args()))