Simplest improvements for Java build, remove chained 'execv' Python calls

* remove chained 'execv' Python calls for compiling Java with with "javac" and "ErrorProne" - replace with single "run_javac.py" script
84bc60b4e87890e6ceea1ef22a2c0936410ff6bb

build/conf/java.conf

@@ -604,9 +604,12 @@ KT_CLASSPATH=${BINDIR}/kt_cp.txt
 COLLECT_KT_CLASSPATH=${WRITER_PY} --file ${BINDIR}/kt_bfg.txt -m --ya-start-command-file ${ext=.jar:MANAGED_PEERS_CLOSURE} --ya-end-command-file
 LINK_KT_CLASSPATH=${MAKE_JAVA_CLASSPATH_FILE} ${BINDIR}/kt_bfg.txt $KT_CLASSPATH && $FS_TOOLS md $KT_CLASSES_DIR
 COMPILE_KT= \
-    ${cwd:ARCADIA_BUILD_ROOT} ${env:"LC_ALL=en_US.UTF-8"} $YMAKE_PYTHON ${input:"build/scripts/with_pathsep_resolve.py"} $YMAKE_PYTHON ${input:"build/scripts/run_javac.py"} --kotlin --sources-list $KT_SRCLIST \
-    $JDK_RESOURCE/bin/java -jar $KOTLIN_COMPILER_RESOURCE_GLOBAL/kotlin-compiler.jar -no-stdlib -module-name $REALPRJNAME -jvm-target ${KOTLIN_JVM_TARGET} \
-    @$KT_SRCLIST -classpath @$KT_CLASSPATH $KOTLINC_FLAGS_VALUE -d $KT_CLASSES_DIR $KOTLINC_OPTS_VALUE
+    ${cwd:ARCADIA_BUILD_ROOT} ${env:"LC_ALL=en_US.UTF-8"} $YMAKE_PYTHON ${input:"build/scripts/run_javac.py"} \
+    ${input;hide:"build/scripts/build_java_with_error_prone2.py"} \
+    ${input;hide:"build/scripts/setup_java_tmpdir.py"} \
+    --kotlin --sources-list $KT_SRCLIST \
+    $JDK_RESOURCE/bin/java -jar $KOTLIN_COMPILER_RESOURCE_GLOBAL/kotlin-compiler.jar -no-stdlib -module-name $REALPRJNAME \
+    -jvm-target ${KOTLIN_JVM_TARGET} @$KT_SRCLIST -classpath @$KT_CLASSPATH $KOTLINC_FLAGS_VALUE -d $KT_CLASSES_DIR $KOTLINC_OPTS_VALUE
 ALL_KT_COMMANDS=
 KT_CLASSPATH_ITEM=
 
@@ -640,7 +643,7 @@ macro KAPT_ANNOTATION_PROCESSOR_CLASSPATH(Args...) {
 
 # tag:java-specific
 JAVAC_CMD=$JDK_RESOURCE/bin/javac
-ERROR_PRONE_JAVAC_CMD=${YMAKE_PYTHON} ${input:"build/scripts/build_java_with_error_prone2.py"} $JDK_RESOURCE/bin/java $JDK_RESOURCE/bin/javac $ERROR_PRONE_RESOURCE/error_prone.jar
+RUN_JAVAC_ARGS=
 
 # tag:java-specific tag:codenav
 JAVA_YNDEXING=no
@@ -690,11 +693,11 @@ macro _PACK_JAR_HELPER(Out) {
 
 # tag:java-specific
 macro _JAVAC_RUN_HELPER(JAVAC_CMD_WITH_ARGS...) {
-    .CMD=${cwd:ARCADIA_BUILD_ROOT} ${env:"LC_ALL=en_US.UTF-8"} $YMAKE_PYTHON ${input:"build/scripts/with_pathsep_resolve.py"} $YMAKE_PYTHON ${input:"build/scripts/setup_java_tmpdir.py"} $YMAKE_PYTHON ${input:"build/scripts/run_javac.py"} --sources-list ${BINDIR}/all-java.srclst ${JAVAC_CMD_WITH_ARGS} @${BINDIR}/all-java.srclst -classpath ${ARCADIA_BUILD_ROOT}/bfg.jar -Xpkginfo:always ${JAVAC_OPTS} $_JAR_ANN_PROC_OPTS($_JAR_ANN_PROCESSORS) -d ${BINDIR}/cls -g -encoding UTF-8
+    .CMD=${cwd:ARCADIA_BUILD_ROOT} ${env:"LC_ALL=en_US.UTF-8"} $YMAKE_PYTHON ${input:"build/scripts/run_javac.py"} ${input;hide:"build/scripts/build_java_with_error_prone2.py"} ${input;hide:"build/scripts/setup_java_tmpdir.py"} --with-setup-java-tmpdir --sources-list ${BINDIR}/all-java.srclst ${RUN_JAVAC_ARGS} ${JAVAC_CMD_WITH_ARGS} @${BINDIR}/all-java.srclst -classpath ${ARCADIA_BUILD_ROOT}/bfg.jar -Xpkginfo:always ${JAVAC_OPTS} $_JAR_ANN_PROC_OPTS($_JAR_ANN_PROCESSORS) -d ${BINDIR}/cls -g -encoding UTF-8
 }
 
 # tag:java-specific
-ERROR_PRONE_JDK16_ADD_OPENS=-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.jvm=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
+ERROR_PRONE_JDK16PLUS_ADD_OPENS=-J--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.jvm=ALL-UNNAMED -J--add-opens=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED
 ERROR_PRONE_2_7_1_FORCED_OPTS=-Xep:InlineMeInliner:OFF -Xep:SameNameButDifferent:OFF
 
 # tag:java-specific
@@ -1034,27 +1037,27 @@ module JAR_LIBRARY: _COMPILABLE_JAR_BASE {
 
     when($ERROR_PRONE_VALUE) {
         PEERDIR+=$ERROR_PRONE_PEERDIR
-        JAVAC_CMD=$ERROR_PRONE_JAVAC_CMD
+        RUN_JAVAC_ARGS=--error-prone $ERROR_PRONE_RESOURCE/error_prone.jar
     }
 
     when($ERROR_PRONE_VALUE && $JDK_REAL_VERSION == "17") {
-        JAVAC_FLAGS_VALUE+=$ERROR_PRONE_JDK16_ADD_OPENS
-        JAVAC_OPTS+=$ERROR_PRONE_JDK16_ADD_OPENS
+        JAVAC_FLAGS_VALUE+=$ERROR_PRONE_JDK16PLUS_ADD_OPENS
+        JAVAC_OPTS+=$ERROR_PRONE_JDK16PLUS_ADD_OPENS
     }
 
     when($ERROR_PRONE_VALUE && $JDK_REAL_VERSION == "20") {
-        JAVAC_FLAGS_VALUE+=$ERROR_PRONE_JDK16_ADD_OPENS
-        JAVAC_OPTS+=$ERROR_PRONE_JDK16_ADD_OPENS
+        JAVAC_FLAGS_VALUE+=$ERROR_PRONE_JDK16PLUS_ADD_OPENS
+        JAVAC_OPTS+=$ERROR_PRONE_JDK16PLUS_ADD_OPENS
     }
 
     when($ERROR_PRONE_VALUE && $JDK_REAL_VERSION == "21") {
-        JAVAC_FLAGS_VALUE+=$ERROR_PRONE_JDK16_ADD_OPENS
-        JAVAC_OPTS+=$ERROR_PRONE_JDK16_ADD_OPENS
+        JAVAC_FLAGS_VALUE+=$ERROR_PRONE_JDK16PLUS_ADD_OPENS
+        JAVAC_OPTS+=$ERROR_PRONE_JDK16PLUS_ADD_OPENS
     }
 
     when($ERROR_PRONE_VALUE && $JDK_REAL_VERSION == "22") {
-        JAVAC_FLAGS_VALUE+=$ERROR_PRONE_JDK16_ADD_OPENS
-        JAVAC_OPTS+=$ERROR_PRONE_JDK16_ADD_OPENS
+        JAVAC_FLAGS_VALUE+=$ERROR_PRONE_JDK16PLUS_ADD_OPENS
+        JAVAC_OPTS+=$ERROR_PRONE_JDK16PLUS_ADD_OPENS
     }
 
     when($ERROR_PRONE_VALUE && $ERROR_PRONE_VERSION == "2.7.1") {

build/scripts/build_java_with_error_prone.py

@@ -1,48 +0,0 @@
-import sys
-import os
-
-ERROR_PRONE_FLAGS = [
-    '-Xep:FunctionalInterfaceMethodChanged:WARN',
-    '-Xep:ReturnValueIgnored:WARN',
-]
-
-JAVA10_EXPORTS = [
-    '--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED',
-    '--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED',
-    '--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED',
-    '--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED',
-    '--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED',
-    '--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED',
-    '--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED',
-    '--add-exports=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED',
-]
-
-
-def just_do_it(argv):
-    java, error_prone_tool, javac_cmd = argv[0], argv[1], argv[2:]
-    if java.endswith('javac') or java.endswith('javac.exe'):
-        for f in javac_cmd:
-            if f.startswith('-Xep'):
-                ERROR_PRONE_FLAGS.append(f)
-        for f in ERROR_PRONE_FLAGS:
-            if f in javac_cmd:
-                javac_cmd.remove(f)
-        os.execv(
-            java,
-            [java]
-            + JAVA10_EXPORTS
-            + ['-processorpath', error_prone_tool, '-XDcompilePolicy=byfile']
-            + [(' '.join(['-Xplugin:ErrorProne'] + ERROR_PRONE_FLAGS))]
-            + javac_cmd,
-        )
-    else:
-        os.execv(
-            java,
-            [java, '-Xbootclasspath/p:' + error_prone_tool, 'com.google.errorprone.ErrorProneCompiler']
-            + ERROR_PRONE_FLAGS
-            + javac_cmd,
-        )
-
-
-if __name__ == '__main__':
-    just_do_it(sys.argv[1:])

build/scripts/build_java_with_error_prone2.py

@@ -22,21 +22,6 @@ JAVA10_EXPORTS = [
 ]
 
 
-def get_java_version(exe):
-    p = subprocess.Popen([exe, '-version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    out, err = p.communicate()
-    for line in (out or '').strip().split("\n") + (err or '').strip().split("\n"):
-        m = re.match(r'java version "(.+)"', line)
-        if m:
-            parts = m.groups()[0].split(".")
-            return parts[1] if parts[0] == "1" else parts[0]
-        m = re.match(r'openjdk version "(\d+).*"', line)
-        if m:
-            parts = m.groups()[0].split(".")
-            return parts[0]
-    return None
-
-
 def get_classpath(cmd):
     for i, part in enumerate(cmd):
         if part == '-classpath':
@@ -58,35 +43,47 @@ def parse_args(argv):
     return parsed + [argv[i + 1 :]]
 
 
+def fix_cmd_line(error_prone_tool, cmd):
+    if not error_prone_tool:
+        return cmd
+    for f in cmd:
+        if f.startswith('-Xep'):
+            ERROR_PRONE_FLAGS.append(f)
+    for f in ERROR_PRONE_FLAGS:
+        if f in cmd:
+            cmd.remove(f)
+    if '-processor' in cmd:
+        classpath = get_classpath(cmd)
+        if classpath:
+            error_prone_tool = error_prone_tool + os.pathsep + classpath
+    return (
+        cmd
+        + JAVA10_EXPORTS
+        + ['-processorpath', error_prone_tool, '-XDcompilePolicy=byfile']
+        + [(' '.join(['-Xplugin:ErrorProne'] + ERROR_PRONE_FLAGS))]
+    )
+
+
+# NOTE: legacy, only for "devtools/ya/jbuild"
 def just_do_it(argv):
     java, javac, error_prone_tool, javac_cmd = parse_args(argv)
-    ver = get_java_version(java)
-    if not ver:
-        raise Exception("Can't determine java version")
-    if int(ver) >= 10:
-        for f in javac_cmd:
-            if f.startswith('-Xep'):
-                ERROR_PRONE_FLAGS.append(f)
-        for f in ERROR_PRONE_FLAGS:
-            if f in javac_cmd:
-                javac_cmd.remove(f)
-        if '-processor' in javac_cmd:
-            classpath = get_classpath(javac_cmd)
-            if classpath:
-                error_prone_tool = error_prone_tool + os.pathsep + classpath
-        cmd = (
-            [javac]
-            + JAVA10_EXPORTS
-            + ['-processorpath', error_prone_tool, '-XDcompilePolicy=byfile']
-            + [(' '.join(['-Xplugin:ErrorProne'] + ERROR_PRONE_FLAGS))]
-            + javac_cmd
-        )
-    else:
-        cmd = (
-            [java, '-Xbootclasspath/p:' + error_prone_tool, 'com.google.errorprone.ErrorProneCompiler']
-            + ERROR_PRONE_FLAGS
-            + javac_cmd
-        )
+    for f in javac_cmd:
+        if f.startswith('-Xep'):
+            ERROR_PRONE_FLAGS.append(f)
+    for f in ERROR_PRONE_FLAGS:
+        if f in javac_cmd:
+            javac_cmd.remove(f)
+    if '-processor' in javac_cmd:
+        classpath = get_classpath(javac_cmd)
+        if classpath:
+            error_prone_tool = error_prone_tool + os.pathsep + classpath
+    cmd = (
+        [javac]
+        + JAVA10_EXPORTS
+        + ['-processorpath', error_prone_tool, '-XDcompilePolicy=byfile']
+        + [(' '.join(['-Xplugin:ErrorProne'] + ERROR_PRONE_FLAGS))]
+        + javac_cmd
+    )
     if platform.system() == 'Windows':
         sys.exit(subprocess.Popen(cmd).wait())
     else:

build/scripts/run_javac.py

@@ -1,17 +1,23 @@
 import sys
 import subprocess
 import optparse
+import os
 import re
 
+import build_java_with_error_prone2 as java_error_prone
+import setup_java_tmpdir as java_tmpdir
+
 
 def parse_args():
     parser = optparse.OptionParser()
     parser.disable_interspersed_args()
     parser.add_option('--sources-list')
+    parser.add_option('--error-prone')
     parser.add_option('--verbose', default=False, action='store_true')
     parser.add_option('--remove-notes', default=False, action='store_true')
     parser.add_option('--ignore-errors', default=False, action='store_true')
     parser.add_option('--kotlin', default=False, action='store_true')
+    parser.add_option('--with-setup-java-tmpdir', default=False, action='store_true')
     return parser.parse_args()
 
 
@@ -45,6 +51,12 @@ def remove_notes(err):
 def main():
     opts, cmd = parse_args()
 
+    if opts.with_setup_java_tmpdir:
+        cmd = java_tmpdir.fix_tmpdir(cmd)
+
+    if opts.error_prone:
+        cmd = java_error_prone.fix_cmd_line(opts.error_prone, cmd)
+
     with open(opts.sources_list) as f:
         input_files = f.read().strip().split()
 

build/scripts/with_pathsep_resolve.py

@@ -4,6 +4,7 @@ import subprocess
 import platform
 
 
+# NOTE: legacy, only for "devtools/ya/jbuild"
 def fix_args(args):
     just_replace_it = False
     for arg in args:

build/scripts/ya.make

@@ -46,7 +46,6 @@ ELSEIF (PY3)
         build_dll_and_java.py
         build_info_gen.py
         build_java_codenav_index.py
-        build_java_with_error_prone.py
         build_java_with_error_prone2.py
         cat.py
         cgo1_wrapper.py