ci: save test artifacts for fail tests (#1501)

.github/actions/s3cmd/action.yml

@@ -54,9 +54,10 @@ runs:
             exit 1
             ;;
         esac
-
         echo "S3_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
         echo "S3_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
+        echo "S3_TEST_ARTIFACTS_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
+        echo "S3_TEST_ARTIFACTS_URL_PREFIX=${{ inputs.s3_endpoint }}/${{ inputs.s3_bucket }}/testing_out_stuff/${{ github.repository }}/${{ github.workflow }}/${{ github.run_id }}/${{ inputs.folder_prefix }}${folder}" >> $GITHUB_ENV
       env:
         s3_key_id: ${{ inputs.s3_key_id }}
         s3_secret_access_key: ${{ inputs.s3_key_secret }}

.github/actions/test_ya/action.yml

@@ -60,6 +60,7 @@ runs:
         echo "LOG_DIR=$TMP_DIR/logs" >> $GITHUB_ENV
         echo "OUT_DIR=$TMP_DIR/out" >> $GITHUB_ENV
         echo "ARTIFACTS_DIR=$TMP_DIR/artifacts" >> $GITHUB_ENV
+        echo "TEST_ARTIFACTS_DIR=$TMP_DIR/test_artifacts" >> $GITHUB_ENV
         echo "REPORTS_ARTIFACTS_DIR=$TMP_DIR/artifacts/test_reports" >> $GITHUB_ENV
         echo "JUNIT_REPORT_XML=$TMP_DIR/junit.xml" >> $GITHUB_ENV
         echo "JUNIT_REPORT_PARTS=$TMP_DIR/junit-split" >> $GITHUB_ENV
@@ -73,7 +74,7 @@ runs:
       shell: bash
       run: |
         rm -rf $TMP_DIR $JUNIT_REPORT_XML $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR
-        mkdir -p $TMP_DIR $OUT_DIR $ARTIFACTS_DIR $LOG_DIR $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR
+        mkdir -p $TMP_DIR $OUT_DIR $ARTIFACTS_DIR $TEST_ARTIFACTS_DIR $LOG_DIR $JUNIT_REPORT_PARTS $REPORTS_ARTIFACTS_DIR
     
     - name: Install Node required for Testmo CLI
       uses: actions/setup-node@v3
@@ -222,7 +223,7 @@ runs:
           ./ya test "${params[@]}" \
             --bazel-remote-put --bazel-remote-username "${{ inputs.bazel_remote_username }}" --bazel-remote-password "${{ inputs.bazel_remote_password }}" -DCONSISTENT_DEBUG \
             --log-file "$LOG_DIR/ya_log_prewarm.txt" --evlog-file "$LOG_DIR/ya_evlog_prewarm.jsonl" \
-            --dist-cache-evict-bins --cache-tests --no-dir-outputs --test-node-output-limit 100000 --drop-graph-result-before-tests || (
+            --dist-cache-evict-bins --cache-tests --no-dir-outputs --test-node-output-limit 1000000 --drop-graph-result-before-tests || (
               RC=$?
               echo "::debug::ya test RC=$RC"
             )
@@ -231,7 +232,7 @@ runs:
         echo "::debug::save tests reports"
         ./ya test "${params[@]}" \
           --stat --log-file "$LOG_DIR/ya_log.txt" --evlog-file "$LOG_DIR/ya_evlog.jsonl" -DCONSISTENT_DEBUG \
-          --cache-tests --dist-cache-evict-bins --no-dir-outputs --test-node-output-limit 100000 --drop-graph-result-before-tests \
+          --cache-tests --dist-cache-evict-bins --no-dir-outputs --test-node-output-limit 1000000 --drop-graph-result-before-tests \
           --junit "$JUNIT_REPORT_XML" --output "$OUT_DIR" || (
             RC=$?
             if [ $RC -ne 0 ]; then
@@ -259,7 +260,8 @@ runs:
           -m .github/config/muted_ya.txt \
           --ya-out "$OUT_DIR" \
           --log-url-prefix "$S3_URL_PREFIX/logs/" \
-          --log-out-dir "$ARTIFACTS_DIR/logs/" \
+          --test-stuff-out "$TEST_ARTIFACTS_DIR/" \
+          --test-stuff-prefix "$S3_TEST_ARTIFACTS_URL_PREFIX/" \
           "$JUNIT_REPORT_XML"
     
         .github/scripts/tests/split-junit.py -o "$JUNIT_REPORT_PARTS" "$JUNIT_REPORT_XML"
@@ -321,6 +323,7 @@ runs:
       run: |
         echo "::group::s3-sync"
         s3cmd sync -r --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$ARTIFACTS_DIR/" "$S3_BUCKET_PATH/"
+        s3cmd sync -r --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$TEST_ARTIFACTS_DIR/" "$S3_TEST_ARTIFACTS_BUCKET_PATH/"
         echo "::endgroup::"
     
     - name: sync logs results to s3

.github/scripts/tests/transform-ya-junit.py

@@ -5,6 +5,7 @@ import json
 import os
 import sys
 import urllib.parse
+import zipfile
 from xml.etree import ElementTree as ET
 from mute_utils import mute_target, pattern_to_re
 from junit_utils import add_junit_link_property, is_faulty_testcase
@@ -53,6 +54,10 @@ class YTestReportTrace:
     def __init__(self, out_root):
         self.out_root = out_root
         self.traces = {}
+        self.logs_dir = None
+
+    def abs_path(self, path):
+        return path.replace("$(BUILD_ROOT)", self.out_root)
 
     def load(self, subdir):
         test_results_dir = os.path.join(self.out_root, f"{subdir}/test-results/")
@@ -61,6 +66,7 @@ class YTestReportTrace:
             log_print(f"Directory {test_results_dir} doesn't exist")
             return
 
+        # find the test result
         for folder in os.listdir(test_results_dir):
             fn = os.path.join(self.out_root, test_results_dir, folder, "ytest.report.trace")
 
@@ -76,6 +82,9 @@ class YTestReportTrace:
                         subtest = event["subtest"]
                         cls = cls.replace("::", ".")
                         self.traces[(cls, subtest)] = event
+                        logs_dir = self.abs_path(event['logs']['logsdir'])
+                        self.logs_dir = logs_dir
+            break
 
     def has(self, cls, name):
         return (cls, name) in self.traces
@@ -93,7 +102,7 @@ class YTestReportTrace:
             if k == "logsdir":
                 continue
 
-            result[k] = path.replace("$(BUILD_ROOT)", self.out_root)
+            result[k] = self.abs_path(path)
 
         return result
 
@@ -135,7 +144,26 @@ def save_log(build_root, fn, out_dir, log_url_prefix, trunc_size):
     return f"{log_url_prefix}{quoted_fpath}"
 
 
-def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_prefix, log_out_dir, log_trunc_size):
+def save_zip(suite_name, out_dir, url_prefix, logs_dir):
+    arc_name = f"{suite_name.replace('/', '-')}.zip"
+
+    arc_fn = os.path.join(out_dir, arc_name)
+
+    zf = zipfile.ZipFile(arc_fn, mode="w", compression=zipfile.ZIP_DEFLATED, compresslevel=9)
+
+    log_print(f"put {logs_dir} into {arc_name}")
+    for root, dirs, files in os.walk(logs_dir):
+        for f in files:
+            filename = os.path.join(root, f)
+            zf.write(filename, os.path.relpath(filename, logs_dir))
+    zf.close()
+
+    quoted_fpath = urllib.parse.quote(arc_name)
+    return f"{url_prefix}{quoted_fpath}"
+
+
+def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_prefix, log_out_dir, log_trunc_size,
+              test_stuff_out, test_stuff_prefix):
     tree = ET.parse(fp)
     root = tree.getroot()
 
@@ -144,11 +172,14 @@ def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_pre
         traces = YTestReportTrace(ya_out_dir)
         traces.load(suite_name)
 
+        has_fail_tests = False
+
         for case in suite.findall("testcase"):
             test_name = case.get("name")
             case.set("classname", suite_name)
 
             is_fail = is_faulty_testcase(case)
+            has_fail_tests |= is_fail
 
             if mute_check(suite_name, test_name):
                 log_print("mute", suite_name, test_name)
@@ -164,6 +195,16 @@ def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_pre
                         url = save_log(ya_out_dir, fn, log_out_dir, log_url_prefix, log_trunc_size)
                         add_junit_link_property(case, name, url)
 
+        if has_fail_tests:
+            if not traces.logs_dir:
+                log_print(f"no logsdir for {suite_name}")
+                continue
+
+            url = save_zip(suite_name, test_stuff_out, test_stuff_prefix, traces.logs_dir)
+
+            for case in suite.findall("testcase"):
+                add_junit_link_property(case, 'logsdir', url)
+
     if save_inplace:
         tree.write(fp.name)
     else:
@@ -187,6 +228,8 @@ def main():
         help="truncate log after specific size, 0 disables truncation",
     )
     parser.add_argument("--ya-out", help="ya make output dir (for searching logs and artifacts)")
+    parser.add_argument('--test-stuff-out', help='output folder for archive testing_out_stuff')
+    parser.add_argument('--test-stuff-prefix', help='url prefix for testing_out_stuff')
     parser.add_argument("in_file", type=argparse.FileType("r"))
 
     args = parser.parse_args()
@@ -204,6 +247,8 @@ def main():
         args.log_url_prefix,
         args.log_out_dir,
         args.log_trunc_size,
+        args.test_stuff_out,
+        args.test_stuff_prefix,
     )