add ya make and test support

add ya make and test support

Pull Request resolved: https://github.com/ydb-platform/ydb/pull/392

.github/actions/build_ya/action.yml

@@ -0,0 +1,68 @@
+name: Build (ya make)
+description: Build targets
+inputs:
+  build_target:
+    required: false
+    description: "build target"
+  sanitizer:
+    required: false
+    description: "sanitizer type (address, memory, thread, undefined, leak)"
+
+  bazel_remote_uri:
+    required: false
+    description: "bazel-remote endpoint"
+  bazel_remote_username:
+    required: false
+    description: "bazel-remote username"
+  bazel_remote_password:
+    required: false
+    description: "bazel-remote password"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Init
+      id: init
+      shell: bash
+      run: |
+        echo "SHELLOPTS=xtrace" >> $GITHUB_ENV
+        export TMP_DIR=$(pwd)/tmp_build
+        echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
+        rm -rf $TMP_DIR && mkdir $TMP_DIR
+    
+    - name: build
+      shell: bash
+      run: |
+        extra_params=()
+
+        if [ ! -z "${{ inputs.build_target }}" ]; then
+          extra_params+=(--target="${{ inputs.build_target }}")
+        fi
+        
+        if [ ! -z "${{ inputs.sanitizer }}" ] && [ "${{ inputs.sanitizer }}" != "none" ]; then
+          extra_params+=(--sanitize="${{ inputs.sanitizer }}")
+        fi
+        
+        if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then
+          extra_params+=(--bazel-remote-store)
+          extra_params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}")
+        fi
+        
+        if [ ! -z "${{ inputs.bazel_remote_username }}" ]; then
+          extra_params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}")
+          extra_params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}")
+          extra_params+=(--bazel-remote-put --add-result .o --yt-replace-result --yt-replace-result-rm-binaries)
+        fi
+
+        ./ya make --build relwithdebinfo --force-build-depends -D'BUILD_LANGUAGES=CPP PY3' -T --stat  \
+          --log-file "$TMP_DIR/ya_log.txt" --evlog-file "$TMP_DIR/ya_evlog.jsonl" \
+          --dump-graph --dump-graph-to-file "$TMP_DIR/ya_graph.json" \
+          "${extra_params[@]}"
+
+    - name: sync logs to s3
+      if: always()
+      shell: bash
+      run: |
+        echo "::group::s3-sync"
+        s3cmd sync --acl-private --no-progress --stats --no-check-md5 "$TMP_DIR/" "$S3_BUCKET_PATH/build_logs/"
+        echo "::endgroup::"

.github/actions/s3cmd/action.yml

@@ -0,0 +1,39 @@
+name: configure s3cmd
+description: configure s3cmd
+inputs:
+  s3_key_id:
+    required: true
+    description: "s3 key id"
+  s3_key_secret:
+    required: true
+    description: "s3 key secret"
+  s3_bucket:
+    required: true
+    description: "s3 bucket"
+  s3_endpoint:
+    required: true
+    description: "s3 endpoint"
+  log_suffix:
+    required: true
+    description: "log suffix"
+runs:
+  using: "composite"
+  steps:
+    - name: configure s3cmd
+      shell: bash
+      run: |
+        export S3CMD_CONFIG=$(mktemp)
+        echo "S3CMD_CONFIG=$S3CMD_CONFIG" >> $GITHUB_ENV
+        cat <<EOF > $S3CMD_CONFIG
+        [default]
+        access_key = ${s3_key_id}
+        secret_key = ${s3_secret_access_key}
+        bucket_location = ru-central1
+        host_base = storage.yandexcloud.net
+        host_bucket = %(bucket)s.storage.yandexcloud.net
+        EOF
+        echo "S3_BUCKET_PATH=s3://${{ inputs.s3_bucket }}/${{ github.repository }}/${{github.workflow}}/${{ github.run_id }}/${{inputs.log_suffix}}" >> $GITHUB_ENV
+        echo "S3_URL_PREFIX=${{inputs.s3_endpoint}}/${{inputs.s3_bucket}}/${{ github.repository }}/${{github.workflow}}/${{github.run_id}}/${{inputs.log_suffix}}" >> $GITHUB_ENV
+      env:
+        s3_key_id: ${{ inputs.s3_key_id }}
+        s3_secret_access_key: ${{ inputs.s3_key_secret }}

.github/actions/test_ya/action.yml

@@ -0,0 +1,184 @@
+name: Run tests (ya make)
+description: Run tests using ya make
+inputs:
+  build_target:
+    required: false
+    description: "build target"
+    
+  sanitizer:
+    required: false
+    description: "sanitizer type (address, memory, thread, undefined, leak)"
+    
+  log_suffix:
+    required: true
+    description: "log suffix"
+  test_threads:
+    required: false
+    default: "28"
+    description: "Test threads count"
+
+  testman_token:
+    required: false
+    description: "test manager auth token"
+  testman_url:
+    required: false
+    description: "test manager endpoint"
+  testman_project_id:
+    required: false
+    description: "test manager project id"
+runs:
+  using: "composite"
+  steps:
+    - name: Init
+      id: init
+      shell: bash
+      run: |
+        echo "SHELLOPTS=xtrace" >> $GITHUB_ENV
+        export TMP_DIR=$(pwd)/tmp
+        echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
+        echo "OUT_DIR=$TMP_DIR/out" >> $GITHUB_ENV
+        echo "ARTIFACTS_DIR=$TMP_DIR/artifacts" >> $GITHUB_ENV
+        echo "JUNIT_REPORT_XML=$TMP_DIR/junit.xml" >> $GITHUB_ENV
+        echo "TESTMO_TOKEN=${{ inputs.testman_token }}" >> $GITHUB_ENV
+        echo "TESTMO_URL=${{ inputs.testman_url }}" >> $GITHUB_ENV
+        echo "SUMMARY_LINKS=$(mktemp)" >> $GITHUB_ENV
+    
+    - name: prepare
+      shell: bash
+      run: |
+        rm -rf $TMP_DIR $JUNIT_REPORT_XML
+        mkdir -p $TMP_DIR $OUT_DIR $ARTIFACTS_DIR
+    
+    
+    - name: Install Node required for Testmo CLI
+      uses: actions/setup-node@v3
+      with:
+        node-version: 19
+    
+    - name: Install Testmo CLI
+      shell: bash
+      run: npm install -g @testmo/testmo-cli
+    - name: Test history run create
+      id: th
+      if: inputs.testman_token
+      shell: bash
+      env:
+        PR_NUMBER: ${{ github.event.number }}
+      run: |
+        RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
+        BRANCH_TAG="$GITHUB_REF_NAME"
+        TESTMO_SOURCE="${{ inputs.log_suffix }}"
+        TESTMO_SOURCE="${TESTMO_SOURCE/_/-}"
+        
+        case $GITHUB_EVENT_NAME in
+        workflow_dispatch)
+         TESTMO_RUN_NAME="${{ github.run_id }} manual"
+         EXTRA_TAG="manual"
+         ;;
+        pull_request | pull_request_target)
+         TESTMO_RUN_NAME="${{ github.run_id }} PR #${PR_NUMBER}"
+         EXTRA_TAG="pr"
+         BRANCH_TAG=""
+         ;;
+        schedule)
+         TESTMO_RUN_NAME="${{ github.run_id }} schedule"
+         EXTRA_TAG="schedule"
+         ;;
+        *)
+         TESTMO_RUN_NAME="${{ github.run_id }}"
+         EXTRA_TAG=""
+         ;;
+        esac
+        
+        testmo automation:resources:add-link --name build --url "$RUN_URL" --resources testmo.json
+        testmo automation:resources:add-field --name git-sha --type string --value "${GITHUB_SHA:0:7}" --resources testmo.json
+        testmo automation:run:create --instance "$TESTMO_URL" --project-id "${{ inputs.testman_project_id }}" --name "$TESTMO_RUN_NAME" \
+          --source "$TESTMO_SOURCE" --resources testmo.json \
+          --tags "$BRANCH_TAG" --tags "$EXTRA_TAG"  | \
+        echo "runid=$(cat)" >> $GITHUB_OUTPUT
+
+    - name: Print test history link
+      shell: bash
+      run: |
+        echo "10 [Test history](${TESTMO_URL}/automation/runs/view/${{steps.th.outputs.runid}})" >> $SUMMARY_LINKS
+
+    - name: ya test
+      shell: bash
+      run: |
+        extra_params=()
+
+        if [ ! -z "${{ inputs.build_target }}" ]; then
+          extra_params+=(--target="${{ inputs.build_target }}")
+        fi
+
+        if [ ! -z "${{ inputs.sanitizer }}" ] && [ "${{ inputs.sanitizer }}" != "none" ]; then
+          extra_params+=(--sanitize="${{ inputs.sanitizer }}")
+        fi
+
+        ./ya test -A --build relwithdebinfo -D'BUILD_LANGUAGES=CPP PY3' --test-threads "${{ inputs.test_threads }}" \
+          --do-not-output-stderrs -T \
+          --junit "$JUNIT_REPORT_XML" --output "$OUT_DIR" "${extra_params[@]}" || (
+            RC=$?
+            if [ $RC == 10 ]; then
+              echo "ya test returned failed tests status, recovering.."
+            else
+              exit $RC
+            fi
+        )
+
+    - name: postprocess junit report
+      shell: bash
+      run: |
+        .github/scripts/tests/transform-ya-junit.py -i \
+          --mu .github/config/muted_test.txt \
+          --mf .github/config/muted_functest.txt \
+          --ya-out "$OUT_DIR" \
+          --log-url-prefix "$S3_URL_PREFIX/logs/" \
+          --log-out-dir "$ARTIFACTS_DIR/logs/" \
+          "$JUNIT_REPORT_XML"
+
+    - name: write tests summary
+      if: always()
+      shell: bash
+      env:
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        
+        cat $SUMMARY_LINKS | python3 -c 'import sys; print(" | ".join([v for _, v in sorted([l.strip().split(" ", 1) for l in sys.stdin], key=lambda a: (int(a[0]), a))]))' >> $GITHUB_STEP_SUMMARY
+        
+        mkdir $ARTIFACTS_DIR/summary/
+        
+        .github/scripts/tests/generate-summary.py \
+        --summary-out-path $ARTIFACTS_DIR/summary/ \
+        --summary-url-prefix $S3_URL_PREFIX/summary/ \
+        "Tests" ya-test.html "$JUNIT_REPORT_XML"
+
+
+    - name: Unit test history upload results
+      if: always() && inputs.testman_token
+      shell: bash
+      run: |
+        testmo automation:run:submit-thread \
+          --instance "$TESTMO_URL" --run-id ${{ steps.th.outputs.runid }} \
+          --results "$JUNIT_REPORT_XML"
+
+    - name: sync test results to s3
+      if: always()
+      shell: bash
+      run: |
+        echo "::group::s3-sync"
+        s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$ARTIFACTS_DIR/" "$S3_BUCKET_PATH/"
+        echo "::endgroup::"
+
+
+    - name: Test history run complete
+      if: always() && inputs.testman_token
+      shell: bash
+      run: |
+        testmo automation:run:complete --instance "$TESTMO_URL" --run-id ${{ steps.th.outputs.runid }}
+
+    - name: check test results
+      shell: bash
+      run: |
+        .github/scripts/tests/fail-checker.py "$JUNIT_REPORT_XML"
+        

.github/scripts/tests/generate-summary.py

@@ -136,6 +136,9 @@ class TestSummary:
         self.is_failed |= line.is_failed
         self.lines.append(line)
 
+    def render_line(self, items):
+        return f"| {' | '.join(items)} |"
+
     def render(self, add_footnote=False):
         github_srv = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
         repo = os.environ.get("GITHUB_REPOSITORY", "ydb-platform/ydb")
@@ -144,25 +147,38 @@ class TestSummary:
 
         footnote = "[^1]" if add_footnote else f'<sup>[?]({footnote_url} "All mute rules are defined here")</sup>'
 
+        columns = [
+            "TESTS", "PASSED", "ERRORS", "FAILED", "SKIPPED", f"MUTED{footnote}"
+        ]
+
+        need_first_column = len(self.lines) > 1
+
+        if need_first_column:
+            columns.insert(0, "")
+
         result = [
-            f"|      | TESTS | PASSED | ERRORS | FAILED | SKIPPED | MUTED{footnote} |",
-            "| :--- | ---:  | -----: | -----: | -----: | ------: | ----: |",
+            self.render_line(columns),
         ]
+
+        if need_first_column:
+            result.append(self.render_line([':---'] + ['---:'] * (len(columns) - 1)))
+        else:
+            result.append(self.render_line(['---:'] * len(columns)))
+
         for line in self.lines:
             report_url = line.report_url
-            result.append(
-                " | ".join(
-                    [
-                        line.title,
-                        render_pm(line.test_count, f"{report_url}", 0),
-                        render_pm(line.passed, f"{report_url}#PASS", 0),
-                        render_pm(line.errors, f"{report_url}#ERROR", 0),
-                        render_pm(line.failed, f"{report_url}#FAIL", 0),
-                        render_pm(line.skipped, f"{report_url}#SKIP", 0),
-                        render_pm(line.muted, f"{report_url}#MUTE", 0),
-                    ]
-                )
-            )
+            row = []
+            if need_first_column:
+                row.append(line.title)
+            row.extend([
+                render_pm(line.test_count, f"{report_url}", 0),
+                render_pm(line.passed, f"{report_url}#PASS", 0),
+                render_pm(line.errors, f"{report_url}#ERROR", 0),
+                render_pm(line.failed, f"{report_url}#FAIL", 0),
+                render_pm(line.skipped, f"{report_url}#SKIP", 0),
+                render_pm(line.muted, f"{report_url}#MUTE", 0),
+            ])
+            result.append(self.render_line(row))
 
         if add_footnote:
             result.append("")

.github/scripts/tests/junit_utils.py

@@ -18,6 +18,13 @@ def add_junit_link_property(testcase, name, url):
 
 def add_junit_property(testcase, name, value):
     props = get_or_create_properties(testcase)
+
+    # remove existing property if exists
+    for item in props.findall("property"):
+        if item.get('name') == name:
+            props.remove(item)
+            break
+
     props.append(ET.Element("property", dict(name=name, value=value)))
 
 
@@ -92,3 +99,7 @@ def iter_xml_files(folder_or_file):
         for suite in suites:
             for case in suite.findall("testcase"):
                 yield fn, suite, case
+
+
+def is_faulty_testcase(testcase):
+    return testcase.find("failure") is not None or testcase.find("error") is not None

.github/scripts/tests/transform-ya-junit.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+import argparse
+import re
+import json
+import os
+import sys
+from xml.etree import ElementTree as ET
+from mute_utils import mute_target, pattern_to_re
+from junit_utils import add_junit_link_property, is_faulty_testcase
+
+
+def log_print(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
+class YaMuteCheck:
+    def __init__(self):
+        self.regexps = set()
+
+    def add_unittest(self, fn):
+        with open(fn, "r") as fp:
+            for line in fp:
+                line = line.strip()
+                path, rest = line.split("/")
+                path = path.replace("-", "/")
+                rest = rest.replace("::", ".")
+                self.populate(f"{path}/{rest}")
+
+    def add_functest(self, fn):
+        with open(fn, "r") as fp:
+            for line in fp:
+                line = line.strip()
+                line = line.replace("::", ".")
+                self.populate(line)
+
+    def populate(self, line):
+        pattern = pattern_to_re(line)
+
+        try:
+            self.regexps.add(re.compile(pattern))
+        except re.error:
+            log_print(f"Unable to compile regex {pattern!r}")
+
+    def __call__(self, suitename, testname):
+        for r in self.regexps:
+            if r.match(f"{suitename}/{testname}"):
+                return True
+        return False
+
+
+class YTestReportTrace:
+    def __init__(self, out_root):
+        self.out_root = out_root
+        self.traces = {}
+
+    def load(self, subdir):
+        test_results_dir = f"{subdir}/test-results/"
+        for folder in os.listdir(os.path.join(self.out_root, test_results_dir)):
+            fn = os.path.join(self.out_root, test_results_dir, folder, "ytest.report.trace")
+
+            if not os.path.isfile(fn):
+                continue
+
+            with open(fn, "r") as fp:
+                for line in fp:
+                    event = json.loads(line.strip())
+                    if event["name"] == "subtest-finished":
+                        event = event["value"]
+                        cls = event["class"]
+                        subtest = event["subtest"]
+                        cls = cls.replace("::", ".")
+                        self.traces[(cls, subtest)] = event
+
+    def has(self, cls, name):
+        return (cls, name) in self.traces
+
+    def get_logs(self, cls, name):
+        trace = self.traces.get((cls, name))
+
+        if not trace:
+            return {}
+
+        logs = trace["logs"]
+
+        result = {}
+        for k, path in logs.items():
+            if k == "logsdir":
+                continue
+
+            result[k] = path.replace("$(BUILD_ROOT)", self.out_root)
+
+        return result
+
+
+def filter_empty_logs(logs):
+    result = {}
+    for k, v in logs.items():
+        if os.stat(v).st_size == 0:
+            continue
+        result[k] = v
+    return result
+
+
+def save_log(build_root, fn, out_dir, log_url_prefix, trunc_size):
+    fpath = os.path.relpath(fn, build_root)
+
+    if out_dir is not None:
+        out_fn = os.path.join(out_dir, fpath)
+        fsize = os.stat(fn).st_size
+
+        out_fn_dir = os.path.dirname(out_fn)
+
+        if not os.path.isdir(out_fn_dir):
+            os.makedirs(out_fn_dir, 0o700)
+
+        if trunc_size and fsize > trunc_size:
+            with open(fn, "rb") as in_fp:
+                in_fp.seek(fsize - trunc_size)
+                log_print(f"truncate {out_fn} to {trunc_size}")
+                with open(out_fn, "wb") as out_fp:
+                    while 1:
+                        buf = in_fp.read(8192)
+                        if not buf:
+                            break
+                        out_fp.write(buf)
+        else:
+            os.symlink(fn, out_fn)
+
+    return f"{log_url_prefix}{fpath}"
+
+
+def transform(fp, mute_check: YaMuteCheck, ya_out_dir, save_inplace, log_url_prefix, log_out_dir, log_trunc_size):
+    tree = ET.parse(fp)
+    root = tree.getroot()
+
+    for suite in root.findall("testsuite"):
+        suite_name = suite.get("name")
+        traces = YTestReportTrace(ya_out_dir)
+        traces.load(suite_name)
+
+        for case in suite.findall("testcase"):
+            test_name = case.get("name")
+            case.set("classname", suite_name)
+
+            is_fail = is_faulty_testcase(case)
+
+            if mute_check(suite_name, test_name):
+                log_print("mute", suite_name, test_name)
+                mute_target(case)
+
+            if is_fail and "." in test_name:
+                test_cls, test_method = test_name.rsplit(".", maxsplit=1)
+                logs = filter_empty_logs(traces.get_logs(test_cls, test_method))
+
+                if logs:
+                    log_print(f"add {list(logs.keys())!r} properties for {test_cls}.{test_method}")
+                    for name, fn in logs.items():
+                        url = save_log(ya_out_dir, fn, log_out_dir, log_url_prefix, log_trunc_size)
+                        add_junit_link_property(case, name, url)
+
+    if save_inplace:
+        tree.write(fp.name)
+    else:
+        ET.indent(root)
+        print(ET.tostring(root, encoding="unicode"))
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-i", action="store_true", dest="save_inplace", default=False, help="modify input file in-place"
+    )
+    parser.add_argument("--mu", help="unittest mute config")
+    parser.add_argument("--mf", help="functional test mute config")
+    parser.add_argument("--log-url-prefix", default="./", help="url prefix for logs")
+    parser.add_argument("--log-out-dir", help="symlink logs to specific directory")
+    parser.add_argument(
+        "--log-truncate-size",
+        dest="log_trunc_size",
+        type=int,
+        default=134217728,
+        help="truncate log after specific size, 0 disables truncation",
+    )
+    parser.add_argument("--ya-out", help="ya make output dir (for searching logs and artifacts)")
+    parser.add_argument("in_file", type=argparse.FileType("r"))
+
+    args = parser.parse_args()
+
+    mute_check = YaMuteCheck()
+
+    if args.mu:
+        mute_check.add_unittest(args.mu)
+
+    if args.mf:
+        mute_check.add_functest(args.mf)
+
+    transform(
+        args.in_file,
+        mute_check,
+        args.ya_out,
+        args.save_inplace,
+        args.log_url_prefix,
+        args.log_out_dir,
+        args.log_trunc_size,
+    )
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/build_and_test_ya.yml

@@ -0,0 +1,72 @@
+name: Ya-Build-and-Test
+
+on:
+  workflow_call:
+    inputs:
+      build_target:
+        type: string
+        default: "ydb/"
+        description: "limit build and test to specific target"
+      sanitizer:
+        type: string
+        default: "none"
+        description: "sanitizer type"
+      runner_kind:
+        type: string
+        required: true
+        description: "self-hosted or provisioned"
+      runner_label:
+        type: string
+        default: "linux"
+        description: "runner label"
+      run_build:
+        type: boolean
+        default: true
+        description: "run build"
+      run_tests:
+        type: boolean
+        default: true
+        description: "run tests"
+      log_suffix:
+        type: string
+        required: true
+        description: "suffix for current build. uses as testmo source and s3 subfolder"
+
+
+jobs:
+  main:
+    name: Build and test
+    runs-on: [ self-hosted, "${{ inputs.runner_kind }}", "${{ inputs.runner_label }}" ]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Prepare s3cmd
+      uses: ./.github/actions/s3cmd
+      with:
+        s3_bucket: ${{ vars.AWS_BUCKET }}
+        s3_endpoint: ${{ vars.AWS_ENDPOINT }}
+        s3_key_id: ${{ secrets.AWS_KEY_ID }}
+        s3_key_secret: ${{ secrets.AWS_KEY_VALUE }}
+        log_suffix: ${{ inputs.log_suffix }}
+
+    - name: Build
+      uses: ./.github/actions/build_ya
+      if: inputs.run_build
+      with:
+        build_target: ${{ inputs.build_target }}
+        sanitizer: ${{ inputs.sanitizer }}
+        bazel_remote_uri: ${{ vars.REMOTE_CACHE_URL_YA || '' }}
+        bazel_remote_username: ${{ secrets.REMOTE_CACHE_USERNAME }}
+        bazel_remote_password: ${{ secrets.REMOTE_CACHE_PASSWORD }}
+
+    - name: Run tests
+      uses: ./.github/actions/test_ya
+      if: inputs.run_tests
+      with:
+        build_target: ${{ inputs.build_target }}
+        sanitizer: ${{ inputs.sanitizer }}
+        log_suffix: ${{ inputs.log_suffix }}
+        testman_token: ${{ secrets.TESTMO_TOKEN }}
+        testman_url: ${{ vars.TESTMO_URL }}
+        testman_project_id: ${{ vars.TESTMO_PROJECT_ID }}

.github/workflows/build_and_test_ya_ondemand.yml

@@ -0,0 +1,107 @@
+name: Ya-Build-and-Test-On-demand
+
+on:
+  workflow_dispatch:
+    inputs:
+      image:
+        type: string
+        default: fd8earpjmhevh8h6ug5o
+        description: "VM image"
+      build_target:
+        type: string
+        default: "ydb/"
+        description: "limit build and test to specific target"
+      sanitizer:
+        type: choice
+        default: "none"
+        description: "sanitizer type"
+        options:
+          - none
+          - address
+          - memory
+          - thread
+          - undefined
+          - leak
+      run_build:
+        type: boolean
+        default: true
+        description: "run build"
+      run_tests:
+        type: boolean
+        default: true
+        description: "run tests"
+
+jobs:
+  provide-runner:
+    name: Start self-hosted YC runner
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-yc-runner.outputs.label }}
+      instance-id: ${{ steps.start-yc-runner.outputs.instance-id }}
+    steps:
+      - name: Start YC runner
+        id: start-yc-runner
+        uses: yc-actions/yc-github-runner@v1
+        with:
+          mode: start
+          yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          folder-id: ${{ secrets.YC_FOLDER }}
+          image-id: ${{ inputs.image }}
+          disk-size: ${{ vars.DISK_SIZE && vars.DISK_SIZE || '1023GB' }}
+          disk-type: network-ssd-nonreplicated
+          cores: 32
+          memory: 128GB
+          core-fraction: 100
+          zone-id: ru-central1-b
+          subnet-id: ${{ secrets.YC_SUBNET }}
+
+  prepare-vm:
+    name: Prepare runner
+    needs: provide-runner
+    runs-on: [ self-hosted, "${{ needs.provide-runner.outputs.label }}" ]
+    steps:
+      - name: Checkout PR
+        uses: actions/checkout@v3
+        if: github.event.pull_request.head.sha != ''
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Checkout
+        uses: actions/checkout@v3
+        if: github.event.pull_request.head.sha == ''
+      - name: Prepare VM
+        uses: ./.github/actions/prepare_vm
+
+  main:
+    uses: ./.github/workflows/build_and_test_ya.yml
+    needs:
+      - provide-runner
+      - prepare-vm
+    with:
+      runner_kind: self-hosted
+      runner_label: ${{ needs.provide-runner.outputs.label }}
+      build_target: ${{ inputs.build_target }}
+      sanitizer: ${{ inputs.sanitizer }}
+      run_build: ${{ inputs.run_build }}
+      run_tests: ${{ inputs.run_tests }}
+      log_suffix: ya-x86-64${{ inputs.sanitizer != 'none' && format('-{1}', inputs.sanitizer) }}
+    secrets: inherit
+
+  release-runner:
+    name: Release self-hosted YC runner if provided on-demand
+    needs:
+      - provide-runner # required to get output from the start-runner job
+      - main # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Stop YC runner
+        uses: yc-actions/yc-github-runner@v1
+        with:
+          mode: stop
+          yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }}
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ needs.provide-runner.outputs.label }}
+          instance-id: ${{ needs.provide-runner.outputs.instance-id }}
+

.github/workflows/build_and_test_ya_provisioned.yml

@@ -0,0 +1,45 @@
+name: Ya-Build-and-Test-Provisioned
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_target:
+        type: string
+        default: "ydb/"
+        description: "limit build and test to specific target"
+      sanitizer:
+        type: choice
+        default: "none"
+        description: "sanitizer type"
+        options:
+          - none
+          - address
+          - memory
+          - thread
+          - undefined
+          - leak
+      runner_label:
+        type: string
+        default: "linux"
+        description: "runner label"
+      run_build:
+        type: boolean
+        default: true
+        description: "run build"
+      run_tests:
+        type: boolean
+        default: true
+        description: "run tests"
+
+jobs:
+  main:
+    uses: ./.github/workflows/build_and_test_ya.yml
+    with:
+      runner_kind: provisioned
+      runner_label: ${{ inputs.runner_label }}
+      build_target: ${{ inputs.build_target }}
+      sanitizer: ${{ inputs.sanitizer }}
+      run_build: ${{ inputs.run_build }}
+      run_tests: ${{ inputs.run_tests }}
+      log_suffix: ya-${{ inputs.sanitizer != 'none' && format('{0}-{1}', inputs.runner_label, inputs.sanitizer) || inputs.runner_label }}
+    secrets: inherit

.mapping.json

@@ -1,7 +1,10 @@
 {
   ".github/actions/build/action.yml":"ydb/github_toplevel/.github/actions/build/action.yml",
+  ".github/actions/build_ya/action.yml":"ydb/github_toplevel/.github/actions/build_ya/action.yml",
   ".github/actions/prepare_vm/action.yaml":"ydb/github_toplevel/.github/actions/prepare_vm/action.yaml",
+  ".github/actions/s3cmd/action.yml":"ydb/github_toplevel/.github/actions/s3cmd/action.yml",
   ".github/actions/test/action.yml":"ydb/github_toplevel/.github/actions/test/action.yml",
+  ".github/actions/test_ya/action.yml":"ydb/github_toplevel/.github/actions/test_ya/action.yml",
   ".github/check_dirs.sh":"ydb/github_toplevel/.github/check_dirs.sh",
   ".github/config/muted_functest.txt":"ydb/github_toplevel/.github/config/muted_functest.txt",
   ".github/config/muted_shard.txt":"ydb/github_toplevel/.github/config/muted_shard.txt",
@@ -26,9 +29,13 @@
   ".github/scripts/tests/mute_utils.py":"ydb/github_toplevel/.github/scripts/tests/mute_utils.py",
   ".github/scripts/tests/pytest-postprocess.py":"ydb/github_toplevel/.github/scripts/tests/pytest-postprocess.py",
   ".github/scripts/tests/templates/summary.html":"ydb/github_toplevel/.github/scripts/tests/templates/summary.html",
+  ".github/scripts/tests/transform-ya-junit.py":"ydb/github_toplevel/.github/scripts/tests/transform-ya-junit.py",
   ".github/workflows/allowed_dirs.yml":"ydb/github_toplevel/.github/workflows/allowed_dirs.yml",
   ".github/workflows/build_and_test_ondemand.yml":"ydb/github_toplevel/.github/workflows/build_and_test_ondemand.yml",
   ".github/workflows/build_and_test_provisioned.yml":"ydb/github_toplevel/.github/workflows/build_and_test_provisioned.yml",
+  ".github/workflows/build_and_test_ya.yml":"ydb/github_toplevel/.github/workflows/build_and_test_ya.yml",
+  ".github/workflows/build_and_test_ya_ondemand.yml":"ydb/github_toplevel/.github/workflows/build_and_test_ya_ondemand.yml",
+  ".github/workflows/build_and_test_ya_provisioned.yml":"ydb/github_toplevel/.github/workflows/build_and_test_ya_provisioned.yml",
   ".github/workflows/docker_publish.yml":"ydb/github_toplevel/.github/workflows/docker_publish.yml",
   ".github/workflows/docs_build.yaml":"ydb/github_toplevel/.github/workflows/docs_build.yaml",
   ".github/workflows/docs_release.yaml":"ydb/github_toplevel/.github/workflows/docs_release.yaml",