ydb/.github/actions/test_ya/action.yml

name: Run build and tests (ya make)
description: Run test targets listed in repository root ya.make (to be created previously)
inputs:
  build_target:
    required: true
  build_preset:
    required: true
    default: "relwithdebinfo"
    description: "relwithdebinfo, release-asan, release-tsan"
  test_type:
    required: false
    type: string
    default: ""
    description: "run only specific test types (or all by default)"
  test_size:
    required: false
    default: "small,medium,large"
    description: "small or small-medium or all"
  test_threads:
    required: false
    default: "56"
    description: "Test threads count"
  link_threads:
    required: false
    default: "12"
    description: "link threads count"
  additional_ya_make_args:
    type: string
    default: ""
  testman_token:
    required: false
    description: "test manager auth token"
  testman_url:
    required: false
    description: "test manager endpoint"
  testman_project_id:
    required: false
    description: "test manager project id"
  put_build_results_to_cache:
    required: false
    default: "true"
  bazel_remote_uri:
    required: false
    description: "bazel-remote endpoint"
  bazel_remote_username:
    required: false
    description: "bazel-remote username"
  bazel_remote_password:
    required: false
    description: "bazel-remote password"
  run_tests:
    type: boolean
    default: true
    description: "run tests"
  test_retry_count:
    type: string
    default: ""
    description: "how many times to retry failed tests"
outputs:
  success:
    value: ${{ steps.build.outputs.status }}
    description: "build success"

runs:
  using: "composite"
  steps:
    - name: Init
      id: init
      shell: bash
      run: |
        set -x

        export TMP_DIR=$(pwd)/tmp
        rm -rf $TMP_DIR
        mkdir -p $TMP_DIR
        echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV

        # The whole dir will be uploaded to s3 with public (=wild internet) ACL
        export PUBLIC_DIR=$TMP_DIR/results
        echo "PUBLIC_DIR=$PUBLIC_DIR" >> $GITHUB_ENV
        export PUBLIC_DIR_URL=$S3_URL_PREFIX
        echo "PUBLIC_DIR_URL=$PUBLIC_DIR_URL" >> $GITHUB_ENV
        mkdir -p $PUBLIC_DIR

        echo "LAST_JUNIT_REPORT_XML=$PUBLIC_DIR/last_junit.xml" >> $GITHUB_ENV
        export TESTMO_URL=${{ inputs.testman_url }}
        echo "TESTMO_URL=$TESTMO_URL" >> $GITHUB_ENV
        echo "SUMMARY_LINKS=$PUBLIC_DIR/summary_links.txt" >> $GITHUB_ENV
        echo "BUILD_PRESET=${{ inputs.build_preset }}" >> $GITHUB_ENV

        python3 -m pip install ydb ydb[yc] codeowners

        if [ ${{ inputs.testman_token }} ]; then
          TESTMO_PROXY_ADDR=127.0.0.1:8888
          openssl req -x509 -newkey rsa:2048 \
            -keyout $TMP_DIR/key.pem -out $TMP_DIR/cert.pem \
            -sha256 -days 1 -nodes -subj "/CN=127.0.0.1"
          
          TESTMO_TOKEN=${{ inputs.testman_token }} ./ydb/ci/testmo-proxy/testmo-proxy.py -l $TESTMO_PROXY_ADDR \
            --cert-file "$TMP_DIR/cert.pem" \
            --cert-key "$TMP_DIR/key.pem" \
            --target-timeout 3,60 \
            --max-request-time 200 \
            "$TESTMO_URL" > $PUBLIC_DIR/testmo_proxy.log 2>&1 &
            
          TESTMO_PROXY_PID=$!
          echo "TESTMO_PROXY_ADDR=$TESTMO_PROXY_ADDR" >> $GITHUB_ENV
          echo "TESTMO_PROXY_PID=$TESTMO_PROXY_PID" >> $GITHUB_ENV

          # testmo rejects self-signed cert without this setting
          echo "NODE_TLS_REJECT_UNAUTHORIZED=0" >> $GITHUB_ENV
        fi

    - name: ya build and test
      id: build
      shell: bash
      run: |
        set -x
        readarray -d ',' -t test_size < <(printf "%s" "${{ inputs.test_size }}")
        readarray -d ',' -t test_type < <(printf "%s" "${{ inputs.test_type }}")

        params=(
          -T
          ${test_size[@]/#/--test-size=} ${test_type[@]/#/--test-type=}
          --stat
          --test-threads "${{ inputs.test_threads }}" --link-threads "${{ inputs.link_threads }}"
          -DUSE_EAT_MY_DATA
        )

        TEST_RETRY_COUNT=${{ inputs.test_retry_count }}
        
        case "$BUILD_PRESET" in
          debug)
            params+=(--build "debug")
            ;;
          relwithdebinfo)
            params+=(--build "relwithdebinfo")
            ;;
          release)  
            params+=(--build "release")
            ;;
          release-clang14)
            params+=(--build "release")
            params+=(--target-platform="CLANG14-LINUX-X86_64")
            params+=(-DLLD_VERSION=16)
            params+=(-DSTRIP)
            ;;
          release-asan)
            params+=(
              --build "release" --sanitize="address"
              -DDEBUGINFO_LINES_ONLY
            )
            if [ $TEST_RETRY_COUNT -z ]; then
              TEST_RETRY_COUNT=1
            fi
            ;;
          release-tsan)
            params+=(
              --build "release" --sanitize="thread"
              -DDEBUGINFO_LINES_ONLY
            )
            if [ $TEST_RETRY_COUNT -z ]; then
              TEST_RETRY_COUNT=1
            fi
            ;;
          release-msan)
            params+=(
              --build "release" --sanitize="memory"
              -DDEBUGINFO_LINES_ONLY
            )
            if [ $TEST_RETRY_COUNT -z ]; then
              TEST_RETRY_COUNT=1
            fi
            ;;
          *)
            echo "Invalid preset: $BUILD_PRESET"
            exit 1
            ;;
        esac

        if [ $TEST_RETRY_COUNT -z ]; then
          # default is 3 for ordinary build and 1 for sanitizer builds
          TEST_RETRY_COUNT=3
        fi
    
        if [ ! -z "${{ inputs.additional_ya_make_args }}" ]; then
          params+=(${{ inputs.additional_ya_make_args }})
        fi

        if [ ! -z "${{ inputs.bazel_remote_uri }}" ]; then
          params+=(--bazel-remote-store)
          params+=(--bazel-remote-base-uri "${{ inputs.bazel_remote_uri }}")
        fi

        if [ "${{ inputs.put_build_results_to_cache }}" = "true" ]; then
          params+=(--bazel-remote-username "${{ inputs.bazel_remote_username }}")
          params+=(--bazel-remote-password "${{ inputs.bazel_remote_password }}")
          params+=(--bazel-remote-put --dist-cache-max-file-size=209715200)
        fi

        if [ true = ${{ inputs.run_tests }} ]; then
          params+=(-A)
          params+=(--retest)
        fi

        params+=(
          --stat -DCONSISTENT_DEBUG --no-dir-outputs
          --test-failure-code 0 --build-all 
          --cache-size 2TB --force-build-depends
        )


        TESTMO_BRANCH_TAG="$GITHUB_REF_NAME"
        TESTMO_ARCH="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"
        TESTMO_PR_NUMBER=${{ github.event.number }}
        
        case "$BUILD_PRESET" in
          relwithdebinfo)
            TESTMO_SOURCE="ya-${TESTMO_ARCH}"
            ;;
          debug)
            TESTMO_SOURCE="ya-${TESTMO_ARCH}-debug"
            ;;
          release-*)
            TESTMO_SOURCE="ya-${TESTMO_ARCH}-${BUILD_PRESET/release-/}"
            ;;
          *)
            echo "Invalid preset: $BUILD_PRESET"
            exit 1
            ;;
        esac

        echo "::debug::get version"
        ./ya --version

        YA_MAKE_OUT_DIR=$TMP_DIR/out

        YA_MAKE_OUTPUT="$PUBLIC_DIR/ya_make_output.log"
        YA_MAKE_OUTPUT_URL="$PUBLIC_DIR_URL/ya_make_output.log"
        echo "20 [Ya make output]($YA_MAKE_OUTPUT_URL)" >> $SUMMARY_LINKS
        echo "YA_MAKE_OUTPUT_URL=$YA_MAKE_OUTPUT_URL" >> $GITHUB_ENV 

        BUILD_FAILED=0

        for RETRY in $(seq 1 $TEST_RETRY_COUNT)
        do
          case $GITHUB_EVENT_NAME in
            workflow_dispatch)
              TESTMO_RUN_NAME="${{ github.run_id }} manual"
              TESTMO_EXTRA_TAG="manual"
              ;;
            pull_request | pull_request_target)
              TESTMO_RUN_NAME="${{ github.run_id }} PR #${TESTMO_PR_NUMBER}"
              TESTMO_EXTRA_TAG="pr"
              TESTMO_BRANCH_TAG=""
              ;;
            schedule)
              TESTMO_RUN_NAME="${{ github.run_id }} schedule"
              TESTMO_EXTRA_TAG="schedule"
              ;;
            push)
              TESTMO_RUN_NAME="${{ github.run_id }} POST"
              TESTMO_EXTRA_TAG="post-commit"
              ;;
            *)
              TESTMO_RUN_NAME="${{ github.run_id }}"
              TESTMO_EXTRA_TAG=""
              ;;
          esac

          echo "TESTMO_RUN_NAME=$TESTMO_RUN_NAME" >> $GITHUB_ENV 
          if [ $RETRY != 1 ]; then
            IS_RETRY=1
            TESTMO_RUN_NAME="$TESTMO_RUN_NAME A$RETRY"
          else
            IS_RETRY=0
          fi

          CURRENT_PUBLIC_DIR_RELATIVE=try_$RETRY
          CURRENT_PUBLIC_DIR=$PUBLIC_DIR/$CURRENT_PUBLIC_DIR_RELATIVE
          mkdir $CURRENT_PUBLIC_DIR

          if [ ${{ inputs.testman_token }} ]; then
            # inititalize testmo session
            TESTMO_RUN_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID"
            TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:resources:add-link --name build --url "$TESTMO_RUN_URL" --resources $CURRENT_PUBLIC_DIR/testmo.json
            TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:resources:add-field --name git-sha --type string --value "${GITHUB_SHA:0:7}" --resources $CURRENT_PUBLIC_DIR/testmo.json
            TESTMO_RUN_ID=$(
              TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:create --instance "https://$TESTMO_PROXY_ADDR" --project-id ${{ inputs.testman_project_id }} \
                --name "$TESTMO_RUN_NAME" --source "$TESTMO_SOURCE" --resources $CURRENT_PUBLIC_DIR/testmo.json \
                --tags "$TESTMO_BRANCH_TAG" --tags "$TESTMO_EXTRA_TAG"
            )
            echo "runid=${TESTMO_RUN_ID}" >> $GITHUB_OUTPUT
            TESTMO_HISTORY_URL="${TESTMO_URL}/automation/runs/view/${TESTMO_RUN_ID}"

            # Replace test history link
            cat $SUMMARY_LINKS | (grep -v "Test history"  || true) > $TMP_DIR/tmp_summary
            mv $TMP_DIR/tmp_summary $SUMMARY_LINKS
            echo "10 [Test history](${TESTMO_HISTORY_URL})" >> $SUMMARY_LINKS
          fi

          CURRENT_MESSAGE="ya make is running..." 
          if [ $IS_RETRY = 0 ]; then
            CURRENT_MESSAGE="$CURRENT_MESSAGE"
            RERUN_FAILED_OPT=""
          else
            CURRENT_MESSAGE="$CURRENT_MESSAGE (failed tests rerun, try $RETRY)"
            RERUN_FAILED_OPT="-X"
          fi

          echo $CURRENT_MESSAGE | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py

          CURRENT_JUNIT_XML_PATH=$CURRENT_PUBLIC_DIR/junit.xml
          set +ex
          (./ya make ${{ inputs.build_target }} "${params[@]}" \
            $RERUN_FAILED_OPT --log-file "$PUBLIC_DIR/ya_log.log"  \
            --evlog-file "$CURRENT_PUBLIC_DIR/ya_evlog.jsonl" \
            --junit "$CURRENT_JUNIT_XML_PATH" --output "$YA_MAKE_OUT_DIR"; echo $? > exit_code) |& cat >> $YA_MAKE_OUTPUT
          set -ex
          RC=`cat exit_code`

          # convert to chromium trace
          # seems analyze-make don't have simple "output" parameter, so change cwd
          ya_dir=$(pwd)
          (cd $CURRENT_PUBLIC_DIR && $ya_dir/ya analyze-make timeline --evlog ya_evlog.jsonl)

          if [ $RC -ne 0 ]; then
            echo "ya make returned $RC, build failed"
            echo "status=failed" >> $GITHUB_OUTPUT
            BUILD_FAILED=1
            break
          fi

          # fix junit files (add links, logs etc)
          # archive unitest reports (orig) 
          gzip -c $CURRENT_JUNIT_XML_PATH > $CURRENT_PUBLIC_DIR/orig_junit.xml.gz

          # postprocess junit report
          .github/scripts/tests/transform-ya-junit.py -i \
            -m .github/config/muted_ya.txt \
            --ya_out "$YA_MAKE_OUT_DIR" \
            --public_dir "$PUBLIC_DIR" \
            --public_dir_url "$PUBLIC_DIR_URL" \
            --log_out_dir "$CURRENT_PUBLIC_DIR_RELATIVE/artifacts/logs/" \
            --test_stuff_out "$CURRENT_PUBLIC_DIR_RELATIVE/test_artifacts/" \
            "$CURRENT_JUNIT_XML_PATH"
          cp $CURRENT_JUNIT_XML_PATH $LAST_JUNIT_REPORT_XML

          TESTS_RESULT=0
          .github/scripts/tests/fail-checker.py "$CURRENT_JUNIT_XML_PATH" --output_path $CURRENT_PUBLIC_DIR/failed_count.txt || TESTS_RESULT=$?
          FAILED_TESTS_COUNT=$(cat $CURRENT_PUBLIC_DIR/failed_count.txt)

          IS_LAST_RETRY=0

          if [ $TESTS_RESULT = 0 ] || [ $RETRY = $TEST_RETRY_COUNT ]; then
            IS_LAST_RETRY=1
          fi

          if [ $FAILED_TESTS_COUNT -gt 500 ]; then 
            IS_LAST_RETRY=1
            TOO_MANY_FAILED="Too many tests failed, NOT going to retry"
            echo $TOO_MANY_FAILED | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py --color red
          fi

          if [ "${{ inputs.run_tests }}" = "true" ]; then
            GITHUB_TOKEN=${{ github.token }} .github/scripts/tests/generate-summary.py \
              --summary_links "$SUMMARY_LINKS" \
              --public_dir "$PUBLIC_DIR" \
              --public_dir_url "$PUBLIC_DIR_URL" \
              --build_preset "$BUILD_PRESET" \
              --status_report_file statusrep.txt \
              --is_retry $IS_RETRY \
              --is_last_retry $IS_LAST_RETRY \
              --comment_color_file summary_color.txt \
              --comment_text_file summary_text.txt \
              "Tests" $CURRENT_PUBLIC_DIR/ya-test.html "$CURRENT_JUNIT_XML_PATH"
          fi

          s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$PUBLIC_DIR/" "$S3_BUCKET_PATH/"

          if [ "${{ inputs.run_tests }}" = "true" ]; then
            cat summary_text.txt | GITHUB_TOKEN="${{ github.token }}" .github/scripts/tests/comment-pr.py --color `cat summary_color.txt`
          fi
          
          # upload tests results to YDB
          ydb_upload_run_name="${TESTMO_RUN_NAME// /"_"}"
          result=`.github/scripts/analytics/upload_tests_results.py --test-results-file ${CURRENT_JUNIT_XML_PATH} --run-timestamp $(date +%s) --commit $(git rev-parse HEAD) --build-type ${BUILD_PRESET} --pull $ydb_upload_run_name --job-name "${{ github.workflow }}" --job-id "${{ github.run_id }}" --branch ${GITHUB_REF_NAME}`

          if [ ${{ inputs.testman_token }} ]; then
            # finish testme session

            # split large junit_report
            export TESTMO_JUNIT_REPORT_PARTS=$TMP_DIR/try_$RETRY/junit-split
            mkdir -p $TESTMO_JUNIT_REPORT_PARTS
            .github/scripts/tests/split-junit.py -o "$TESTMO_JUNIT_REPORT_PARTS" "$CURRENT_JUNIT_XML_PATH"
            # archive unitest reports (transformed)
            tar -C $TESTMO_JUNIT_REPORT_PARTS/.. -czf $PUBLIC_DIR/junit_parts.xml.tar.gz $(basename $TESTMO_JUNIT_REPORT_PARTS)
            
            TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:submit-thread \
              --instance "https://$TESTMO_PROXY_ADDR" --run-id "$TESTMO_RUN_ID" \
              --results "$TESTMO_JUNIT_REPORT_PARTS/*.xml"

            TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id $TESTMO_RUN_ID || true
            echo "runid=" >> $GITHUB_OUTPUT
          fi

          if [ $IS_LAST_RETRY = 1 ]; then
            break
          fi
        done;

        if [ $BUILD_FAILED = 0 ]; then
          echo "status=true" >> $GITHUB_OUTPUT
        fi
        

    - name: comment-build-status
      if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
      shell: bash
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        set -x
        
        if [ "${{ steps.build.outputs.status }}" == "failed" ]; then
          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
            -d '{"state":"failure","description":"The check has been failed","context":"build_${{inputs.build_preset}}"}'
          echo "Build failed. see the [logs]($YA_MAKE_OUTPUT_URL)." | .github/scripts/tests/comment-pr.py --color red
        else
          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
            https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
            -d '{"state":"success","description":"The check has been completed successfully","context":"build_${{inputs.build_preset}}"}'
          echo "Build successful." | .github/scripts/tests/comment-pr.py --color green
        fi
    

    - name: Clean up unfinished testmo sessions
      if: always()
      shell: bash
      run: |
        if [ ${{ steps.build.outputs.runid }} ]; then
          TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id ${{ steps.build.outputs.runid }} || true
        fi
        if [ ${{ inputs.testman_token }} ]; then
          kill $TESTMO_PROXY_PID
        fi
    - name: analyze tests results
      shell: bash
      env:
        GITHUB_TOKEN: ${{ github.token }}
      run: |
        set -x
        if [ true = ${{ inputs.run_tests }} ]; then
          teststatus=$(cat statusrep.txt)
          if [[ $teststatus == "success" ]];then
            testmessage="The check has been completed successfully"
          else
            testmessage="The check has been failed"
          fi
          curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${{github.token}}" -H "X-GitHub-Api-Version: 2022-11-28" \
              https://api.github.com/repos/${{github.repository}}/statuses/${{github.event.pull_request.head.sha}} \
              -d '{"state":"'$teststatus'","description":"'"$testmessage"'","context":"test_${{inputs.build_preset}}"}'
          
          if [[ $teststatus != "success" ]];then
            echo "status=failed" >> $GITHUB_OUTPUT
          fi
        fi
    
    - name: check test results
      if: inputs.run_tests
      shell: bash
      run: |
        .github/scripts/tests/fail-checker.py "$LAST_JUNIT_REPORT_XML"

    - name: sync results to s3 and publish links
      if: always()
      shell: bash
      run: |
        set -x
        echo "::group::s3-sync"
        .github/scripts/Indexer/indexer.py -r "$PUBLIC_DIR/"
        echo "00 [Workflow artifacts](${S3_URL_PREFIX}/index.html)" >> $SUMMARY_LINKS
        s3cmd sync --follow-symlinks --acl-public --no-progress --stats --no-check-md5 "$PUBLIC_DIR/" "$S3_BUCKET_PATH/"
        cat $SUMMARY_LINKS | python3 -c 'import sys; print(" | ".join([v for _, v in sorted([l.strip().split(" ", 1) for l in sys.stdin], key=lambda a: (int(a[0]), a))]))' >> $GITHUB_STEP_SUMMARY
        echo "::endgroup::"

    - name: show free space
      if: always()
      shell: bash
      run: df -h