ci: add more details to PR comment about build and tests (#1054)

.github/actions/build_ya/action.yml

@@ -28,13 +28,19 @@ runs:
     - name: Init
       id: init
       shell: bash
+      env:
+        build_preset: ${{ inputs.build_preset }}
       run: |
         echo "SHELLOPTS=xtrace" >> $GITHUB_ENV
         export TMP_DIR=$(pwd)/tmp_build
         echo "TMP_DIR=$TMP_DIR" >> $GITHUB_ENV
         rm -rf $TMP_DIR && mkdir $TMP_DIR
-    
+        
+        echo "BUILD_PRESET=$build_preset" >> $GITHUB_ENV
+        echo "GITHUB_TOKEN=${{ github.token }}" >> $GITHUB_ENV
+
     - name: build
+      id: build
       shell: bash
       run: |
         extra_params=()
@@ -85,12 +91,18 @@ runs:
         echo "::debug::get version"
         ./ya --version
         
+        echo "Build **{platform_name}-${BUILD_PRESET}** is running..." | .github/scripts/tests/comment-pr.py
+        
+        # to be sure
+        set -o pipefail
+        
         ./ya make -k --build "${build_type}" --force-build-depends -D'BUILD_LANGUAGES=CPP PY3 PY2 GO' -T --stat -DCONSISTENT_DEBUG \
           --log-file "$TMP_DIR/ya_log.txt" --evlog-file "$TMP_DIR/ya_evlog.jsonl" \
           --cache-size 512G --link-threads "${{ inputs.link_threads }}" \
-          "${extra_params[@]}" || (
+          "${extra_params[@]}" |& tee $TMP_DIR/ya_make.log || (
               RC=$?
               echo "::debug::ya make RC=$RC"
+              echo "status=failed" >> $GITHUB_OUTPUT
           )
 
     - name: sync logs to s3
@@ -98,9 +110,22 @@ runs:
       shell: bash
       run: |
         echo "::group::s3-sync"
-        s3cmd sync --acl-private --no-progress --stats --no-check-md5 "$TMP_DIR/" "$S3_BUCKET_PATH/build_logs/"
+        s3cmd sync --acl-private --exclude="ya_make.log" --no-progress --stats --no-check-md5 "$TMP_DIR/" "$S3_BUCKET_PATH/build_logs/"
+        s3cmd sync --acl-public --no-progress --stats --no-check-md5 "$TMP_DIR/ya_make.log" "$S3_BUCKET_PATH/build_logs/"
         echo "::endgroup::"
 
+    - name: comment-build-status
+      if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
+      shell: bash
+      run: |
+        log_url="$S3_URL_PREFIX/build_logs/ya_make.log"
+
+        if [ "${{ steps.build.outputs.status }}" == "failed" ]; then
+          echo "Build failed. see the [build logs]($log_url)." | .github/scripts/tests/comment-pr.py --fail
+        else
+          echo "Build successful." | .github/scripts/tests/comment-pr.py --ok
+        fi
+
     - name: show free space
       if: always()
       shell: bash

.github/actions/test_ya/action.yml

@@ -66,6 +66,8 @@ runs:
         echo "TESTMO_TOKEN=${{ inputs.testman_token }}" >> $GITHUB_ENV
         echo "TESTMO_URL=${{ inputs.testman_url }}" >> $GITHUB_ENV
         echo "SUMMARY_LINKS=$(mktemp)" >> $GITHUB_ENV
+        echo "GITHUB_TOKEN=${{ github.token }}" >> $GITHUB_ENV
+        echo "BUILD_PRESET=${{ inputs.build_preset }}" >> $GITHUB_ENV
     
     - name: prepare
       shell: bash
@@ -93,7 +95,6 @@ runs:
         BRANCH_TAG="$GITHUB_REF_NAME"
         ARCH="${{ runner.arch == 'X64' && 'x86-64' || runner.arch == 'ARM64' && 'arm64' || 'unknown' }}"
         
-        BUILD_PRESET="${{ inputs.build_preset }}"
         case "$BUILD_PRESET" in
           relwithdebinfo)
             TESTMO_SOURCE="ya-${ARCH}"
@@ -105,7 +106,7 @@ runs:
             TESTMO_SOURCE="ya-${ARCH}-${BUILD_PRESET/release-/}"
             ;;
           *)
-            echo "Invalid preset: ${{ inputs.build_preset }}"
+            echo "Invalid preset: $BUILD_PRESET"
             exit 1
             ;;
         esac
@@ -170,7 +171,7 @@ runs:
         )
         
         # FIXME: copy-paste from build_ya
-        case "${{ inputs.build_preset }}" in
+        case "$BUILD_PRESET" in
           debug)
             params+=(--build "debug")
             ;;
@@ -196,7 +197,7 @@ runs:
             )
             ;;
           *)
-            echo "Invalid preset: ${{ inputs.build_preset }}"
+            echo "Invalid preset: $BUILD_PRESET"
             exit 1
             ;;
         esac
@@ -213,6 +214,8 @@ runs:
         echo "::debug::get version"
         ./ya --version
         
+        echo "Tests are running..." | .github/scripts/tests/comment-pr.py
+        
         if [ ! -z "${{ inputs.bazel_remote_username }}" ]; then
           echo "::debug::start tests"
 
@@ -300,20 +303,16 @@ runs:
     - name: write tests summary
       shell: bash
       if: always()
-      env:
-        GITHUB_TOKEN: ${{ github.token }}
       run: |
         mkdir $ARTIFACTS_DIR/summary/
         
         cat $SUMMARY_LINKS | python3 -c 'import sys; print(" | ".join([v for _, v in sorted([l.strip().split(" ", 1) for l in sys.stdin], key=lambda a: (int(a[0]), a))]))' >> $GITHUB_STEP_SUMMARY
         
-        platform_name=$(uname | tr '[:upper:]' '[:lower:]')-$(arch)
-        
         .github/scripts/tests/generate-summary.py \
           --summary-out-path $ARTIFACTS_DIR/summary/ \
           --summary-url-prefix $S3_URL_PREFIX/summary/ \
           --test-history-url $TEST_HISTORY_URL \
-          --build-preset "${platform_name}-${{ inputs.build_preset }}" \
+          --build-preset "$BUILD_PRESET" \
           "Tests" ya-test.html "$JUNIT_REPORT_XML"
 
     - name: sync test results to s3

.github/scripts/tests/comment-pr.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+import os
+import json
+import argparse
+from github import Github, Auth as GithubAuth
+from github.PullRequest import PullRequest
+from gh_status import update_pr_comment_text
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--rewrite", dest="rewrite", action="store_true")
+    parser.add_argument("--color", dest="color", default="white")
+    parser.add_argument("--fail", dest="fail", action="store_true")
+    parser.add_argument("--ok", dest="ok", action="store_true")
+    parser.add_argument("text", type=argparse.FileType("r"), nargs="?", default="-")
+
+    args = parser.parse_args()
+    color = args.color
+
+    if args.ok:
+        color = 'green'
+    elif args.fail:
+        color = 'red'
+
+    build_preset = os.environ["BUILD_PRESET"]
+
+    gh = Github(auth=GithubAuth.Token(os.environ["GITHUB_TOKEN"]))
+
+    with open(os.environ["GITHUB_EVENT_PATH"]) as fp:
+        event = json.load(fp)
+
+    pr = gh.create_from_raw_data(PullRequest, event["pull_request"])
+
+    update_pr_comment_text(pr, build_preset, color, args.text.read().rstrip(), args.rewrite)
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/tests/generate-summary.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import argparse
 import dataclasses
+import datetime
 import os
 import re
 import json
@@ -12,6 +13,7 @@ from operator import attrgetter
 from typing import List, Optional, Dict
 from jinja2 import Environment, FileSystemLoader, StrictUndefined
 from junit_utils import get_property_value, iter_xml_files
+from gh_status import update_pr_comment_text
 
 
 class TestStatus(Enum):
@@ -155,7 +157,7 @@ class TestSummary:
         github_srv = os.environ.get("GITHUB_SERVER_URL", "https://github.com")
         repo = os.environ.get("GITHUB_REPOSITORY", "ydb-platform/ydb")
 
-        footnote_url = f"{github_srv}/{repo}/tree/main/.github/config"
+        footnote_url = f"{github_srv}/{repo}/tree/main/.github/config/muted_ya.txt"
 
         footnote = "[^1]" if add_footnote else f'<sup>[?]({footnote_url} "All mute rules are defined here")</sup>'
 
@@ -287,18 +289,20 @@ def gen_summary(summary_url_prefix, summary_out_folder, paths):
     return summary
 
 
-def get_comment_text(pr: PullRequest, summary: TestSummary, build_preset: str, test_history_url: str):
+def get_comment_text(pr: PullRequest, summary: TestSummary, test_history_url: str):
     if summary.is_empty:
         return [
-            f":red_circle: **{build_preset}**: Test run completed, no test results found for commit {pr.head.sha}. "
+            f"Test run completed, no test results found for commit {pr.head.sha}. "
             f"Please check build logs."
         ]
     elif summary.is_failed:
-        result = f":red_circle: **{build_preset}**: some tests FAILED"
+        result = f"Some tests failed, follow the links below."
     else:
-        result = f":green_circle: **{build_preset}**: all tests PASSED"
+        result = f"Tests successful."
 
-    body = [f"{result} for commit {pr.head.sha}."]
+    body = [
+        result
+    ]
 
     if test_history_url:
         body.append("")
@@ -309,36 +313,6 @@ def get_comment_text(pr: PullRequest, summary: TestSummary, build_preset: str, t
     return body
 
 
-def update_pr_comment(run_number: int, pr: PullRequest, summary: TestSummary, build_preset: str, test_history_url: str):
-    header = f"<!-- status pr={pr.number}, run={{}} -->"
-    header_re = re.compile(header.format(r"(\d+)"))
-
-    comment = body = None
-
-    for c in pr.get_issue_comments():
-        if matches := header_re.match(c.body):
-            comment = c
-            if int(matches[1]) == run_number:
-                body = [c.body, "", "---", ""]
-
-    if body is None:
-        body = [
-            header.format(run_number),
-            "> [!NOTE]",
-            "> This is an automated comment that will be appended during run.",
-            "",
-        ]
-
-    body.extend(get_comment_text(pr, summary, build_preset, test_history_url))
-
-    body = "\n".join(body)
-
-    if comment is None:
-        pr.create_issue_comment(body)
-    else:
-        comment.edit(body)
-
-
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--summary-out-path", required=True)
@@ -364,9 +338,16 @@ def main():
         with open(os.environ["GITHUB_EVENT_PATH"]) as fp:
             event = json.load(fp)
 
-        run_number = int(os.environ.get("GITHUB_RUN_NUMBER"))
         pr = gh.create_from_raw_data(PullRequest, event["pull_request"])
-        update_pr_comment(run_number, pr, summary, args.build_preset, args.test_history_url)
+
+        text = get_comment_text(pr, summary, args.test_history_url)
+
+        if summary.is_empty | summary.is_failed:
+            color = 'red'
+        else:
+            color = 'green'
+
+        update_pr_comment_text(pr, args.build_preset, color, text='\n'.join(text), rewrite=False)
 
 
 if __name__ == "__main__":

.github/scripts/tests/gh_status.py

@@ -0,0 +1,44 @@
+import datetime
+import platform
+from github.PullRequest import PullRequest
+
+
+def get_timestamp():
+    return datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S UTC")
+
+
+def get_platform_name():
+    return f'{platform.system().lower()}-{platform.machine()}'
+
+
+def update_pr_comment_text(pr: PullRequest, build_preset: str, color: str, text: str, rewrite: bool):
+    platform_name = get_platform_name()
+    header = f"<!-- status pr={pr.number}, preset={platform_name}-{build_preset} -->"
+
+    body = comment = None
+    for c in pr.get_issue_comments():
+        if c.body.startswith(header):
+            print(f"found comment id={c.id}")
+            comment = c
+            if not rewrite:
+                body = [c.body]
+            break
+
+    if body is None:
+        body = [header]
+
+    indicator = f":{color}_circle:"
+    body.append(f"{indicator} `{get_timestamp()}` {text}")
+
+    body = "\n".join(body)
+
+    if '{platform_name}' in body:
+        # input can contain '{platform_name}'
+        body = body.replace('{platform_name}', platform_name)
+
+    if comment is None:
+        print(f"post new comment")
+        pr.create_issue_comment(body)
+    else:
+        print(f"edit comment")
+        comment.edit(body)

.github/workflows/build_and_test_ya.yml

@@ -49,7 +49,9 @@ on:
       put_build_results_to_cache:
         type: boolean
         default: true
-
+defaults:
+  run:
+    shell: bash
 jobs:
   main:
     name: Build and test ${{ inputs.build_preset }}
@@ -63,7 +65,20 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v3
       if: github.event.pull_request.head.sha == ''
-
+    
+    - name: comment-build-start
+      if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
+      shell: bash
+      env:
+        BUILD_PRESET: ${{ inputs.build_preset }}
+        GITHUB_TOKEN: ${{ github.token }}
+      run: |
+        jobs_url="https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs"
+        # tricky: we are searching job with name that contains build_preset
+        check_url=$(curl -s $jobs_url | jq --arg n "$BUILD_PRESET" -r '.jobs[] | select(.name | contains($n)) | .html_url')
+        
+        echo "Pre-commit [check]($check_url) for ${{ github.event.pull_request.head.sha }} has started." | .github/scripts/tests/comment-pr.py --rewrite
+    
     - name: Prepare s3cmd
       uses: ./.github/actions/s3cmd
       with:
@@ -101,3 +116,10 @@ jobs:
         bazel_remote_password: ${{ inputs.put_build_results_to_cache && secrets.REMOTE_CACHE_PASSWORD || '' }}
         link_threads: ${{ inputs.link_threads }}
         test_threads: ${{ inputs.test_threads }}
+    
+    - name: comment-if-cancel
+      if: cancelled() && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target')
+      env:
+        BUILD_PRESET: ${{ inputs.build_preset }}
+        GITHUB_TOKEN: ${{ github.token }}
+      run:  echo "Check cancelled" | .github/scripts/tests/comment-pr.py --color black