Update `codecov_ats` workflow (#57587)

These changes update the `codecov_ats` workflow.
The main objective is to derive 2 lists from this:

* ATS_TESTS_TO_RUN - list of tests to be executed
* ATS_TESTS_TO_SKIP - list of tests to be skipped

Learn more about [Automated Test
Selection](https://docs.codecov.com/docs/automated-test-selection)

There's a summary of the run so it's easy to checkout what happened and
how many tests were selected.

The workflow may fail, but will still produce the 2 lists of tests mentioned above.
The debug step afterwards is for us to be able to check what tests are being selected.

The actual testing is NOT done in this workflow.

.github/workflows/codecov_ats.yml

@@ -13,6 +13,12 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    # the default default is:
+    #      bash --noprofile --norc -eo pipefail {0}
+    shell: bash --noprofile --norc -eo pipefail -ux {0}
+
 jobs:
     files-changed:
         name: detect what files changed
@@ -35,19 +41,26 @@ jobs:
               token: ${{ github.token }}
               filters: .github/file-filters.yml
     coverage-ats:
-        # Temporary test
         if: needs.files-changed.outputs.backend == 'true'
         needs: files-changed
-        timeout-minutes: 40
         runs-on: ubuntu-latest
+        # Map a step output to a job output
+        outputs:
+          ATS_TESTS_TO_RUN: ${{ steps.codecov_automated_test_selection.outputs.ATS_TESTS_TO_RUN }}
+          ATS_TESTS_TO_SKIP: ${{ steps.codecov_automated_test_selection.outputs.ATS_TESTS_TO_SKIP }}
         steps:
         - uses: actions/checkout@v3
           with:
+            # fetch-depth: 0 - Use if the BASE_COMMIT on codecov_automated_test_selection is at unknown depth
+            #                   (i.e. git merge-base ${{ github.sha }}^ origin/main)
+            # fetch-depth: 2 - Use if the BASE_COMMIT on codecov_automated_test_selection is at known depth
+            #                   (i.e. git rev-parse ${{ github.sha }}^)
             fetch-depth: 0
         - name: Set up Python 3.10.10
           uses: actions/setup-python@v4
           with:
             python-version: "3.10.10"
+        # We need the setup to collect the list of tests properly
         - name: Setup sentry env
           uses: ./.github/actions/setup-sentry
           id: setup
@@ -60,7 +73,7 @@ jobs:
             pg-version: 14
         - name: Download Codecov CLI
           run: |
-            pip install --extra-index-url https://pypi.org/simple --no-cache-dir pytest codecov-cli
+            pip install --extra-index-url https://pypi.org/simple --no-cache-dir codecov-cli>=0.4.0
         # Creates the commit and report objects in codecov
         - name: Codecov startup
           run: |
@@ -80,13 +93,77 @@ jobs:
           env:
             CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
             CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }}
-        # Run label analysis IN DRY MODE (no tests will actually run)
-        - name: Label Analysis
-          continue-on-error: true
+        # Run Automated Test Selection in dry mode to get the list of tests to run
+        # The base commit will be the parent commit (apparently commits on master don't exist in codecov)
+        - name: Codecov Automated Test Selection
+          id: codecov_automated_test_selection
           run: |
             BASE_COMMIT=$(git merge-base ${{ github.sha }}^ origin/master)
             echo $BASE_COMMIT
-            codecovcli --codecov-yml-path=codecov.yml label-analysis --dry-run --token=${CODECOV_STATIC_TOKEN} --base-sha=${BASE_COMMIT}
+            output=$(codecovcli --codecov-yml-path=codecov.yml label-analysis --dry-run --token=${CODECOV_STATIC_TOKEN} --base-sha=${BASE_COMMIT}) || true
+            if [ -n "${output}" ];
+            then
+
+              echo ATS_TESTS_TO_RUN=$(jq <<< $output '.runner_options + .ats_tests_to_run | @json | @sh' --raw-output) >> "$GITHUB_OUTPUT"
+              echo ATS_TESTS_TO_SKIP=$(jq <<< $output '.runner_options + .ats_tests_to_skip | @json | @sh' --raw-output) >> "$GITHUB_OUTPUT"
+
+              testcount() { jq <<< $output ".$1 | length"; }
+              run_count=$(testcount ats_tests_to_run)
+              skip_count=$(testcount ats_tests_to_skip)
+              tee <<< "Selected $run_count / $(($run_count + $skip_count)) tests to run" "$GITHUB_STEP_SUMMARY"
+            else
+              tee <<< "ATS failed. Can't get list of tests to run. Fallback to all tests" "$GITHUB_STEP_SUMMARY"
+              # We need not forget to add the search options in the fallback command, otherwise pytest might run more tests than expected
+              # These search options match what's defined in codecov.yml:105
+              echo 'ATS_TESTS_TO_RUN<<EOF' >> $GITHUB_OUTPUT
+              jq -c @json <<< '[
+                "--cov-context=test",
+                "tests/sentry",
+                "tests/integration",
+                "--ignore=tests/sentry/eventstream/kafka",
+                "--ignore=tests/sentry/post_process_forwarder",
+                "--ignore=tests/sentry/snuba",
+                "--ignore=tests/sentry/search/events",
+                "--ignore=tests/sentry/ingest/ingest_consumer/test_ingest_consumer_kafka.py",
+                "--ignore=tests/sentry/region_to_control/test_region_to_control_kafka.py"
+              ]' >> $GITHUB_OUTPUT
+              echo 'EOF' >> $GITHUB_OUTPUT
+              echo ATS_TESTS_TO_SKIP="'[]'" >> "$GITHUB_OUTPUT"
+              echo "::error ATS failed"
+              exit 1
+            fi
           env:
             CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
             CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }}
+    # The actual running of tests would come here, after the labels are available
+    # Something like pytest <options> $ATS_TESTS_TO_RUN
+    debug:
+      runs-on: ubuntu-latest
+      needs: coverage-ats
+      if: ${{ always() }}
+      steps:
+        - name: Debug ATS_TESTS_TO_RUN
+          run: |
+            : ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_RUN }}
+            length_of_tests=$(jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_RUN }} 'length')
+            # The 1st value doesn't count, it's '--cov-context=test' (hence -gt 1)
+            if [ $length_of_tests -gt 1 ]; then
+              echo "Running $length_of_tests tests"
+              # --raw-output0 doesn't work.
+              jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_RUN }} 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo
+            else
+              echo "No tests to run"
+            fi
+        - name: Debug ATS_TESTS_TO_SKIP
+          run: |
+            : ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }}
+            ATS_TESTS_TO_SKIP='${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }}'
+            length_of_tests=$(jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }} 'length')
+            # The 1st value doesn't count, it's '--cov-context=test'
+            if [ $length_of_tests -gt 1 ]; then
+              echo "Running $length_of_tests tests"
+              # --raw-output0 doesn't work.
+              jq <<< ${{ needs.coverage-ats.outputs.ATS_TESTS_TO_SKIP }} 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo
+            else
+              echo "No tests to run"
+            fi