name: IGNORE ME codecov-ats on: pull_request: env: CLI_VERSION: v0.1.5 # Cancel in progress workflows on pull_requests. # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true defaults: run: # the default default is: # bash --noprofile --norc -eo pipefail {0} shell: bash --noprofile --norc -eo pipefail -ux {0} jobs: files-changed: name: detect what files changed runs-on: ubuntu-20.04 timeout-minutes: 3 # Map a step output to a job output outputs: api_docs: ${{ steps.changes.outputs.api_docs }} backend: ${{ steps.changes.outputs.backend_all }} backend_dependencies: ${{ steps.changes.outputs.backend_dependencies }} backend_any_type: ${{ steps.changes.outputs.backend_any_type }} migration_lockfile: ${{ steps.changes.outputs.migration_lockfile }} steps: - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Check for backend file changes uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 id: changes with: token: ${{ github.token }} filters: .github/file-filters.yml coverage-ats: if: needs.files-changed.outputs.backend == 'true' needs: files-changed runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: # fetch-depth: 0 - Use if the BASE_COMMIT on codecov_automated_test_selection is at unknown depth # (i.e. git merge-base ${{ github.sha }}^ origin/main) # fetch-depth: 2 - Use if the BASE_COMMIT on codecov_automated_test_selection is at known depth # (i.e. git rev-parse ${{ github.sha }}^) fetch-depth: 0 - name: Set up Python 3.10.10 uses: actions/setup-python@v4 with: python-version: "3.10.10" # We need the setup to collect the list of tests properly - name: Setup sentry env uses: ./.github/actions/setup-sentry id: setup with: snuba: true # Right now, we run so few bigtable related tests that the # overhead of running bigtable in all backend tests # is way smaller than the time it would take to run in its own job. bigtable: true pg-version: 14 - name: Download Codecov CLI run: | pip install --extra-index-url https://pypi.org/simple --no-cache-dir codecov-cli>=0.4.1 # Creates the commit and report objects in codecov - name: Codecov startup run: | codecovcli create-commit codecovcli create-report env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} # Sends static analysis information to codecov - name: Static Analysis run: | codecovcli static-analysis --token=${CODECOV_STATIC_TOKEN} \ --folders-to-exclude .artifacts \ --folders-to-exclude .github \ --folders-to-exclude .venv \ --folders-to-exclude static \ --folders-to-exclude bin env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }} # Run Automated Test Selection in dry mode to get the list of tests to run # The base commit will be the parent commit (apparently commits on master don't exist in codecov) - name: Codecov Automated Test Selection id: codecov_automated_test_selection run: | # Directory for the artifacts from this step mkdir .artifacts/codecov_ats # This is the base for the git diff BASE..HEAD BASE_COMMIT=$(git merge-base ${{ github.sha }}^ origin/master) # Get list of tests to run from Codecov output=$(codecovcli --codecov-yml-path=codecov.yml label-analysis --dry-run --token=${CODECOV_STATIC_TOKEN} --base-sha=${BASE_COMMIT}) || true # Post processing and validation if [ -n "${output}" ]; then jq <<< $output '.runner_options + .ats_tests_to_run | @json' --raw-output > .artifacts/codecov_ats/tests_to_run.json jq <<< $output '.runner_options + .ats_tests_to_skip | @json' --raw-output > .artifacts/codecov_ats/tests_to_skip.json testcount() { jq <<< $output ".$1 | length"; } run_count=$(testcount ats_tests_to_run) skip_count=$(testcount ats_tests_to_skip) # Parse any potential errors that made ATS fallback to running all tests # And surface them ats_fallback_reason=$(jq <<< "$output" '.ats_fallback_reason') if [ "$ats_fallback_reason" == "null" ]; then ats_success=true else ats_success=false fi tee <<< \ "{\"ats_success\": $ats_success, \"error\": $ats_fallback_reason, \"tests_to_run\": $run_count, \"tests_analyzed\": $((run_count+skip_count))}" \ "$GITHUB_STEP_SUMMARY" \ ".artifacts/codecov_ats/result.json" else # We need not forget to add the search options in the fallback command, otherwise pytest might run more tests than expected # These search options match what's defined in codecov.yml:105 jq '@json' --raw-output <<< '[ "--cov-context=test", "tests/sentry", "tests/integration", "--ignore=tests/sentry/eventstream/kafka", "--ignore=tests/sentry/post_process_forwarder", "--ignore=tests/sentry/snuba", "--ignore=tests/sentry/search/events", "--ignore=tests/sentry/ingest/ingest_consumer/test_ingest_consumer_kafka.py", "--ignore=tests/sentry/region_to_control/test_region_to_control_kafka.py" ]' > .artifacts/codecov_ats/tests_to_skip.json echo '[]' > .artifacts/codecov_ats/tests_to_run.json # If we reached this point it means that ATS failed with some error tee <<< '{"ats_success": false, "error": "exception_raised"}' "$GITHUB_STEP_SUMMARY" ".artifacts/codecov_ats/result.json" fi env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }} - uses: actions/upload-artifact@v3 with: name: codecov_ats path: .artifacts/codecov_ats if-no-files-found: error # The actual running of tests would come here, after the labels are available # Something like pytest $ATS_TESTS_TO_RUN debug: runs-on: ubuntu-latest needs: - coverage-ats - files-changed # Avoids running this job if it's a frontend change # It would fail if the coverage-ats step didn't run if: needs.files-changed.outputs.backend == 'true' steps: - uses: actions/download-artifact@v3 with: name: codecov_ats path: .artifacts - name: Debug ATS_TESTS_TO_RUN run: | length_of_tests=$(cat .artifacts/tests_to_run.json | jq 'length') # The 1st value doesn't count, it's '--cov-context=test' (hence -gt 1) if [ $length_of_tests -gt 1 ]; then echo "Running $length_of_tests tests" # --raw-output0 doesn't work. cat .artifacts/tests_to_run.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo 'pytest' else echo "No tests to run" fi - name: Debug ATS_TESTS_TO_SKIP run: | length_of_tests=$(cat .artifacts/tests_to_skip.json | jq 'length') # The 1st value doesn't count, it's '--cov-context=test' if [ $length_of_tests -gt 1 ]; then echo "Running $length_of_tests tests" # --raw-output0 doesn't work. cat .artifacts/tests_to_skip.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo 'pytest' else echo "No tests to run" fi