codecov_ats.yml 8.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187
  1. name: IGNORE ME codecov-ats
  2. on:
  3. pull_request:
  4. env:
  5. CLI_VERSION: v0.1.5
  6. # Cancel in progress workflows on pull_requests.
  7. # https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
  8. concurrency:
  9. group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  10. cancel-in-progress: true
  11. defaults:
  12. run:
  13. # the default default is:
  14. # bash --noprofile --norc -eo pipefail {0}
  15. shell: bash --noprofile --norc -eo pipefail -ux {0}
  16. jobs:
  17. files-changed:
  18. name: detect what files changed
  19. runs-on: ubuntu-20.04
  20. timeout-minutes: 3
  21. # Map a step output to a job output
  22. outputs:
  23. api_docs: ${{ steps.changes.outputs.api_docs }}
  24. backend: ${{ steps.changes.outputs.backend_all }}
  25. backend_dependencies: ${{ steps.changes.outputs.backend_dependencies }}
  26. backend_any_type: ${{ steps.changes.outputs.backend_any_type }}
  27. migration_lockfile: ${{ steps.changes.outputs.migration_lockfile }}
  28. steps:
  29. - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
  30. - name: Check for backend file changes
  31. uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
  32. id: changes
  33. with:
  34. token: ${{ github.token }}
  35. filters: .github/file-filters.yml
  36. coverage-ats:
  37. if: needs.files-changed.outputs.backend == 'true'
  38. needs: files-changed
  39. runs-on: ubuntu-latest
  40. steps:
  41. - uses: actions/checkout@v3
  42. with:
  43. # fetch-depth: 0 - Use if the BASE_COMMIT on codecov_automated_test_selection is at unknown depth
  44. # (i.e. git merge-base ${{ github.sha }}^ origin/main)
  45. # fetch-depth: 2 - Use if the BASE_COMMIT on codecov_automated_test_selection is at known depth
  46. # (i.e. git rev-parse ${{ github.sha }}^)
  47. fetch-depth: 0
  48. - name: Set up Python 3.10.10
  49. uses: actions/setup-python@v4
  50. with:
  51. python-version: "3.10.10"
  52. # We need the setup to collect the list of tests properly
  53. - name: Setup sentry env
  54. uses: ./.github/actions/setup-sentry
  55. id: setup
  56. with:
  57. snuba: true
  58. # Right now, we run so few bigtable related tests that the
  59. # overhead of running bigtable in all backend tests
  60. # is way smaller than the time it would take to run in its own job.
  61. bigtable: true
  62. pg-version: 14
  63. - name: Download Codecov CLI
  64. run: |
  65. pip install --extra-index-url https://pypi.org/simple --no-cache-dir codecov-cli>=0.4.1
  66. # Creates the commit and report objects in codecov
  67. - name: Codecov startup
  68. run: |
  69. codecovcli create-commit
  70. codecovcli create-report
  71. env:
  72. CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
  73. # Sends static analysis information to codecov
  74. - name: Static Analysis
  75. run: |
  76. codecovcli static-analysis --token=${CODECOV_STATIC_TOKEN} \
  77. --folders-to-exclude .artifacts \
  78. --folders-to-exclude .github \
  79. --folders-to-exclude .venv \
  80. --folders-to-exclude static \
  81. --folders-to-exclude bin
  82. env:
  83. CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
  84. CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }}
  85. # Run Automated Test Selection in dry mode to get the list of tests to run
  86. # The base commit will be the parent commit (apparently commits on master don't exist in codecov)
  87. - name: Codecov Automated Test Selection
  88. id: codecov_automated_test_selection
  89. run: |
  90. # Directory for the artifacts from this step
  91. mkdir .artifacts/codecov_ats
  92. # This is the base for the git diff BASE..HEAD
  93. BASE_COMMIT=$(git merge-base ${{ github.sha }}^ origin/master)
  94. # Get list of tests to run from Codecov
  95. output=$(codecovcli --codecov-yml-path=codecov.yml label-analysis --dry-run --token=${CODECOV_STATIC_TOKEN} --base-sha=${BASE_COMMIT}) || true
  96. # Post processing and validation
  97. if [ -n "${output}" ];
  98. then
  99. jq <<< $output '.runner_options + .ats_tests_to_run | @json' --raw-output > .artifacts/codecov_ats/tests_to_run.json
  100. jq <<< $output '.runner_options + .ats_tests_to_skip | @json' --raw-output > .artifacts/codecov_ats/tests_to_skip.json
  101. testcount() { jq <<< $output ".$1 | length"; }
  102. run_count=$(testcount ats_tests_to_run)
  103. skip_count=$(testcount ats_tests_to_skip)
  104. # Parse any potential errors that made ATS fallback to running all tests
  105. # And surface them
  106. ats_fallback_reason=$(jq <<< "$output" '.ats_fallback_reason')
  107. if [ "$ats_fallback_reason" == "null" ]; then
  108. ats_success=true
  109. else
  110. ats_success=false
  111. fi
  112. tee <<< \
  113. "{\"ats_success\": $ats_success, \"error\": $ats_fallback_reason, \"tests_to_run\": $run_count, \"tests_analyzed\": $((run_count+skip_count))}" \
  114. "$GITHUB_STEP_SUMMARY" \
  115. ".artifacts/codecov_ats/result.json"
  116. else
  117. # We need not forget to add the search options in the fallback command, otherwise pytest might run more tests than expected
  118. # These search options match what's defined in codecov.yml:105
  119. jq '@json' --raw-output <<< '[
  120. "--cov-context=test",
  121. "tests/sentry",
  122. "tests/integration",
  123. "--ignore=tests/sentry/eventstream/kafka",
  124. "--ignore=tests/sentry/post_process_forwarder",
  125. "--ignore=tests/sentry/snuba",
  126. "--ignore=tests/sentry/search/events",
  127. "--ignore=tests/sentry/ingest/ingest_consumer/test_ingest_consumer_kafka.py",
  128. "--ignore=tests/sentry/region_to_control/test_region_to_control_kafka.py"
  129. ]' > .artifacts/codecov_ats/tests_to_skip.json
  130. echo '[]' > .artifacts/codecov_ats/tests_to_run.json
  131. # If we reached this point it means that ATS failed with some error
  132. tee <<< '{"ats_success": false, "error": "exception_raised"}' "$GITHUB_STEP_SUMMARY" ".artifacts/codecov_ats/result.json"
  133. fi
  134. env:
  135. CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
  136. CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }}
  137. - uses: actions/upload-artifact@v3
  138. with:
  139. name: codecov_ats
  140. path: .artifacts/codecov_ats
  141. if-no-files-found: error
  142. # The actual running of tests would come here, after the labels are available
  143. # Something like pytest <options> $ATS_TESTS_TO_RUN
  144. debug:
  145. runs-on: ubuntu-latest
  146. needs:
  147. - coverage-ats
  148. - files-changed
  149. # Avoids running this job if it's a frontend change
  150. # It would fail if the coverage-ats step didn't run
  151. if: needs.files-changed.outputs.backend == 'true'
  152. steps:
  153. - uses: actions/download-artifact@v3
  154. with:
  155. name: codecov_ats
  156. path: .artifacts
  157. - name: Debug ATS_TESTS_TO_RUN
  158. run: |
  159. length_of_tests=$(cat .artifacts/tests_to_run.json | jq 'length')
  160. # The 1st value doesn't count, it's '--cov-context=test' (hence -gt 1)
  161. if [ $length_of_tests -gt 1 ]; then
  162. echo "Running $length_of_tests tests"
  163. # --raw-output0 doesn't work.
  164. cat .artifacts/tests_to_run.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo 'pytest'
  165. else
  166. echo "No tests to run"
  167. fi
  168. - name: Debug ATS_TESTS_TO_SKIP
  169. run: |
  170. length_of_tests=$(cat .artifacts/tests_to_skip.json | jq 'length')
  171. # The 1st value doesn't count, it's '--cov-context=test'
  172. if [ $length_of_tests -gt 1 ]; then
  173. echo "Running $length_of_tests tests"
  174. # --raw-output0 doesn't work.
  175. cat .artifacts/tests_to_skip.json | jq 'join("\u0000")' --raw-output | tr -d '\n' | xargs -r0 echo 'pytest'
  176. else
  177. echo "No tests to run"
  178. fi