ref: combine relay into the main sentry test suite (#56229)

by combining suites we can better balance the test execution and manage
actions workers more efficiently
<!-- Describe your PR here. -->

.github/workflows/backend.yml

@@ -237,46 +237,6 @@ jobs:
         run: |
           ./.github/workflows/scripts/migration-check.sh
 
-  relay:
-    if: needs.files-changed.outputs.backend == 'true'
-    needs: files-changed
-    name: relay test
-    runs-on: ubuntu-20.04
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
-        with:
-          # Avoid codecov error message related to SHA resolution:
-          # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891
-          fetch-depth: '2'
-
-      - name: Setup sentry env
-        uses: ./.github/actions/setup-sentry
-        id: setup
-        with:
-          snuba: true
-          kafka: true
-
-      - name: Pull relay image
-        run: |
-          # pull relay we'll run and kill it for each test
-          docker pull us.gcr.io/sentryio/relay:nightly
-          docker ps -a
-
-      - name: Run test
-        env:
-          SENTRY_FORCE_SILOED_TESTS: 1
-        run: |
-          make test-relay-integration
-
-      # Upload coverage data even if running the tests step fails since
-      # it reduces large coverage fluctuations
-      - name: Handle artifacts
-        if: ${{ always() }}
-        uses: ./.github/actions/artifacts
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-
   symbolicator:
     if: needs.files-changed.outputs.backend == 'true'
     needs: files-changed
@@ -376,7 +336,6 @@ jobs:
         files-changed,
         requirements,
         migration,
-        relay,
         symbolicator,
         typing,
       ]

Makefile

@@ -127,10 +127,12 @@ test-js-ci: node-version-check
 
 test-python-ci: create-db
 	@echo "--> Running CI Python tests"
-	pytest tests/integration tests/sentry tests/sentry_plugins \
+	pytest \
+		tests/integration \
 		tests/minimetrics \
-		--ignore tests/sentry/ingest/ingest_consumer/test_ingest_consumer_kafka.py \
-		--ignore tests/sentry/region_to_control/test_region_to_control_kafka.py \
+		tests/relay_integration \
+		tests/sentry \
+		tests/sentry_plugins \
 		--cov . --cov-report="xml:.artifacts/python.coverage.xml"
 	@echo ""
 
@@ -171,6 +173,7 @@ test-acceptance: node-version-check
 	@$(WEBPACK)
 	make run-acceptance
 
+# XXX: this is called by `getsentry/relay`
 test-relay-integration:
 	@echo "--> Running Relay integration tests"
 	pytest \