test: Only tests marked @pytest.mark.snuba_ci run in Snuba CI (#56174)

It is no longer sustainable to run every single Sentry test module that
happens to use Snuba in Snuba's CI. Vast majority of them don't even
query Snuba in any way. Only a small subset of tests marked
@pytest.mark.snuba_ci will be run going forward.

.github/workflows/backend.yml

@@ -74,13 +74,13 @@ jobs:
       fail-fast: false
       matrix:
         # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
-        instance: [0, 1, 2, 3, 4, 5, 6]
+        instance: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
         pg-version: ['14']
 
     env:
       # XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`.
       # If this increases, make sure to also increase `flags.backend.after_n_builds` in `codecov.yml`.
-      MATRIX_INSTANCE_TOTAL: 7
+      MATRIX_INSTANCE_TOTAL: 11
 
     steps:
       - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
@@ -93,6 +93,7 @@ jobs:
         uses: ./.github/actions/setup-sentry
         id: setup
         with:
+          kafka: true
           snuba: true
           # Right now, we run so few bigtable related tests that the
           # overhead of running bigtable in all backend tests
@@ -298,49 +299,6 @@ jobs:
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
-  snuba:
-    if: needs.files-changed.outputs.backend == 'true'
-    needs: files-changed
-    name: snuba test
-    runs-on: ubuntu-20.04
-    timeout-minutes: 45
-    strategy:
-      # This helps not having to run multiple jobs because one fails, thus, reducing resource usage
-      # and reducing the risk that one of many runs would turn red again (read: intermittent tests)
-      fail-fast: false
-      matrix:
-        # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
-        instance: [0, 1, 2, 3]
-    env:
-      # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
-      MATRIX_INSTANCE_TOTAL: 4
-
-    steps:
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
-        with:
-          # Avoid codecov error message related to SHA resolution:
-          # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891
-          fetch-depth: '2'
-
-      - name: Setup sentry env
-        uses: ./.github/actions/setup-sentry
-        id: setup
-        with:
-          snuba: true
-          kafka: true
-
-      - name: Run snuba test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
-        run: |
-          make test-snuba
-
-      # Upload coverage data even if running the tests step fails since
-      # it reduces large coverage fluctuations
-      - name: Handle artifacts
-        if: ${{ always() }}
-        uses: ./.github/actions/artifacts
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-
   symbolicator:
     if: needs.files-changed.outputs.backend == 'true'
     needs: files-changed
@@ -442,7 +400,6 @@ jobs:
         migration,
         plugins,
         relay,
-        snuba,
         symbolicator,
         typing,
       ]

Makefile

@@ -128,26 +128,20 @@ test-js-ci: node-version-check
 test-python-ci: create-db
 	@echo "--> Running CI Python tests"
 	pytest tests/integration tests/sentry \
-		--ignore tests/sentry/eventstream/kafka \
-		--ignore tests/sentry/post_process_forwarder \
-		--ignore tests/sentry/snuba \
-		--ignore tests/sentry/search/events \
 		--ignore tests/sentry/ingest/ingest_consumer/test_ingest_consumer_kafka.py \
 		--ignore tests/sentry/region_to_control/test_region_to_control_kafka.py \
 		--cov . --cov-report="xml:.artifacts/python.coverage.xml"
 	@echo ""
 
+
 test-snuba: create-db
 	@echo "--> Running snuba tests"
-	pytest tests/snuba \
-		tests/sentry/eventstream/kafka \
-		tests/sentry/post_process_forwarder \
-		tests/sentry/snuba \
-		tests/sentry/search/events \
-		tests/sentry/event_manager \
+	pytest tests \
+		-m snuba_ci \
 		-vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml"
 	@echo ""
 
+
 test-tools:
 	@echo "--> Running tools tests"
 	pytest -c /dev/null --confcutdir tests/tools tests/tools -vv --cov=tools --cov=tests/tools --cov-report="xml:.artifacts/tools.coverage.xml"

pyproject.toml

@@ -23,6 +23,7 @@ addopts = "-ra --tb=short --strict-markers -p no:celery"
 # currently we have a few relative imports that don't work with that.
 markers = [
     "snuba: test requires access to snuba",
+    "snuba_ci: test is run in snuba ci",
     "sentry_metrics: test requires access to sentry metrics",
     "symbolicator: test requires access to symbolicator",
 ]

src/sentry/post_process_forwarder/post_process_forwarder.py

@@ -5,6 +5,7 @@ from abc import ABC, abstractmethod
 from enum import Enum
 from typing import Any, Literal, Mapping, Optional, Union
 
+import pytest
 from arroyo import configure_metrics
 from arroyo.backends.kafka import KafkaConsumer, KafkaPayload
 from arroyo.backends.kafka.configuration import build_kafka_consumer_configuration
@@ -33,6 +34,7 @@ class PostProcessForwarderType(str, Enum):
     ISSUE_PLATFORM = "search_issues"
 
 
+@pytest.mark.snuba_ci
 class PostProcessForwarder:
     """
     The `dispatch_function` should take a message and dispatch the post_process_group

tests/sentry/snuba/test_query_subscription_consumer.py

@@ -26,6 +26,7 @@ from sentry.testutils.cases import TestCase
 from sentry.utils import json
 
 
+@pytest.mark.snuba_ci
 class BaseQuerySubscriptionTest:
     @cached_property
     def topic(self):

tests/sentry/snuba/test_subscriptions.py

@@ -17,6 +17,7 @@ from sentry.testutils.cases import TestCase
 pytestmark = pytest.mark.sentry_metrics
 
 
+@pytest.mark.snuba_ci
 class CreateSnubaQueryTest(TestCase):
     def test(self):
         query_type = SnubaQuery.Type.ERROR

tests/snuba/rules/conditions/test_event_frequency.py

@@ -4,6 +4,7 @@ from datetime import timedelta, timezone
 from unittest.mock import patch
 from uuid import uuid4
 
+import pytest
 from django.utils.timezone import now
 from freezegun import freeze_time
 
@@ -20,6 +21,7 @@ from sentry.testutils.silo import region_silo_test
 from sentry.utils.samples import load_data
 
 
+@pytest.mark.snuba_ci
 class FrequencyConditionMixin:
     def increment(self, event, count, environment=None, timestamp=None):
         raise NotImplementedError