Removed cardinality limiting from indexer (#67535)

<!-- Describe your PR here. -->
[Relay already applies cardinality
limits](https://github.com/getsentry/relay/issues/2717), so we can
safely remove cardinality limiting from the indexer
<!--

  Sentry employees and contractors can delete or ignore the following.

-->

### Legal Boilerplate

Look, I get it. The entity doing business as "Sentry" was incorporated
in the State of Delaware in 2015 as Functional Software, Inc. and is
gonna need some rights from me in order to utilize my contributions in
this here PR. So here's the deal: I retain all rights, title and
interest in and to my contributions, and by keeping this boilerplate
intact I confirm that Sentry can use, modify, copy, and redistribute my
contributions, under Sentry's choice of terms.

---------

Co-authored-by: Rachel Chen <rachelchen@PL6VFX9HP4.attlocal.net>

src/sentry/sentry_metrics/consumers/indexer/processing.py

@@ -22,7 +22,6 @@ from sentry.sentry_metrics.consumers.indexer.tags_validator import (
     ReleaseHealthTagsValidator,
 )
 from sentry.sentry_metrics.indexer.base import StringIndexer
-from sentry.sentry_metrics.indexer.limiters.cardinality import cardinality_limiter_factory
 from sentry.sentry_metrics.indexer.mock import MockIndexer
 from sentry.sentry_metrics.indexer.postgres.postgres_v2 import PostgresIndexer
 from sentry.utils import metrics, sdk
@@ -126,19 +125,6 @@ class MessageProcessor:
 
         sdk.set_measurement("indexer_batch.payloads.len", len(batch.parsed_payloads_by_meta))
 
-        with metrics.timer("metrics_consumer.check_cardinality_limits"), sentry_sdk.start_span(
-            op="check_cardinality_limits"
-        ):
-            cardinality_limiter = cardinality_limiter_factory.get_ratelimiter(self._config)
-            cardinality_limiter_state = cardinality_limiter.check_cardinality_limits(
-                self._config.use_case_id, batch.parsed_payloads_by_meta
-            )
-
-        sdk.set_measurement(
-            "cardinality_limiter.keys_to_remove.len", len(cardinality_limiter_state.keys_to_remove)
-        )
-        batch.filter_messages(cardinality_limiter_state.keys_to_remove)
-
         extracted_strings = batch.extract_strings()
 
         sdk.set_measurement("org_strings.len", len(extracted_strings))
@@ -153,10 +139,4 @@ class MessageProcessor:
 
         sdk.set_measurement("new_messages.len", len(results.data))
 
-        with metrics.timer("metrics_consumer.apply_cardinality_limits"), sentry_sdk.start_span(
-            op="apply_cardinality_limits"
-        ):
-            # TODO: move to separate thread
-            cardinality_limiter.apply_cardinality_limits(cardinality_limiter_state)
-
         return results

src/sentry/sentry_metrics/indexer/limiters/cardinality.py

@@ -1,208 +0,0 @@
-from __future__ import annotations
-
-import dataclasses
-from collections import defaultdict
-from collections.abc import Mapping, MutableMapping, Sequence
-from typing import TypedDict
-
-from sentry import options
-from sentry.ratelimits.cardinality import (
-    CardinalityLimiter,
-    GrantedQuota,
-    Quota,
-    RedisCardinalityLimiter,
-    RequestedQuota,
-    Timestamp,
-)
-from sentry.sentry_metrics.configuration import MetricsIngestConfiguration, UseCaseKey
-from sentry.sentry_metrics.consumers.indexer.common import BrokerMeta
-from sentry.sentry_metrics.use_case_id_registry import (
-    USE_CASE_ID_CARDINALITY_LIMIT_QUOTA_OPTIONS,
-    UseCaseID,
-)
-from sentry.utils import metrics
-from sentry.utils.hashlib import hash_values
-from sentry.utils.options import sample_modulo
-
-OrgId = int
-
-
-@dataclasses.dataclass(frozen=True)
-class CardinalityLimiterState:
-    _cardinality_limiter: CardinalityLimiter
-    _metric_path_key: UseCaseKey
-    _grants: Sequence[GrantedQuota] | None
-    _timestamp: Timestamp | None
-    keys_to_remove: Sequence[BrokerMeta]
-
-
-def _build_quota_key(use_case_id: UseCaseID, org_id: OrgId) -> str:
-    return f"metrics-indexer-cardinality-{use_case_id.value}-org-{org_id}"
-
-
-@metrics.wraps("sentry_metrics.indexer.construct_quotas")
-def _construct_quotas(use_case_id: UseCaseID) -> Quota | None:
-    """
-    Construct write limit's quotas based on current sentry options.
-
-    This value can potentially cached globally as long as it is invalidated
-    when sentry.options are.
-    """
-
-    if use_case_id in USE_CASE_ID_CARDINALITY_LIMIT_QUOTA_OPTIONS:
-        quota_args = options.get(USE_CASE_ID_CARDINALITY_LIMIT_QUOTA_OPTIONS[use_case_id])
-
-    else:
-        quota_args = options.get(
-            "sentry-metrics.cardinality-limiter.limits.generic-metrics.per-org"
-        )
-
-    if not quota_args:
-        raise ValueError("quotas cannot be empty")
-    if len(quota_args) > 1:
-        raise ValueError("multiple quotas are actually unsupported")
-
-    return Quota(**quota_args[0])
-
-
-class InboundMessage(TypedDict):
-    # Note: This is only the subset of fields we access in this file.
-    org_id: int
-    name: str
-    tags: dict[str, str]
-    # now that all messages are getting a use_case_id
-    # field via message processing, we can add it here
-    use_case_id: UseCaseID
-
-
-class TimeseriesCardinalityLimiter:
-    def __init__(self, namespace: str, rate_limiter: CardinalityLimiter) -> None:
-        self.namespace = namespace
-        self.backend: CardinalityLimiter = rate_limiter
-
-    def check_cardinality_limits(
-        self, metric_path_key: UseCaseKey, messages: Mapping[BrokerMeta, InboundMessage]
-    ) -> CardinalityLimiterState:
-        request_hashes = defaultdict(set)
-        hash_to_meta: Mapping[str, dict[int, BrokerMeta]] = defaultdict(dict)
-        prefix_to_quota = {}
-
-        # this works by applying one cardinality limiter rollout option
-        # for each metric path. ultimately, this can be moved into the
-        # loop below to make rollout options occur on a per use case-basis
-        rollout_option = {
-            UseCaseKey.PERFORMANCE: "sentry-metrics.cardinality-limiter.orgs-rollout-rate",
-            UseCaseKey.RELEASE_HEALTH: "sentry-metrics.cardinality-limiter-rh.orgs-rollout-rate",
-        }[metric_path_key]
-
-        for key, message in messages.items():
-            org_id = message["org_id"]
-            if not sample_modulo(rollout_option, org_id):
-                continue
-
-            message_hash = int(
-                hash_values(
-                    [
-                        message["name"],
-                        message["tags"],
-                    ]
-                ),
-                16,
-            )
-            prefix = _build_quota_key(message["use_case_id"], org_id)
-            hash_to_meta[prefix][message_hash] = key
-            request_hashes[prefix].add(message_hash)
-            configured_quota = _construct_quotas(message["use_case_id"])
-
-            # since we might have some use cases that are covered by
-            # a quota and some that are not, only add entries that
-            # are covered by a quota
-            if configured_quota is not None:
-                prefix_to_quota[prefix] = configured_quota
-
-        requested_quotas = []
-
-        grants = None
-        timestamp = None
-
-        # if none of the use cases are covered by a quota
-        if len(prefix_to_quota) == 0:
-            return CardinalityLimiterState(
-                _cardinality_limiter=self.backend,
-                _metric_path_key=metric_path_key,
-                _grants=grants,
-                _timestamp=timestamp,
-                keys_to_remove=[],
-            )
-
-        for prefix, hashes in request_hashes.items():
-            quota = prefix_to_quota.get(prefix)
-
-            if quota is not None:
-                requested_quotas.append(
-                    RequestedQuota(prefix=prefix, unit_hashes=hashes, quota=quota)
-                )
-
-        timestamp, grants = self.backend.check_within_quotas(requested_quotas)
-
-        keys_to_remove = hash_to_meta
-        # make sure that hash_to_broker_meta is no longer used, as the underlying
-        # dict will be mutated
-        del hash_to_meta
-
-        for grant in grants:
-            for hash in grant.granted_unit_hashes:
-                del keys_to_remove[grant.request.prefix][hash]
-
-            substrings = grant.request.prefix.split("-")
-            grant_use_case_id = substrings[3]
-
-            metrics.incr(
-                "sentry_metrics.indexer.process_messages.dropped_message",
-                amount=len(keys_to_remove[grant.request.prefix]),
-                tags={
-                    "reason": "cardinality_limit",
-                    "use_case_id": grant_use_case_id,
-                },
-            )
-
-        return CardinalityLimiterState(
-            _cardinality_limiter=self.backend,
-            _metric_path_key=metric_path_key,
-            _grants=grants,
-            _timestamp=timestamp,
-            keys_to_remove=[key for grant in keys_to_remove.values() for key in grant.values()],
-        )
-
-    def apply_cardinality_limits(self, state: CardinalityLimiterState) -> None:
-        if state._grants is not None and state._timestamp is not None:
-            state._cardinality_limiter.use_quotas(state._grants, state._timestamp)
-
-
-class TimeseriesCardinalityLimiterFactory:
-    """
-    The TimeseriesCardinalityLimiterFactory is in charge of initializing the
-    TimeseriesCardinalityLimiter based on a configuration's namespace and
-    options. Ideally this logic would live in the initialization of the
-    backends (postgres, etc) but since each backend supports
-    multiple use cases dynamically we just keep the mapping of rate limiters in
-    this factory.
-
-    [Copied from sentry.sentry_metrics.indexer.limiters.writes]
-    """
-
-    def __init__(self) -> None:
-        self.rate_limiters: MutableMapping[str, TimeseriesCardinalityLimiter] = {}
-
-    def get_ratelimiter(self, config: MetricsIngestConfiguration) -> TimeseriesCardinalityLimiter:
-        namespace = config.cardinality_limiter_namespace
-        if namespace not in self.rate_limiters:
-            limiter = TimeseriesCardinalityLimiter(
-                namespace, RedisCardinalityLimiter(**config.cardinality_limiter_cluster_options)
-            )
-            self.rate_limiters[namespace] = limiter
-
-        return self.rate_limiters[namespace]
-
-
-cardinality_limiter_factory = TimeseriesCardinalityLimiterFactory()

tests/sentry/sentry_metrics/limiters/test_cardinality_limiter.py

@@ -1,457 +0,0 @@
-import time
-from collections.abc import Sequence
-
-import pytest
-from arroyo import Partition, Topic
-
-from sentry.ratelimits.cardinality import (
-    CardinalityLimiter,
-    GrantedQuota,
-    RequestedQuota,
-    Timestamp,
-)
-from sentry.sentry_metrics.configuration import UseCaseKey
-from sentry.sentry_metrics.consumers.indexer.common import BrokerMeta
-from sentry.sentry_metrics.indexer.limiters.cardinality import (
-    TimeseriesCardinalityLimiter,
-    _build_quota_key,
-)
-from sentry.sentry_metrics.use_case_id_registry import UseCaseID
-from sentry.testutils.helpers.options import override_options
-
-
-@pytest.fixture(autouse=True)
-def rollout_all_orgs_generic_metrics(set_sentry_option):
-    with set_sentry_option("sentry-metrics.cardinality-limiter.orgs-rollout-rate", 1.0):
-        yield
-
-
-class MockCardinalityLimiter(CardinalityLimiter):
-    def __init__(self):
-        self.grant_hashes = {}
-        # self.assert_quota: Optional[Quota] = None
-        self.assert_requests: Sequence[RequestedQuota] | None = None
-
-    def check_within_quotas(
-        self, requests: Sequence[RequestedQuota], timestamp: Timestamp | None = None
-    ) -> tuple[Timestamp, Sequence[GrantedQuota]]:
-        if timestamp is None:
-            timestamp = int(time.time())
-        else:
-            timestamp = int(timestamp)
-
-        if self.assert_requests is not None:
-            assert requests == self.assert_requests
-
-        grants = []
-        granted = {request.prefix: 0 for request in requests}
-
-        for request in requests:
-            # assert request.quota == self.assert_quota
-            prefix = request.prefix
-
-            granted_hashes = set()
-            for hash in request.unit_hashes:
-                if granted[prefix] < self.grant_hashes[prefix]:
-                    granted[prefix] += 1
-                    granted_hashes.add(hash)
-
-            # reached_quotas is incorrect, but we don't necessarily need it for testing
-            grants.append(
-                GrantedQuota(
-                    request=request, granted_unit_hashes=granted_hashes, reached_quota=None
-                )
-            )
-
-        return timestamp, grants
-
-    def use_quotas(
-        self,
-        grants: Sequence[GrantedQuota],
-        timestamp: Timestamp,
-    ) -> None:
-        pass
-
-
-def test_reject_all():
-    with override_options(
-        {
-            "sentry-metrics.cardinality-limiter.limits.performance.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.spans.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.custom.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}
-            ],
-        },
-    ):
-        backend = MockCardinalityLimiter()
-        backend.grant_hashes = {
-            _build_quota_key(UseCaseID.TRANSACTIONS, 1): 0,
-            _build_quota_key(UseCaseID.SPANS, 1): 0,
-            _build_quota_key(UseCaseID.CUSTOM, 1): 0,
-        }
-
-        limiter = TimeseriesCardinalityLimiter("", backend)
-
-        result = limiter.check_cardinality_limits(
-            UseCaseKey.PERFORMANCE,
-            {
-                BrokerMeta(Partition(Topic("topic"), 0), 0): {
-                    "org_id": 1,
-                    "name": "foo",
-                    "tags": {},
-                    "use_case_id": UseCaseID.TRANSACTIONS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 1): {
-                    "org_id": 1,
-                    "name": "bar",
-                    "tags": {},
-                    "use_case_id": UseCaseID.SPANS,
-                },
-            },
-        )
-
-        assert result.keys_to_remove == [
-            BrokerMeta(Partition(Topic("topic"), 0), 0),
-            BrokerMeta(Partition(Topic("topic"), 0), 1),
-        ]
-
-
-def test_reject_all_with_default():
-    with override_options(
-        {
-            "sentry-metrics.cardinality-limiter.limits.performance.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.spans.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.custom.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.generic-metrics.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}
-            ],
-        },
-    ):
-        backend = MockCardinalityLimiter()
-        backend.grant_hashes = {
-            _build_quota_key(UseCaseID.TRANSACTIONS, 1): 0,
-            _build_quota_key(UseCaseID.SPANS, 1): 0,
-            _build_quota_key(UseCaseID.CUSTOM, 1): 0,
-            _build_quota_key(UseCaseID.ESCALATING_ISSUES, 1): 0,
-        }
-
-        # backend.grant_hashes = 0
-        limiter = TimeseriesCardinalityLimiter("", backend)
-
-        result = limiter.check_cardinality_limits(
-            UseCaseKey.PERFORMANCE,
-            {
-                BrokerMeta(Partition(Topic("topic"), 0), 0): {
-                    "org_id": 1,
-                    "name": "foo",
-                    "tags": {},
-                    "use_case_id": UseCaseID.TRANSACTIONS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 1): {
-                    "org_id": 1,
-                    "name": "bar",
-                    "tags": {},
-                    "use_case_id": UseCaseID.SPANS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 2): {
-                    "org_id": 1,
-                    "name": "boo",
-                    "tags": {},
-                    "use_case_id": UseCaseID.ESCALATING_ISSUES,
-                },
-            },
-        )
-
-        assert result.keys_to_remove == [
-            BrokerMeta(Partition(Topic("topic"), 0), 0),
-            BrokerMeta(Partition(Topic("topic"), 0), 1),
-            BrokerMeta(Partition(Topic("topic"), 0), 2),
-        ]
-
-
-def test_reject_partial():
-    with override_options(
-        {
-            "sentry-metrics.cardinality-limiter.limits.performance.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 2}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.spans.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.custom.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}
-            ],
-        },
-    ):
-        backend = MockCardinalityLimiter()
-        backend.grant_hashes = {
-            _build_quota_key(UseCaseID.TRANSACTIONS, 1): 2,
-            _build_quota_key(UseCaseID.SPANS, 1): 0,
-            _build_quota_key(UseCaseID.CUSTOM, 1): 0,
-        }
-        limiter = TimeseriesCardinalityLimiter("", backend)
-
-        result = limiter.check_cardinality_limits(
-            UseCaseKey.PERFORMANCE,
-            {
-                BrokerMeta(Partition(Topic("topic"), 0), 0): {
-                    "org_id": 1,
-                    "name": "foo",
-                    "tags": {},
-                    "use_case_id": UseCaseID.TRANSACTIONS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 1): {
-                    "org_id": 1,
-                    "name": "bar",
-                    "tags": {},
-                    "use_case_id": UseCaseID.TRANSACTIONS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 2): {
-                    "org_id": 1,
-                    "name": "baz",
-                    "tags": {},
-                    "use_case_id": UseCaseID.SPANS,
-                },
-            },
-        )
-
-        assert result.keys_to_remove == [BrokerMeta(Partition(Topic("topic"), 0), 2)]
-
-
-def test_reject_partial_again():
-    with override_options(
-        {
-            "sentry-metrics.cardinality-limiter.limits.performance.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 2}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.spans.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 2}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.custom.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}
-            ],
-        },
-    ):
-        backend = MockCardinalityLimiter()
-        backend.grant_hashes = {
-            _build_quota_key(UseCaseID.TRANSACTIONS, 1): 2,
-            _build_quota_key(UseCaseID.SPANS, 1): 2,
-            _build_quota_key(UseCaseID.CUSTOM, 1): 0,
-        }
-        limiter = TimeseriesCardinalityLimiter("", backend)
-
-        result = limiter.check_cardinality_limits(
-            UseCaseKey.PERFORMANCE,
-            {
-                BrokerMeta(Partition(Topic("topic"), 0), 0): {
-                    "org_id": 1,
-                    "name": "foo",
-                    "tags": {},
-                    "use_case_id": UseCaseID.TRANSACTIONS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 1): {
-                    "org_id": 1,
-                    "name": "bar",
-                    "tags": {},
-                    "use_case_id": UseCaseID.TRANSACTIONS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 2): {
-                    "org_id": 1,
-                    "name": "baz",
-                    "tags": {},
-                    "use_case_id": UseCaseID.SPANS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 3): {
-                    "org_id": 1,
-                    "name": "boo",
-                    "tags": {},
-                    "use_case_id": UseCaseID.CUSTOM,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 4): {
-                    "org_id": 1,
-                    "name": "bye",
-                    "tags": {},
-                    "use_case_id": UseCaseID.SPANS,
-                },
-            },
-        )
-
-        assert result.keys_to_remove == [BrokerMeta(Partition(Topic("topic"), 0), 3)]
-
-
-def test_accept_all():
-    with override_options(
-        {
-            "sentry-metrics.cardinality-limiter.limits.performance.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 100}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.spans.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 100}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.custom.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 100}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.generic-metrics.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 100}
-            ],
-        },
-    ):
-        backend = MockCardinalityLimiter()
-        backend.grant_hashes = {
-            _build_quota_key(UseCaseID.TRANSACTIONS, 1): 100,
-            _build_quota_key(UseCaseID.SPANS, 1): 100,
-            _build_quota_key(UseCaseID.CUSTOM, 1): 100,
-            _build_quota_key(UseCaseID.ESCALATING_ISSUES, 1): 100,
-        }
-        limiter = TimeseriesCardinalityLimiter("", backend)
-
-        result = limiter.check_cardinality_limits(
-            UseCaseKey.PERFORMANCE,
-            {
-                BrokerMeta(Partition(Topic("topic"), 0), 0): {
-                    "org_id": 1,
-                    "name": "foo",
-                    "tags": {},
-                    "use_case_id": UseCaseID.TRANSACTIONS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 1): {
-                    "org_id": 1,
-                    "name": "bar",
-                    "tags": {},
-                    "use_case_id": UseCaseID.TRANSACTIONS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 2): {
-                    "org_id": 1,
-                    "name": "baz",
-                    "tags": {},
-                    "use_case_id": UseCaseID.SPANS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 3): {
-                    "org_id": 1,
-                    "name": "bazz",
-                    "tags": {},
-                    "use_case_id": UseCaseID.ESCALATING_ISSUES,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 4): {
-                    "org_id": 1,
-                    "name": "bye",
-                    "tags": {},
-                    "use_case_id": UseCaseID.CUSTOM,
-                },
-            },
-        )
-
-        assert not result.keys_to_remove
-
-
-def test_sample_rate_zero(set_sentry_option):
-    """
-    Assert that with a rollout rate of zero, no quotas are applied.
-    """
-    with override_options(
-        {
-            "sentry-metrics.cardinality-limiter.limits.performance.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 10}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.spans.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 10}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.custom.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 10}
-            ],
-        },
-    ), set_sentry_option("sentry-metrics.cardinality-limiter.orgs-rollout-rate", 0.0):
-        backend = MockCardinalityLimiter()
-        backend.grant_hashes = {
-            _build_quota_key(UseCaseID.TRANSACTIONS, 1): 10,
-            _build_quota_key(UseCaseID.SPANS, 1): 10,
-            _build_quota_key(UseCaseID.CUSTOM, 1): 10,
-        }
-        limiter = TimeseriesCardinalityLimiter("", backend)
-
-        result = limiter.check_cardinality_limits(
-            UseCaseKey.PERFORMANCE,
-            {
-                BrokerMeta(Partition(Topic("topic"), 0), 0): {
-                    "org_id": 1,
-                    "name": "foo",
-                    "tags": {},
-                    "use_case_id": UseCaseID.TRANSACTIONS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 1): {
-                    "org_id": 1,
-                    "name": "bar",
-                    "tags": {},
-                    "use_case_id": UseCaseID.SPANS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 2): {
-                    "org_id": 1,
-                    "name": "baz",
-                    "tags": {},
-                    "use_case_id": UseCaseID.CUSTOM,
-                },
-            },
-        )
-
-        assert not result.keys_to_remove
-
-        # Right now we do not call the limiter with an empty list of requests.
-        # Hence, `_grants` is `None`.
-        assert result._grants is None
-
-
-def test_sample_rate_half(set_sentry_option):
-    with override_options(
-        {
-            "sentry-metrics.cardinality-limiter.limits.performance.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 2}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.spans.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 1}
-            ],
-            "sentry-metrics.cardinality-limiter.limits.custom.per-org": [
-                {"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}
-            ],
-        },
-    ), set_sentry_option("sentry-metrics.cardinality-limiter.orgs-rollout-rate", 0.5):
-
-        backend = MockCardinalityLimiter()
-        backend.grant_hashes = {
-            _build_quota_key(UseCaseID.TRANSACTIONS, 1): 0,
-            _build_quota_key(UseCaseID.SPANS, 1): 0,
-            _build_quota_key(UseCaseID.CUSTOM, 1): 0,
-        }
-        # backend.assert_quota = Quota(window_seconds=3600, granularity_seconds=60, limit=0)
-        limiter = TimeseriesCardinalityLimiter("", backend)
-
-        result = limiter.check_cardinality_limits(
-            UseCaseKey.PERFORMANCE,
-            {
-                BrokerMeta(Partition(Topic("topic"), 0), 0): {
-                    "org_id": 1,
-                    "name": "foo",
-                    "tags": {},
-                    "use_case_id": UseCaseID.TRANSACTIONS,
-                },
-                BrokerMeta(Partition(Topic("topic"), 0), 1): {
-                    "org_id": 99,
-                    "name": "bar",
-                    "tags": {},
-                    "use_case_id": UseCaseID.TRANSACTIONS,
-                },
-            },
-        )
-
-        # We are sampling org_id=1 into cardinality limiting. Because our quota is
-        # zero, only that org's metrics are dropped.
-        assert result.keys_to_remove == [BrokerMeta(Partition(Topic("topic"), 0), 0)]

tests/sentry/sentry_metrics/test_gen_metrics_multiprocess_steps.py

@@ -16,7 +16,6 @@ from arroyo.dlq import InvalidMessage
 from arroyo.processing.strategies import MessageRejected
 from arroyo.types import BrokerValue, Message, Partition, Topic, Value
 
-from sentry.ratelimits.cardinality import CardinalityLimiter
 from sentry.sentry_metrics.aggregation_option_registry import get_aggregation_options
 from sentry.sentry_metrics.configuration import IndexerStorage, UseCaseKey, get_ingest_config
 from sentry.sentry_metrics.consumers.indexer.batch import valid_metric_name
@@ -26,10 +25,6 @@ from sentry.sentry_metrics.consumers.indexer.common import (
     MetricsBatchBuilder,
 )
 from sentry.sentry_metrics.consumers.indexer.processing import MessageProcessor
-from sentry.sentry_metrics.indexer.limiters.cardinality import (
-    TimeseriesCardinalityLimiter,
-    cardinality_limiter_factory,
-)
 from sentry.sentry_metrics.indexer.mock import MockIndexer, RawSimpleIndexer
 from sentry.sentry_metrics.use_case_id_registry import UseCaseID
 from sentry.utils import json
@@ -590,60 +585,6 @@ def test_process_messages_rate_limited(caplog: Any, settings: Any) -> None:
     assert "dropped_message" in caplog.text
 
 
-@pytest.mark.django_db
-def test_process_messages_cardinality_limited(
-    caplog: Any, settings: Any, monkeypatch: Any, set_sentry_option: Callable[..., Any]
-) -> None:
-    """
-    Test that the message processor correctly calls the cardinality limiter.
-    """
-    settings.SENTRY_METRICS_INDEXER_DEBUG_LOG_SAMPLE_RATE = 1.0
-
-    # set any limit at all to ensure we actually use the underlying rate limiter
-    with (
-        set_sentry_option(
-            "sentry-metrics.cardinality-limiter.limits.performance.per-org",
-            [{"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}],
-        ),
-        set_sentry_option("sentry-metrics.cardinality-limiter.orgs-rollout-rate", 1.0),
-    ):
-
-        class MockCardinalityLimiter(CardinalityLimiter):
-            def check_within_quotas(self, requested_quotas):
-                # Grant nothing, limit everything
-                return 123, []
-
-            def use_quotas(self, grants, timestamp):
-                pass
-
-        monkeypatch.setitem(
-            cardinality_limiter_factory.rate_limiters,
-            "performance",
-            TimeseriesCardinalityLimiter("performance", MockCardinalityLimiter()),
-        )
-
-        message_payloads = counter_payloads + distribution_payloads + set_payloads
-        message_batch = [
-            Message(
-                BrokerValue(
-                    KafkaPayload(None, json.dumps(payload).encode("utf-8"), []),
-                    Partition(Topic("topic"), 0),
-                    i + 1,
-                    datetime.now(),
-                )
-            )
-            for i, payload in enumerate(message_payloads)
-        ]
-
-        last = message_batch[-1]
-        outer_message = Message(Value(message_batch, last.committable))
-
-        with caplog.at_level(logging.ERROR):
-            new_batch = MESSAGE_PROCESSOR.process_messages(outer_message=outer_message)
-
-        compare_message_batches_ignoring_metadata(new_batch, [])
-
-
 def test_valid_metric_name() -> None:
     assert valid_metric_name("") is True
     assert valid_metric_name("blah") is True

tests/sentry/sentry_metrics/test_rh_metrics_multiprocess_steps.py

@@ -15,7 +15,6 @@ from arroyo.dlq import InvalidMessage
 from arroyo.processing.strategies import MessageRejected
 from arroyo.types import BrokerValue, Message, Partition, Topic, Value
 
-from sentry.ratelimits.cardinality import CardinalityLimiter
 from sentry.sentry_metrics.configuration import IndexerStorage, UseCaseKey, get_ingest_config
 from sentry.sentry_metrics.consumers.indexer.batch import valid_metric_name
 from sentry.sentry_metrics.consumers.indexer.common import (
@@ -24,10 +23,6 @@ from sentry.sentry_metrics.consumers.indexer.common import (
     MetricsBatchBuilder,
 )
 from sentry.sentry_metrics.consumers.indexer.processing import MessageProcessor
-from sentry.sentry_metrics.indexer.limiters.cardinality import (
-    TimeseriesCardinalityLimiter,
-    cardinality_limiter_factory,
-)
 from sentry.sentry_metrics.indexer.mock import MockIndexer, RawSimpleIndexer
 from sentry.sentry_metrics.use_case_id_registry import UseCaseID
 from sentry.snuba.metrics.naming_layer.mri import SessionMRI
@@ -530,57 +525,6 @@ def test_process_messages_rate_limited(caplog, settings) -> None:
     assert "dropped_message" in caplog.text
 
 
-@pytest.mark.django_db
-def test_process_messages_cardinality_limited(
-    caplog, settings, monkeypatch, set_sentry_option
-) -> None:
-    """
-    Test that the message processor correctly calls the cardinality limiter.
-    """
-    settings.SENTRY_METRICS_INDEXER_DEBUG_LOG_SAMPLE_RATE = 1.0
-
-    # set any limit at all to ensure we actually use the underlying rate limiter
-    with set_sentry_option(
-        "sentry-metrics.cardinality-limiter.limits.releasehealth.per-org",
-        [{"window_seconds": 3600, "granularity_seconds": 60, "limit": 0}],
-    ), set_sentry_option("sentry-metrics.cardinality-limiter-rh.orgs-rollout-rate", 1.0):
-
-        class MockCardinalityLimiter(CardinalityLimiter):
-            def check_within_quotas(self, requested_quotas):
-                # Grant nothing, limit everything
-                return 123, []
-
-            def use_quotas(self, grants, timestamp):
-                pass
-
-        monkeypatch.setitem(
-            cardinality_limiter_factory.rate_limiters,
-            "releasehealth",
-            TimeseriesCardinalityLimiter("releasehealth", MockCardinalityLimiter()),
-        )
-
-        message_payloads = [counter_payload, distribution_payload, set_payload]
-        message_batch = [
-            Message(
-                BrokerValue(
-                    KafkaPayload(None, json.dumps(payload).encode("utf-8"), []),
-                    Partition(Topic("topic"), 0),
-                    i + 1,
-                    datetime.now(),
-                )
-            )
-            for i, payload in enumerate(message_payloads)
-        ]
-
-        last = message_batch[-1]
-        outer_message = Message(Value(message_batch, last.committable))
-
-        with caplog.at_level(logging.ERROR):
-            new_batch = MESSAGE_PROCESSOR.process_messages(outer_message=outer_message)
-
-        compare_message_batches_ignoring_metadata(new_batch, [])
-
-
 def test_valid_metric_name() -> None:
     assert valid_metric_name("") is True
     assert valid_metric_name("blah") is True