feat(cardinality): Add cardinality limits to project configs (#63562)

requirements-base.txt

@@ -63,7 +63,7 @@ rfc3986-validator>=0.1.1
 sentry-arroyo>=2.15.3
 sentry-kafka-schemas>=0.1.38
 sentry-redis-tools>=0.1.7
-sentry-relay>=0.8.41
+sentry-relay>=0.8.44
 sentry-sdk>=1.39.2
 snuba-sdk>=2.0.20
 simplejson>=3.17.6

requirements-dev-frozen.txt

@@ -180,7 +180,7 @@ sentry-forked-django-stubs==4.2.7.post1
 sentry-forked-djangorestframework-stubs==3.14.5.post1
 sentry-kafka-schemas==0.1.38
 sentry-redis-tools==0.1.7
-sentry-relay==0.8.41
+sentry-relay==0.8.44
 sentry-sdk==1.39.2
 sentry-usage-accountant==0.0.10
 simplejson==3.17.6

requirements-frozen.txt

@@ -121,7 +121,7 @@ s3transfer==0.10.0
 sentry-arroyo==2.15.3
 sentry-kafka-schemas==0.1.38
 sentry-redis-tools==0.1.7
-sentry-relay==0.8.41
+sentry-relay==0.8.44
 sentry-sdk==1.39.2
 sentry-usage-accountant==0.0.10
 simplejson==3.17.6

src/sentry/relay/config/__init__.py

@@ -18,7 +18,7 @@ from typing import (
 import sentry_sdk
 from sentry_sdk import Hub, capture_exception
 
-from sentry import features, killswitches, quotas, utils
+from sentry import features, killswitches, options, quotas, utils
 from sentry.constants import HEALTH_CHECK_GLOBS, ObjectStatus
 from sentry.datascrubbing import get_datascrubbing_settings, get_pii_config
 from sentry.dynamic_sampling import generate_rules
@@ -44,6 +44,7 @@ from sentry.relay.config.metric_extraction import (
     get_metric_extraction_config,
 )
 from sentry.relay.utils import to_camel_case_name
+from sentry.sentry_metrics.use_case_id_registry import USE_CASE_ID_CARDINALITY_LIMIT_QUOTA_OPTIONS
 from sentry.utils import metrics
 from sentry.utils.http import get_origins
 from sentry.utils.options import sample_modulo
@@ -194,6 +195,55 @@ def get_quotas(project: Project, keys: Optional[Sequence[ProjectKey]] = None) ->
         return computed_quotas
 
 
+class SlidingWindow(TypedDict):
+    windowSeconds: int
+    granularitySeconds: int
+
+
+class CardinalityLimit(TypedDict):
+    id: str
+    window: SlidingWindow
+    limit: int
+    scope: Literal["organization"]
+    namespace: Optional[str]
+
+
+def get_metrics_config() -> Mapping[str, Any]:
+    metrics_config = {}
+
+    cardinality_limits: List[CardinalityLimit] = []
+    cardinality_options = {
+        "unsupported": "sentry-metrics.cardinality-limiter.limits.generic-metrics.per-org"
+    }
+    cardinality_options.update(
+        (namespace.value, option)
+        for namespace, option in USE_CASE_ID_CARDINALITY_LIMIT_QUOTA_OPTIONS.items()
+    )
+    for namespace, option_name in cardinality_options.items():
+        option = options.get(option_name)
+        if not option or not len(option) == 1:
+            # Multiple quotas are not supported
+            continue
+
+        quota = option[0]
+
+        cardinality_limits.append(
+            {
+                "id": namespace,
+                "window": {
+                    "windowSeconds": quota["window_seconds"],
+                    "granularitySeconds": quota["granularity_seconds"],
+                },
+                "limit": quota["limit"],
+                "scope": "organization",
+                "namespace": namespace,
+            }
+        )
+    metrics_config["cardinalityLimits"] = cardinality_limits
+
+    return metrics_config
+
+
 def get_project_config(
     project: Project, full_config: bool = True, project_keys: Optional[Sequence[ProjectKey]] = None
 ) -> "ProjectConfig":
@@ -364,6 +414,8 @@ def _get_project_config(
 
     config["breakdownsV2"] = project.get_option("sentry:breakdowns")
 
+    add_experimental_config(config, "metrics", get_metrics_config)
+
     if _should_extract_transaction_metrics(project):
         add_experimental_config(
             config,

tests/sentry/relay/snapshots/test_config/test_get_project_config/full_config/REGION.pysnap

@@ -1,6 +1,4 @@
 ---
-created: '2023-12-19T08:38:08.254426Z'
-creator: sentry
 source: tests/sentry/relay/test_config.py
 ---
 config:
@@ -101,6 +99,43 @@ config:
   groupingConfig:
     enhancements: eJybzDRxc15qeXFJZU6qlZGBkbGugaGuoeEEAHJMCAM
     id: newstyle:2023-01-11
+  metrics:
+    cardinalityLimits:
+    - id: unsupported
+      limit: 10000
+      namespace: unsupported
+      scope: organization
+      window:
+        granularitySeconds: 600
+        windowSeconds: 3600
+    - id: transactions
+      limit: 10000
+      namespace: transactions
+      scope: organization
+      window:
+        granularitySeconds: 600
+        windowSeconds: 3600
+    - id: sessions
+      limit: 10000
+      namespace: sessions
+      scope: organization
+      window:
+        granularitySeconds: 600
+        windowSeconds: 3600
+    - id: spans
+      limit: 10000
+      namespace: spans
+      scope: organization
+      window:
+        granularitySeconds: 600
+        windowSeconds: 3600
+    - id: custom
+      limit: 10000
+      namespace: custom
+      scope: organization
+      window:
+        granularitySeconds: 600
+        windowSeconds: 3600
   piiConfig:
     applications:
       $string:

tests/sentry/relay/snapshots/test_config/test_project_config_cardinality_limits/REGION.pysnap

@@ -0,0 +1,39 @@
+---
+source: tests/sentry/relay/test_config.py
+---
+cardinalityLimits:
+- id: unsupported
+  limit: 50
+  namespace: unsupported
+  scope: organization
+  window:
+    granularitySeconds: 500
+    windowSeconds: 5000
+- id: transactions
+  limit: 10
+  namespace: transactions
+  scope: organization
+  window:
+    granularitySeconds: 100
+    windowSeconds: 1000
+- id: sessions
+  limit: 20
+  namespace: sessions
+  scope: organization
+  window:
+    granularitySeconds: 200
+    windowSeconds: 2000
+- id: spans
+  limit: 30
+  namespace: spans
+  scope: organization
+  window:
+    granularitySeconds: 300
+    windowSeconds: 3000
+- id: custom
+  limit: 40
+  namespace: custom
+  scope: organization
+  window:
+    granularitySeconds: 400
+    windowSeconds: 4000

tests/sentry/relay/test_config.py

@@ -1050,3 +1050,33 @@ def test_performance_calculate_score_with_optional_lcp_and_cls(default_project):
                 "value": "Opera",
             },
         }
+
+
+@django_db_all
+@region_silo_test
+def test_project_config_cardinality_limits(default_project, insta_snapshot):
+    with override_options(
+        {
+            "sentry-metrics.cardinality-limiter.limits.performance.per-org": [
+                {"window_seconds": 1000, "granularity_seconds": 100, "limit": 10}
+            ],
+            "sentry-metrics.cardinality-limiter.limits.releasehealth.per-org": [
+                {"window_seconds": 2000, "granularity_seconds": 200, "limit": 20}
+            ],
+            "sentry-metrics.cardinality-limiter.limits.spans.per-org": [
+                {"window_seconds": 3000, "granularity_seconds": 300, "limit": 30}
+            ],
+            "sentry-metrics.cardinality-limiter.limits.custom.per-org": [
+                {"window_seconds": 4000, "granularity_seconds": 400, "limit": 40}
+            ],
+            "sentry-metrics.cardinality-limiter.limits.generic-metrics.per-org": [
+                {"window_seconds": 5000, "granularity_seconds": 500, "limit": 50}
+            ],
+        },
+    ):
+        project_cfg = get_project_config(default_project, full_config=True)
+
+        cfg = project_cfg.to_dict()
+        _validate_project_config(cfg["config"])
+
+        insta_snapshot(cfg["config"]["metrics"])