Revert "feat(metrics): persist span attribute extraction rules in django (#73421)"

This reverts commit d34bf95cb16b3f7ad2c38aca63b5f93df75fd72c.

Co-authored-by: shellmayr <6788060+shellmayr@users.noreply.github.com>

fixtures/backup/model_dependencies/detailed.json

@@ -5862,57 +5862,6 @@
       ]
     ]
   },
-  "sentry.spanattributeextractionrulecondition": {
-    "dangling": false,
-    "foreign_keys": {
-      "config": {
-        "kind": "FlexibleForeignKey",
-        "model": "sentry.spanattributeextractionruleconfig",
-        "nullable": false
-      },
-      "created_by_id": {
-        "kind": "HybridCloudForeignKey",
-        "model": "sentry.user",
-        "nullable": true
-      }
-    },
-    "model": "sentry.spanattributeextractionrulecondition",
-    "relocation_dependencies": [],
-    "relocation_scope": "Organization",
-    "silos": [
-      "Region"
-    ],
-    "table_name": "sentry_spanattributeextractionrulecondition",
-    "uniques": []
-  },
-  "sentry.spanattributeextractionruleconfig": {
-    "dangling": false,
-    "foreign_keys": {
-      "created_by_id": {
-        "kind": "HybridCloudForeignKey",
-        "model": "sentry.user",
-        "nullable": true
-      },
-      "project": {
-        "kind": "FlexibleForeignKey",
-        "model": "sentry.project",
-        "nullable": false
-      }
-    },
-    "model": "sentry.spanattributeextractionruleconfig",
-    "relocation_dependencies": [],
-    "relocation_scope": "Organization",
-    "silos": [
-      "Region"
-    ],
-    "table_name": "sentry_spanattributeextractionruleconfig",
-    "uniques": [
-      [
-        "project",
-        "span_attribute"
-      ]
-    ]
-  },
   "sentry.stringindexer": {
     "dangling": false,
     "foreign_keys": {

fixtures/backup/model_dependencies/flat.json

@@ -810,14 +810,6 @@
   "sentry.snubaqueryeventtype": [
     "sentry.snubaquery"
   ],
-  "sentry.spanattributeextractionrulecondition": [
-    "sentry.spanattributeextractionruleconfig",
-    "sentry.user"
-  ],
-  "sentry.spanattributeextractionruleconfig": [
-    "sentry.project",
-    "sentry.user"
-  ],
   "sentry.stringindexer": [
     "sentry.organization"
   ],

fixtures/backup/model_dependencies/sorted.json

@@ -104,8 +104,6 @@
   "uptime.projectuptimesubscription",
   "sentry.useroption",
   "sentry.useremail",
-  "sentry.spanattributeextractionruleconfig",
-  "sentry.spanattributeextractionrulecondition",
   "sentry.snubaquery",
   "sentry.sentryapp",
   "sentry.rule",

fixtures/backup/model_dependencies/truncate.json

@@ -104,8 +104,6 @@
   "uptime_projectuptimesubscription",
   "sentry_useroption",
   "sentry_useremail",
-  "sentry_spanattributeextractionruleconfig",
-  "sentry_spanattributeextractionrulecondition",
   "sentry_snubaquery",
   "sentry_sentryapp",
   "sentry_rule",

migrations_lockfile.txt

@@ -10,6 +10,6 @@ hybridcloud: 0016_add_control_cacheversion
 nodestore: 0002_nodestore_no_dictfield
 remote_subscriptions: 0002_remove_separate_remote_subscription
 replays: 0004_index_together
-sentry: 0732_add_span_attribute_extraction_rules
+sentry: 0731_add_insight_project_flags
 social_auth: 0002_default_auto_field
 uptime: 0002_remove_separate_remote_subscription

src/sentry/api/endpoints/project_metrics_extraction_rules.py

@@ -1,5 +1,6 @@
-import sentry_sdk
-from django.db import router, transaction
+from collections.abc import Sequence
+from typing import Any
+
 from rest_framework.request import Request
 from rest_framework.response import Response
 
@@ -10,16 +11,15 @@ from sentry.api.base import region_silo_endpoint
 from sentry.api.bases import ProjectEndpoint
 from sentry.api.paginator import OffsetPaginator
 from sentry.api.serializers import serialize
-from sentry.api.serializers.models.metrics_extraction_rules import (
-    SpanAttributeExtractionRuleConfigSerializer,
-)
+from sentry.api.serializers.models.metrics_extraction_rules import MetricsExtractionRuleSerializer
 from sentry.models.project import Project
-from sentry.sentry_metrics.configuration import HARD_CODED_UNITS
-from sentry.sentry_metrics.models import (
-    SpanAttributeExtractionRuleCondition,
-    SpanAttributeExtractionRuleConfig,
+from sentry.sentry_metrics.extraction_rules import (
+    MetricsExtractionRule,
+    create_metrics_extraction_rules,
+    delete_metrics_extraction_rules,
+    get_metrics_extraction_rules,
+    update_metrics_extraction_rules,
 )
-from sentry.tasks.relay import schedule_invalidate_project_config
 
 
 @region_silo_endpoint
@@ -42,22 +42,15 @@ class ProjectMetricsExtractionRulesEndpoint(ProjectEndpoint):
         if not self.has_feature(project.organization, request):
             return Response(status=404)
 
-        config_update = request.data.get("metricsExtractionRules") or []
-        if len(config_update) == 0:
+        rules_update = request.data.get("metricsExtractionRules") or []
+        if len(rules_update) == 0:
             return Response(status=204)
 
         try:
-            with transaction.atomic(router.db_for_write(SpanAttributeExtractionRuleConfig)):
-                for obj in config_update:
-                    SpanAttributeExtractionRuleConfig.objects.filter(
-                        project=project, span_attribute=obj["spanAttribute"]
-                    ).delete()
-                schedule_invalidate_project_config(
-                    project_id=project.id, trigger="span_attribute_extraction_configs"
-                )
+            state_update = self._generate_deleted_rule_objects(rules_update)
+            delete_metrics_extraction_rules(project, state_update)
         except Exception as e:
-            sentry_sdk.capture_exception()
-            return Response(status=400, data={"detail": str(e)})
+            return Response(status=500, data={"detail": str(e)})
 
         return Response(status=204)
 
@@ -67,18 +60,16 @@ class ProjectMetricsExtractionRulesEndpoint(ProjectEndpoint):
             return Response(status=404)
 
         try:
-            configs = SpanAttributeExtractionRuleConfig.objects.filter(project=project)
-
+            extraction_rules = get_metrics_extraction_rules(project)
         except Exception as e:
             return Response(status=500, data={"detail": str(e)})
 
-        # TODO(metrics): do real pagination using the database
         return self.paginate(
             request,
-            queryset=list(configs),
+            queryset=extraction_rules,
             paginator_cls=OffsetPaginator,
             on_results=lambda x: serialize(
-                x, user=request.user, serializer=SpanAttributeExtractionRuleConfigSerializer()
+                x, user=request.user, serializer=MetricsExtractionRuleSerializer()
             ),
             default_per_page=1000,
             max_per_page=1000,
@@ -90,88 +81,62 @@ class ProjectMetricsExtractionRulesEndpoint(ProjectEndpoint):
         if not self.has_feature(project.organization, request):
             return Response(status=404)
 
-        config_update = request.data.get("metricsExtractionRules")
+        rules_update = request.data.get("metricsExtractionRules")
 
-        if not config_update:
+        if not rules_update or len(rules_update) == 0:
             return Response(
                 status=400,
                 data={"detail": "Please specify the metric extraction rule to be created."},
             )
+
         try:
-            configs = []
-            with transaction.atomic(router.db_for_write(SpanAttributeExtractionRuleConfig)):
-                for obj in config_update:
-                    configs.append(
-                        SpanAttributeExtractionRuleConfig.from_dict(obj, request.user.id, project)
-                    )
-                schedule_invalidate_project_config(
-                    project_id=project.id, trigger="span_attribute_extraction_configs"
-                )
-
-        except Exception:
-            sentry_sdk.capture_exception()
-            return Response(
-                status=400,
+            state_update = self._generate_updated_rule_objects(rules_update)
+            persisted_rules = create_metrics_extraction_rules(project, state_update)
+            updated_rules = serialize(
+                persisted_rules, request.user, MetricsExtractionRuleSerializer()
             )
+        except Exception as e:
+            return Response(status=400, data={"detail": str(e)})
 
-        persisted_config = serialize(
-            configs,
-            request.user,
-            SpanAttributeExtractionRuleConfigSerializer(),
-        )
-        return Response(data=persisted_config, status=200)
+        return Response(status=200, data=updated_rules)
 
     def put(self, request: Request, project: Project) -> Response:
         """PUT to modify an existing extraction rule."""
         if not self.has_feature(project.organization, request):
             return Response(status=404)
 
-        config_update = request.data.get("metricsExtractionRules")
-        if not config_update:
+        rules_update = request.data.get("metricsExtractionRules")
+
+        if not rules_update or len(rules_update) == 0:
             return Response(status=200)
 
         try:
-            with transaction.atomic(router.db_for_write(SpanAttributeExtractionRuleConfig)):
-                configs = []
-                for obj in config_update:
-                    config = SpanAttributeExtractionRuleConfig.objects.get(
-                        project=project, span_attribute=obj["spanAttribute"]
-                    )
-                    config.aggregates = obj["aggregates"]
-                    config.unit = HARD_CODED_UNITS.get(obj["spanAttribute"], obj["unit"])
-                    config.tags = obj["tags"]
-                    config.save()
-                    config.refresh_from_db()
-                    configs.append(config)
-
-                    # delete conditions not present in update
-                    included_conditions = [x["id"] for x in obj["conditions"]]
-                    SpanAttributeExtractionRuleCondition.objects.filter(config=config).exclude(
-                        id__in=included_conditions
-                    ).delete()
-
-                    for condition in obj["conditions"]:
-                        condition_id = condition["id"] if "id" in condition else None
-                        SpanAttributeExtractionRuleCondition.objects.update_or_create(
-                            id=condition_id,
-                            config=config,
-                            defaults={
-                                "value": condition["value"],
-                                "created_by_id": request.user.id,
-                            },
-                        )
-                schedule_invalidate_project_config(
-                    project_id=project.id, trigger="span_attribute_extraction_configs"
-                )
-
-        except Exception:
-            sentry_sdk.capture_exception()
-            return Response(status=400)
-
-        persisted_config = serialize(
-            configs,
-            request.user,
-            SpanAttributeExtractionRuleConfigSerializer(),
-        )
+            state_update = self._generate_updated_rule_objects(rules_update)
+            persisted_rules = update_metrics_extraction_rules(project, state_update)
+            updated_rules = serialize(
+                persisted_rules, request.user, MetricsExtractionRuleSerializer()
+            )
+        except Exception as e:
+            return Response(status=400, data={"detail": str(e)})
+
+        return Response(status=200, data=updated_rules)
+
+    def _generate_updated_rule_objects(
+        self, updated_rules: list[dict[str, Any]]
+    ) -> dict[str, MetricsExtractionRule]:
+        state_update = {}
+        for updated_rule in updated_rules:
+            rule = MetricsExtractionRule.from_dict(updated_rule)
+            mri = rule.generate_mri()
+            state_update[mri] = rule
+
+        return state_update
+
+    def _generate_deleted_rule_objects(
+        self, updated_rules: list[dict[str, Any]]
+    ) -> Sequence[MetricsExtractionRule]:
+        state_update = []
+        for updated_rule in updated_rules:
+            state_update.append(MetricsExtractionRule.from_dict(updated_rule))
 
-        return Response(data=persisted_config, status=200)
+        return state_update

src/sentry/api/serializers/models/metrics_extraction_rules.py

@@ -1,12 +1,12 @@
-from typing import Any
-
 from sentry.api.serializers import Serializer
-from sentry.sentry_metrics.models import SpanAttributeExtractionRuleConfig
 
 
 class MetricsExtractionRuleSerializer(Serializer):
     def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+        Serializer.__init__(self, *args, **kwargs)
+
+    def get_attrs(self, item_list, user):
+        return {item: item.__dict__ for item in item_list}
 
     def serialize(self, obj, attrs, user):
         return {
@@ -15,25 +15,4 @@ class MetricsExtractionRuleSerializer(Serializer):
             "unit": attrs.get("unit"),
             "tags": list(attrs.get("tags") or set()),
             "conditions": list(attrs.get("conditions") or set()),
-            "id": attrs.get("id"),
-        }
-
-
-class SpanAttributeExtractionRuleConfigSerializer(Serializer):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def serialize(self, obj, attrs, user):
-        return {
-            "spanAttribute": obj.span_attribute,
-            "aggregates": list(obj.aggregates),
-            "unit": obj.unit,
-            "tags": list(obj.tags),
-            "conditions": self._serialize_conditions(obj),
         }
-
-    def _serialize_conditions(self, obj: SpanAttributeExtractionRuleConfig) -> list[dict[str, Any]]:
-        return [
-            {"id": condition.id, "value": condition.value, "mris": condition.generate_mris()}
-            for condition in obj.conditions.all()
-        ]

src/sentry/backup/comparators.py

@@ -838,8 +838,6 @@ def get_default_comparators() -> dict[str, list[JSONScrubbingComparator]]:
             ],
             "sentry.sentryappinstallation": [DateUpdatedComparator("date_updated")],
             "sentry.servicehook": [HashObfuscatingComparator("secret")],
-            "sentry.spanattributeextractionruleconfig": [DateUpdatedComparator("date_updated")],
-            "sentry.spanattributeextractionrulecondition": [DateUpdatedComparator("date_updated")],
             "sentry.team": [
                 # TODO(getsentry/sentry#66247): Remove once self-hosted 24.4.0 is released.
                 IgnoredComparator("org_role"),

src/sentry/conf/server.py

@@ -399,7 +399,6 @@ INSTALLED_APPS: tuple[str, ...] = (
     "sentry.replays",
     "sentry.release_health",
     "sentry.search",
-    "sentry.sentry_metrics",
     "sentry.sentry_metrics.indexer.postgres.apps.Config",
     "sentry.snuba",
     "sentry.lang.java.apps.Config",

src/sentry/migrations/0732_add_span_attribute_extraction_rules.py

@@ -1,100 +0,0 @@
-# Generated by Django 5.0.6 on 2024-07-01 21:11
-
-import django.db.models.deletion
-import django.utils.timezone
-from django.db import migrations, models
-
-import sentry.db.models.fields.array
-import sentry.db.models.fields.bounded
-import sentry.db.models.fields.foreignkey
-import sentry.db.models.fields.hybrid_cloud_foreign_key
-from sentry.new_migrations.migrations import CheckedMigration
-
-
-class Migration(CheckedMigration):
-    # This flag is used to mark that a migration shouldn't be automatically run in production.
-    # This should only be used for operations where it's safe to run the migration after your
-    # code has deployed. So this should not be used for most operations that alter the schema
-    # of a table.
-    # Here are some things that make sense to mark as post deployment:
-    # - Large data migrations. Typically we want these to be run manually so that they can be
-    #   monitored and not block the deploy for a long period of time while they run.
-    # - Adding indexes to large tables. Since this can take a long time, we'd generally prefer to
-    #   run this outside deployments so that we don't block them. Note that while adding an index
-    #   is a schema change, it's completely safe to run the operation after the code has deployed.
-    # Once deployed, run these manually via: https://develop.sentry.dev/database-migrations/#migration-deployment
-
-    is_post_deployment = False
-
-    dependencies = [
-        ("sentry", "0731_add_insight_project_flags"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="SpanAttributeExtractionRuleConfig",
-            fields=[
-                (
-                    "id",
-                    sentry.db.models.fields.bounded.BoundedBigAutoField(
-                        primary_key=True, serialize=False
-                    ),
-                ),
-                ("date_updated", models.DateTimeField(default=django.utils.timezone.now)),
-                ("date_added", models.DateTimeField(default=django.utils.timezone.now, null=True)),
-                (
-                    "created_by_id",
-                    sentry.db.models.fields.hybrid_cloud_foreign_key.HybridCloudForeignKey(
-                        "sentry.User", db_index=True, null=True, on_delete="SET_NULL"
-                    ),
-                ),
-                ("span_attribute", models.CharField(max_length=1000)),
-                ("unit", models.CharField(default="none", max_length=100)),
-                ("tags", sentry.db.models.fields.array.ArrayField(null=True)),
-                ("aggregates", sentry.db.models.fields.array.ArrayField(null=True)),
-                (
-                    "project",
-                    sentry.db.models.fields.foreignkey.FlexibleForeignKey(
-                        db_constraint=False,
-                        on_delete=django.db.models.deletion.CASCADE,
-                        to="sentry.project",
-                    ),
-                ),
-            ],
-            options={
-                "db_table": "sentry_spanattributeextractionruleconfig",
-                "unique_together": {("project", "span_attribute")},
-            },
-        ),
-        migrations.CreateModel(
-            name="SpanAttributeExtractionRuleCondition",
-            fields=[
-                (
-                    "id",
-                    sentry.db.models.fields.bounded.BoundedBigAutoField(
-                        primary_key=True, serialize=False
-                    ),
-                ),
-                ("date_updated", models.DateTimeField(default=django.utils.timezone.now)),
-                ("date_added", models.DateTimeField(default=django.utils.timezone.now, null=True)),
-                (
-                    "created_by_id",
-                    sentry.db.models.fields.hybrid_cloud_foreign_key.HybridCloudForeignKey(
-                        "sentry.User", db_index=True, null=True, on_delete="SET_NULL"
-                    ),
-                ),
-                ("value", models.CharField(blank=True, max_length=1000, null=True)),
-                (
-                    "config",
-                    sentry.db.models.fields.foreignkey.FlexibleForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        related_name="conditions",
-                        to="sentry.spanattributeextractionruleconfig",
-                    ),
-                ),
-            ],
-            options={
-                "db_table": "sentry_spanattributeextractionrulecondition",
-            },
-        ),
-    ]