Make event tags hstore based instead of relational tables

events/migrations/0001_initial.py

@@ -3,7 +3,6 @@
 from django.db import migrations, models
 import django.db.models.deletion
 import uuid
-from issues.migrations.sql.add_event_tags_function import ADD_EVENT_TAGS
 from issues.migrations.sql.triggers import UPDATE_ISSUE_TRIGGER
 
 
@@ -102,9 +101,6 @@ class Migration(migrations.Migration):
 
     operations = [
         migrations.SeparateDatabaseAndState(state_operations=state_operations),
-        migrations.RunSQL(
-            sql=ADD_EVENT_TAGS, reverse_sql="DROP FUNCTION IF EXISTS add_event_tags;",
-        ),
         migrations.RunSQL(
             sql=UPDATE_ISSUE_TRIGGER,
             reverse_sql="DROP TRIGGER IF EXISTS event_issue_update on issues_event; DROP FUNCTION IF EXISTS update_issue;",

events/migrations/0003_auto_20210116_2110.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.1.5 on 2021-01-16 21:10
+
+import django.contrib.postgres.fields.hstore
+from django.contrib.postgres.operations import HStoreExtension
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("events", "0002_auto_20201229_1643"),
+    ]
+
+    operations = [
+        migrations.RemoveField(model_name="event", name="tags",),
+        HStoreExtension(),
+        migrations.AddField(
+            model_name="event",
+            name="tags",
+            field=django.contrib.postgres.fields.hstore.HStoreField(default=dict),
+        ),
+        migrations.DeleteModel(name="EventTag",),
+        migrations.DeleteModel(name="EventTagKey",),
+        migrations.RunSQL(
+            sql="DROP FUNCTION IF EXISTS add_event_tags;",
+            reverse_sql="DROP FUNCTION IF EXISTS add_event_tags;",
+        ),
+    ]

events/models.py

@@ -1,24 +1,10 @@
 import uuid
 from django.db import models
 from django.db.models import Q
+from django.contrib.postgres.fields import HStoreField
 from user_reports.models import UserReport
 
 
-class EventTagKey(models.Model):
-    key = models.CharField(max_length=255, unique=True)
-
-    def __str__(self):
-        return self.key
-
-
-class EventTag(models.Model):
-    key = models.ForeignKey(EventTagKey, on_delete=models.CASCADE)
-    value = models.CharField(max_length=225)
-
-    class Meta:
-        unique_together = ("key", "value")
-
-
 class EventManager(models.Manager):
     def for_organization(self, organization):
         return (
@@ -53,7 +39,7 @@ class Event(models.Model):
 
     created = models.DateTimeField(auto_now_add=True, db_index=True)
     data = models.JSONField()
-    tags = models.ManyToManyField(EventTag, blank=True)
+    tags = HStoreField(default=dict)
     release = models.ForeignKey(
         "releases.Release", blank=True, null=True, on_delete=models.SET_NULL
     )
@@ -82,7 +68,7 @@ class Event(models.Model):
         event = self.data
         event["event_id"] = self.event_id_hex
         event["project"] = self.issue.project_id
-        event["tags"] = self.tags.all().values_list("key__key", "value")
+        event["tags"] = self.tags.items()
         if self.timestamp:
             event["datetime"] = self.timestamp.isoformat().replace("+00:00", "Z")
         if self.release:

events/serializers.py

@@ -13,7 +13,7 @@ from issues.serializers import BaseBreadcrumbsSerializer
 from environments.models import Environment
 from releases.models import Release
 from glitchtip.serializers import FlexibleDateTimeField
-from .models import Event, EventTagKey
+from .models import Event
 from .event_tag_processors import TAG_PROCESSORS
 from .event_context_processors import EVENT_CONTEXT_PROCESSORS
 
@@ -154,7 +154,7 @@ class StoreDefaultSerializer(SentrySDKEventSerializer):
                             frame["in_app"] = False
         return exception
 
-    def generate_tags(self, event: Event, data: Dict, tags: List[Tuple[str, str]] = []):
+    def generate_tags(self, data: Dict, tags: List[Tuple[str, str]] = []):
         """
         Determine tag relational data
 
@@ -167,26 +167,7 @@ class StoreDefaultSerializer(SentrySDKEventSerializer):
                 tags.append((processor.tag, value))
         if data.get("tags"):
             tags += [(k, v) for k, v in data["tags"].items()]
-        self.save_tags(event, tags)
-
-    def save_tags(self, event, tags: List[Tuple[str, str]]):
-        """ Commit tags to database """
-        tag_key_values = []
-        # Get tag keys with 1 query (New ones are rarely created)
-        event_tag_keys = EventTagKey.objects.filter(key__in=[tag[0] for tag in tags])
-        for tag, value in tags:
-            tag_key = next((x for x in event_tag_keys if x.key == tag), None)
-            if tag_key is None:  # If there's a new tag key, create it
-                tag_key, _ = EventTagKey.objects.get_or_create(key=tag)
-            tag_key_values.append((tag_key.id, value))
-
-        # add_event_tags adds event tag value (if necessary) and into event.tags
-        if tag_key_values:
-            with connection.cursor() as cursor:
-                cursor.execute(
-                    "select add_event_tags(%s::uuid, %s::tag_key_value[]);",
-                    [event.event_id, tag_key_values],
-                )
+        return tags
 
     def annotate_contexts(self, event):
         """
@@ -260,6 +241,14 @@ class StoreDefaultSerializer(SentrySDKEventSerializer):
             if data.get("release"):
                 release = self.get_release(data["release"], project)
 
+            tags = []
+            if environment:
+                tags.append(("environment", environment.name))
+            if release:
+                tags.append(("release", release.version))
+            tags = self.generate_tags(data, tags)
+            tags = {tag[0]: tag[1] for tag in tags}
+
             json_data = {
                 "breadcrumbs": breadcrumbs,
                 "contexts": contexts,
@@ -288,6 +277,7 @@ class StoreDefaultSerializer(SentrySDKEventSerializer):
             params = {
                 "event_id": data["event_id"],
                 "issue": issue,
+                "tags": tags,
                 "timestamp": data.get("timestamp"),
                 "data": sanitize_bad_postgres_json(json_data),
                 "release": release,
@@ -305,13 +295,6 @@ class StoreDefaultSerializer(SentrySDKEventSerializer):
 
         issue.check_for_status_update()
 
-        tags = []
-        if environment:
-            tags.append(("environment", environment.name))
-        if release:
-            tags.append(("release", release.version))
-        self.generate_tags(event, data, tags)
-
         return event
 
 

events/tests/tests.py

@@ -87,13 +87,13 @@ class EventStoreTestCase(APITestCase):
     def test_performance(self):
         with open("events/test_data/py_hi_event.json") as json_file:
             data = json.load(json_file)
-        with self.assertNumQueries(20):
+        with self.assertNumQueries(14):
             res = self.client.post(self.url, data, format="json")
         self.assertEqual(res.status_code, 200)
 
         # Second event should have less queries
         data["event_id"] = "6600a066e64b4caf8ed7ec5af64ac4bb"
-        with self.assertNumQueries(9):
+        with self.assertNumQueries(7):
             res = self.client.post(self.url, data, format="json")
         self.assertEqual(res.status_code, 200)
 
@@ -217,18 +217,17 @@ class EventStoreTestCase(APITestCase):
         self.assertTrue(event.release)
         self.assertEqual(event_json.get("release"), event.release.version)
         self.assertIn(
-            event.release.version,
-            event_json.get("tags").values_list("value", flat=True),
+            event.release.version, dict(event_json.get("tags")).values(),
         )
 
     def test_client_tags(self):
         with open("events/test_data/py_hi_event.json") as json_file:
             data = json.load(json_file)
-            data["tags"] = {"test_tag": "the value"}
+        data["tags"] = {"test_tag": "the value"}
         self.client.post(self.url, data, format="json")
         event = Event.objects.first()
         event_json = event.event_json()
         self.assertIn(
-            "the value", event_json.get("tags").values_list("value", flat=True),
+            "the value", tuple(event_json.get("tags"))[1],
         )
 

glitchtip/settings.py

@@ -134,6 +134,7 @@ INSTALLED_APPS = [
     "corsheaders",
     "django_celery_results",
     "django_filters",
+    "django_extensions",
     "debug_toolbar",
     "rest_framework",
     "drf_yasg",

issues/migrations/0004_auto_20200804_0053.py

@@ -1,7 +1,6 @@
 # Generated by Django 3.0.8 on 2020-08-04 00:53
 
 from django.db import migrations
-from .sql.add_event_tags_function import ADD_EVENT_TAGS
 
 
 class Migration(migrations.Migration):
@@ -10,8 +9,4 @@ class Migration(migrations.Migration):
         ("issues", "0003_auto_20200731_1531"),
     ]
 
-    operations = [
-        migrations.RunSQL(
-            sql=ADD_EVENT_TAGS, reverse_sql="DROP FUNCTION IF EXISTS add_event_tags;",
-        ),
-    ]
+    operations = []

issues/migrations/0010_auto_20210117_1543.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.1.5 on 2021-01-17 15:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('issues', '0009_auto_20201229_1622'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='issue',
+            name='type',
+            field=models.PositiveSmallIntegerField(choices=[(0, 'default'), (1, 'error'), (2, 'csp'), (3, 'transaction')], default=0),
+        ),
+    ]

issues/migrations/sql/add_event_tags_function.py

@@ -1,29 +0,0 @@
-ADD_EVENT_TAGS = """
-DROP TYPE IF EXISTS tag_key_value CASCADE;
-CREATE TYPE tag_key_value AS (key_id integer, value text);
-
-CREATE OR REPLACE FUNCTION add_event_tags(IN event_id uuid, IN tags tag_key_value[]) RETURNS void as $$
-DECLARE
-  event_tag tag_key_value;
-BEGIN
-  FOREACH event_tag IN ARRAY tags
-  LOOP
-    WITH e AS (
-        INSERT INTO events_eventtag (key_id, "value") VALUES
-        (event_tag.key_id, event_tag.value)
-        ON CONFLICT (key_id, value) DO NOTHING
-        RETURNING id
-    ), y as (
-    SELECT id FROM e
-        UNION
-            SELECT id
-            FROM events_eventtag
-            WHERE key_id=event_tag.key_id AND value=event_tag.value
-    )
-    INSERT INTO events_event_tags (event_id, eventtag_id)
-    select event_id, id
-    FROM y;
-  END LOOP;
-END
-$$ LANGUAGE plpgsql;
-"""

issues/serializers.py

@@ -4,18 +4,10 @@ from user_reports.serializers import UserReportSerializer
 from sentry.interfaces.stacktrace import get_context
 from glitchtip.serializers import FlexibleDateTimeField
 from releases.serializers import ReleaseSerializer
-from events.models import Event, EventTag
+from events.models import Event
 from .models import Issue, EventType, EventStatus
 
 
-class EventTagSerializer(serializers.ModelSerializer):
-    key = serializers.StringRelatedField()
-
-    class Meta:
-        model = EventTag
-        fields = ("key", "value")
-
-
 class EventUserSerializer(serializers.Serializer):
     username = serializers.CharField(allow_null=True)
     name = serializers.CharField(allow_null=True)
@@ -107,13 +99,18 @@ class EventEntriesSerializer(serializers.Serializer):
         return entries
 
 
+class EventTagField(serializers.HStoreField):
+    def to_representation(self, obj):
+        return [{"key": tag[0], "value": tag[1]} for tag in obj.items()]
+
+
 class EventSerializer(serializers.ModelSerializer):
     eventID = serializers.CharField(source="event_id_hex")
     id = serializers.CharField(source="event_id_hex")
     dateCreated = serializers.DateTimeField(source="timestamp")
     dateReceived = serializers.DateTimeField(source="created")
     entries = EventEntriesSerializer(source="data")
-    tags = EventTagSerializer(many=True)
+    tags = EventTagField()
     user = EventUserSerializer()
 
     class Meta: