ref: Extract GroupTagKey interactions out to TagStorage abstraction (#6251)

src/sentry/api/endpoints/group_details.py

@@ -7,7 +7,7 @@ from uuid import uuid4
 from django.utils import timezone
 from rest_framework.response import Response
 
-from sentry import tsdb
+from sentry import tsdb, tagstore
 from sentry.api import client
 from sentry.api.base import DocSection
 from sentry.api.bases import GroupEndpoint
@@ -19,7 +19,6 @@ from sentry.models import (
     GroupHash,
     GroupSeen,
     GroupStatus,
-    GroupTagKey,
     Release,
     User,
     UserReport,
@@ -211,9 +210,7 @@ class GroupDetailsEndpoint(GroupEndpoint):
         if last_release:
             last_release = self._get_release_info(request, group, last_release)
 
-        tags = list(GroupTagKey.objects.filter(
-            group_id=group.id,
-        )[:100])
+        tags = tagstore.get_group_tag_keys(group.id, limit=100)
 
         participants = list(
             User.objects.filter(

src/sentry/api/endpoints/group_tagkey_details.py

@@ -9,7 +9,7 @@ from sentry.api.base import DocSection
 from sentry.api.bases.group import GroupEndpoint
 from sentry.api.exceptions import ResourceDoesNotExist
 from sentry.api.serializers import serialize
-from sentry.models import (GroupTagKey, GroupTagValue, Group)
+from sentry.models import (GroupTagValue, Group)
 from sentry.utils.apidocs import scenario
 
 
@@ -46,11 +46,8 @@ class GroupTagKeyDetailsEndpoint(GroupEndpoint):
             raise ResourceDoesNotExist
 
         try:
-            group_tag_key = GroupTagKey.objects.get(
-                group_id=group.id,
-                key=lookup_key,
-            )
-        except GroupTagKey.DoesNotExist:
+            group_tag_key = tagstore.get_group_tag_key(group.id, lookup_key)
+        except tagstore.GroupTagKeyNotFound:
             raise ResourceDoesNotExist
 
         total_values = GroupTagValue.get_value_count(group.id, lookup_key)

src/sentry/api/endpoints/group_tags.py

@@ -8,14 +8,12 @@ from collections import defaultdict
 from sentry import tagstore
 from sentry.api.bases.group import GroupEndpoint
 from sentry.api.serializers import serialize
-from sentry.models import GroupTagValue, GroupTagKey
+from sentry.models import GroupTagValue
 
 
 class GroupTagsEndpoint(GroupEndpoint):
     def get(self, request, group):
-        grouptagkeys = list(GroupTagKey.objects.filter(
-            group_id=group.id
-        ).values_list('key', flat=True))
+        grouptagkeys = [gtk.key for gtk in tagstore.get_group_tag_keys(group.id)]
 
         tag_keys = tagstore.get_tag_keys(group.project_id, grouptagkeys)
 

src/sentry/api/serializers/models/group.py

@@ -9,12 +9,12 @@ from django.core.urlresolvers import reverse
 from django.db.models import Q
 from django.utils import timezone
 
-from sentry import tsdb
+from sentry import tagstore, tsdb
 from sentry.api.serializers import Serializer, register, serialize
 from sentry.constants import LOG_LEVELS, StatsPeriod
 from sentry.models import (
     Group, GroupAssignee, GroupBookmark, GroupMeta, GroupResolution, GroupSeen, GroupSnooze,
-    GroupStatus, GroupSubscription, GroupSubscriptionReason, GroupTagKey, User, UserOption,
+    GroupStatus, GroupSubscription, GroupSubscriptionReason, User, UserOption,
     UserOptionValue
 )
 from sentry.utils.db import attach_foreignkey
@@ -124,12 +124,7 @@ class GroupSerializer(Serializer):
             ).select_related('user')
         )
 
-        user_counts = dict(
-            GroupTagKey.objects.filter(
-                group_id__in=[g.id for g in item_list],
-                key='sentry:user',
-            ).values_list('group_id', 'values_seen')
-        )
+        user_counts = tagstore.get_values_seen([g.id for g in item_list], 'sentry:user')
 
         ignore_items = {g.group_id: g for g in GroupSnooze.objects.filter(
             group__in=item_list,

src/sentry/models/group.py

@@ -325,14 +325,8 @@ class Group(Model):
         return self._oldest_event
 
     def get_tags(self):
-        from sentry.models import GroupTagKey
         if not hasattr(self, '_tag_cache'):
-            group_tags = GroupTagKey.objects.filter(
-                group_id=self.id,
-                project_id=self.project_id,
-            )
-
-            group_tags = list(group_tags.values_list('key', flat=True))
+            group_tags = [gtk.key for gtk in tagstore.get_group_tag_keys(self.id)]
 
             tag_keys = dict(
                 (t.key, t) for t in tagstore.get_tag_keys(self.project_id, group_tags)
@@ -447,9 +441,4 @@ class Group(Model):
         )
 
     def count_users_seen(self):
-        from sentry.models import GroupTagKey
-
-        return GroupTagKey.objects.filter(
-            group_id=self.id,
-            key='sentry:user',
-        ).aggregate(t=models.Sum('values_seen'))['t'] or 0
+        return tagstore.get_values_seen(self.id, 'sentry:user')[self.id]

src/sentry/tagstore/base.py

@@ -30,11 +30,36 @@ class TagKeyStatus(object):
 
 class TagStorage(Service):
     __all__ = (
-        'is_valid_key', 'is_valid_value', 'is_reserved_key', 'prefix_reserved_key',
-        'get_standardized_key', 'create_tag_key', 'get_or_create_tag_key',
-        'create_tag_value', 'get_or_create_tag_value', 'get_tag_key', 'get_tag_keys',
-        'get_tag_value', 'get_tag_values', 'delete_tag_key', 'incr_values_seen',
-        'incr_times_seen', 'get_group_event_ids', 'get_tag_value_qs'
+        'is_valid_key',
+        'is_valid_value',
+        'is_reserved_key',
+        'prefix_reserved_key',
+        'get_standardized_key',
+
+        'create_tag_key',
+        'get_or_create_tag_key',
+        'create_tag_value',
+        'get_or_create_tag_value',
+        'create_group_tag_key',
+        'get_or_create_group_tag_key',
+
+        'get_tag_key',
+        'get_tag_keys',
+        'get_tag_value',
+        'get_tag_values',
+        'get_group_tag_key',
+        'get_group_tag_keys',
+
+        'get_group_event_ids',
+        'get_tag_value_qs',
+
+        'delete_tag_key',
+        'delete_group_tag_key',
+        'delete_all_group_tag_keys',
+
+        'get_values_seen',
+        'incr_values_seen',
+        'incr_times_seen',
     )
 
     def is_valid_key(self, key):
@@ -82,14 +107,27 @@ class TagStorage(Service):
         """
         raise NotImplementedError
 
+    def create_group_tag_key(self, project_id, group_id, key, **kwargs):
+        """
+        >>> create_group_tag_key(1, 2, "key1")
+        """
+        raise NotImplementedError
+
+    def get_or_create_group_tag_key(self, project_id, group_id, key, **kwargs):
+        """
+        >>> get_or_create_group_tag_key(1, 2, "key1")
+        """
+        raise NotImplementedError
+
     def get_tag_key(self, project_id, key, status=TagKeyStatus.VISIBLE):
         """
         >>> get_tag_key(1, "key1")
         """
         raise NotImplementedError
 
-    def get_tag_keys(self, project_id, keys=None, status=TagKeyStatus.VISIBLE):
+    def get_tag_keys(self, project_ids, keys=None, status=TagKeyStatus.VISIBLE):
         """
+        >>> get_tag_key([1, 2], ["key1", "key2"])
         >>> get_tag_key(1, ["key1", "key2"])
         """
         raise NotImplementedError
@@ -103,6 +141,20 @@ class TagStorage(Service):
     def get_tag_values(self, project_ids, key, values=None):
         """
         >>> get_tag_values([1, 2], "key1", ["value1, "value2"])
+        >>> get_tag_values(1, "key1", ["value1, "value2"])
+        """
+        raise NotImplementedError
+
+    def get_group_tag_key(self, group_id, key):
+        """
+        >>> get_group_tag_key(1, "key1")
+        """
+        raise NotImplementedError
+
+    def get_group_tag_keys(self, group_ids, keys=None, limit=None):
+        """
+        >>> get_group_tag_keys([1, 2], ["key1", "key2"])
+        >>> get_group_tag_keys(1, ["key1", "key2"])
         """
         raise NotImplementedError
 
@@ -112,6 +164,18 @@ class TagStorage(Service):
         """
         raise NotImplementedError
 
+    def delete_group_tag_key(self, group_id, key):
+        """
+        >>> delete_group_tag_key(1, "key1")
+        """
+        raise NotImplementedError
+
+    def delete_all_group_tag_keys(self, group_id):
+        """
+        >>> delete_all_group_tag_keys(1)
+        """
+        raise NotImplementedError
+
     def incr_values_seen(self, project_id, key, count=1):
         """
         >>> incr_values_seen(1, "key1")
@@ -135,3 +199,9 @@ class TagStorage(Service):
         >>> get_tag_value_qs(1, 'environment', query='prod')
         """
         raise NotImplementedError
+
+    def get_values_seen(self, group_ids, key):
+        """
+        get_values_seen([1, 2], 'key1')
+        """
+        raise NotImplementedError

src/sentry/tagstore/exceptions.py

@@ -15,3 +15,7 @@ class TagKeyNotFound(Exception):
 
 class TagValueNotFound(Exception):
     pass
+
+
+class GroupTagKeyNotFound(Exception):
+    pass

src/sentry/tagstore/legacy/backend.py

@@ -10,13 +10,14 @@ from __future__ import absolute_import
 
 import six
 
+from collections import defaultdict, Iterable
 from django.db.models import Q
 from operator import or_
 from six.moves import reduce
 
 from sentry import buffer
 from sentry.tagstore import TagKeyStatus
-from sentry.models import TagKey, TagValue, EventTag
+from sentry.models import GroupTagKey, TagKey, TagValue, EventTag
 from sentry.tagstore.base import TagStorage
 from sentry.utils.cache import cache
 from sentry.tasks.deletion import delete_tag_key
@@ -36,6 +37,14 @@ class LegacyTagStorage(TagStorage):
         return TagValue.objects.get_or_create(
             project_id=project_id, key=key, value=value, defaults=kwargs)
 
+    def create_group_tag_key(self, project_id, group_id, key, **kwargs):
+        return GroupTagKey.objects.create(project_id=project_id, group_id=group_id,
+                                          key=key, **kwargs)
+
+    def get_or_create_group_tag_key(self, project_id, group_id, key, **kwargs):
+        return GroupTagKey.objects.get_or_create(project_id=project_id, group_id=group_id,
+                                                 key=key, defaults=kwargs)
+
     def get_tag_key(self, project_id, key, status=TagKeyStatus.VISIBLE):
         from sentry.tagstore.exceptions import TagKeyNotFound
 
@@ -52,28 +61,34 @@ class LegacyTagStorage(TagStorage):
         except TagKey.DoesNotExist:
             raise TagKeyNotFound
 
-    def _get_tag_keys_cache_key(self, project_id, status):
-        return 'filterkey:all:%s:%s' % (project_id, status)
+    def _get_tag_keys_cache_key(self, project_ids, status):
+        if isinstance(project_ids, Iterable):
+            project_ids = "-".join(sorted(project_ids))
+        return 'filterkey:all:%s:%s' % (project_ids, status)
+
+    def get_tag_keys(self, project_ids, keys=None, status=TagKeyStatus.VISIBLE):
+        def _get_base_qs():
+            if isinstance(project_ids, six.integer_types):
+                qs = TagKey.objects.filter(project_id=project_ids)
+            else:
+                qs = TagKey.objects.filter(project_id__in=project_ids)
+
+            if status:
+                qs = qs.filter(status=status)
+
+            return qs
 
-    def get_tag_keys(self, project_id, keys=None, status=TagKeyStatus.VISIBLE):
         if not keys:
             # TODO: cache invalidation via post_save/post_delete signals much like BaseManager
-            key = self._get_tag_keys_cache_key(project_id, status)
+            key = self._get_tag_keys_cache_key(project_ids, status)
             result = cache.get(key)
             if result is None:
-                qs = TagKey.objects.filter(project_id=project_id)
-
-                if status:
-                    qs = qs.filter(status=status)
-
+                qs = _get_base_qs()
                 result = list(qs.order_by('-values_seen')[:20])
                 cache.set(key, result, 60)
             return result
 
-        qs = TagKey.objects.filter(
-            project_id=project_id,
-            key__in=keys,
-        )
+        qs = _get_base_qs()
 
         if status:
             qs = qs.filter(status=status)
@@ -95,10 +110,10 @@ class LegacyTagStorage(TagStorage):
     def get_tag_values(self, project_ids, key, values=None):
         qs = TagValue.objects.filter(key=key)
 
-        if isinstance(project_ids, list):
-            qs = qs.filter(project_id__in=project_ids)
-        else:
+        if isinstance(project_ids, six.integer_types):
             qs = qs.filter(project_id=project_ids)
+        else:
+            qs = qs.filter(project_id__in=project_ids)
 
         qs = TagValue.objects.filter(
             project_id__in=project_ids,
@@ -110,6 +125,34 @@ class LegacyTagStorage(TagStorage):
 
         return list(qs)
 
+    def get_group_tag_key(self, group_id, key):
+        from sentry.tagstore.exceptions import GroupTagKeyNotFound
+
+        try:
+            return GroupTagKey.objects.get(
+                group_id=group_id,
+                key=key,
+            )
+        except GroupTagKey.DoesNotExist:
+            raise GroupTagKeyNotFound
+
+    def get_group_tag_keys(self, group_ids, keys=None, limit=None):
+        if isinstance(group_ids, six.integer_types):
+            qs = GroupTagKey.objects.filter(group_id=group_ids)
+        else:
+            qs = GroupTagKey.objects.filter(group_id__in=group_ids)
+
+        if keys is not None:
+            if isinstance(keys, six.text_type):
+                qs = qs.filter(key=keys)
+            else:
+                qs = qs.filter(key__in=keys)
+
+        if limit is not None:
+            qs = qs[:limit]
+
+        return list(qs)
+
     def delete_tag_key(self, project_id, key):
         tagkey = self.get_tag_key(project_id, key, status=None)
 
@@ -123,6 +166,17 @@ class LegacyTagStorage(TagStorage):
 
         return (updated, tagkey)
 
+    def delete_group_tag_key(self, group_id, key):
+        GroupTagKey.objects.filter(
+            group_id=group_id,
+            key=key
+        ).delete()
+
+    def delete_all_group_tag_keys(self, group_id):
+        GroupTagKey.objects.filter(
+            group_id=group_id,
+        ).delete()
+
     def incr_values_seen(self, project_id, key, count=1):
         buffer.incr(TagKey, {
             'values_seen': count,
@@ -206,3 +260,13 @@ class LegacyTagStorage(TagStorage):
             queryset = queryset.filter(value__contains=query)
 
         return queryset
+
+    def get_values_seen(self, group_ids, key):
+        if isinstance(group_ids, six.integer_types):
+            qs = GroupTagKey.objects.filter(group_id=group_ids)
+        else:
+            qs = GroupTagKey.objects.filter(group_id__in=group_ids)
+
+        return defaultdict(int, qs.filter(
+            key=key,
+        ).values_list('group_id', 'values_seen'))

src/sentry/tasks/unmerge.py

@@ -6,6 +6,7 @@ from collections import defaultdict
 from django.db import transaction
 from django.db.models import F
 
+from sentry import tagstore
 from sentry.app import tsdb
 from sentry.constants import DEFAULT_LOGGER_NAME, LOG_LEVELS_MAP
 from sentry.event_manager import (
@@ -13,7 +14,7 @@ from sentry.event_manager import (
 )
 from sentry.models import (
     Activity, Environment, Event, EventMapping, EventTag, EventUser, Group, GroupHash, GroupRelease,
-    GroupTagKey, GroupTagValue, Project, Release, UserReport
+    GroupTagValue, Project, Release, UserReport
 )
 from sentry.similarity import features
 from sentry.tasks.base import instrumented_task
@@ -237,9 +238,7 @@ def migrate_events(caches, project, source_id, destination_id, fingerprints, eve
 
 
 def truncate_denormalizations(group):
-    GroupTagKey.objects.filter(
-        group_id=group.id,
-    ).delete()
+    tagstore.delete_all_group_tag_keys(group.id)
 
     GroupTagValue.objects.filter(
         group_id=group.id,
@@ -288,7 +287,7 @@ def collect_tag_data(events):
 def repair_tag_data(caches, project, events):
     for group_id, keys in collect_tag_data(events).items():
         for key, values in keys.items():
-            GroupTagKey.objects.get_or_create(
+            tagstore.get_or_create_group_tag_key(
                 project_id=project.id,
                 group_id=group_id,
                 key=key,
@@ -465,7 +464,7 @@ def repair_denormalizations(caches, project, events):
 
 
 def update_tag_value_counts(id_list):
-    instances = GroupTagKey.objects.filter(group_id__in=id_list)
+    instances = tagstore.get_group_tag_keys(id_list)
     for instance in instances:
         instance.update(
             values_seen=GroupTagValue.objects.filter(

src/sentry/utils/javascript.py

@@ -14,10 +14,10 @@ from django.core.urlresolvers import reverse
 from django.utils import timezone
 from django.utils.html import escape
 
-from sentry import tsdb
+from sentry import tsdb, tagstore
 from sentry.app import env
 from sentry.models import (
-    Group, GroupBookmark, GroupMeta, GroupTagKey, GroupSeen, GroupStatus
+    Group, GroupBookmark, GroupMeta, GroupSeen, GroupStatus
 )
 from sentry.templatetags.sentry_plugins import get_legacy_annotations
 from sentry.utils import json
@@ -127,10 +127,7 @@ class GroupTransformer(Transformer):
         else:
             historical_data = {}
 
-        user_tagkeys = GroupTagKey.objects.filter(
-            group_id__in=[o.id for o in objects],
-            key='sentry:user',
-        )
+        user_tagkeys = tagstore.get_group_tag_keys([o.id for o in objects], 'sentry:user')
         user_counts = {}
         for user_tagkey in user_tagkeys:
             user_counts[user_tagkey.group_id] = user_tagkey.values_seen