ref(api): Add GroupSerializer that supports multiple projects and environments

src/sentry/api/serializers/models/group.py

@@ -19,6 +19,7 @@ from sentry.models import (
     GroupShare, GroupStatus, GroupSubscription, GroupSubscriptionReason, Integration, User, UserOption,
     UserOptionValue
 )
+from sentry.tagstore.snuba.backend import SnubaTagStorage
 from sentry.utils.db import attach_foreignkey
 from sentry.utils.http import absolute_uri
 from sentry.utils.safe import safe_execute
@@ -35,10 +36,16 @@ SUBSCRIPTION_REASON_MAP = {
 disabled = object()
 
 
-@register(Group)
-class GroupSerializer(Serializer):
-    def __init__(self, environment_func=None):
-        self.environment_func = environment_func if environment_func is not None else lambda: None
+class GroupSerializerBase(Serializer):
+    def _get_seen_stats(self, item_list, user):
+        """
+        Returns a dictionary keyed by item that includes:
+            - times_seen
+            - first_seen
+            - last_seen
+            - user_count
+        """
+        raise NotImplementedError
 
     def _get_subscriptions(self, item_list, user):
         """
@@ -147,42 +154,6 @@ class GroupSerializer(Serializer):
         }
         resolved_assignees = Actor.resolve_dict(assignees)
 
-        try:
-            environment = self.environment_func()
-        except Environment.DoesNotExist:
-            user_counts = {}
-            first_seen = {}
-            last_seen = {}
-            times_seen = {}
-        else:
-            project_id = item_list[0].project_id
-            item_ids = [g.id for g in item_list]
-            user_counts = tagstore.get_groups_user_counts(
-                [project_id],
-                item_ids,
-                environment_ids=environment and [environment.id],
-            )
-            first_seen = {}
-            last_seen = {}
-            times_seen = {}
-            if environment is not None:
-                environment_tagvalues = tagstore.get_group_list_tag_value(
-                    [project_id],
-                    item_ids,
-                    [environment.id],
-                    'environment',
-                    environment.name,
-                )
-                for item_id, value in environment_tagvalues.items():
-                    first_seen[item_id] = value.first_seen
-                    last_seen[item_id] = value.last_seen
-                    times_seen[item_id] = value.times_seen
-            else:
-                for item in item_list:
-                    first_seen[item.id] = item.first_seen
-                    last_seen[item.id] = item.last_seen
-                    times_seen[item.id] = item.times_seen
-
         ignore_items = {g.group_id: g for g in GroupSnooze.objects.filter(
             group__in=item_list,
         )}
@@ -243,6 +214,9 @@ class GroupSerializer(Serializer):
         ).values_list('group_id', 'uuid'))
 
         result = {}
+
+        seen_stats = self._get_seen_stats(item_list, user)
+
         for item in item_list:
             active_date = item.active_at or item.first_seen
 
@@ -289,17 +263,15 @@ class GroupSerializer(Serializer):
                 'subscription': subscriptions[item.id],
                 'has_seen': seen_groups.get(item.id, active_date) > active_date,
                 'annotations': annotations,
-                'user_count': user_counts.get(item.id, 0),
                 'ignore_until': ignore_item,
                 'ignore_actor': ignore_actor,
                 'resolution': resolution,
                 'resolution_type': resolution_type,
                 'resolution_actor': resolution_actor,
                 'share_id': share_ids.get(item.id),
-                'times_seen': times_seen.get(item.id, 0),
-                'first_seen': first_seen.get(item.id),  # TODO: missing?
-                'last_seen': last_seen.get(item.id),
             }
+
+            result[item].update(seen_stats.get(item, {}))
         return result
 
     def serialize(self, obj, attrs, user):
@@ -414,6 +386,60 @@ class GroupSerializer(Serializer):
         }
 
 
+@register(Group)
+class GroupSerializer(GroupSerializerBase):
+    def __init__(self, environment_func=None):
+        self.environment_func = environment_func if environment_func is not None else lambda: None
+
+    def _get_seen_stats(self, item_list, user):
+        try:
+            environment = self.environment_func()
+        except Environment.DoesNotExist:
+            user_counts = {}
+            first_seen = {}
+            last_seen = {}
+            times_seen = {}
+        else:
+            project_id = item_list[0].project_id
+            item_ids = [g.id for g in item_list]
+            user_counts = tagstore.get_groups_user_counts(
+                [project_id],
+                item_ids,
+                environment_ids=environment and [environment.id],
+            )
+            first_seen = {}
+            last_seen = {}
+            times_seen = {}
+            if environment is not None:
+                environment_tagvalues = tagstore.get_group_list_tag_value(
+                    [project_id],
+                    item_ids,
+                    [environment.id],
+                    'environment',
+                    environment.name,
+                )
+                for item_id, value in environment_tagvalues.items():
+                    first_seen[item_id] = value.first_seen
+                    last_seen[item_id] = value.last_seen
+                    times_seen[item_id] = value.times_seen
+            else:
+                for item in item_list:
+                    first_seen[item.id] = item.first_seen
+                    last_seen[item.id] = item.last_seen
+                    times_seen[item.id] = item.times_seen
+
+        attrs = {}
+        for item in item_list:
+            attrs[item] = {
+                'times_seen': times_seen.get(item.id, 0),
+                'first_seen': first_seen.get(item.id),  # TODO: missing?
+                'last_seen': last_seen.get(item.id),
+                'user_count': user_counts.get(item.id, 0),
+            }
+
+        return attrs
+
+
 class StreamGroupSerializer(GroupSerializer):
     STATS_PERIOD_CHOICES = {
         '14d': StatsPeriod(14, timedelta(hours=24)),
@@ -500,3 +526,49 @@ class SharedGroupSerializer(GroupSerializer):
         result = super(SharedGroupSerializer, self).serialize(obj, attrs, user)
         del result['annotations']
         return result
+
+
+class GroupSerializerSnuba(GroupSerializerBase):
+    def __init__(self, environment_ids=None):
+        self.environment_ids = environment_ids
+
+    def _get_seen_stats(self, item_list, user):
+        tagstore = SnubaTagStorage()
+        project_ids = list(set([item.project_id for item in item_list]))
+        group_ids = [item.id for item in item_list]
+        user_counts = tagstore.get_groups_user_counts(
+            project_ids,
+            group_ids,
+            environment_ids=self.environment_ids,
+        )
+
+        first_seen = {}
+        last_seen = {}
+        times_seen = {}
+        if self.environment_ids is None:
+            # use issue fields
+            for item in item_list:
+                first_seen[item.id] = item.first_seen
+                last_seen[item.id] = item.last_seen
+                times_seen[item.id] = item.times_seen
+        else:
+            seen_data = tagstore.get_group_seen_values_for_environments(
+                project_ids,
+                group_ids,
+                self.environment_ids,
+            )
+            for item_id, value in seen_data.items():
+                first_seen[item_id] = value['first_seen']
+                last_seen[item_id] = value['last_seen']
+                times_seen[item_id] = value['times_seen']
+
+        attrs = {}
+        for item in item_list:
+            attrs[item] = {
+                'times_seen': times_seen.get(item.id, 0),
+                'first_seen': first_seen.get(item.id),
+                'last_seen': last_seen.get(item.id),
+                'user_count': user_counts.get(item.id, 0),
+            }
+
+        return attrs

src/sentry/tagstore/snuba/backend.py

@@ -286,6 +286,34 @@ class SnubaTagStorage(TagStorage):
             ) for issue, data in six.iteritems(result)
         }
 
+    def get_group_seen_values_for_environments(self, project_ids, group_id_list, environment_ids):
+        # Get the total times seen, first seen, and last seen across multiple environments
+
+        # TODO(jess): this is mostly copy paste from above
+        # also, this is temporary and will probably need to be updated to
+        # filter correctly based on date filters -- waiting on some product decisions
+        start, end = self.get_time_range()
+        filters = {
+            'project_id': project_ids,
+            'issue': group_id_list,
+        }
+        conditions = None
+        if environment_ids:
+            filters['environment'] = environment_ids
+
+        aggregations = [
+            ['count()', '', 'times_seen'],
+            ['min', SEEN_COLUMN, 'first_seen'],
+            ['max', SEEN_COLUMN, 'last_seen'],
+        ]
+
+        result = snuba.query(start, end, ['issue'], conditions, filters, aggregations,
+                             referrer='tagstore.get_group_seen_values_for_environments')
+
+        return {
+            issue: fix_tag_value_data(data) for issue, data in six.iteritems(result)
+        }
+
     def get_group_tag_value_count(self, project_id, group_id, environment_id, key):
         start, end = self.get_time_range()
         tag = u'tags[{}]'.format(key)

tests/snuba/api/serializers/test_group.py

@@ -0,0 +1,327 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import
+
+import six
+
+from datetime import timedelta
+
+from django.utils import timezone
+from mock import patch
+
+from sentry.api.serializers import serialize
+from sentry.api.serializers.models.group import GroupSerializerSnuba
+from sentry.models import (
+    GroupLink, GroupResolution, GroupSnooze, GroupStatus,
+    GroupSubscription, UserOption, UserOptionValue
+)
+from sentry.testutils import APITestCase, SnubaTestCase
+
+
+class GroupSerializerSnubaTest(APITestCase, SnubaTestCase):
+    def setUp(self):
+        super(GroupSerializerSnubaTest, self).setUp()
+        self.min_ago = timezone.now() - timedelta(minutes=1)
+        self.day_ago = timezone.now() - timedelta(days=1)
+        self.week_ago = timezone.now() - timedelta(days=7)
+
+    def test_is_ignored_with_expired_snooze(self):
+        now = timezone.now().replace(microsecond=0)
+
+        user = self.create_user()
+        group = self.create_group(
+            status=GroupStatus.IGNORED,
+        )
+        GroupSnooze.objects.create(
+            group=group,
+            until=now - timedelta(minutes=1),
+        )
+
+        result = serialize(group, user, serializer=GroupSerializerSnuba())
+        assert result['status'] == 'unresolved'
+        assert result['statusDetails'] == {}
+
+    def test_is_ignored_with_valid_snooze(self):
+        now = timezone.now().replace(microsecond=0)
+
+        user = self.create_user()
+        group = self.create_group(
+            status=GroupStatus.IGNORED,
+        )
+        snooze = GroupSnooze.objects.create(
+            group=group,
+            until=now + timedelta(minutes=1),
+        )
+
+        result = serialize(group, user, serializer=GroupSerializerSnuba())
+        assert result['status'] == 'ignored'
+        assert result['statusDetails']['ignoreCount'] == snooze.count
+        assert result['statusDetails']['ignoreWindow'] == snooze.window
+        assert result['statusDetails']['ignoreUserCount'] == snooze.user_count
+        assert result['statusDetails']['ignoreUserWindow'] == snooze.user_window
+        assert result['statusDetails']['ignoreUntil'] == snooze.until
+        assert result['statusDetails']['actor'] is None
+
+    def test_is_ignored_with_valid_snooze_and_actor(self):
+        now = timezone.now().replace(microsecond=0)
+
+        user = self.create_user()
+        group = self.create_group(
+            status=GroupStatus.IGNORED,
+        )
+        GroupSnooze.objects.create(
+            group=group,
+            until=now + timedelta(minutes=1),
+            actor_id=user.id,
+        )
+
+        result = serialize(group, user, serializer=GroupSerializerSnuba())
+        assert result['status'] == 'ignored'
+        assert result['statusDetails']['actor']['id'] == six.text_type(user.id)
+
+    def test_resolved_in_next_release(self):
+        release = self.create_release(project=self.project, version='a')
+        user = self.create_user()
+        group = self.create_group(
+            status=GroupStatus.RESOLVED,
+        )
+        GroupResolution.objects.create(
+            group=group,
+            release=release,
+            type=GroupResolution.Type.in_next_release,
+        )
+
+        result = serialize(group, user, serializer=GroupSerializerSnuba())
+        assert result['status'] == 'resolved'
+        assert result['statusDetails'] == {'inNextRelease': True, 'actor': None}
+
+    def test_resolved_in_release(self):
+        release = self.create_release(project=self.project, version='a')
+        user = self.create_user()
+        group = self.create_group(
+            status=GroupStatus.RESOLVED,
+        )
+        GroupResolution.objects.create(
+            group=group,
+            release=release,
+            type=GroupResolution.Type.in_release,
+        )
+
+        result = serialize(group, user, serializer=GroupSerializerSnuba())
+        assert result['status'] == 'resolved'
+        assert result['statusDetails'] == {'inRelease': 'a', 'actor': None}
+
+    def test_resolved_with_actor(self):
+        release = self.create_release(project=self.project, version='a')
+        user = self.create_user()
+        group = self.create_group(
+            status=GroupStatus.RESOLVED,
+        )
+        GroupResolution.objects.create(
+            group=group,
+            release=release,
+            type=GroupResolution.Type.in_release,
+            actor_id=user.id,
+        )
+
+        result = serialize(group, user, serializer=GroupSerializerSnuba())
+        assert result['status'] == 'resolved'
+        assert result['statusDetails']['actor']['id'] == six.text_type(user.id)
+
+    def test_resolved_in_commit(self):
+        repo = self.create_repo(project=self.project)
+        commit = self.create_commit(repo=repo)
+        user = self.create_user()
+        group = self.create_group(
+            status=GroupStatus.RESOLVED,
+        )
+        GroupLink.objects.create(
+            group_id=group.id,
+            project_id=group.project_id,
+            linked_id=commit.id,
+            linked_type=GroupLink.LinkedType.commit,
+            relationship=GroupLink.Relationship.resolves,
+        )
+
+        result = serialize(group, user, serializer=GroupSerializerSnuba())
+        assert result['status'] == 'resolved'
+        assert result['statusDetails']['inCommit']['id'] == commit.key
+
+    @patch('sentry.models.Group.is_over_resolve_age')
+    def test_auto_resolved(self, mock_is_over_resolve_age):
+        mock_is_over_resolve_age.return_value = True
+
+        user = self.create_user()
+        group = self.create_group(
+            status=GroupStatus.UNRESOLVED,
+        )
+
+        result = serialize(group, user, serializer=GroupSerializerSnuba())
+        assert result['status'] == 'resolved'
+        assert result['statusDetails'] == {'autoResolved': True}
+
+    def test_subscribed(self):
+        user = self.create_user()
+        group = self.create_group()
+
+        GroupSubscription.objects.create(
+            user=user,
+            group=group,
+            project=group.project,
+            is_active=True,
+        )
+
+        result = serialize(group, user, serializer=GroupSerializerSnuba())
+        assert result['isSubscribed']
+        assert result['subscriptionDetails'] == {
+            'reason': 'unknown',
+        }
+
+    def test_explicit_unsubscribed(self):
+        user = self.create_user()
+        group = self.create_group()
+
+        GroupSubscription.objects.create(
+            user=user,
+            group=group,
+            project=group.project,
+            is_active=False,
+        )
+
+        result = serialize(group, user, serializer=GroupSerializerSnuba())
+        assert not result['isSubscribed']
+        assert not result['subscriptionDetails']
+
+    def test_implicit_subscribed(self):
+        user = self.create_user()
+        group = self.create_group()
+
+        combinations = (
+            # ((default, project), (subscribed, details))
+            ((None, None), (True, None)),
+            ((UserOptionValue.all_conversations, None), (True, None)),
+            ((UserOptionValue.all_conversations, UserOptionValue.all_conversations), (True, None)),
+            ((UserOptionValue.all_conversations, UserOptionValue.participating_only), (False, None)),
+            ((UserOptionValue.all_conversations, UserOptionValue.no_conversations),
+             (False, {'disabled': True})),
+            ((UserOptionValue.participating_only, None), (False, None)),
+            ((UserOptionValue.participating_only, UserOptionValue.all_conversations), (True, None)),
+            ((UserOptionValue.participating_only, UserOptionValue.participating_only), (False, None)),
+            ((UserOptionValue.participating_only, UserOptionValue.no_conversations),
+             (False, {'disabled': True})),
+            ((UserOptionValue.no_conversations, None), (False, {'disabled': True})),
+            ((UserOptionValue.no_conversations, UserOptionValue.all_conversations), (True, None)),
+            ((UserOptionValue.no_conversations, UserOptionValue.participating_only), (False, None)),
+            ((UserOptionValue.no_conversations, UserOptionValue.no_conversations),
+             (False, {'disabled': True})),
+        )
+
+        def maybe_set_value(project, value):
+            if value is not None:
+                UserOption.objects.set_value(
+                    user=user,
+                    project=project,
+                    key='workflow:notifications',
+                    value=value,
+                )
+            else:
+                UserOption.objects.unset_value(
+                    user=user,
+                    project=project,
+                    key='workflow:notifications',
+                )
+
+        for options, (is_subscribed, subscription_details) in combinations:
+            default_value, project_value = options
+            UserOption.objects.clear_local_cache()
+            maybe_set_value(None, default_value)
+            maybe_set_value(group.project, project_value)
+            result = serialize(group, user, serializer=GroupSerializerSnuba())
+            assert result['isSubscribed'] is is_subscribed
+            assert result.get('subscriptionDetails') == subscription_details
+
+    def test_global_no_conversations_overrides_group_subscription(self):
+        user = self.create_user()
+        group = self.create_group()
+
+        GroupSubscription.objects.create(
+            user=user,
+            group=group,
+            project=group.project,
+            is_active=True,
+        )
+
+        UserOption.objects.set_value(
+            user=user,
+            project=None,
+            key='workflow:notifications',
+            value=UserOptionValue.no_conversations,
+        )
+
+        result = serialize(group, user, serializer=GroupSerializerSnuba())
+        assert not result['isSubscribed']
+        assert result['subscriptionDetails'] == {
+            'disabled': True,
+        }
+
+    def test_project_no_conversations_overrides_group_subscription(self):
+        user = self.create_user()
+        group = self.create_group()
+
+        GroupSubscription.objects.create(
+            user=user,
+            group=group,
+            project=group.project,
+            is_active=True,
+        )
+
+        UserOption.objects.set_value(
+            user=user,
+            project=group.project,
+            key='workflow:notifications',
+            value=UserOptionValue.no_conversations,
+        )
+
+        result = serialize(group, user, serializer=GroupSerializerSnuba())
+        assert not result['isSubscribed']
+        assert result['subscriptionDetails'] == {
+            'disabled': True,
+        }
+
+    def test_no_user_unsubscribed(self):
+        group = self.create_group()
+
+        result = serialize(group, serializer=GroupSerializerSnuba())
+        assert not result['isSubscribed']
+
+    def test_seen_stats(self):
+        group = self.create_group(first_seen=self.week_ago, times_seen=5)
+
+        # should use group columns when no environments arg passed
+        result = serialize(group, serializer=GroupSerializerSnuba())
+        assert result['count'] == '5'
+        assert result['lastSeen'] == group.last_seen
+        assert result['firstSeen'] == group.first_seen
+
+        environment = self.create_environment(project=group.project)
+        environment2 = self.create_environment(project=group.project)
+
+        self.create_event(
+            'a' * 32, group=group, datetime=self.day_ago, tags={'environment': environment.name}
+        )
+        self.create_event(
+            'b' * 32, group=group, datetime=self.min_ago, tags={'environment': environment.name}
+        )
+        self.create_event(
+            'c' * 32, group=group, datetime=self.min_ago, tags={'environment': environment2.name}
+        )
+
+        result = serialize(
+            group, serializer=GroupSerializerSnuba(
+                environment_ids=[environment.id, environment2.id])
+        )
+        assert result['count'] == '3'
+        # result is rounded down to nearest second
+        assert result['lastSeen'] == self.min_ago - timedelta(microseconds=self.min_ago.microsecond)
+        assert result['firstSeen'] == self.day_ago - \
+            timedelta(microseconds=self.day_ago.microsecond)