sentry/tests/snuba/search/test_backend.py

import uuid
from datetime import datetime, timedelta
from hashlib import md5
from unittest import mock

import pytest
import pytz
from django.utils import timezone

from sentry import options
from sentry.api.issue_search import convert_query_values, issue_search_config, parse_search_query
from sentry.exceptions import InvalidSearchQuery
from sentry.models import (
    Environment,
    Group,
    GroupAssignee,
    GroupBookmark,
    GroupEnvironment,
    GroupHistoryStatus,
    GroupStatus,
    GroupSubscription,
    Integration,
    record_group_history,
)
from sentry.models.groupowner import GroupOwner
from sentry.search.snuba.backend import (
    CdcEventsDatasetSnubaSearchBackend,
    EventsDatasetSnubaSearchBackend,
)
from sentry.search.snuba.executors import InvalidQueryForExecutor
from sentry.testutils import SnubaTestCase, TestCase, xfail_if_not_postgres
from sentry.testutils.helpers.datetime import before_now, iso_format
from sentry.testutils.helpers.faux import Any
from sentry.types.issues import GroupType
from sentry.utils.snuba import SENTRY_SNUBA_MAP, Dataset, SnubaError


def date_to_query_format(date):
    return date.strftime("%Y-%m-%dT%H:%M:%S")


class EventsSnubaSearchTest(TestCase, SnubaTestCase):
    @property
    def backend(self):
        return EventsDatasetSnubaSearchBackend()

    def setUp(self):
        super().setUp()
        self.base_datetime = (datetime.utcnow() - timedelta(days=3)).replace(tzinfo=pytz.utc)

        event1_timestamp = iso_format(self.base_datetime - timedelta(days=21))
        self.event1 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group1"],
                "event_id": "a" * 32,
                "message": "foo. Also, this message is intended to be greater than 256 characters so that we can put some unique string identifier after that point in the string. The purpose of this is in order to verify we are using snuba to search messages instead of Postgres (postgres truncates at 256 characters and clickhouse does not). santryrox.",
                "environment": "production",
                "tags": {"server": "example.com", "sentry:user": "event1@example.com"},
                "timestamp": event1_timestamp,
                "stacktrace": {"frames": [{"module": "group1"}]},
            },
            project_id=self.project.id,
        )
        self.event3 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group1"],
                "event_id": "c" * 32,
                "message": "group1",
                "environment": "production",
                "tags": {"server": "example.com", "sentry:user": "event3@example.com"},
                "timestamp": iso_format(self.base_datetime),
                "stacktrace": {"frames": [{"module": "group1"}]},
            },
            project_id=self.project.id,
        )

        self.group1 = Group.objects.get(id=self.event1.group.id)
        assert self.group1.id == self.event1.group.id
        assert self.group1.id == self.event3.group.id

        assert self.group1.first_seen == self.event1.datetime
        assert self.group1.last_seen == self.event3.datetime

        self.group1.times_seen = 5
        self.group1.status = GroupStatus.UNRESOLVED
        self.group1.update(type=GroupType.ERROR.value)
        self.group1.save()
        self.store_group(self.group1)

        self.event2 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group2"],
                "event_id": "b" * 32,
                "timestamp": iso_format(self.base_datetime - timedelta(days=20)),
                "message": "bar",
                "stacktrace": {"frames": [{"module": "group2"}]},
                "environment": "staging",
                "tags": {
                    "server": "example.com",
                    "url": "http://example.com",
                    "sentry:user": "event2@example.com",
                },
            },
            project_id=self.project.id,
        )

        self.group2 = Group.objects.get(id=self.event2.group.id)
        assert self.group2.id == self.event2.group.id
        assert self.group2.first_seen == self.group2.last_seen == self.event2.datetime

        self.group2.status = GroupStatus.RESOLVED
        self.group2.times_seen = 10
        self.group2.update(type=GroupType.ERROR.value)
        self.group2.save()
        self.store_group(self.group2)

        GroupBookmark.objects.create(user=self.user, group=self.group2, project=self.group2.project)

        GroupAssignee.objects.create(user=self.user, group=self.group2, project=self.group2.project)

        GroupSubscription.objects.create(
            user=self.user, group=self.group1, project=self.group1.project, is_active=True
        )

        GroupSubscription.objects.create(
            user=self.user, group=self.group2, project=self.group2.project, is_active=False
        )

        self.environments = {
            "production": self.event1.get_environment(),
            "staging": self.event2.get_environment(),
        }

    def store_event(self, data, *args, **kwargs):
        event = super().store_event(data, *args, **kwargs)
        environment_name = data.get("environment")
        if environment_name:
            GroupEnvironment.objects.filter(
                group_id=event.group_id,
                environment__name=environment_name,
                first_seen__gt=event.datetime,
            ).update(first_seen=event.datetime)
        return event

    def set_up_multi_project(self):
        self.project2 = self.create_project(organization=self.project.organization)
        self.event_p2 = self.store_event(
            data={
                "event_id": "a" * 32,
                "fingerprint": ["put-me-in-groupP2"],
                "timestamp": iso_format(self.base_datetime - timedelta(days=21)),
                "message": "foo",
                "stacktrace": {"frames": [{"module": "group_p2"}]},
                "tags": {"server": "example.com"},
                "environment": "production",
            },
            project_id=self.project2.id,
        )

        self.group_p2 = Group.objects.get(id=self.event_p2.group.id)
        self.group_p2.times_seen = 6
        self.group_p2.last_seen = self.base_datetime - timedelta(days=1)
        self.group_p2.save()
        self.store_group(self.group_p2)

    def create_group_with_integration_external_issue(self, environment="production"):
        event = self.store_event(
            data={
                "fingerprint": ["linked_group1"],
                "event_id": uuid.uuid4().hex,
                "timestamp": iso_format(self.base_datetime),
                "environment": environment,
            },
            project_id=self.project.id,
        )
        integration = Integration.objects.create(provider="example", name="Example")
        integration.add_organization(event.group.organization, self.user)
        self.create_integration_external_issue(
            group=event.group,
            integration=integration,
            key="APP-123",
        )
        return event.group

    def create_group_with_platform_external_issue(self, environment="production"):
        event = self.store_event(
            data={
                "fingerprint": ["linked_group2"],
                "event_id": uuid.uuid4().hex,
                "timestamp": iso_format(self.base_datetime),
                "environment": environment,
            },
            project_id=self.project.id,
        )
        self.create_platform_external_issue(
            group=event.group,
            service_type="sentry-app",
            display_name="App#issue-1",
            web_url="https://example.com/app/issues/1",
        )
        return event.group

    def build_search_filter(self, query, projects=None, user=None, environments=None):
        user = user if user is not None else self.user
        projects = projects if projects is not None else [self.project]
        return convert_query_values(parse_search_query(query), projects, user, environments)

    def make_query(
        self,
        projects=None,
        search_filter_query=None,
        environments=None,
        sort_by="date",
        limit=None,
        count_hits=False,
        date_from=None,
        date_to=None,
    ):
        search_filters = []
        projects = projects if projects is not None else [self.project]
        if search_filter_query is not None:
            search_filters = self.build_search_filter(
                search_filter_query, projects, environments=environments
            )

        kwargs = {}
        if limit is not None:
            kwargs["limit"] = limit

        return self.backend.query(
            projects,
            search_filters=search_filters,
            environments=environments,
            count_hits=count_hits,
            sort_by=sort_by,
            date_from=date_from,
            date_to=date_to,
            **kwargs,
        )

    def run_test_query_in_syntax(
        self, query, expected_groups, expected_negative_groups=None, environments=None
    ):
        results = self.make_query(search_filter_query=query, environments=environments)
        sort_key = lambda result: result.id
        print("results", results.results)
        assert sorted(results, key=sort_key) == sorted(expected_groups, key=sort_key)

        if expected_negative_groups is not None:
            results = self.make_query(search_filter_query=f"!{query}")
            assert sorted(results, key=sort_key) == sorted(expected_negative_groups, key=sort_key)

    def test_query(self):
        results = self.make_query(search_filter_query="foo")
        assert set(results) == {self.group1}

        results = self.make_query(search_filter_query="bar")
        assert set(results) == {self.group2}

    def test_query_multi_project(self):
        self.set_up_multi_project()
        results = self.make_query([self.project, self.project2], search_filter_query="foo")
        assert set(results) == {self.group1, self.group_p2}

    def test_query_with_environment(self):
        results = self.make_query(
            environments=[self.environments["production"]], search_filter_query="foo"
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            environments=[self.environments["production"]], search_filter_query="bar"
        )
        assert set(results) == set()

        results = self.make_query(
            environments=[self.environments["staging"]], search_filter_query="bar"
        )
        assert set(results) == {self.group2}

    def test_query_for_text_in_long_message(self):
        results = self.make_query(
            [self.project],
            environments=[self.environments["production"]],
            search_filter_query="santryrox",
        )

        assert set(results) == {self.group1}

    def test_multi_environments(self):
        self.set_up_multi_project()
        results = self.make_query(
            [self.project, self.project2],
            environments=[self.environments["production"], self.environments["staging"]],
        )
        assert set(results) == {self.group1, self.group2, self.group_p2}

    def test_query_with_environment_multi_project(self):
        self.set_up_multi_project()
        results = self.make_query(
            [self.project, self.project2],
            environments=[self.environments["production"]],
            search_filter_query="foo",
        )
        assert set(results) == {self.group1, self.group_p2}

        results = self.make_query(
            [self.project, self.project2],
            environments=[self.environments["production"]],
            search_filter_query="bar",
        )
        assert set(results) == set()

    def test_sort(self):
        results = self.make_query(sort_by="date")
        assert list(results) == [self.group1, self.group2]

        results = self.make_query(sort_by="new")
        assert list(results) == [self.group2, self.group1]

        results = self.make_query(sort_by="freq")
        assert list(results) == [self.group1, self.group2]

        results = self.make_query(sort_by="priority")
        assert list(results) == [self.group1, self.group2]

        results = self.make_query(sort_by="user")
        assert list(results) == [self.group1, self.group2]

    def test_sort_with_environment(self):
        for dt in [
            self.group1.first_seen + timedelta(days=1),
            self.group1.first_seen + timedelta(days=2),
            self.group1.last_seen + timedelta(days=1),
        ]:
            self.store_event(
                data={
                    "fingerprint": ["put-me-in-group2"],
                    "timestamp": iso_format(dt),
                    "stacktrace": {"frames": [{"module": "group2"}]},
                    "environment": "production",
                    "message": "group2",
                },
                project_id=self.project.id,
            )

        results = self.make_query(environments=[self.environments["production"]], sort_by="date")
        assert list(results) == [self.group2, self.group1]
        results = self.make_query(environments=[self.environments["production"]], sort_by="new")
        assert list(results) == [self.group2, self.group1]

        results = self.make_query(environments=[self.environments["production"]], sort_by="freq")
        assert list(results) == [self.group2, self.group1]

        results = self.make_query(
            environments=[self.environments["production"]], sort_by="priority"
        )
        assert list(results) == [self.group2, self.group1]

        results = self.make_query(environments=[self.environments["production"]], sort_by="user")
        assert list(results) == [self.group1, self.group2]

    def test_status(self):
        results = self.make_query(search_filter_query="is:unresolved")
        assert set(results) == {self.group1}

        results = self.make_query(search_filter_query="is:resolved")
        assert set(results) == {self.group2}

        event_3 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group3"],
                "event_id": "c" * 32,
                "timestamp": iso_format(self.base_datetime - timedelta(days=20)),
            },
            project_id=self.project.id,
        )
        group_3 = event_3.group
        group_3.status = GroupStatus.MUTED
        group_3.save()

        self.run_test_query_in_syntax(
            "status:[unresolved, resolved]", [self.group1, self.group2], [group_3]
        )
        self.run_test_query_in_syntax(
            "status:[resolved, muted]", [self.group2, group_3], [self.group1]
        )

    def test_category(self):
        with self.feature("organizations:performance-issues"):
            results = self.make_query(search_filter_query="category:error")
        assert set(results) == {self.group1, self.group2}

        event_3 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group3"],
                "event_id": "c" * 32,
                "timestamp": iso_format(self.base_datetime - timedelta(days=20)),
            },
            project_id=self.project.id,
        )
        group_3 = event_3.group
        group_3.update(type=GroupType.PERFORMANCE_N_PLUS_ONE.value)
        with self.feature("organizations:performance-issues"):
            results = self.make_query(search_filter_query="category:performance")
        assert set(results) == {group_3}

        with self.feature("organizations:performance-issues"):
            results = self.make_query(search_filter_query="category:[error, performance]")
        assert set(results) == {self.group1, self.group2, group_3}

        with pytest.raises(InvalidSearchQuery):
            with self.feature("organizations:performance-issues"):
                self.make_query(search_filter_query="category:hellboy")

    def test_type(self):
        with self.feature("organizations:performance-issues"):
            results = self.make_query(search_filter_query="type:error")
        assert set(results) == {self.group1, self.group2}

        event_3 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group3"],
                "event_id": "c" * 32,
                "timestamp": iso_format(self.base_datetime - timedelta(days=20)),
                "type": GroupType.PERFORMANCE_N_PLUS_ONE.value,
            },
            project_id=self.project.id,
        )
        group_3 = event_3.group
        group_3.update(type=GroupType.PERFORMANCE_N_PLUS_ONE.value)

        with self.feature("organizations:performance-issues"):
            results = self.make_query(search_filter_query="type:performance_n_plus_one")
        assert set(results) == {group_3}

        event_4 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group4"],
                "event_id": "d" * 32,
                "timestamp": iso_format(self.base_datetime - timedelta(days=20)),
            },
            project_id=self.project.id,
        )
        group_4 = event_4.group
        group_4.update(type=GroupType.PERFORMANCE_SLOW_SPAN.value)
        with self.feature("organizations:performance-issues"):
            results = self.make_query(search_filter_query="type:performance_slow_span")
        assert set(results) == {group_4}

        with self.feature("organizations:performance-issues"):
            results = self.make_query(
                search_filter_query="type:[performance_slow_span, performance_n_plus_one, error]"
            )
        assert set(results) == {self.group1, self.group2, group_3, group_4}

        with pytest.raises(InvalidSearchQuery):
            with self.feature("organizations:performance-issues"):
                self.make_query(search_filter_query="type:performance_i_dont_exist")

    def test_status_with_environment(self):
        results = self.make_query(
            environments=[self.environments["production"]], search_filter_query="is:unresolved"
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            environments=[self.environments["staging"]], search_filter_query="is:resolved"
        )
        assert set(results) == {self.group2}

        results = self.make_query(
            environments=[self.environments["production"]], search_filter_query="is:resolved"
        )
        assert set(results) == set()

    def test_tags(self):
        results = self.make_query(search_filter_query="environment:staging")
        assert set(results) == {self.group2}

        results = self.make_query(search_filter_query="environment:example.com")
        assert set(results) == set()

        results = self.make_query(search_filter_query="has:environment")
        assert set(results) == {self.group2, self.group1}

        results = self.make_query(search_filter_query="environment:staging server:example.com")
        assert set(results) == {self.group2}

        results = self.make_query(search_filter_query='url:"http://example.com"')
        assert set(results) == {self.group2}

        results = self.make_query(search_filter_query="environment:staging has:server")
        assert set(results) == {self.group2}

        results = self.make_query(search_filter_query="environment:staging server:bar.example.com")
        assert set(results) == set()

    def test_tags_with_environment(self):
        results = self.make_query(
            environments=[self.environments["production"]], search_filter_query="server:example.com"
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            environments=[self.environments["staging"]], search_filter_query="server:example.com"
        )
        assert set(results) == {self.group2}

        results = self.make_query(
            environments=[self.environments["staging"]], search_filter_query="has:server"
        )
        assert set(results) == {self.group2}

        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query='url:"http://example.com"',
        )
        assert set(results) == set()

        results = self.make_query(
            environments=[self.environments["staging"]],
            search_filter_query='url:"http://example.com"',
        )
        assert set(results) == {self.group2}

        results = self.make_query(
            environments=[self.environments["staging"]],
            search_filter_query="server:bar.example.com",
        )
        assert set(results) == set()

    def test_bookmarked_by(self):
        results = self.make_query(search_filter_query="bookmarks:%s" % self.user.username)
        assert set(results) == {self.group2}

    def test_bookmarked_by_in_syntax(self):
        self.run_test_query_in_syntax(
            f"bookmarks:[{self.user.username}]", [self.group2], [self.group1]
        )
        user_2 = self.create_user()
        GroupBookmark.objects.create(user=user_2, group=self.group1, project=self.group2.project)
        self.run_test_query_in_syntax(
            f"bookmarks:[{self.user.username}, {user_2.username}]", [self.group2, self.group1], []
        )

    def test_bookmarked_by_with_environment(self):
        results = self.make_query(
            environments=[self.environments["staging"]],
            search_filter_query="bookmarks:%s" % self.user.username,
        )
        assert set(results) == {self.group2}

        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query="bookmarks:%s" % self.user.username,
        )
        assert set(results) == set()

    def test_search_filter_query_with_custom_priority_tag(self):
        priority = "high"
        self.store_event(
            data={
                "fingerprint": ["put-me-in-group2"],
                "timestamp": iso_format(self.group2.first_seen + timedelta(days=1)),
                "stacktrace": {"frames": [{"module": "group2"}]},
                "message": "group2",
                "tags": {"priority": priority},
            },
            project_id=self.project.id,
        )

        results = self.make_query(search_filter_query="priority:%s" % priority)

        assert set(results) == {self.group2}

    def test_search_filter_query_with_custom_priority_tag_and_priority_sort(self):
        priority = "high"
        for i in range(1, 3):
            self.store_event(
                data={
                    "fingerprint": ["put-me-in-group1"],
                    "timestamp": iso_format(self.group2.last_seen + timedelta(days=i)),
                    "stacktrace": {"frames": [{"module": "group1"}]},
                    "message": "group1",
                    "tags": {"priority": priority},
                },
                project_id=self.project.id,
            )
        self.store_event(
            data={
                "fingerprint": ["put-me-in-group2"],
                "timestamp": iso_format(self.group2.last_seen + timedelta(days=2)),
                "stacktrace": {"frames": [{"module": "group2"}]},
                "message": "group2",
                "tags": {"priority": priority},
            },
            project_id=self.project.id,
        )
        results = self.make_query(search_filter_query="priority:%s" % priority, sort_by="priority")
        assert list(results) == [self.group1, self.group2]

    def test_search_tag_overlapping_with_internal_fields(self):
        # Using a tag of email overlaps with the promoted user.email column in events.
        # We don't want to bypass public schema limits in issue search.
        self.store_event(
            data={
                "fingerprint": ["put-me-in-group2"],
                "timestamp": iso_format(self.group2.first_seen + timedelta(days=1)),
                "stacktrace": {"frames": [{"module": "group2"}]},
                "message": "group2",
                "tags": {"email": "tags@example.com"},
            },
            project_id=self.project.id,
        )
        results = self.make_query(search_filter_query="email:tags@example.com")
        assert set(results) == {self.group2}

    def test_project(self):
        results = self.make_query([self.create_project(name="other")])
        assert set(results) == set()

    def test_pagination(self):
        for options_set in [
            {"snuba.search.min-pre-snuba-candidates": None},
            {"snuba.search.min-pre-snuba-candidates": 500},
        ]:
            with self.options(options_set):
                results = self.backend.query([self.project], limit=1, sort_by="date")
                assert set(results) == {self.group1}
                assert not results.prev.has_results
                assert results.next.has_results

                results = self.backend.query(
                    [self.project], cursor=results.next, limit=1, sort_by="date"
                )
                assert set(results) == {self.group2}
                assert results.prev.has_results
                assert not results.next.has_results

                # note: previous cursor
                results = self.backend.query(
                    [self.project], cursor=results.prev, limit=1, sort_by="date"
                )
                assert set(results) == {self.group1}
                assert results.prev.has_results
                assert results.next.has_results

                # note: previous cursor, paging too far into 0 results
                results = self.backend.query(
                    [self.project], cursor=results.prev, limit=1, sort_by="date"
                )
                assert set(results) == set()
                assert not results.prev.has_results
                assert results.next.has_results

                results = self.backend.query(
                    [self.project], cursor=results.next, limit=1, sort_by="date"
                )
                assert set(results) == {self.group1}
                assert results.prev.has_results
                assert results.next.has_results

                results = self.backend.query(
                    [self.project], cursor=results.next, limit=1, sort_by="date"
                )
                assert set(results) == {self.group2}
                assert results.prev.has_results
                assert not results.next.has_results

                results = self.backend.query(
                    [self.project], cursor=results.next, limit=1, sort_by="date"
                )
                assert set(results) == set()
                assert results.prev.has_results
                assert not results.next.has_results

    def test_pagination_with_environment(self):
        for dt in [
            self.group1.first_seen + timedelta(days=1),
            self.group1.first_seen + timedelta(days=2),
            self.group1.last_seen + timedelta(days=1),
        ]:
            self.store_event(
                data={
                    "fingerprint": ["put-me-in-group2"],
                    "timestamp": iso_format(dt),
                    "environment": "production",
                    "message": "group2",
                    "stacktrace": {"frames": [{"module": "group2"}]},
                },
                project_id=self.project.id,
            )

        results = self.backend.query(
            [self.project],
            environments=[self.environments["production"]],
            sort_by="date",
            limit=1,
            count_hits=True,
        )
        assert list(results) == [self.group2]
        assert results.hits == 2

        results = self.backend.query(
            [self.project],
            environments=[self.environments["production"]],
            sort_by="date",
            limit=1,
            cursor=results.next,
            count_hits=True,
        )
        assert list(results) == [self.group1]
        assert results.hits == 2

        results = self.backend.query(
            [self.project],
            environments=[self.environments["production"]],
            sort_by="date",
            limit=1,
            cursor=results.next,
            count_hits=True,
        )
        assert list(results) == []
        assert results.hits == 2

    def test_age_filter(self):
        results = self.make_query(
            search_filter_query="firstSeen:>=%s" % date_to_query_format(self.group2.first_seen)
        )
        assert set(results) == {self.group2}

        results = self.make_query(
            search_filter_query="firstSeen:<=%s"
            % date_to_query_format(self.group1.first_seen + timedelta(minutes=1))
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            search_filter_query="firstSeen:>=%s firstSeen:<=%s"
            % (
                date_to_query_format(self.group1.first_seen),
                date_to_query_format(self.group1.first_seen + timedelta(minutes=1)),
            )
        )
        assert set(results) == {self.group1}

    def test_age_filter_with_environment(self):
        # add time instead to make it greater than or less than as needed.
        group1_first_seen = GroupEnvironment.objects.get(
            environment=self.environments["production"], group=self.group1
        ).first_seen

        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query="firstSeen:>=%s" % date_to_query_format(group1_first_seen),
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query="firstSeen:<=%s" % date_to_query_format(group1_first_seen),
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query="firstSeen:>%s" % date_to_query_format(group1_first_seen),
        )
        assert set(results) == set()
        self.store_event(
            data={
                "fingerprint": ["put-me-in-group1"],
                "timestamp": iso_format(group1_first_seen + timedelta(days=1)),
                "message": "group1",
                "stacktrace": {"frames": [{"module": "group1"}]},
                "environment": "development",
            },
            project_id=self.project.id,
        )

        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query="firstSeen:>%s" % date_to_query_format(group1_first_seen),
        )
        assert set(results) == set()

        results = self.make_query(
            environments=[Environment.objects.get(name="development")],
            search_filter_query="firstSeen:>%s" % date_to_query_format(group1_first_seen),
        )
        assert set(results) == {self.group1}

    def test_times_seen_filter(self):
        results = self.make_query([self.project], search_filter_query="times_seen:2")
        assert set(results) == {self.group1}

        results = self.make_query([self.project], search_filter_query="times_seen:>=2")
        assert set(results) == {self.group1}

        results = self.make_query([self.project], search_filter_query="times_seen:<=1")
        assert set(results) == {self.group2}

    def test_last_seen_filter(self):
        results = self.make_query(
            search_filter_query="lastSeen:>=%s" % date_to_query_format(self.group1.last_seen)
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            search_filter_query="lastSeen:>=%s lastSeen:<=%s"
            % (
                date_to_query_format(self.group1.last_seen),
                date_to_query_format(self.group1.last_seen + timedelta(minutes=1)),
            )
        )
        assert set(results) == {self.group1}

    def test_last_seen_filter_with_environment(self):
        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query="lastSeen:>=%s" % date_to_query_format(self.group1.last_seen),
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query="lastSeen:<=%s" % date_to_query_format(self.group1.last_seen),
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query="lastSeen:>%s" % date_to_query_format(self.group1.last_seen),
        )
        assert set(results) == set()

        self.store_event(
            data={
                "fingerprint": ["put-me-in-group1"],
                "timestamp": iso_format(self.group1.last_seen + timedelta(days=1)),
                "message": "group1",
                "stacktrace": {"frames": [{"module": "group1"}]},
                "environment": "development",
            },
            project_id=self.project.id,
        )

        self.group1.update(last_seen=self.group1.last_seen + timedelta(days=1))

        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query="lastSeen:>%s" % date_to_query_format(self.group1.last_seen),
        )
        assert set(results) == set()

        results = self.make_query(
            environments=[Environment.objects.get(name="development")],
            search_filter_query="lastSeen:>%s" % date_to_query_format(self.group1.last_seen),
        )
        assert set(results) == set()

        results = self.make_query(
            environments=[Environment.objects.get(name="development")],
            search_filter_query="lastSeen:>=%s" % date_to_query_format(self.group1.last_seen),
        )
        assert set(results) == {self.group1}

    def test_date_filter(self):
        results = self.make_query(
            date_from=self.event2.datetime,
            search_filter_query="timestamp:>=%s" % date_to_query_format(self.event2.datetime),
        )
        assert set(results) == {self.group1, self.group2}

        results = self.make_query(
            date_to=self.event1.datetime + timedelta(minutes=1),
            search_filter_query="timestamp:<=%s"
            % date_to_query_format(self.event1.datetime + timedelta(minutes=1)),
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            date_from=self.event1.datetime,
            date_to=self.event2.datetime + timedelta(minutes=1),
            search_filter_query="timestamp:>=%s timestamp:<=%s"
            % (
                date_to_query_format(self.event1.datetime),
                date_to_query_format(self.event2.datetime + timedelta(minutes=1)),
            ),
        )
        assert set(results) == {self.group1, self.group2}

        # Test with `Z` utc marker, should be equivalent
        results = self.make_query(
            date_from=self.event1.datetime,
            date_to=self.event2.datetime + timedelta(minutes=1),
            search_filter_query="timestamp:>=%s timestamp:<=%s"
            % (
                date_to_query_format(self.event1.datetime) + "Z",
                date_to_query_format(self.event2.datetime + timedelta(minutes=1)) + "Z",
            ),
        )
        assert set(results) == {self.group1, self.group2}

    def test_date_filter_with_environment(self):
        results = self.backend.query(
            [self.project],
            environments=[self.environments["production"]],
            date_from=self.event2.datetime,
        )
        assert set(results) == {self.group1}

        results = self.backend.query(
            [self.project],
            environments=[self.environments["production"]],
            date_to=self.event1.datetime + timedelta(minutes=1),
        )
        assert set(results) == {self.group1}

        results = self.backend.query(
            [self.project],
            environments=[self.environments["staging"]],
            date_from=self.event1.datetime,
            date_to=self.event2.datetime + timedelta(minutes=1),
        )
        assert set(results) == {self.group2}

    def test_linked(self):
        linked_group1 = self.create_group_with_integration_external_issue()
        linked_group2 = self.create_group_with_platform_external_issue()

        results = self.make_query(search_filter_query="is:unlinked")
        assert set(results) == {self.group1, self.group2}

        results = self.make_query(search_filter_query="is:linked")
        assert set(results) == {linked_group1, linked_group2}

    def test_linked_with_only_integration_external_issue(self):
        linked_group = self.create_group_with_integration_external_issue()

        results = self.make_query(search_filter_query="is:unlinked")
        assert set(results) == {self.group1, self.group2}

        results = self.make_query(search_filter_query="is:linked")
        assert set(results) == {linked_group}

    def test_linked_with_only_platform_external_issue(self):
        linked_group = self.create_group_with_platform_external_issue()

        results = self.make_query(search_filter_query="is:unlinked")
        assert set(results) == {self.group1, self.group2}

        results = self.make_query(search_filter_query="is:linked")
        assert set(results) == {linked_group}

    def test_linked_with_environment(self):
        linked_group1 = self.create_group_with_integration_external_issue(environment="production")
        linked_group2 = self.create_group_with_platform_external_issue(environment="staging")

        results = self.make_query(
            environments=[self.environments["production"]], search_filter_query="is:unlinked"
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            environments=[self.environments["staging"]], search_filter_query="is:unlinked"
        )
        assert set(results) == {self.group2}

        results = self.make_query(
            environments=[self.environments["production"]], search_filter_query="is:linked"
        )
        assert set(results) == {linked_group1}

        results = self.make_query(
            environments=[self.environments["staging"]], search_filter_query="is:linked"
        )
        assert set(results) == {linked_group2}

    def test_unassigned(self):
        results = self.make_query(search_filter_query="is:unassigned")
        assert set(results) == {self.group1}

        results = self.make_query(search_filter_query="is:assigned")
        assert set(results) == {self.group2}

    def test_unassigned_with_environment(self):
        results = self.make_query(
            environments=[self.environments["production"]], search_filter_query="is:unassigned"
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            environments=[self.environments["staging"]], search_filter_query="is:assigned"
        )
        assert set(results) == {self.group2}

        results = self.make_query(
            environments=[self.environments["production"]], search_filter_query="is:assigned"
        )
        assert set(results) == set()

    def test_assigned_to(self):
        results = self.make_query(search_filter_query="assigned:%s" % self.user.username)
        assert set(results) == {self.group2}

        # test team assignee
        ga = GroupAssignee.objects.get(
            user=self.user, group=self.group2, project=self.group2.project
        )
        ga.update(team=self.team, user=None)
        assert GroupAssignee.objects.get(id=ga.id).user is None

        results = self.make_query(search_filter_query="assigned:%s" % self.user.username)
        assert set(results) == {self.group2}

        # test when there should be no results
        other_user = self.create_user()
        results = self.make_query(search_filter_query="assigned:%s" % other_user.username)
        assert set(results) == set()

        owner = self.create_user()
        self.create_member(
            organization=self.project.organization, user=owner, role="owner", teams=[]
        )

        # test that owners don't see results for all teams
        results = self.make_query(search_filter_query="assigned:%s" % owner.username)
        assert set(results) == set()

    def test_assigned_to_in_syntax(self):
        group_3 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group3"],
                "event_id": "c" * 32,
                "timestamp": iso_format(self.base_datetime - timedelta(days=20)),
            },
            project_id=self.project.id,
        ).group
        group_3.status = GroupStatus.MUTED
        group_3.save()
        other_user = self.create_user()
        self.run_test_query_in_syntax(
            f"assigned:[{self.user.username}, {other_user.username}]",
            [self.group2],
            [self.group1, group_3],
        )

        GroupAssignee.objects.create(project=self.project, group=group_3, user=other_user)
        self.run_test_query_in_syntax(
            f"assigned:[{self.user.username}, {other_user.username}]",
            [self.group2, group_3],
            [self.group1],
        )

        self.run_test_query_in_syntax(
            f"assigned:[#{self.team.slug}, {other_user.username}]",
            [group_3],
            [self.group1, self.group2],
        )

        ga_2 = GroupAssignee.objects.get(
            user=self.user, group=self.group2, project=self.group2.project
        )
        ga_2.update(team=self.team, user=None)
        self.run_test_query_in_syntax(
            f"assigned:[{self.user.username}, {other_user.username}]",
            [self.group2, group_3],
            [self.group1],
        )
        self.run_test_query_in_syntax(
            f"assigned:[#{self.team.slug}, {other_user.username}]",
            [self.group2, group_3],
            [self.group1],
        )

        self.run_test_query_in_syntax(
            f"assigned:[me, none, {other_user.username}]",
            [self.group1, self.group2, group_3],
            [],
        )

    def test_assigned_or_suggested_in_syntax(self):
        Group.objects.all().delete()
        group = self.store_event(
            data={
                "timestamp": iso_format(before_now(seconds=180)),
                "fingerprint": ["group-1"],
            },
            project_id=self.project.id,
        ).group
        group1 = self.store_event(
            data={
                "timestamp": iso_format(before_now(seconds=185)),
                "fingerprint": ["group-2"],
            },
            project_id=self.project.id,
        ).group
        group2 = self.store_event(
            data={
                "timestamp": iso_format(before_now(seconds=190)),
                "fingerprint": ["group-3"],
            },
            project_id=self.project.id,
        ).group

        assigned_group = self.store_event(
            data={
                "timestamp": iso_format(before_now(seconds=195)),
                "fingerprint": ["group-4"],
            },
            project_id=self.project.id,
        ).group

        assigned_to_other_group = self.store_event(
            data={
                "timestamp": iso_format(before_now(seconds=195)),
                "fingerprint": ["group-5"],
            },
            project_id=self.project.id,
        ).group

        self.run_test_query_in_syntax(
            "assigned_or_suggested:[me]",
            [],
            [group, group1, group2, assigned_group, assigned_to_other_group],
        )

        GroupOwner.objects.create(
            group=assigned_to_other_group,
            project=self.project,
            organization=self.organization,
            type=0,
            team_id=None,
            user_id=self.user.id,
        )
        GroupOwner.objects.create(
            group=group,
            project=self.project,
            organization=self.organization,
            type=0,
            team_id=None,
            user_id=self.user.id,
        )
        self.run_test_query_in_syntax(
            "assigned_or_suggested:[me]",
            [group, assigned_to_other_group],
            [group1, group2, assigned_group],
        )

        # Because assigned_to_other_event is assigned to self.other_user, it should not show up in assigned_or_suggested search for anyone but self.other_user. (aka. they are now the only owner)
        other_user = self.create_user("other@user.com", is_superuser=False)
        GroupAssignee.objects.create(
            group=assigned_to_other_group,
            project=self.project,
            user=other_user,
        )
        self.run_test_query_in_syntax(
            "assigned_or_suggested:[me]",
            [group],
            [group1, group2, assigned_group, assigned_to_other_group],
        )

        self.run_test_query_in_syntax(
            f"assigned_or_suggested:[{other_user.email}]",
            [assigned_to_other_group],
            [group, group1, group2, assigned_group],
        )

        GroupAssignee.objects.create(group=assigned_group, project=self.project, user=self.user)
        self.run_test_query_in_syntax(
            f"assigned_or_suggested:[{self.user.email}]",
            [assigned_group, group],
        )

        GroupOwner.objects.create(
            group=group,
            project=self.project,
            organization=self.organization,
            type=0,
            team_id=self.team.id,
            user_id=None,
        )
        self.run_test_query_in_syntax(
            f"assigned_or_suggested:[#{self.team.slug}]",
            [group],
        )

        self.run_test_query_in_syntax(
            "assigned_or_suggested:[me, none]",
            [group, group1, group2, assigned_group],
            [assigned_to_other_group],
        )

        not_me = self.create_user(email="notme@sentry.io")
        GroupOwner.objects.create(
            group=group2,
            project=self.project,
            organization=self.organization,
            type=0,
            team_id=None,
            user_id=not_me.id,
        )
        self.run_test_query_in_syntax(
            "assigned_or_suggested:[me, none]",
            [group, group1, assigned_group],
            [assigned_to_other_group, group2],
        )
        GroupOwner.objects.filter(group=group, user=self.user).delete()
        self.run_test_query_in_syntax(
            f"assigned_or_suggested:[me, none, #{self.team.slug}]",
            [group, group1, assigned_group],
            [assigned_to_other_group, group2],
        )
        self.run_test_query_in_syntax(
            f"assigned_or_suggested:[me, none, #{self.team.slug}, {not_me.email}]",
            [group, group1, assigned_group, group2],
            [assigned_to_other_group],
        )

    def test_assigned_to_with_environment(self):
        results = self.make_query(
            environments=[self.environments["staging"]],
            search_filter_query="assigned:%s" % self.user.username,
        )
        assert set(results) == {self.group2}

        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query="assigned:%s" % self.user.username,
        )
        assert set(results) == set()

    def test_subscribed_by(self):
        results = self.make_query(
            [self.group1.project], search_filter_query="subscribed:%s" % self.user.username
        )
        assert set(results) == {self.group1}

    def test_subscribed_by_in_syntax(self):
        self.run_test_query_in_syntax(
            f"subscribed:[{self.user.username}]", [self.group1], [self.group2]
        )
        user_2 = self.create_user()
        GroupSubscription.objects.create(
            user=user_2, group=self.group2, project=self.project, is_active=True
        )
        self.run_test_query_in_syntax(
            f"subscribed:[{self.user.username}, {user_2.username}]", [self.group1, self.group2], []
        )

    def test_subscribed_by_with_environment(self):
        results = self.make_query(
            [self.group1.project],
            environments=[self.environments["production"]],
            search_filter_query="subscribed:%s" % self.user.username,
        )
        assert set(results) == {self.group1}

        results = self.make_query(
            [self.group1.project],
            environments=[self.environments["staging"]],
            search_filter_query="subscribed:%s" % self.user.username,
        )
        assert set(results) == set()

    @mock.patch("sentry.utils.snuba.raw_query")
    def test_snuba_not_called_optimization(self, query_mock):
        assert self.make_query(search_filter_query="status:unresolved").results == [self.group1]
        assert not query_mock.called

        assert (
            self.make_query(
                search_filter_query="last_seen:>%s" % date_to_query_format(timezone.now()),
                sort_by="date",
            ).results
            == []
        )
        assert query_mock.called

    @mock.patch("sentry.utils.snuba.raw_query")
    def test_optimized_aggregates(self, query_mock):
        # TODO this test is annoyingly fragile and breaks in hard-to-see ways
        # any time anything about the snuba query changes
        query_mock.return_value = {"data": [], "totals": {"total": 0}}

        DEFAULT_LIMIT = 100
        chunk_growth = options.get("snuba.search.chunk-growth-rate")
        limit = int(DEFAULT_LIMIT * chunk_growth)

        common_args = {
            "arrayjoin": None,
            "dataset": Dataset.Events,
            "start": Any(datetime),
            "end": Any(datetime),
            "filter_keys": {
                "project_id": [self.project.id],
                "group_id": [self.group1.id, self.group2.id],
            },
            "referrer": "search",
            "groupby": ["group_id"],
            "conditions": [[["positionCaseInsensitive", ["message", "'foo'"]], "!=", 0]],
            "selected_columns": [],
            "limit": limit,
            "offset": 0,
            "totals": True,
            "turbo": False,
            "sample": 1,
        }

        self.make_query(search_filter_query="status:unresolved")
        assert not query_mock.called

        self.make_query(
            search_filter_query="last_seen:>=%s foo" % date_to_query_format(timezone.now()),
            sort_by="date",
        )
        query_mock.call_args[1]["aggregations"].sort()
        assert query_mock.call_args == mock.call(
            orderby=["-last_seen", "group_id"],
            aggregations=[
                ["multiply(toUInt64(max(timestamp)), 1000)", "", "last_seen"],
                ["uniq", "group_id", "total"],
            ],
            having=[["last_seen", ">=", Any(int)]],
            **common_args,
        )

        self.make_query(search_filter_query="foo", sort_by="priority")
        query_mock.call_args[1]["aggregations"].sort()
        assert query_mock.call_args == mock.call(
            orderby=["-priority", "group_id"],
            aggregations=[
                ["count()", "", "times_seen"],
                ["multiply(toUInt64(max(timestamp)), 1000)", "", "last_seen"],
                ["toUInt64(plus(multiply(log(times_seen), 600), last_seen))", "", "priority"],
                ["uniq", "group_id", "total"],
            ],
            having=[],
            **common_args,
        )

        self.make_query(search_filter_query="times_seen:5 foo", sort_by="freq")
        query_mock.call_args[1]["aggregations"].sort()
        assert query_mock.call_args == mock.call(
            orderby=["-times_seen", "group_id"],
            aggregations=[["count()", "", "times_seen"], ["uniq", "group_id", "total"]],
            having=[["times_seen", "=", 5]],
            **common_args,
        )

        self.make_query(search_filter_query="foo", sort_by="user")
        query_mock.call_args[1]["aggregations"].sort()
        assert query_mock.call_args == mock.call(
            orderby=["-user_count", "group_id"],
            aggregations=[
                ["uniq", "group_id", "total"],
                ["uniq", "tags[sentry:user]", "user_count"],
            ],
            having=[],
            **common_args,
        )

    def test_pre_and_post_filtering(self):
        prev_max_pre = options.get("snuba.search.max-pre-snuba-candidates")
        options.set("snuba.search.max-pre-snuba-candidates", 1)
        try:
            # normal queries work as expected
            results = self.make_query(search_filter_query="foo")
            assert set(results) == {self.group1}
            results = self.make_query(search_filter_query="bar")
            assert set(results) == {self.group2}

            # no candidate matches in Sentry, immediately return empty paginator
            results = self.make_query(search_filter_query="NO MATCHES IN SENTRY")
            assert set(results) == set()

            # too many candidates, skip pre-filter, requires >1 postfilter queries
            results = self.make_query()
            assert set(results) == {self.group1, self.group2}
        finally:
            options.set("snuba.search.max-pre-snuba-candidates", prev_max_pre)

    def test_optimizer_enabled(self):
        prev_optimizer_enabled = options.get("snuba.search.pre-snuba-candidates-optimizer")
        options.set("snuba.search.pre-snuba-candidates-optimizer", True)

        try:
            results = self.make_query(
                search_filter_query="server:example.com",
                environments=[self.environments["production"]],
            )
            assert set(results) == {self.group1}
        finally:
            options.set("snuba.search.pre-snuba-candidates-optimizer", prev_optimizer_enabled)

    def test_search_out_of_range(self):
        the_date = datetime(2000, 1, 1, 0, 0, 0, tzinfo=pytz.utc)
        results = self.make_query(
            search_filter_query=f"event.timestamp:>{the_date} event.timestamp:<{the_date}",
            date_from=the_date,
            date_to=the_date,
        )
        assert set(results) == set()

    def test_hits_estimate(self):
        # 400 Groups/Events
        # Every 3rd one is Unresolved
        # Every 2nd one has tag match=1
        for i in range(400):
            event = self.store_event(
                data={
                    "event_id": md5(f"event {i}".encode()).hexdigest(),
                    "fingerprint": [f"put-me-in-group{i}"],
                    "timestamp": iso_format(self.base_datetime - timedelta(days=21)),
                    "message": f"group {i} event",
                    "stacktrace": {"frames": [{"module": f"module {i}"}]},
                    "tags": {"match": f"{i % 2}"},
                    "environment": "production",
                },
                project_id=self.project.id,
            )

            group = event.group
            group.times_seen = 5
            group.status = GroupStatus.UNRESOLVED if i % 3 == 0 else GroupStatus.RESOLVED
            group.save()
            self.store_group(group)

        # Sample should estimate there are roughly 66 overall matching groups
        # based on a random sample of 100 (or $sample_size) of the total 200
        # snuba matches, of which 33% should pass the postgres filter.
        with self.options(
            {
                # Too small to pass all django candidates down to snuba
                "snuba.search.max-pre-snuba-candidates": 5,
                "snuba.search.hits-sample-size": 50,
            }
        ):
            first_results = self.make_query(
                search_filter_query="is:unresolved match:1", limit=10, count_hits=True
            )

            # Deliberately do not assert that the value is within some margin
            # of error, as this will fail tests at some rate corresponding to
            # our confidence interval.
            assert first_results.hits > 10

            # When searching for the same tags, we should get the same set of
            # hits as the sampling is based on the hash of the query.
            second_results = self.make_query(
                search_filter_query="is:unresolved match:1", limit=10, count_hits=True
            )

            assert first_results.results == second_results.results

            # When using a different search, we should get a different sample
            # but still should have some hits.
            third_results = self.make_query(
                search_filter_query="is:unresolved match:0", limit=10, count_hits=True
            )

            assert third_results.hits > 10
            assert third_results.results != second_results.results

    def test_regressed_in_release(self):
        # expect no groups within the results since there are no releases
        results = self.make_query(search_filter_query="regressed_in_release:fake")
        assert set(results) == set()

        # expect no groups even though there is a release; since no group regressed in this release
        release_1 = self.create_release()

        results = self.make_query(search_filter_query="regressed_in_release:%s" % release_1.version)
        assert set(results) == set()

        # Create a new event so that we get a group in this release
        group = self.store_event(
            data={
                "release": release_1.version,
            },
            project_id=self.project.id,
        ).group

        # # Should still be no group since we didn't regress in this release
        results = self.make_query(search_filter_query="regressed_in_release:%s" % release_1.version)
        assert set(results) == set()

        record_group_history(group, GroupHistoryStatus.REGRESSED, release=release_1)
        results = self.make_query(search_filter_query="regressed_in_release:%s" % release_1.version)
        assert set(results) == {group}

        # Make sure this works correctly with multiple releases
        release_2 = self.create_release()
        group_2 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group9001"],
                "event_id": "a" * 32,
                "release": release_2.version,
            },
            project_id=self.project.id,
        ).group
        record_group_history(group_2, GroupHistoryStatus.REGRESSED, release=release_2)

        results = self.make_query(search_filter_query="regressed_in_release:%s" % release_1.version)
        assert set(results) == {group}
        results = self.make_query(search_filter_query="regressed_in_release:%s" % release_2.version)
        assert set(results) == {group_2}

    def test_first_release(self):

        # expect no groups within the results since there are no releases

        results = self.make_query(search_filter_query="first_release:%s" % "fake")
        assert set(results) == set()

        # expect no groups even though there is a release; since no group
        # is attached to a release

        release_1 = self.create_release(self.project)

        results = self.make_query(search_filter_query="first_release:%s" % release_1.version)
        assert set(results) == set()

        # Create a new event so that we get a group in this release
        group = self.store_event(
            data={
                "fingerprint": ["put-me-in-group9001"],
                "event_id": "a" * 32,
                "message": "hello",
                "environment": "production",
                "tags": {"server": "example.com"},
                "release": release_1.version,
                "stacktrace": {"frames": [{"module": "group1"}]},
            },
            project_id=self.project.id,
        ).group

        results = self.make_query(search_filter_query="first_release:%s" % release_1.version)
        assert set(results) == {group}

    def test_first_release_in_syntax(self):
        # expect no groups within the results since there are no releases
        self.run_test_query_in_syntax("first_release:[fake, fake2]", [])

        # expect no groups even though there is a release; since no group
        # is attached to a release
        release_1 = self.create_release(self.project)
        release_2 = self.create_release(self.project)

        self.run_test_query_in_syntax(
            f"first_release:[{release_1.version}, {release_2.version}]", []
        )

        # Create a new event so that we get a group in this release
        group = self.store_event(
            data={
                "fingerprint": ["put-me-in-group9001"],
                "event_id": "a" * 32,
                "message": "hello",
                "environment": "production",
                "tags": {"server": "example.com"},
                "release": release_1.version,
                "stacktrace": {"frames": [{"module": "group1"}]},
            },
            project_id=self.project.id,
        ).group

        self.run_test_query_in_syntax(
            f"first_release:[{release_1.version}, {release_2.version}]",
            [group],
            [self.group1, self.group2],
        )

        # Create a new event so that we get a group in this release
        group_2 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group9002"],
                "event_id": "a" * 32,
                "message": "hello",
                "environment": "production",
                "tags": {"server": "example.com"},
                "release": release_2.version,
                "stacktrace": {"frames": [{"module": "group1"}]},
            },
            project_id=self.project.id,
        ).group

        self.run_test_query_in_syntax(
            f"first_release:[{release_1.version}, {release_2.version}]",
            [group, group_2],
        )

    def test_first_release_environments(self):
        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query="first_release:fake",
        )
        assert set(results) == set()

        release = self.create_release(self.project)
        group_env = GroupEnvironment.get_or_create(
            group_id=self.group1.id, environment_id=self.environments["production"].id
        )[0]

        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query=f"first_release:{release.version}",
        )
        assert set(results) == set()

        group_env.first_release = release
        group_env.save()

        results = self.make_query(
            environments=[self.environments["production"]],
            search_filter_query=f"first_release:{release.version}",
        )
        assert set(results) == {self.group1}

    def test_first_release_environments_in_syntax(self):
        self.run_test_query_in_syntax(
            "first_release:[fake, fake2]",
            [],
            [self.group1, self.group2],
            environments=[self.environments["production"]],
        )

        release = self.create_release(self.project)
        group_1_env = GroupEnvironment.objects.get(
            group_id=self.group1.id, environment_id=self.environments["production"].id
        )
        group_1_env.update(first_release=release)

        self.run_test_query_in_syntax(
            f"first_release:[{release.version}, fake2]",
            [self.group1],
            [self.group2],
            environments=[self.environments["production"]],
        )

        group_2_env = GroupEnvironment.objects.get(
            group_id=self.group2.id, environment_id=self.environments["staging"].id
        )
        group_2_env.update(first_release=release)
        self.run_test_query_in_syntax(
            f"first_release:[{release.version}, fake2]",
            [self.group1, self.group2],
            [],
            environments=[self.environments["production"], self.environments["staging"]],
        )

        # Make sure we don't get duplicate groups
        GroupEnvironment.objects.create(
            group_id=self.group1.id,
            environment_id=self.environments["staging"].id,
            first_release=release,
        )
        self.run_test_query_in_syntax(
            f"first_release:[{release.version}, fake2]",
            [self.group1, self.group2],
            [],
            environments=[self.environments["production"], self.environments["staging"]],
        )

    def test_query_enclosed_in_quotes(self):
        results = self.make_query(search_filter_query='"foo"')
        assert set(results) == {self.group1}

        results = self.make_query(search_filter_query='"bar"')
        assert set(results) == {self.group2}

    @xfail_if_not_postgres("Wildcard searching only supported in Postgres")
    def test_wildcard(self):
        escaped_event = self.store_event(
            data={
                "fingerprint": ["hello-there"],
                "event_id": "f" * 32,
                "message": "somet[hing]",
                "environment": "production",
                "tags": {"server": "example.net"},
                "timestamp": iso_format(self.base_datetime),
                "stacktrace": {"frames": [{"module": "group1"}]},
            },
            project_id=self.project.id,
        )
        # Note: Adding in `environment:production` so that we make sure we query
        # in both snuba and postgres
        results = self.make_query(search_filter_query="environment:production so*t")
        assert set(results) == {escaped_event.group}
        # Make sure it's case insensitive
        results = self.make_query(search_filter_query="environment:production SO*t")
        assert set(results) == {escaped_event.group}
        results = self.make_query(search_filter_query="environment:production so*zz")
        assert set(results) == set()
        results = self.make_query(search_filter_query="environment:production [hing]")
        assert set(results) == {escaped_event.group}
        results = self.make_query(search_filter_query="environment:production s*]")
        assert set(results) == {escaped_event.group}
        results = self.make_query(search_filter_query="environment:production server:example.*")
        assert set(results) == {self.group1, escaped_event.group}
        results = self.make_query(search_filter_query="environment:production !server:*net")
        assert set(results) == {self.group1}
        # TODO: Disabling tests that use [] syntax for the moment. Re-enable
        # these if we decide to add back in, or remove if this comment has been
        # here a while.
        # results = self.make_query(
        #     search_filter_query='environment:production [s][of][mz]',
        # )
        # assert set(results) == set([escaped_event.group])
        # results = self.make_query(
        #     search_filter_query='environment:production [z][of][mz]',
        # )
        # assert set(results) == set()

    def test_null_tags(self):
        tag_event = self.store_event(
            data={
                "fingerprint": ["hello-there"],
                "event_id": "f" * 32,
                "message": "something",
                "environment": "production",
                "tags": {"server": "example.net"},
                "timestamp": iso_format(self.base_datetime),
                "stacktrace": {"frames": [{"module": "group1"}]},
            },
            project_id=self.project.id,
        )
        no_tag_event = self.store_event(
            data={
                "fingerprint": ["hello-there-2"],
                "event_id": "5" * 32,
                "message": "something",
                "environment": "production",
                "timestamp": iso_format(self.base_datetime),
                "stacktrace": {"frames": [{"module": "group2"}]},
            },
            project_id=self.project.id,
        )
        results = self.make_query(search_filter_query="environment:production !server:*net")
        assert set(results) == {self.group1, no_tag_event.group}
        results = self.make_query(search_filter_query="environment:production server:*net")
        assert set(results) == {tag_event.group}
        results = self.make_query(search_filter_query="environment:production !server:example.net")
        assert set(results) == {self.group1, no_tag_event.group}
        results = self.make_query(search_filter_query="environment:production server:example.net")
        assert set(results) == {tag_event.group}
        results = self.make_query(search_filter_query="environment:production has:server")
        assert set(results) == {self.group1, tag_event.group}
        results = self.make_query(search_filter_query="environment:production !has:server")
        assert set(results) == {no_tag_event.group}

    def test_null_promoted_tags(self):
        tag_event = self.store_event(
            data={
                "fingerprint": ["hello-there"],
                "event_id": "f" * 32,
                "message": "something",
                "environment": "production",
                "tags": {"logger": "csp"},
                "timestamp": iso_format(self.base_datetime),
                "stacktrace": {"frames": [{"module": "group1"}]},
            },
            project_id=self.project.id,
        )
        no_tag_event = self.store_event(
            data={
                "fingerprint": ["hello-there-2"],
                "event_id": "5" * 32,
                "message": "something",
                "environment": "production",
                "timestamp": iso_format(self.base_datetime),
                "stacktrace": {"frames": [{"module": "group2"}]},
            },
            project_id=self.project.id,
        )
        results = self.make_query(search_filter_query="environment:production !logger:*sp")
        assert set(results) == {self.group1, no_tag_event.group}
        results = self.make_query(search_filter_query="environment:production logger:*sp")
        assert set(results) == {tag_event.group}
        results = self.make_query(search_filter_query="environment:production !logger:csp")
        assert set(results) == {self.group1, no_tag_event.group}
        results = self.make_query(search_filter_query="environment:production logger:csp")
        assert set(results) == {tag_event.group}
        results = self.make_query(search_filter_query="environment:production has:logger")
        assert set(results) == {tag_event.group}
        results = self.make_query(search_filter_query="environment:production !has:logger")
        assert set(results) == {self.group1, no_tag_event.group}

    def test_sort_multi_project(self):
        self.set_up_multi_project()
        results = self.make_query([self.project, self.project2], sort_by="date")
        assert list(results) == [self.group1, self.group_p2, self.group2]

        results = self.make_query([self.project, self.project2], sort_by="new")
        assert list(results) == [self.group2, self.group_p2, self.group1]

        results = self.make_query([self.project, self.project2], sort_by="freq")
        assert list(results) == [self.group1, self.group_p2, self.group2]

        results = self.make_query([self.project, self.project2], sort_by="priority")
        assert list(results) == [self.group1, self.group2, self.group_p2]

        results = self.make_query([self.project, self.project2], sort_by="user")
        assert list(results) == [self.group1, self.group2, self.group_p2]

    def test_sort_trend(self):
        start = self.group1.first_seen - timedelta(days=1)
        end = before_now(days=1).replace(tzinfo=pytz.utc)
        middle = start + ((end - start) / 2)
        self.store_event(
            data={
                "fingerprint": ["put-me-in-group1"],
                "event_id": "2" * 32,
                "message": "something",
                "timestamp": iso_format(self.base_datetime),
            },
            project_id=self.project.id,
        )
        self.store_event(
            data={
                "fingerprint": ["put-me-in-group1"],
                "event_id": "3" * 32,
                "message": "something",
                "timestamp": iso_format(self.base_datetime),
            },
            project_id=self.project.id,
        )

        fewer_events_group = self.store_event(
            data={
                "fingerprint": ["put-me-in-group4"],
                "event_id": "4" * 32,
                "message": "something",
                "timestamp": iso_format(middle - timedelta(days=1)),
            },
            project_id=self.project.id,
        ).group
        self.store_event(
            data={
                "fingerprint": ["put-me-in-group4"],
                "event_id": "5" * 32,
                "message": "something",
                "timestamp": iso_format(middle - timedelta(days=1)),
            },
            project_id=self.project.id,
        )
        self.store_event(
            data={
                "fingerprint": ["put-me-in-group4"],
                "event_id": "6" * 32,
                "message": "something",
                "timestamp": iso_format(self.base_datetime),
            },
            project_id=self.project.id,
        )

        no_before_group = self.store_event(
            data={
                "fingerprint": ["put-me-in-group5"],
                "event_id": "3" * 32,
                "message": "something",
                "timestamp": iso_format(self.base_datetime),
            },
            project_id=self.project.id,
        ).group
        no_after_group = self.store_event(
            data={
                "fingerprint": ["put-me-in-group6"],
                "event_id": "4" * 32,
                "message": "something",
                "timestamp": iso_format(middle - timedelta(days=1)),
            },
            project_id=self.project.id,
        ).group

        self.set_up_multi_project()
        results = self.make_query([self.project], sort_by="trend", date_from=start, date_to=end)
        assert results[:2] == [self.group1, fewer_events_group]
        # These will be arbitrarily ordered since their trend values are all 0
        assert set(results[2:]) == {self.group2, no_before_group, no_after_group}

    def test_in_syntax_is_invalid(self):
        with pytest.raises(InvalidSearchQuery, match='"in" syntax invalid for "is" search'):
            self.make_query(search_filter_query="is:[unresolved, resolved]")

    def test_first_release_any_or_no_environments(self):
        # test scenarios for tickets:
        # SEN-571
        # ISSUE-432

        # given the following setup:
        #
        # groups table:
        # group    first_release
        # A        1
        # B        1
        # C        2
        #
        # groupenvironments table:
        # group    environment    first_release
        # A        staging        1
        # A        production     2
        #
        # when querying by first release, the appropriate set of groups should be displayed:
        #
        #     first_release: 1
        #         env=[]: A, B
        #         env=[production, staging]: A
        #         env=[staging]: A
        #         env=[production]: nothing
        #
        #     first_release: 2
        #         env=[]: A, C
        #         env=[production, staging]: A
        #         env=[staging]: nothing
        #         env=[production]: A

        # create an issue/group whose events that occur in 2 distinct environments

        group_a_event_1 = self.store_event(
            data={
                "fingerprint": ["group_a"],
                "event_id": "aaa" + ("1" * 29),
                "environment": "example_staging",
                "release": "release_1",
            },
            project_id=self.project.id,
        )

        group_a_event_2 = self.store_event(
            data={
                "fingerprint": ["group_a"],
                "event_id": "aaa" + ("2" * 29),
                "environment": "example_production",
                "release": "release_2",
            },
            project_id=self.project.id,
        )

        group_a = group_a_event_1.group

        # get the environments for group_a

        prod_env = group_a_event_2.get_environment()
        staging_env = group_a_event_1.get_environment()

        # create an issue/group whose event that occur in no environments
        # but will be tied to release release_1

        group_b_event_1 = self.store_event(
            data={
                "fingerprint": ["group_b"],
                "event_id": "bbb" + ("1" * 29),
                "release": "release_1",
            },
            project_id=self.project.id,
        )
        assert group_b_event_1.get_environment().name == ""  # has no environment

        group_b = group_b_event_1.group

        # create an issue/group whose event that occur in no environments
        # but will be tied to release release_2

        group_c_event_1 = self.store_event(
            data={
                "fingerprint": ["group_c"],
                "event_id": "ccc" + ("1" * 29),
                "release": "release_2",
            },
            project_id=self.project.id,
        )
        assert group_c_event_1.get_environment().name == ""  # has no environment

        group_c = group_c_event_1.group

        # query by release release_1

        results = self.make_query(search_filter_query="first_release:%s" % "release_1")
        assert set(results) == {group_a, group_b}

        results = self.make_query(
            environments=[staging_env, prod_env],
            search_filter_query="first_release:%s" % "release_1",
        )
        assert set(results) == {group_a}

        results = self.make_query(
            environments=[staging_env], search_filter_query="first_release:%s" % "release_1"
        )
        assert set(results) == {group_a}

        results = self.make_query(
            environments=[prod_env], search_filter_query="first_release:%s" % "release_1"
        )
        assert set(results) == set()

        # query by release release_2

        results = self.make_query(search_filter_query="first_release:%s" % "release_2")
        assert set(results) == {group_a, group_c}

        results = self.make_query(
            environments=[staging_env, prod_env],
            search_filter_query="first_release:%s" % "release_2",
        )
        assert set(results) == {group_a}

        results = self.make_query(
            environments=[staging_env], search_filter_query="first_release:%s" % "release_2"
        )
        assert set(results) == set()

        results = self.make_query(
            environments=[prod_env], search_filter_query="first_release:%s" % "release_2"
        )
        assert set(results) == {group_a}

    def test_all_fields_do_not_error(self):
        # Just a sanity check to make sure that all fields can be successfully
        # searched on without returning type errors and other schema related
        # issues.
        def test_query(query):
            try:
                self.make_query(search_filter_query=query)
            except SnubaError as e:
                self.fail(f"Query {query} errored. Error info: {e}")

        for key in SENTRY_SNUBA_MAP:
            if key in ["project.id", "issue.id", "performance.issue_ids"]:
                continue
            test_query("has:%s" % key)
            test_query("!has:%s" % key)
            if key == "error.handled":
                val = 1
            elif key in issue_search_config.numeric_keys:
                val = "123"
            elif key in issue_search_config.date_keys:
                val = self.base_datetime.isoformat()
            elif key in issue_search_config.boolean_keys:
                val = "true"
            elif key in {"trace.span", "trace.parent_span"}:
                val = "abcdef1234abcdef"
                test_query(f"!{key}:{val}")
            else:
                val = "abadcafedeadbeefdeaffeedabadfeed"
                test_query(f"!{key}:{val}")

            test_query(f"{key}:{val}")


class CdcEventsSnubaSearchTest(TestCase, SnubaTestCase):
    @property
    def backend(self):
        return CdcEventsDatasetSnubaSearchBackend()

    def setUp(self):
        super().setUp()
        self.base_datetime = (datetime.utcnow() - timedelta(days=3)).replace(tzinfo=pytz.utc)

        self.event1 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group1"],
                "event_id": "a" * 32,
                "environment": "production",
                "timestamp": iso_format(self.base_datetime - timedelta(days=21)),
                "tags": {"sentry:user": "user1"},
            },
            project_id=self.project.id,
        )
        self.env1 = self.event1.get_environment()
        self.group1 = self.event1.group
        self.event3 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group1"],
                "environment": "staging",
                "timestamp": iso_format(self.base_datetime),
                "tags": {"sentry:user": "user2"},
            },
            project_id=self.project.id,
        )

        self.event2 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group2"],
                "timestamp": iso_format(self.base_datetime - timedelta(days=20)),
                "environment": "staging",
                "tags": {"sentry:user": "user1"},
            },
            project_id=self.project.id,
        )
        self.group2 = self.event2.group
        self.env2 = self.event2.get_environment()

    def build_search_filter(self, query, projects=None, user=None, environments=None):
        user = user if user is not None else self.user
        projects = projects if projects is not None else [self.project]
        return convert_query_values(parse_search_query(query), projects, user, environments)

    def make_query(
        self,
        projects=None,
        search_filter_query=None,
        environments=None,
        sort_by="date",
        limit=None,
        count_hits=False,
        date_from=None,
        date_to=None,
        cursor=None,
    ):
        search_filters = []
        projects = projects if projects is not None else [self.project]
        if search_filter_query is not None:
            search_filters = self.build_search_filter(
                search_filter_query, projects, environments=environments
            )

        kwargs = {}
        if limit is not None:
            kwargs["limit"] = limit

        return self.backend.query(
            projects,
            search_filters=search_filters,
            environments=environments,
            count_hits=count_hits,
            sort_by=sort_by,
            date_from=date_from,
            date_to=date_to,
            cursor=cursor,
            **kwargs,
        )

    def run_test(
        self,
        search_filter_query,
        expected_groups,
        expected_hits,
        projects=None,
        environments=None,
        sort_by="date",
        limit=None,
        count_hits=False,
        date_from=None,
        date_to=None,
        cursor=None,
    ):
        results = self.make_query(
            projects=projects,
            search_filter_query=search_filter_query,
            environments=environments,
            sort_by=sort_by,
            limit=limit,
            count_hits=count_hits,
            date_from=date_from,
            date_to=date_to,
            cursor=cursor,
        )
        assert list(results) == expected_groups
        assert results.hits == expected_hits
        return results

    def test(self):
        self.run_test("is:unresolved", [self.group1, self.group2], None)

    def test_invalid(self):
        with pytest.raises(InvalidQueryForExecutor):
            self.make_query(search_filter_query="is:unresolved abc:123")

    def test_resolved_group(self):
        self.group2.status = GroupStatus.RESOLVED
        self.group2.save()
        self.store_group(self.group2)
        self.run_test("is:unresolved", [self.group1], None)
        self.run_test("is:resolved", [self.group2], None)
        self.run_test("is:unresolved is:resolved", [], None)

    def test_environment(self):
        self.run_test("is:unresolved", [self.group1], None, environments=[self.env1])
        self.run_test("is:unresolved", [self.group1, self.group2], None, environments=[self.env2])

    def test_sort_times_seen(self):
        self.run_test(
            "is:unresolved",
            [self.group1, self.group2],
            None,
            sort_by="freq",
            date_from=self.base_datetime - timedelta(days=30),
        )

        self.store_event(
            data={
                "fingerprint": ["put-me-in-group2"],
                "timestamp": iso_format(self.base_datetime - timedelta(days=15)),
            },
            project_id=self.project.id,
        )
        self.store_event(
            data={
                "fingerprint": ["put-me-in-group2"],
                "timestamp": iso_format(self.base_datetime - timedelta(days=10)),
                "tags": {"sentry:user": "user2"},
            },
            project_id=self.project.id,
        )

        self.run_test(
            "is:unresolved",
            [self.group2, self.group1],
            None,
            sort_by="freq",
            # Change the date range to bust the
            date_from=self.base_datetime - timedelta(days=29),
        )

    def test_sort_first_seen(self):
        self.run_test(
            "is:unresolved",
            [self.group2, self.group1],
            None,
            sort_by="new",
            date_from=self.base_datetime - timedelta(days=30),
        )

        group3 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group3"],
                "timestamp": iso_format(self.base_datetime + timedelta(days=1)),
            },
            project_id=self.project.id,
        ).group

        self.run_test(
            "is:unresolved",
            [group3, self.group2, self.group1],
            None,
            sort_by="new",
            # Change the date range to bust the
            date_from=self.base_datetime - timedelta(days=29),
        )

    def test_sort_user(self):
        self.run_test(
            "is:unresolved",
            [self.group1, self.group2],
            None,
            sort_by="user",
            date_from=self.base_datetime - timedelta(days=30),
        )

        self.store_event(
            data={
                "fingerprint": ["put-me-in-group2"],
                "timestamp": iso_format(self.base_datetime + timedelta(days=1)),
                "tags": {"sentry:user": "user2"},
            },
            project_id=self.project.id,
        )
        self.store_event(
            data={
                "fingerprint": ["put-me-in-group2"],
                "timestamp": iso_format(self.base_datetime + timedelta(days=1)),
                "tags": {"sentry:user": "user2"},
            },
            project_id=self.project.id,
        )
        self.store_event(
            data={
                "fingerprint": ["put-me-in-group1"],
                "timestamp": iso_format(self.base_datetime + timedelta(days=1)),
                "tags": {"sentry:user": "user1"},
            },
            project_id=self.project.id,
        )
        self.store_event(
            data={
                "fingerprint": ["put-me-in-group1"],
                "timestamp": iso_format(self.base_datetime + timedelta(days=1)),
                "tags": {"sentry:user": "user1"},
            },
            project_id=self.project.id,
        )
        # Test group with no users, which can return a null count
        group3 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group3"],
                "timestamp": iso_format(self.base_datetime + timedelta(days=1)),
            },
            project_id=self.project.id,
        ).group

        self.run_test(
            "is:unresolved",
            [self.group2, self.group1, group3],
            None,
            sort_by="user",
            # Change the date range to bust the
            date_from=self.base_datetime - timedelta(days=29),
        )

    def test_sort_priority(self):
        self.run_test(
            "is:unresolved",
            [self.group1, self.group2],
            None,
            sort_by="priority",
            date_from=self.base_datetime - timedelta(days=30),
        )

    def test_cursor(self):
        group3 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group3"],
                "timestamp": iso_format(self.base_datetime + timedelta(days=1)),
                "tags": {"sentry:user": "user2"},
            },
            project_id=self.project.id,
        ).group
        group4 = self.store_event(
            data={
                "fingerprint": ["put-me-in-group7"],
                "timestamp": iso_format(self.base_datetime + timedelta(days=2)),
                "tags": {"sentry:user": "user2"},
            },
            project_id=self.project.id,
        ).group

        results = self.run_test("is:unresolved", [group4], 4, limit=1, count_hits=True)
        results = self.run_test(
            "is:unresolved", [group3], 4, limit=1, cursor=results.next, count_hits=True
        )
        results = self.run_test(
            "is:unresolved", [group4], 4, limit=1, cursor=results.prev, count_hits=True
        )
        self.run_test(
            "is:unresolved", [group3, self.group1], 4, limit=2, cursor=results.next, count_hits=True
        )

    def test_rechecking(self):
        self.group2.status = GroupStatus.RESOLVED
        self.group2.save()
        # Explicitly avoid calling `store_group` here. This means that Clickhouse will still see
        # this group as `UNRESOLVED` and it will be returned in the snuba results. This group
        # should still be filtered out by our recheck.
        self.run_test("is:unresolved", [self.group1], None)