feat(test) Add acceptance test that is backed by snuba (#12596)

Add a `requires_snuba` pytest marker so that we can integrate snuba
based tests into our acceptance tests and gracefully handle snuba not
being available. I've refactored the other service availability checks
to only make one connect call in an attempt to keep tests faster.

Snuba does not share our acceptance test framework's mocked now. We need
to use dates closer to the real time when interacting with snuba.

Fixes SEN-246

.travis.yml

@@ -169,8 +169,9 @@ matrix:
 
     - python: 2.7
       name: 'Acceptance'
-      env: TEST_SUITE=acceptance
+      env: TEST_SUITE=acceptance USE_SNUBA=1
       services:
+        - docker
         - memcached
         - redis-server
         - postgresql
@@ -195,8 +196,9 @@ matrix:
     # XXX(markus): Remove after rust interfaces are done
     - python: 2.7
       name: 'Acceptance (Rust Interface Renormalization)'
-      env: TEST_SUITE=acceptance SENTRY_TEST_USE_RUST_INTERFACE_RENORMALIZATION=1 PERCY_ENABLE=0
+      env: TEST_SUITE=acceptance USE_SNUBA=1 SENTRY_TEST_USE_RUST_INTERFACE_RENORMALIZATION=1 PERCY_ENABLE=0
       services:
+        - docker
         - memcached
         - redis-server
         - postgresql
@@ -268,7 +270,7 @@ matrix:
     # snuba in testing
     - python: 2.7
       name: 'Snuba Integration'
-      env: TEST_SUITE=snuba SENTRY_TAGSTORE=sentry.tagstore.snuba.SnubaTagStorage SENTRY_ZOOKEEPER_HOSTS=localhost:2181 SENTRY_KAFKA_HOSTS=localhost:9092
+      env: TEST_SUITE=snuba USE_SNUBA=1 SENTRY_ZOOKEEPER_HOSTS=localhost:2181 SENTRY_KAFKA_HOSTS=localhost:9092
       services:
         - docker
         - memcached
@@ -290,7 +292,7 @@ matrix:
     # XXX(markus): Remove after rust interfaces are done
     - python: 2.7
       name: 'Snuba Integration (Rust Interface Renormalization)'
-      env: TEST_SUITE=snuba SENTRY_TAGSTORE=sentry.tagstore.snuba.SnubaTagStorage SENTRY_ZOOKEEPER_HOSTS=localhost:2181 SENTRY_KAFKA_HOSTS=localhost:9092 SENTRY_TEST_USE_RUST_INTERFACE_RENORMALIZATION=1
+      env: TEST_SUITE=snuba USE_SNUBA=1 SENTRY_ZOOKEEPER_HOSTS=localhost:2181 SENTRY_KAFKA_HOSTS=localhost:9092 SENTRY_TEST_USE_RUST_INTERFACE_RENORMALIZATION=1
       services:
         - docker
         - memcached

setup.cfg

@@ -4,6 +4,8 @@ addopts = --tb=native -p no:doctest -p no:warnings
 norecursedirs = bin dist docs htmlcov script hooks node_modules .* {args}
 looponfailroots = src tests
 selenium_driver = chrome
+markers =
+    snuba: mark a test as requiring snuba
 
 [flake8]
 ignore = F999,E501,E128,E124,E402,W503,E731,C901

src/sentry/testutils/cases.py

@@ -11,7 +11,8 @@ from __future__ import absolute_import
 __all__ = (
     'TestCase', 'TransactionTestCase', 'APITestCase', 'TwoFactorAPITestCase', 'AuthProviderTestCase', 'RuleTestCase',
     'PermissionTestCase', 'PluginTestCase', 'CliTestCase', 'AcceptanceTestCase',
-    'IntegrationTestCase', 'UserReportEnvironmentTestCase', 'SnubaTestCase', 'IntegrationRepositoryTestCase',
+    'IntegrationTestCase', 'UserReportEnvironmentTestCase', 'SnubaTestCase',
+    'IntegrationRepositoryTestCase',
     'ReleaseCommitPatchTest', 'SetRefsTestCase', 'OrganizationDashboardWidgetTestCase'
 )
 
@@ -66,6 +67,8 @@ from sentry.utils import json
 from sentry.utils.auth import SSO_SESSION_KEY
 
 from .fixtures import Fixtures
+from .factories import Factories
+from .skips import requires_snuba
 from .helpers import (
     AuthProvider, Feature, get_auth_header, TaskRunner, override_options, parse_queries
 )
@@ -831,9 +834,20 @@ class IntegrationTestCase(TestCase):
         assert 'window.opener.postMessage(' in resp.content
 
 
-class SnubaTestCase(TestCase):
+@pytest.mark.snuba
+@requires_snuba
+class SnubaTestCase(BaseTestCase):
+    """
+    Mixin for enabling test case classes to talk to snuba
+    Useful when you are working on acceptance tests or integration
+    tests that require snuba.
+    """
+
     def setUp(self):
         super(SnubaTestCase, self).setUp()
+        self.init_snuba()
+
+    def init_snuba(self):
         self.snuba_eventstream = SnubaEventStream()
         self.snuba_tagstore = SnubaCompatibilityTagStorage()
         assert requests.post(settings.SENTRY_SNUBA + '/tests/drop').status_code == 200
@@ -849,7 +863,7 @@ class SnubaTestCase(TestCase):
             mock.patch('sentry.tagstore.incr_group_tag_value_times_seen',
                        self.snuba_tagstore.incr_group_tag_value_times_seen),
         ):
-            return super(SnubaTestCase, self).store_event(*args, **kwargs)
+            return Factories.store_event(*args, **kwargs)
 
     def __wrap_event(self, event, data, primary_hash):
         # TODO: Abstract and combine this with the stream code in
@@ -877,8 +891,7 @@ class SnubaTestCase(TestCase):
         world all test events would go through the full regular pipeline.
         """
         # XXX: Use `store_event` instead of this!
-
-        event = super(SnubaTestCase, self).create_event(*args, **kwargs)
+        event = Factories.create_event(*args, **kwargs)
 
         data = event.data.data
         tags = dict(data.get('tags', []))

src/sentry/testutils/skips.py

@@ -7,30 +7,41 @@ sentry.testutils.skips
 """
 from __future__ import absolute_import
 
+from django.conf import settings
+from six.moves.urllib.parse import urlparse
 import os
 import socket
 import pytest
 
 
+_service_status = {}
+
+
 def riak_is_available():
+    if 'riak' in _service_status:
+        return _service_status['riak']
     try:
         socket.create_connection(('127.0.0.1', 8098), 1.0)
     except socket.error:
-        return False
+        _service_status['riak'] = False
     else:
-        return True
+        _service_status['riak'] = True
+    return _service_status['riak']
 
 
 requires_riak = pytest.mark.skipif(not riak_is_available(), reason="requires riak server running")
 
 
 def cassandra_is_available():
+    if 'cassandra' in _service_status:
+        return _service_status['cassandra']
     try:
         socket.create_connection(('127.0.0.1', 9042), 1.0)
     except socket.error:
-        return False
+        _service_status['cassandra'] = False
     else:
-        return True
+        _service_status['cassandra'] = True
+    return _service_status['cassandra']
 
 
 requires_cassandra = pytest.mark.skipif(
@@ -38,6 +49,24 @@ requires_cassandra = pytest.mark.skipif(
 )
 
 
+def snuba_is_available():
+    if 'snuba' in _service_status:
+        return _service_status['snuba']
+    try:
+        parsed = urlparse(settings.SENTRY_SNUBA)
+        socket.create_connection((parsed.host, parsed.port), 1.0)
+    except socket.error:
+        _service_status['snuba'] = False
+    else:
+        _service_status['snuba'] = True
+    return _service_status['snuba']
+
+
+requires_snuba = pytest.mark.skipif(
+    not snuba_is_available, reason='requires snuba server running'
+)
+
+
 def xfail_if_not_postgres(reason):
     def decorator(function):
         return pytest.mark.xfail(

src/sentry/utils/pytest/sentry.py

@@ -129,6 +129,12 @@ def pytest_configure(config):
         }
     }
 
+    if os.environ.get('USE_SNUBA', False):
+        settings.SENTRY_SEARCH = 'sentry.search.snuba.SnubaSearchBackend'
+        settings.SENTRY_TAGSTORE = 'sentry.tagstore.snuba.SnubaCompatibilityTagStorage'
+        settings.SENTRY_TSDB = 'sentry.tsdb.redissnuba.RedisSnubaTSDB'
+        settings.SENTRY_EVENTSTREAM = 'sentry.eventstream.snuba.SnubaEventStream'
+
     if not hasattr(settings, 'SENTRY_OPTIONS'):
         settings.SENTRY_OPTIONS = {}
 
@@ -241,9 +247,10 @@ def register_extensions():
 
 
 def pytest_runtest_teardown(item):
-    from sentry import tsdb
-    # TODO(dcramer): this only works if this is the correct tsdb backend
-    tsdb.flush()
+    if not os.environ.get('USE_SNUBA', False):
+        from sentry import tsdb
+        # TODO(dcramer): this only works if this is the correct tsdb backend
+        tsdb.flush()
 
     # XXX(dcramer): only works with DummyNewsletter
     from sentry import newsletter

tests/acceptance/test_dashboard.py

@@ -2,13 +2,13 @@ from __future__ import absolute_import
 
 from django.utils import timezone
 
-from sentry.testutils import AcceptanceTestCase
+from sentry.testutils import AcceptanceTestCase, SnubaTestCase
 from sentry.models import GroupAssignee, Release, Environment, Deploy, ReleaseProjectEnvironment, OrganizationOnboardingTask, OnboardingTask, OnboardingTaskStatus
-from sentry.utils.samples import create_sample_event
+from sentry.utils.samples import load_data
 from datetime import datetime
 
 
-class DashboardTest(AcceptanceTestCase):
+class DashboardTest(AcceptanceTestCase, SnubaTestCase):
     def setUp(self):
         super(DashboardTest, self).setUp()
         self.user = self.create_user('foo@example.com')
@@ -65,11 +65,15 @@ class DashboardTest(AcceptanceTestCase):
         self.browser.snapshot('org dash no issues')
 
     def test_one_issue(self):
-        event = create_sample_event(
-            project=self.project,
-            platform='python',
-            event_id='d964fdbd649a4cf8bfc35d18082b6b0e',
-            timestamp=1452683305,
+        self.init_snuba()
+
+        event_data = load_data('python')
+        event_data['event_id'] = 'd964fdbd649a4cf8bfc35d18082b6b0e'
+        event_data['timestamp'] = 1452683305
+        event = self.store_event(
+            project_id=self.project.id,
+            data=event_data,
+            assert_no_errors=False
         )
         event.group.update(
             first_seen=datetime(2018, 1, 12, 3, 8, 25, tzinfo=timezone.utc),

tests/acceptance/test_issue_details.py

@@ -6,19 +6,11 @@ from datetime import datetime
 from django.conf import settings
 from django.utils import timezone
 
-from sentry.testutils import AcceptanceTestCase
-from sentry.utils.samples import create_sample_event as _create_sample_event
+from sentry.testutils import AcceptanceTestCase, SnubaTestCase
+from sentry.utils.samples import load_data
 
 
-def create_sample_event(*args, **kwargs):
-    event = _create_sample_event(*args, **kwargs)
-    # Prevent Percy screenshot from constantly changing
-    event.datetime = datetime(2017, 9, 6, 0, 0)
-    event.save()
-    return event
-
-
-class IssueDetailsTest(AcceptanceTestCase):
+class IssueDetailsTest(AcceptanceTestCase, SnubaTestCase):
     def setUp(self):
         super(IssueDetailsTest, self).setUp()
         self.user = self.create_user('foo@example.com')
@@ -33,13 +25,15 @@ class IssueDetailsTest(AcceptanceTestCase):
         self.dismiss_assistant()
 
     def create_sample_event(self, platform, default=None, sample_name=None):
-        event = create_sample_event(
-            project=self.project,
-            platform=platform,
-            default=default,
-            sample_name=sample_name,
-            event_id='d964fdbd649a4cf8bfc35d18082b6b0e'
-        )
+        event_data = load_data(platform, default=default, sample_name=sample_name)
+        event_data['event_id'] = 'd964fdbd649a4cf8bfc35d18082b6b0e'
+        event = self.store_event(
+            data=event_data,
+            project_id=self.project.id,
+            assert_no_errors=False,
+        )
+        event.datetime = datetime(2017, 9, 6, 0, 0)
+        event.save()
         event.group.update(
             first_seen=datetime(2015, 8, 13, 3, 8, 25, tzinfo=timezone.utc),
             last_seen=datetime(2016, 1, 13, 3, 8, 25, tzinfo=timezone.utc),

tests/acceptance/test_organization_group_index.py

@@ -1,11 +1,18 @@
 from __future__ import absolute_import
 
+import pytz
+
+from datetime import datetime, timedelta
 from django.utils import timezone
 
-from sentry.testutils import AcceptanceTestCase
+from sentry.testutils import AcceptanceTestCase, SnubaTestCase
+from mock import patch
+
+
+event_time = (datetime.utcnow() - timedelta(days=3)).replace(tzinfo=pytz.utc)
 
 
-class OrganizationGroupIndexTest(AcceptanceTestCase):
+class OrganizationGroupIndexTest(AcceptanceTestCase, SnubaTestCase):
     def setUp(self):
         super(OrganizationGroupIndexTest, self).setUp()
         self.user = self.create_user('foo@example.com')
@@ -43,5 +50,37 @@ class OrganizationGroupIndexTest(AcceptanceTestCase):
             self.browser.wait_until_test_id('empty-state')
             self.browser.snapshot('organization issues no results')
 
+    @patch('django.utils.timezone.now')
+    def test_with_results(self, mock_now):
+        mock_now.return_value = datetime.utcnow().replace(tzinfo=pytz.utc)
+        self.store_event(
+            data={
+                'event_id': 'a' * 32,
+                'message': 'oh no',
+                'timestamp': event_time.isoformat()[:19],
+                'fingerprint': ['group-1']
+            },
+            project_id=self.project.id
+        )
+        self.store_event(
+            data={
+                'event_id': 'b' * 32,
+                'message': 'oh snap',
+                'timestamp': event_time.isoformat()[:19],
+                'fingerprint': ['group-2']
+            },
+            project_id=self.project.id
+        )
+        with self.feature(['organizations:sentry10', 'organizations:discover']):
+            self.browser.get(self.path)
+            self.wait_until_loaded()
+            self.browser.wait_until('.event-issue-header')
+            self.browser.snapshot('organization issues with issues')
+
+            groups = self.browser.find_elements_by_class_name('event-issue-header')
+            assert len(groups) == 2
+            assert 'oh snap' in groups[0].text
+            assert 'oh no' in groups[1].text
+
     def wait_until_loaded(self):
         self.browser.wait_until_not('.loading')

tests/acceptance/test_project_overview.py

@@ -1,11 +1,15 @@
 from __future__ import absolute_import
 
+import pytz
+
+from datetime import datetime
 from django.utils import timezone
+from mock import patch
 
-from sentry.testutils import AcceptanceTestCase
+from sentry.testutils import AcceptanceTestCase, SnubaTestCase
 
 
-class ProjectOverviewTest(AcceptanceTestCase):
+class ProjectOverviewTest(AcceptanceTestCase, SnubaTestCase):
     def setUp(self):
         super(ProjectOverviewTest, self).setUp()
         self.user = self.create_user('foo@example.com')
@@ -22,11 +26,18 @@ class ProjectOverviewTest(AcceptanceTestCase):
         self.path = u'/{}/{}/dashboard/'.format(
             self.org.slug, self.project.slug)
 
-    def test_with_issues(self):
-        self.project.update(first_event=timezone.now())
-        self.create_group(
-            project=self.project,
-            message='Foo bar',
+    @patch('django.utils.timezone.now')
+    def test_with_issues(self, mock_now):
+        mock_now.return_value = datetime.utcnow().replace(tzinfo=pytz.utc)
+
+        self.store_event(
+            data={
+                'message': 'Foo bar',
+                'level': 'error',
+                'timestamp': timezone.now().isoformat()[:19]
+            },
+            project_id=self.project.id,
+            assert_no_errors=False
         )
         self.browser.get(self.path)
         self.browser.wait_until('.chart-wrapper')

tests/acceptance/test_project_release_tracking_settings.py

@@ -1,10 +1,9 @@
 from __future__ import absolute_import
 
-from sentry import tagstore
-from sentry.testutils import AcceptanceTestCase
+from sentry.testutils import AcceptanceTestCase, SnubaTestCase
 
 
-class ProjectReleaseTrackingSettingsTest(AcceptanceTestCase):
+class ProjectReleaseTrackingSettingsTest(AcceptanceTestCase, SnubaTestCase):
     def setUp(self):
         super(ProjectReleaseTrackingSettingsTest, self).setUp()
         self.user = self.create_user('foo@example.com')
@@ -25,12 +24,20 @@ class ProjectReleaseTrackingSettingsTest(AcceptanceTestCase):
             teams=[self.team],
         )
 
-        tagstore.create_tag_key(project_id=self.project.id, environment_id=None, key="Foo")
-
         self.login_as(self.user)
         self.path1 = u'/{}/{}/settings/release-tracking/'.format(self.org.slug, self.project.slug)
 
     def test_tags_list(self):
+        self.store_event(
+            data={
+                'event_id': 'a' * 32,
+                'message': 'oh no',
+                'environment': 'prod',
+                'release': 'first',
+                'tags': {'Foo': 'value'},
+            },
+            project_id=self.project.id,
+        )
         self.browser.get(self.path1)
         self.browser.wait_until_not('.loading-indicator')
         self.browser.snapshot('project settings - release tracking')