ref(tests) Try to improve consistency of store_event() (#20959)

We've hit a few scenarios where store_event() isn't resulting in a fully
persisted event. Ensuring that the event can be fetched from eventstore
should help eliminate a failure scenario.

Add a wait method for events. We don't want to invisibly add time to all tests.
Instead we can use the wait method on tests that are having timing
challenges. Add the new wait to an acceptance test that has been flaky lately.

src/sentry/testutils/cases.py

@@ -26,6 +26,7 @@ import os.path
 import pytest
 import requests
 import six
+import time
 import inspect
 from uuid import uuid4
 from contextlib import contextmanager
@@ -53,6 +54,7 @@ from rest_framework.test import APITestCase as BaseAPITestCase
 from six.moves.urllib.parse import urlencode
 
 from sentry import auth
+from sentry import eventstore
 from sentry.auth.providers.dummy import DummyProvider
 from sentry.auth.superuser import (
     Superuser,
@@ -680,6 +682,26 @@ class SnubaTestCase(BaseTestCase):
                 self.store_group(stored_group)
             return stored_event
 
+    def wait_for_event_count(self, project_id, total, attempts=2):
+        """
+        Wait until the event count reaches the provided value or until attempts is reached.
+
+        Useful when you're storing several events and need to ensure that snuba/clickhouse
+        state has settled.
+        """
+        # Verify that events have settled in snuba's storage.
+        # While snuba is synchronous, clickhouse isn't entirely synchronous.
+        attempt = 0
+        snuba_filter = eventstore.Filter(project_ids=[project_id])
+        while attempt < attempts:
+            events = eventstore.get_events(snuba_filter)
+            if len(events) >= total:
+                break
+            attempt += 1
+            time.sleep(0.05)
+        if attempt == attempts:
+            assert False, "Could not ensure event was persisted within 10 attempts"
+
     def store_session(self, session):
         assert (
             requests.post(

tests/acceptance/test_performance_summary.py

@@ -5,7 +5,7 @@ import pytz
 from six.moves.urllib.parse import urlencode
 from mock import patch
 
-from sentry.testutils import AcceptanceTestCase
+from sentry.testutils import AcceptanceTestCase, SnubaTestCase
 from sentry.testutils.helpers.datetime import before_now, iso_format
 from sentry.utils.samples import load_data
 
@@ -19,7 +19,7 @@ def make_event(event_data):
     return event_data
 
 
-class PerformanceSummaryTest(AcceptanceTestCase):
+class PerformanceSummaryTest(AcceptanceTestCase, SnubaTestCase):
     def setUp(self):
         super(PerformanceSummaryTest, self).setUp()
         self.org = self.create_organization(owner=self.user, name="Rowdy Tiger")
@@ -43,6 +43,7 @@ class PerformanceSummaryTest(AcceptanceTestCase):
         # Create a transaction
         event = make_event(load_data("transaction", timestamp=before_now(minutes=1)))
         self.store_event(data=event, project_id=self.project.id)
+        self.wait_for_event_count(self.project.id, 1)
 
         self.store_event(
             data={

tests/snuba/api/endpoints/test_organization_group_index.py

@@ -505,7 +505,11 @@ class GroupListTest(APITestCase, SnubaTestCase):
 
         self.login_as(user=self.user)
         groups = []
+
         for day in days:
+            patched_params_update.side_effect = [
+                (self.organization.id, {"project": [self.project.id]})
+            ]
             group = self.store_event(
                 data={
                     "timestamp": iso_format(before_now(days=day)),