fix(tests): Fix dnd backend test flakes (#37916)

This PR fixes 3 major flakes:

Fixes SENTRY-TESTS-3J5: Just sort the project id order

Fixes SENTRY-TESTS-3HQ: Flakes because we calculate the retention
in the test once and the value returned in the response is calculated
a little while after. We don't need to test for seconds granularity
so replacing seconds to 0.

Fixes SENTRY-TESTS-3J0: Successively calling before_now results in some flakes
particularly in tests that are calculating aggregates
on transaction.duration. Introduced a load_data method
that takes a datetime object timestamp and a timedelta duration
calculates the offset based on timestamp to get start_timestamp.

tests/sentry/api/endpoints/test_organization_dashboard_details.py

@@ -12,7 +12,7 @@ from sentry.models import (
 )
 from sentry.models.project import Project
 from sentry.testutils import OrganizationDashboardWidgetTestCase
-from sentry.testutils.helpers.datetime import iso_format
+from sentry.testutils.helpers.datetime import before_now, iso_format
 
 
 class OrganizationDashboardDetailsTestCase(OrganizationDashboardWidgetTestCase):
@@ -186,9 +186,9 @@ class OrganizationDashboardDetailsGetTest(OrganizationDashboardDetailsTestCase):
         assert not response.data["utc"]
 
     def test_response_truncates_with_retention(self):
-        start = iso_format(datetime.now() - timedelta(days=3))
-        end = iso_format(datetime.now() - timedelta(days=2))
-        expected_adjusted_retention_start = iso_format(datetime.now() - timedelta(days=1))
+        start = before_now(days=3)
+        end = before_now(days=2)
+        expected_adjusted_retention_start = before_now(days=1)
         filters = {"start": start, "end": end}
         dashboard = Dashboard.objects.create(
             title="Dashboard With Filters",
@@ -201,7 +201,9 @@ class OrganizationDashboardDetailsGetTest(OrganizationDashboardDetailsTestCase):
             response = self.do_request("get", self.url(dashboard.id))
 
         assert response.data["expired"]
-        assert iso_format(response.data["start"]) == expected_adjusted_retention_start
+        assert iso_format(response.data["start"].replace(second=0)) == iso_format(
+            expected_adjusted_retention_start.replace(second=0)
+        )
 
 
 class OrganizationDashboardDetailsDeleteTest(OrganizationDashboardDetailsTestCase):
@@ -1364,7 +1366,7 @@ class OrganizationDashboardDetailsPutTest(OrganizationDashboardDetailsTestCase):
 
         response = self.do_request("put", self.url(self.dashboard.id), data=data)
         assert response.status_code == 200, response.data
-        assert response.data["projects"] == [project1.id, project2.id]
+        assert sorted(response.data["projects"]) == [project1.id, project2.id]
         assert response.data["environment"] == ["alpha"]
         assert response.data["period"] == "7d"
         assert response.data["filters"]["release"] == ["v1"]

tests/snuba/api/endpoints/test_organization_events.py

@@ -1,5 +1,6 @@
 import math
 from base64 import b64encode
+from datetime import timedelta
 from unittest import mock
 
 import pytest
@@ -57,6 +58,19 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         with self.feature(features):
             return self.client_get(self.reverse_url(), query, format="json")
 
+    def load_data(self, platform="transaction", timestamp=None, duration=None, **kwargs):
+        if timestamp is None:
+            timestamp = before_now(minutes=1)
+
+        start_timestamp = None
+        if duration is not None:
+            start_timestamp = timestamp - duration
+            start_timestamp = start_timestamp - timedelta(
+                microseconds=start_timestamp.microsecond % 1000
+            )
+
+        return load_data(platform, timestamp=timestamp, start_timestamp=start_timestamp, **kwargs)
+
     def test_no_projects(self):
         response = self.do_request({})
 
@@ -747,7 +761,7 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert "project.id" not in response.data["data"][0]
 
     def test_error_handled_condition(self):
-        prototype = load_data("android-ndk")
+        prototype = self.load_data(platform="android-ndk")
         events = (
             ("a" * 32, "not handled", False),
             ("b" * 32, "was handled", True),
@@ -785,7 +799,7 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
             assert [1] == response.data["data"][1]["error.handled"]
 
     def test_error_unhandled_condition(self):
-        prototype = load_data("android-ndk")
+        prototype = self.load_data(platform="android-ndk")
         events = (
             ("a" * 32, "not handled", False),
             ("b" * 32, "was handled", True),
@@ -1298,10 +1312,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
             ("three", 3000),
         ]
         for idx, event in enumerate(events):
-            data = load_data(
-                "transaction",
+            data = self.load_data(
                 timestamp=before_now(minutes=(10 + idx)),
-                start_timestamp=before_now(minutes=(10 + idx), milliseconds=event[1]),
+                duration=timedelta(milliseconds=event[1]),
             )
             data["event_id"] = f"{idx}" * 32
             data["transaction"] = f"/count_miserable/horribilis/{idx}"
@@ -1380,10 +1393,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
             ("three", 3000),
         ]
         for idx, event in enumerate(events):
-            data = load_data(
-                "transaction",
+            data = self.load_data(
                 timestamp=before_now(minutes=(10 + idx)),
-                start_timestamp=before_now(minutes=(10 + idx), milliseconds=event[1]),
+                duration=timedelta(milliseconds=event[1]),
             )
             data["event_id"] = f"{idx}" * 32
             data["transaction"] = f"/count_miserable/horribilis/{event[0]}"
@@ -1443,8 +1455,7 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
             3000,
         ]
         for idx, lcp in enumerate(lcps):
-            data = load_data(
-                "transaction",
+            data = self.load_data(
                 timestamp=before_now(minutes=(10 + idx)),
             )
             data["event_id"] = f"{idx}" * 32
@@ -1456,11 +1467,7 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
             self.store_event(data, project_id=self.project.id)
 
         # Shouldn't count towards misery
-        data = load_data(
-            "transaction",
-            timestamp=before_now(minutes=(10)),
-            start_timestamp=before_now(minutes=(10)),
-        )
+        data = self.load_data(timestamp=before_now(minutes=10), duration=timedelta(milliseconds=0))
         data["transaction"] = "/misery/new/"
         data["user"] = {"email": "7@example.com"}
         data["measurements"] = {}
@@ -1496,10 +1503,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
             ("three", 3000),
         ]
         for idx, event in enumerate(events):
-            data = load_data(
-                "transaction",
+            data = self.load_data(
                 timestamp=before_now(minutes=(10 + idx)),
-                start_timestamp=before_now(minutes=(10 + idx), milliseconds=event[1]),
+                duration=timedelta(milliseconds=event[1]),
             )
             data["event_id"] = f"{idx}" * 32
             data["transaction"] = f"/user_misery/{idx}"
@@ -1535,8 +1541,7 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
             3000,
         ]
         for idx, lcp in enumerate(lcps):
-            data = load_data(
-                "transaction",
+            data = self.load_data(
                 timestamp=before_now(minutes=(10 + idx)),
             )
             data["event_id"] = f"{idx}" * 32
@@ -1548,10 +1553,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
             self.store_event(data, project_id=self.project.id)
 
         # Shouldn't count towards apdex
-        data = load_data(
-            "transaction",
-            timestamp=before_now(minutes=(10)),
-            start_timestamp=before_now(minutes=(10)),
+        data = self.load_data(
+            timestamp=before_now(minutes=10),
+            duration=timedelta(milliseconds=0),
         )
         data["transaction"] = "/apdex/new/"
         data["user"] = {"email": "7@example.com"}
@@ -1595,10 +1599,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
             ("three", 3000),
         ]
         for idx, event in enumerate(events):
-            data = load_data(
-                "transaction",
+            data = self.load_data(
                 timestamp=before_now(minutes=(10 + idx)),
-                start_timestamp=before_now(minutes=(10 + idx), milliseconds=event[1]),
+                duration=timedelta(milliseconds=event[1]),
             )
             data["event_id"] = f"{idx}" * 32
             data["transaction"] = f"/apdex/new/{event[0]}"
@@ -1654,10 +1657,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
             ("three", 3000),
         ]
         for idx, event in enumerate(events):
-            data = load_data(
-                "transaction",
+            data = self.load_data(
                 timestamp=before_now(minutes=(10 + idx)),
-                start_timestamp=before_now(minutes=(10 + idx), milliseconds=event[1]),
+                duration=timedelta(milliseconds=event[1]),
             )
             data["event_id"] = f"{idx}" * 32
             data["transaction"] = f"/count_miserable/horribilis/{event[0]}"
@@ -1705,10 +1707,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
             ("four", 4000),
         ]
         for idx, event in enumerate(events):
-            data = load_data(
-                "transaction",
+            data = self.load_data(
                 timestamp=before_now(minutes=(10 + idx)),
-                start_timestamp=before_now(minutes=(10 + idx), milliseconds=event[1]),
+                duration=timedelta(milliseconds=event[1]),
             )
             data["event_id"] = f"{idx}" * 32
             data["transaction"] = f"/count_miserable/horribilis/{idx}"
@@ -1789,10 +1790,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
             ("four", 4000),
         ]
         for idx, event in enumerate(events):
-            data = load_data(
-                "transaction",
+            data = self.load_data(
                 timestamp=before_now(minutes=(10 + idx)),
-                start_timestamp=before_now(minutes=(10 + idx), milliseconds=event[1]),
+                duration=timedelta(milliseconds=event[1]),
             )
             data["event_id"] = f"{idx}" * 32
             data["transaction"] = f"/count_miserable/horribilis/{idx}"
@@ -1810,7 +1810,7 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
 
         project2 = self.create_project()
 
-        data = load_data("transaction", timestamp=before_now(minutes=1))
+        data = self.load_data(timestamp=before_now(minutes=1))
         data["transaction"] = "/count_miserable/horribilis/project2"
         data["user"] = {"email": "project2@example.com"}
         self.store_event(data, project_id=project2.id)
@@ -1994,18 +1994,16 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert data[0]["count_unique(user)"] == 2
 
     def test_aggregation_alias_comparison(self):
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=5),
+            duration=timedelta(seconds=5),
         )
         data["transaction"] = "/aggregates/1"
         self.store_event(data, project_id=self.project.id)
 
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=3),
+            duration=timedelta(seconds=3),
         )
         data["transaction"] = "/aggregates/2"
         event = self.store_event(data, project_id=self.project.id)
@@ -2024,18 +2022,16 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert data[0]["p95()"] == 3000
 
     def test_auto_aggregations(self):
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=5),
+            duration=timedelta(seconds=5),
         )
         data["transaction"] = "/aggregates/1"
         self.store_event(data, project_id=self.project.id)
 
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=3),
+            duration=timedelta(seconds=3),
         )
         data["transaction"] = "/aggregates/2"
         event = self.store_event(data, project_id=self.project.id)
@@ -2172,18 +2168,16 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert data[0]["issue.id"] == event.group_id
 
     def test_percentile_function(self):
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=5),
+            duration=timedelta(seconds=5),
         )
         data["transaction"] = "/aggregates/1"
         event1 = self.store_event(data, project_id=self.project.id)
 
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=3),
+            duration=timedelta(seconds=3),
         )
         data["transaction"] = "/aggregates/2"
         event2 = self.store_event(data, project_id=self.project.id)
@@ -2204,18 +2198,16 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert data[1]["percentile(transaction.duration, 0.95)"] == 3000
 
     def test_percentile_function_as_condition(self):
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=5),
+            duration=timedelta(seconds=5),
         )
         data["transaction"] = "/aggregates/1"
         event1 = self.store_event(data, project_id=self.project.id)
 
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=3),
+            duration=timedelta(seconds=3),
         )
         data["transaction"] = "/aggregates/2"
         self.store_event(data, project_id=self.project.id)
@@ -2234,18 +2226,16 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert data[0]["percentile(transaction.duration, 0.95)"] == 5000
 
     def test_epm_function(self):
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=5),
+            duration=timedelta(seconds=5),
         )
         data["transaction"] = "/aggregates/1"
         event1 = self.store_event(data, project_id=self.project.id)
 
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=3),
+            duration=timedelta(seconds=3),
         )
         data["transaction"] = "/aggregates/2"
         event2 = self.store_event(data, project_id=self.project.id)
@@ -2374,7 +2364,7 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert data[0]["count()"] == 0
 
     def test_stack_wildcard_condition(self):
-        data = load_data("javascript")
+        data = self.load_data(platform="javascript")
         data["timestamp"] = self.ten_mins_ago
         self.store_event(data=data, project_id=self.project.id)
 
@@ -2385,7 +2375,7 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert response.data["meta"]["fields"]["message"] == "string"
 
     def test_email_wildcard_condition(self):
-        data = load_data("javascript")
+        data = self.load_data(platform="javascript")
         data["timestamp"] = self.ten_mins_ago
         self.store_event(data=data, project_id=self.project.id)
 
@@ -2948,10 +2938,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert 1 == data["count()"]
 
     def test_aggregate_negation(self):
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=5),
+            duration=timedelta(seconds=5),
         )
         self.store_event(data, project_id=self.project.id)
 
@@ -2977,18 +2966,16 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert len(data) == 0
 
     def test_all_aggregates_in_columns(self):
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=11),
-            start_timestamp=before_now(minutes=11, seconds=5),
+            duration=timedelta(seconds=5),
         )
         data["transaction"] = "/failure_rate/1"
         self.store_event(data, project_id=self.project.id)
 
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=5),
+            duration=timedelta(seconds=5),
         )
         data["transaction"] = "/failure_rate/1"
         data["contexts"]["trace"]["status"] = "unauthenticated"
@@ -3199,18 +3186,16 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert data[0]["user_misery()"] == 0
 
     def test_all_aggregates_in_query(self):
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=11),
-            start_timestamp=before_now(minutes=11, seconds=5),
+            duration=timedelta(seconds=5),
         )
         data["transaction"] = "/failure_rate/1"
         self.store_event(data, project_id=self.project.id)
 
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=5),
+            duration=timedelta(seconds=5),
         )
         data["transaction"] = "/failure_rate/2"
         data["contexts"]["trace"]["status"] = "unauthenticated"
@@ -3329,18 +3314,16 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert data[0]["apdex(400)"] == 0
 
     def test_functions_in_orderby(self):
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=11),
-            start_timestamp=before_now(minutes=11, seconds=5),
+            duration=timedelta(seconds=5),
         )
         data["transaction"] = "/failure_rate/1"
         self.store_event(data, project_id=self.project.id)
 
-        data = load_data(
-            "transaction",
+        data = self.load_data(
             timestamp=before_now(minutes=10),
-            start_timestamp=before_now(minutes=10, seconds=5),
+            duration=timedelta(seconds=5),
         )
         data["transaction"] = "/failure_rate/2"
         data["contexts"]["trace"]["status"] = "unauthenticated"
@@ -3676,18 +3659,18 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
 
     def test_in_query_events_stack(self):
         test_js = self.store_event(
-            load_data(
-                "javascript",
+            self.load_data(
+                platform="javascript",
                 timestamp=before_now(minutes=10),
-                start_timestamp=before_now(minutes=10, seconds=5),
+                duration=timedelta(seconds=5),
             ),
             project_id=self.project.id,
         )
         test_java = self.store_event(
-            load_data(
-                "java",
+            self.load_data(
+                platform="java",
                 timestamp=before_now(minutes=10),
-                start_timestamp=before_now(minutes=10, seconds=5),
+                duration=timedelta(seconds=5),
             ),
             project_id=self.project.id,
         )
@@ -3756,8 +3739,8 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert len(data) == 0
 
     def test_context_fields_between_datasets(self):
-        event_data = load_data("android")
-        transaction_data = load_data("transaction")
+        event_data = self.load_data(platform="android")
+        transaction_data = self.load_data()
         event_data["spans"] = transaction_data["spans"]
         event_data["contexts"]["trace"] = transaction_data["contexts"]["trace"]
         event_data["type"] = "transaction"
@@ -3805,8 +3788,8 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
                 assert results[0][field] == expected, field + str(datum)
 
     def test_http_fields_between_datasets(self):
-        event_data = load_data("android")
-        transaction_data = load_data("transaction")
+        event_data = self.load_data(platform="android")
+        transaction_data = self.load_data()
         event_data["spans"] = transaction_data["spans"]
         event_data["contexts"]["trace"] = transaction_data["contexts"]["trace"]
         event_data["type"] = "transaction"
@@ -4540,7 +4523,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert response.data["data"][0]["apdex(300)"] == 0
 
     def test_equation_simple(self):
-        event_data = load_data("transaction", timestamp=before_now(minutes=10))
+        event_data = self.load_data(
+            timestamp=before_now(minutes=10),
+        )
         event_data["breakdowns"]["span_ops"]["ops.http"]["value"] = 1500
         self.store_event(data=event_data, project_id=self.project.id)
 
@@ -4629,10 +4614,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
     def test_count_if(self):
         unicode_phrase1 = "\u716e\u6211\u66f4\u591a\u7684\u98df\u7269\uff0c\u6211\u9913\u4e86"
         for i in range(5):
-            data = load_data(
-                "transaction",
+            data = self.load_data(
                 timestamp=before_now(minutes=(10 + i)),
-                start_timestamp=before_now(minutes=(10 + i), milliseconds=100 if i < 3 else 200),
+                duration=timedelta(milliseconds=100 if i < 3 else 200),
             )
             data["tags"] = {
                 "sub_customer.is-Enterprise-42": "yes" if i == 0 else "no",
@@ -4686,10 +4670,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
 
     def test_count_if_filter(self):
         for i in range(5):
-            data = load_data(
-                "transaction",
+            data = self.load_data(
                 timestamp=before_now(minutes=(10 + i)),
-                start_timestamp=before_now(minutes=(10 + i), milliseconds=100 if i < 3 else 200),
+                duration=timedelta(milliseconds=100 if i < 3 else 200),
             )
             data["tags"] = {"sub_customer.is-Enterprise-42": "yes" if i == 0 else "no"}
             self.store_event(data, project_id=self.project.id)
@@ -5067,7 +5050,9 @@ class OrganizationEventsEndpointTest(APITestCase, SnubaTestCase):
         assert response.status_code == 400, response.content
 
     def test_tag_that_looks_like_aggregate(self):
-        data = load_data("transaction", timestamp=before_now(minutes=1))
+        data = self.load_data(
+            timestamp=before_now(minutes=1),
+        )
         data["tags"] = {"p95": "<5k"}
         self.store_event(data, project_id=self.project.id)