ref(release-health): Reflect Relay changes in test harness (#35883)

getsentry/relay#1316 changes the way we extract user metrics for release
health. Reflect these changes in the test harness.

src/sentry/testutils/cases.py

@@ -1089,8 +1089,6 @@ class SessionMetricsTestCase(SnubaTestCase):
             self._push_metric(
                 session, "counter", SessionMRI.SESSION, {"session.status": "init"}, +1
             )
-            if not user_is_nil:
-                self._push_metric(session, "set", SessionMRI.USER, {"session.status": "init"}, user)
 
         status = session["status"]
 
@@ -1101,6 +1099,8 @@ class SessionMetricsTestCase(SnubaTestCase):
                 self._push_metric(
                     session, "set", SessionMRI.USER, {"session.status": "errored"}, user
                 )
+        elif not user_is_nil:
+            self._push_metric(session, "set", SessionMRI.USER, {}, user)
 
         if status in ("abnormal", "crashed"):  # fatal
             self._push_metric(
@@ -1109,7 +1109,7 @@ class SessionMetricsTestCase(SnubaTestCase):
             if not user_is_nil:
                 self._push_metric(session, "set", SessionMRI.USER, {"session.status": status}, user)
 
-        if status != "ok":  # terminal
+        if status == "exited":
             if session["duration"] is not None:
                 self._push_metric(
                     session,
@@ -1119,11 +1119,6 @@ class SessionMetricsTestCase(SnubaTestCase):
                     session["duration"],
                 )
 
-        # Also extract user for non-init healthy sessions
-        # (see # https://github.com/getsentry/relay/pull/1275)
-        if session["seq"] > 0 and status in ("ok", "exited") and not user_is_nil:
-            self._push_metric(session, "set", SessionMRI.USER, {"session.status": "ok"}, user)
-
     def bulk_store_sessions(self, sessions):
         for session in sessions:
             self.store_session(session)

tests/snuba/api/endpoints/test_organization_sessions.py

@@ -58,11 +58,13 @@ def make_duration(kwargs):
 
 def make_session(project, **kwargs):
     return dict(
-        TEMPLATE,
-        session_id=uuid4().hex,
-        org_id=project.organization_id,
-        project_id=project.id,
-        duration=make_duration(kwargs),
+        dict(
+            TEMPLATE,
+            session_id=uuid4().hex,
+            org_id=project.organization_id,
+            project_id=project.id,
+            duration=make_duration(kwargs),
+        ),
         **kwargs,
     )
 
@@ -692,6 +694,106 @@ class OrganizationSessionsEndpointTest(APITestCase, SnubaTestCase):
         "p99(session.duration)": 79159.99999999999,
     }
 
+    @freeze_time(MOCK_DATETIME)
+    def test_users_groupby_status_advanced(self):
+        project = self.create_project()
+
+        user1 = uuid4().hex
+        session1 = uuid4().hex
+
+        user2 = uuid4().hex
+        session2a = uuid4().hex
+        session2b = uuid4().hex
+
+        user3 = uuid4().hex
+        session3 = uuid4().hex
+
+        self.store_session(
+            make_session(project, session_id=session1, distinct_id=user1, status="ok")
+        )
+        self.store_session(
+            make_session(
+                project, session_id=session1, distinct_id=user1, seq=1, errors=1, status="errored"
+            )
+        )
+        self.store_session(
+            make_session(project, session_id=session1, distinct_id=user1, seq=2, status="crashed")
+        )
+
+        self.store_session(
+            make_session(project, session_id=session2a, distinct_id=user2, status="ok")
+        )
+        self.store_session(
+            make_session(project, session_id=session2b, distinct_id=user2, status="ok")
+        )
+        self.store_session(
+            make_session(project, session_id=session2b, distinct_id=user2, status="abnormal")
+        )
+
+        self.store_session(
+            make_session(
+                project, session_id=session3, distinct_id=user3, errors=123, status="errored"
+            )
+        )
+
+        # Add some extra healthy users:
+        for _ in range(3):
+            user = uuid4().hex
+            self.store_session(make_session(project, distinct_id=user))
+
+        # First, check if totals make sense:
+        response = self.do_request(
+            {
+                "project": [project.id],
+                "statsPeriod": "1d",
+                "interval": "1d",
+                "field": ["count_unique(user)"],
+            }
+        )
+        assert response.status_code == 200, response.content
+        assert result_sorted(response.data)["groups"] == [
+            {
+                "by": {},
+                "series": {"count_unique(user)": [6]},
+                "totals": {"count_unique(user)": 6},
+            },
+        ]
+
+        # Then check if grouping makes sense:
+        response = self.do_request(
+            {
+                "project": [project.id],
+                "statsPeriod": "1d",
+                "interval": "1d",
+                "field": ["count_unique(user)"],
+                "groupBy": ["session.status"],
+            }
+        )
+        assert response.status_code == 200, response.content
+        assert result_sorted(response.data)["groups"] == [
+            {
+                "by": {"session.status": "abnormal"},
+                "series": {"count_unique(user)": [1]},
+                "totals": {"count_unique(user)": 1},
+            },
+            {
+                "by": {"session.status": "crashed"},
+                "series": {"count_unique(user)": [1]},
+                "totals": {"count_unique(user)": 1},
+            },
+            {
+                "by": {"session.status": "errored"},
+                "series": {"count_unique(user)": [1]},
+                "totals": {"count_unique(user)": 1},
+            },
+            {
+                # user
+                "by": {"session.status": "healthy"},
+                "series": {"count_unique(user)": [3]},
+                "totals": {"count_unique(user)": 3},
+            },
+        ]
+
     @freeze_time(MOCK_DATETIME)
     def test_duration_percentiles(self):
         response = self.do_request(