from __future__ import annotations
import time
from datetime import UTC, datetime, timedelta
from unittest import mock
import pytest
from django.utils import timezone
from sentry.release_health.base import OverviewStat
from sentry.release_health.metrics import MetricsReleaseHealthBackend
from sentry.sentry_metrics.use_case_id_registry import UseCaseID
from sentry.snuba.sessions import _make_stats
from sentry.testutils.cases import BaseMetricsTestCase, TestCase
pytestmark = pytest.mark.sentry_metrics
def format_timestamp(dt):
if not isinstance(dt, datetime):
dt = datetime.fromtimestamp(dt)
return dt.strftime("%Y-%m-%dT%H:%M:%S+00:00")
def make_24h_stats(ts, adjust_start=False):
ret_val = _make_stats(datetime.fromtimestamp(ts, UTC), 3600, 24)
if adjust_start:
# HACK this adds another interval at the beginning in accordance with the new way of calculating intervals
# https://www.notion.so/sentry/Metrics-Layer-get_intervals-bug-dce140607d054201a5e6629b070cb969
ret_val.insert(0, [ret_val[0][0] - 3600, 0])
return ret_val
class SnubaSessionsTest(TestCase, BaseMetricsTestCase):
backend = MetricsReleaseHealthBackend()
def setUp(self):
super().setUp()
self.received = time.time()
self.session_started = time.time() // 60 * 60
self.session_release = "foo@1.0.0"
self.session_crashed_release = "foo@2.0.0"
session_1 = "5d52fd05-fcc9-4bf3-9dc9-267783670341"
session_2 = "5e910c1a-6941-460e-9843-24103fb6a63c"
session_3 = "a148c0c5-06a2-423b-8901-6b43b812cf82"
user_1 = "39887d89-13b2-4c84-8c23-5d13d2102666"
self.store_session(
self.build_session(
distinct_id=user_1,
session_id=session_1,
status="exited",
release=self.session_release,
environment="prod",
started=self.session_started,
received=self.received,
)
)
self.store_session(
self.build_session(
distinct_id=user_1,
session_id=session_2,
release=self.session_release,
environment="prod",
duration=None,
started=self.session_started,
received=self.received,
)
)
self.store_session(
self.build_session(
distinct_id=user_1,
session_id=session_2,
seq=1,
duration=30,
status="exited",
release=self.session_release,
environment="prod",
started=self.session_started,
received=self.received,
)
)
self.store_session(
self.build_session(
distinct_id=user_1,
session_id=session_3,
status="crashed",
release=self.session_crashed_release,
environment="prod",
started=self.session_started,
received=self.received,
)
)
def test_get_oldest_health_data_for_releases(self):
data = self.backend.get_oldest_health_data_for_releases(
[(self.project.id, self.session_release)]
)
assert data == {
(self.project.id, self.session_release): format_timestamp(
self.session_started // 3600 * 3600
)
}
def test_check_has_health_data(self):
data = self.backend.check_has_health_data(
[(self.project.id, self.session_release), (self.project.id, "dummy-release")]
)
assert data == {(self.project.id, self.session_release)}
def test_check_has_health_data_without_releases_should_include_sessions_lte_90_days(self):
"""
Test that ensures that `check_has_health_data` returns a set of projects that has health
data within the last 90d if only a list of project ids is provided and any project with
session data earlier than 90 days should be included
"""
project2 = self.create_project(
name="Bar2",
slug="bar2",
teams=[self.team],
fire_project_created=True,
organization=self.organization,
)
self.store_session(
self.build_session(
**{
"project_id": project2.id,
"org_id": project2.organization_id,
"status": "exited",
}
)
)
data = self.backend.check_has_health_data([self.project.id, project2.id])
assert data == {self.project.id, project2.id}
def test_check_has_health_data_does_not_crash_when_sending_projects_list_as_set(self):
data = self.backend.check_has_health_data({self.project.id})
assert data == {self.project.id}
def test_get_project_releases_by_stability(self):
# Add an extra session with a different `distinct_id` so that sorting by users
# is stable
self.store_session(
self.build_session(
release=self.session_release,
environment="prod",
started=self.session_started,
received=self.received,
)
)
for scope in "sessions", "users":
data = self.backend.get_project_releases_by_stability(
[self.project.id], offset=0, limit=100, scope=scope, stats_period="24h"
)
assert data == [
(self.project.id, self.session_release),
(self.project.id, self.session_crashed_release),
]
def test_get_project_releases_by_stability_for_crash_free_sort(self):
"""
Test that ensures that using crash free rate sort options, returns a list of ASC releases
according to the chosen crash_free sort option
"""
# add another user to session_release to make sure that they are sorted correctly
self.store_session(
self.build_session(
status="exited",
release=self.session_release,
environment="prod",
started=self.session_started,
received=self.received,
)
)
for scope in "crash_free_sessions", "crash_free_users":
data = self.backend.get_project_releases_by_stability(
[self.project.id], offset=0, limit=100, scope=scope, stats_period="24h"
)
assert data == [
(self.project.id, self.session_crashed_release),
(self.project.id, self.session_release),
]
def test_get_project_releases_by_stability_for_releases_with_users_data(self):
"""
Test that ensures if releases contain no users data, then those releases should not be
returned on `users` and `crash_free_users` sorts
"""
self.store_session(
self.build_session(
distinct_id=None,
release="release-with-no-users",
environment="prod",
started=self.session_started,
received=self.received,
)
)
data = self.backend.get_project_releases_by_stability(
[self.project.id], offset=0, limit=100, scope="users", stats_period="24h"
)
assert set(data) == {
(self.project.id, self.session_release),
(self.project.id, self.session_crashed_release),
}
data = self.backend.get_project_releases_by_stability(
[self.project.id], offset=0, limit=100, scope="crash_free_users", stats_period="24h"
)
assert set(data) == {
(self.project.id, self.session_crashed_release),
(self.project.id, self.session_release),
}
def test_get_release_adoption(self):
data = self.backend.get_release_adoption(
[
(self.project.id, self.session_release),
(self.project.id, self.session_crashed_release),
(self.project.id, "dummy-release"),
]
)
assert data == {
(self.project.id, self.session_release): {
"sessions_24h": 2,
"users_24h": 1,
"adoption": 100.0,
"sessions_adoption": 66.66666666666666,
"project_sessions_24h": 3,
"project_users_24h": 1,
},
(self.project.id, self.session_crashed_release): {
"sessions_24h": 1,
"users_24h": 1,
"adoption": 100.0,
"sessions_adoption": 33.33333333333333,
"project_sessions_24h": 3,
"project_users_24h": 1,
},
}
def test_get_release_adoption_lowered(self):
self.store_session(
self.build_session(
release=self.session_crashed_release,
environment="prod",
status="crashed",
started=self.session_started,
received=self.received,
)
)
data = self.backend.get_release_adoption(
[
(self.project.id, self.session_release),
(self.project.id, self.session_crashed_release),
(self.project.id, "dummy-release"),
]
)
assert data == {
(self.project.id, self.session_release): {
"sessions_24h": 2,
"users_24h": 1,
"adoption": 50.0,
"sessions_adoption": 50.0,
"project_sessions_24h": 4,
"project_users_24h": 2,
},
(self.project.id, self.session_crashed_release): {
"sessions_24h": 2,
"users_24h": 2,
"adoption": 100.0,
"sessions_adoption": 50.0,
"project_sessions_24h": 4,
"project_users_24h": 2,
},
}
def test_fetching_release_sessions_time_bounds_for_different_release(self):
"""
Test that ensures only session bounds for releases are calculated according
to their respective release
"""
# Same release session
self.store_session(
self.build_session(
release=self.session_release,
environment="prod",
status="exited",
started=self.session_started - 3600 * 2,
received=self.received - 3600 * 2,
)
)
# Different release session
self.store_session(
self.build_session(
release=self.session_crashed_release,
environment="prod",
status="crashed",
started=self.session_started - 3600 * 2,
received=self.received - 3600 * 2,
)
)
expected_formatted_lower_bound = (
datetime.fromtimestamp(self.session_started - 3600 * 2)
.replace(minute=0)
.isoformat()[:19]
+ "Z"
)
expected_formatted_upper_bound = (
datetime.fromtimestamp(self.session_started).replace(minute=0).isoformat()[:19] + "Z"
)
# Test for self.session_release
data = self.backend.get_release_sessions_time_bounds(
project_id=self.project.id,
release=self.session_release,
org_id=self.organization.id,
environments=["prod"],
)
assert data == {
"sessions_lower_bound": expected_formatted_lower_bound,
"sessions_upper_bound": expected_formatted_upper_bound,
}
# Test for self.session_crashed_release
data = self.backend.get_release_sessions_time_bounds(
project_id=self.project.id,
release=self.session_crashed_release,
org_id=self.organization.id,
environments=["prod"],
)
assert data == {
"sessions_lower_bound": expected_formatted_lower_bound,
"sessions_upper_bound": expected_formatted_upper_bound,
}
def test_fetching_release_sessions_time_bounds_for_different_release_with_no_sessions(self):
"""
Test that ensures if no sessions are available for a specific release then the bounds
should be returned as None
"""
data = self.backend.get_release_sessions_time_bounds(
project_id=self.project.id,
release="different_release",
org_id=self.organization.id,
environments=["prod"],
)
assert data == {
"sessions_lower_bound": None,
"sessions_upper_bound": None,
}
def test_get_crash_free_breakdown(self):
start = timezone.now() - timedelta(days=4)
# it should work with and without environments
for environments in [None, ["prod"]]:
data = self.backend.get_crash_free_breakdown(
project_id=self.project.id,
release=self.session_release,
start=start,
environments=environments,
)
# Last returned date is generated within function, should be close to now:
last_date = data[-1]["date"]
assert timezone.now() - last_date < timedelta(seconds=1)
assert data == [
{
"crash_free_sessions": None,
"crash_free_users": None,
"date": start + timedelta(days=1),
"total_sessions": 0,
"total_users": 0,
},
{
"crash_free_sessions": None,
"crash_free_users": None,
"date": start + timedelta(days=2),
"total_sessions": 0,
"total_users": 0,
},
{
"crash_free_sessions": 100.0,
"crash_free_users": 100.0,
"total_sessions": 2,
"total_users": 1,
"date": mock.ANY, # tested above
},
]
data = self.backend.get_crash_free_breakdown(
project_id=self.project.id,
release=self.session_crashed_release,
start=start,
environments=["prod"],
)
assert data == [
{
"crash_free_sessions": None,
"crash_free_users": None,
"date": start + timedelta(days=1),
"total_sessions": 0,
"total_users": 0,
},
{
"crash_free_sessions": None,
"crash_free_users": None,
"date": start + timedelta(days=2),
"total_sessions": 0,
"total_users": 0,
},
{
"crash_free_sessions": 0.0,
"crash_free_users": 0.0,
"total_sessions": 1,
"total_users": 1,
"date": mock.ANY,
},
]
data = self.backend.get_crash_free_breakdown(
project_id=self.project.id,
release="non-existing",
start=start,
environments=["prod"],
)
assert data == [
{
"crash_free_sessions": None,
"crash_free_users": None,
"date": start + timedelta(days=1),
"total_sessions": 0,
"total_users": 0,
},
{
"crash_free_sessions": None,
"crash_free_users": None,
"date": start + timedelta(days=2),
"total_sessions": 0,
"total_users": 0,
},
{
"crash_free_sessions": None,
"crash_free_users": None,
"total_sessions": 0,
"total_users": 0,
"date": mock.ANY,
},
]
def test_basic_release_model_adoptions(self):
"""
Test that the basic (project,release) data is returned
"""
proj_id = self.project.id
data = self.backend.get_changed_project_release_model_adoptions([proj_id])
assert set(data) == {(proj_id, "foo@1.0.0"), (proj_id, "foo@2.0.0")}
def test_old_release_model_adoptions(self):
"""
Test that old entries (older that 72 h) are not returned
"""
_100h = 100 * 60 * 60 # 100 hours in seconds
proj_id = self.project.id
self.store_session(
self.build_session(
release="foo@3.0.0",
environment="prod",
status="crashed",
started=self.session_started - _100h,
received=self.received - 3600 * 2,
)
)
data = self.backend.get_changed_project_release_model_adoptions([proj_id])
assert set(data) == {(proj_id, "foo@1.0.0"), (proj_id, "foo@2.0.0")}
def test_multi_proj_release_model_adoptions(self):
"""Test that the api works with multiple projects"""
proj_id = self.project.id
new_proj_id = proj_id + 1
self.store_session(
self.build_session(
project_id=new_proj_id,
release="foo@3.0.0",
environment="prod",
status="crashed",
started=self.session_started,
received=self.received - 3600 * 2,
)
)
data = self.backend.get_changed_project_release_model_adoptions([proj_id, new_proj_id])
assert set(data) == {
(proj_id, "foo@1.0.0"),
(proj_id, "foo@2.0.0"),
(new_proj_id, "foo@3.0.0"),
}
@staticmethod
def _add_timestamps_to_series(series, start: datetime):
one_day = 24 * 60 * 60
day0 = one_day * int(start.timestamp() / one_day)
def ts(days: int) -> int:
return day0 + days * one_day
return [(ts(i + 1), data) for i, data in enumerate(series)]
def _test_get_project_release_stats(
self, stat: OverviewStat, release: str, expected_series, expected_totals
):
end = timezone.now()
start = end - timedelta(days=4)
stats, totals = self.backend.get_project_release_stats(
self.project.id,
release=release,
stat=stat,
rollup=86400,
start=start,
end=end,
)
# one system returns lists instead of tuples
normed = [(ts, data) for ts, data in stats]
assert normed == self._add_timestamps_to_series(expected_series, start)
assert totals == expected_totals
def test_get_project_release_stats_users(self):
self._test_get_project_release_stats(
"users",
self.session_release,
[
{
"duration_p50": None,
"duration_p90": None,
"users": 0,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"users": 0,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"users": 0,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 0,
},
{
"duration_p50": 45.0,
"duration_p90": 57.0,
"users": 1,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 1,
},
],
{
"users": 1,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 1,
},
)
def test_get_project_release_stats_users_crashed(self):
self._test_get_project_release_stats(
"users",
self.session_crashed_release,
[
{
"duration_p50": None,
"duration_p90": None,
"users": 0,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"users": 0,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"users": 0,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"users": 1,
"users_abnormal": 0,
"users_crashed": 1,
"users_errored": 0,
"users_healthy": 0,
},
],
{
"users": 1,
"users_abnormal": 0,
"users_crashed": 1,
"users_errored": 0,
"users_healthy": 0,
},
)
def test_get_project_release_stats_sessions(self):
self._test_get_project_release_stats(
"sessions",
self.session_release,
[
{
"duration_p50": None,
"duration_p90": None,
"sessions": 0,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"sessions": 0,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"sessions": 0,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 0,
},
{
"duration_p50": 45.0,
"duration_p90": 57.0,
"sessions": 2,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 2,
},
],
{
"sessions": 2,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 2,
},
)
def test_get_project_release_stats_sessions_crashed(self):
self._test_get_project_release_stats(
"sessions",
self.session_crashed_release,
[
{
"duration_p50": None,
"duration_p90": None,
"sessions": 0,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"sessions": 0,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"sessions": 0,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"sessions": 1,
"sessions_abnormal": 0,
"sessions_crashed": 1,
"sessions_errored": 0,
"sessions_healthy": 0,
},
],
{
"sessions": 1,
"sessions_abnormal": 0,
"sessions_crashed": 1,
"sessions_errored": 0,
"sessions_healthy": 0,
},
)
def test_get_project_release_stats_no_sessions(self):
"""
Test still returning correct data when no sessions are available
:return:
"""
self._test_get_project_release_stats(
"sessions",
"INEXISTENT-RELEASE",
[
{
"duration_p50": None,
"duration_p90": None,
"sessions": 0,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"sessions": 0,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"sessions": 0,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"sessions": 0,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 0,
},
],
{
"sessions": 0,
"sessions_abnormal": 0,
"sessions_crashed": 0,
"sessions_errored": 0,
"sessions_healthy": 0,
},
)
def test_get_project_release_stats_no_users(self):
self._test_get_project_release_stats(
"users",
"INEXISTENT-RELEASE",
[
{
"duration_p50": None,
"duration_p90": None,
"users": 0,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"users": 0,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"users": 0,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 0,
},
{
"duration_p50": None,
"duration_p90": None,
"users": 0,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 0,
},
],
{
"users": 0,
"users_abnormal": 0,
"users_crashed": 0,
"users_errored": 0,
"users_healthy": 0,
},
)
class GetCrashFreeRateTestCase(TestCase, BaseMetricsTestCase):
"""
TestClass that tests that `get_current_and_previous_crash_free_rates` returns the correct
`currentCrashFreeRate` and `previousCrashFreeRate` for each project
TestData:
Project 1:
In the last 24h -> 2 Exited Sessions / 2 Total Sessions -> 100% Crash free rate
In the previous 24h (>24h & <48h) -> 2 Exited + 1 Crashed Sessions / 3 Sessions -> 66.7%
Project 2:
In the last 24h -> 1 Exited + 1 Crashed / 2 Total Sessions -> 50% Crash free rate
In the previous 24h (>24h & <48h) -> 0 Sessions -> None
Project 3:
In the last 24h -> 0 Sessions -> None
In the previous 24h (>24h & <48h) -> 4 Exited + 1 Crashed / 5 Total Sessions -> 80%
"""
backend = MetricsReleaseHealthBackend()
def setUp(self):
super().setUp()
self.session_started = time.time() // 60 * 60
self.session_started_gt_24_lt_48 = self.session_started - 30 * 60 * 60
self.project2 = self.create_project(
name="Bar2",
slug="bar2",
teams=[self.team],
fire_project_created=True,
organization=self.organization,
)
self.project3 = self.create_project(
name="Bar3",
slug="bar3",
teams=[self.team],
fire_project_created=True,
organization=self.organization,
)
# Project 1
for _ in range(0, 2):
self.store_session(
self.build_session(
**{
"project_id": self.project.id,
"org_id": self.project.organization_id,
"status": "exited",
}
)
)
for idx in range(0, 3):
status = "exited"
if idx == 2:
status = "crashed"
self.store_session(
self.build_session(
**{
"project_id": self.project.id,
"org_id": self.project.organization_id,
"status": status,
"started": self.session_started_gt_24_lt_48,
}
)
)
# Project 2
for i in range(0, 2):
status = "exited"
if i == 1:
status = "crashed"
self.store_session(
self.build_session(
**{
"project_id": self.project2.id,
"org_id": self.project2.organization_id,
"status": status,
}
)
)
# Project 3
for i in range(0, 5):
status = "exited"
if i == 4:
status = "crashed"
self.store_session(
self.build_session(
**{
"project_id": self.project3.id,
"org_id": self.project3.organization_id,
"status": status,
"started": self.session_started_gt_24_lt_48,
}
)
)
def test_get_current_and_previous_crash_free_rates(self):
now = timezone.now().replace(minute=15, second=23)
last_24h_start = now - 24 * timedelta(hours=1)
last_48h_start = now - 2 * 24 * timedelta(hours=1)
data = self.backend.get_current_and_previous_crash_free_rates(
org_id=self.organization.id,
project_ids=[self.project.id, self.project2.id, self.project3.id],
current_start=last_24h_start,
current_end=now,
previous_start=last_48h_start,
previous_end=last_24h_start,
rollup=3600,
)
assert data == {
self.project.id: {
"currentCrashFreeRate": 100,
"previousCrashFreeRate": 66.66666666666667,
},
self.project2.id: {"currentCrashFreeRate": 50.0, "previousCrashFreeRate": None},
self.project3.id: {"currentCrashFreeRate": None, "previousCrashFreeRate": 80.0},
}
def test_get_current_and_previous_crash_free_rates_with_zero_sessions(self):
now = timezone.now().replace(minute=15, second=23)
last_48h_start = now - 2 * 24 * timedelta(hours=1)
last_72h_start = now - 3 * 24 * timedelta(hours=1)
last_96h_start = now - 4 * 24 * timedelta(hours=1)
data = self.backend.get_current_and_previous_crash_free_rates(
org_id=self.organization.id,
project_ids=[self.project.id],
current_start=last_72h_start,
current_end=last_48h_start,
previous_start=last_96h_start,
previous_end=last_72h_start,
rollup=3600,
)
assert data == {
self.project.id: {
"currentCrashFreeRate": None,
"previousCrashFreeRate": None,
},
}
def test_extract_crash_free_rate_from_result_groups(self):
result_groups = [
{"by": {"project_id": 1}, "totals": {"rate": 0.66}},
{"by": {"project_id": 2}, "totals": {"rate": 0.8}},
]
crash_free_rates = self.backend._extract_crash_free_rates_from_result_groups(result_groups)
assert crash_free_rates[1] == 0.66 * 100
assert crash_free_rates[2] == 0.8 * 100
def test_extract_crash_free_rate_from_result_groups_with_none(self):
result_groups = [
{"by": {"project_id": 1}, "totals": {"rate": 0.66}},
{"by": {"project_id": 2}, "totals": {"rate": None}},
]
crash_free_rates = self.backend._extract_crash_free_rates_from_result_groups(result_groups)
assert crash_free_rates[1] == 0.66 * 100
assert crash_free_rates[2] is None
def test_extract_crash_free_rates_from_result_groups_only_none(self):
result_groups = [
{"by": {"project_id": 2}, "totals": {"rate": None}},
]
crash_free_rates = self.backend._extract_crash_free_rates_from_result_groups(result_groups)
assert crash_free_rates[2] is None
class GetProjectReleasesCountTest(TestCase, BaseMetricsTestCase):
backend = MetricsReleaseHealthBackend()
def test_empty(self):
# Test no errors when no session data
org = self.create_organization()
proj = self.create_project(organization=org)
assert (
self.backend.get_project_releases_count(
org.id, [proj.id], "crash_free_users", stats_period="14d"
)
== 0
)
def test_with_other_metrics(self):
if not self.backend.is_metrics_based():
return
assert isinstance(self, BaseMetricsTestCase)
# Test no errors when no session data
org = self.create_organization()
proj = self.create_project(organization=org)
# Insert a different set metric:
for value in 1, 2, 3:
self.store_metric(
org_id=org.id,
project_id=proj.id,
name="foobarbaz", # any other metric ID
timestamp=int(time.time()),
tags={},
type="set",
value=value,
use_case_id=UseCaseID.SESSIONS,
)
assert (
self.backend.get_project_releases_count(
org.id, [proj.id], "crash_free_users", stats_period="14d"
)
== 0
)
def test(self):
project_release_1 = self.create_release(self.project)
other_project = self.create_project()
other_project_release_1 = self.create_release(other_project)
self.bulk_store_sessions(
[
self.build_session(
environment=self.environment.name, release=project_release_1.version
),
self.build_session(
environment="staging",
project_id=other_project.id,
release=other_project_release_1.version,
),
]
)
assert (
self.backend.get_project_releases_count(
self.organization.id, [self.project.id], "sessions"
)
== 1
)
assert (
self.backend.get_project_releases_count(
self.organization.id, [self.project.id], "users"
)
== 1
)
assert (
self.backend.get_project_releases_count(
self.organization.id, [self.project.id, other_project.id], "sessions"
)
== 2
)
assert (
self.backend.get_project_releases_count(
self.organization.id,
[self.project.id, other_project.id],
"users",
)
== 2
)
assert (
self.backend.get_project_releases_count(
self.organization.id,
[self.project.id, other_project.id],
"sessions",
environments=[self.environment.name],
)
== 1
)
class CheckReleasesHaveHealthDataTest(TestCase, BaseMetricsTestCase):
backend = MetricsReleaseHealthBackend()
def run_test(self, expected, projects, releases, start=None, end=None):
if not start:
start = datetime.now() - timedelta(days=1)
if not end:
end = datetime.now()
assert self.backend.check_releases_have_health_data(
self.organization.id,
[p.id for p in projects],
[r.version for r in releases],
start,
end,
) == {v.version for v in expected}
def test_empty(self):
# Test no errors when no session data
project_release_1 = self.create_release(self.project)
self.run_test([], [self.project], [project_release_1])
def test(self):
other_project = self.create_project()
release_1 = self.create_release(
self.project, version="1", additional_projects=[other_project]
)
release_2 = self.create_release(other_project, version="2")
self.bulk_store_sessions(
[
self.build_session(release=release_1),
self.build_session(project_id=other_project, release=release_1),
self.build_session(project_id=other_project, release=release_2),
]
)
self.run_test([release_1], [self.project], [release_1])
self.run_test([release_1], [self.project], [release_1, release_2])
self.run_test([release_1], [other_project], [release_1])
self.run_test([release_1, release_2], [other_project], [release_1, release_2])
self.run_test([release_1, release_2], [self.project, other_project], [release_1, release_2])
class CheckNumberOfSessions(TestCase, BaseMetricsTestCase):
backend = MetricsReleaseHealthBackend()
def setUp(self):
super().setUp()
self.dev_env = self.create_environment(name="development", project=self.project)
self.prod_env = self.create_environment(name="production", project=self.project)
self.test_env = self.create_environment(name="test", project=self.project)
self.another_project = self.create_project()
self.third_project = self.create_project()
# now_dt should be set to 17:40 of some day not in the future and (system time - now_dt)
# must be less than 90 days for the metrics DB TTL
ONE_DAY_AGO = timezone.now() - timedelta(days=1)
self.now_dt = ONE_DAY_AGO.replace(hour=17, minute=40, second=0)
self._5_min_ago_dt = self.now_dt - timedelta(minutes=5)
self._30_min_ago_dt = self.now_dt - timedelta(minutes=30)
self._1_h_ago_dt = self.now_dt - timedelta(hours=1)
self._2_h_ago_dt = self.now_dt - timedelta(hours=2)
self._3_h_ago_dt = self.now_dt - timedelta(hours=3)
self.now = self.now_dt.timestamp()
self._5_min_ago = self._5_min_ago_dt.timestamp()
self._30_min_ago = self._30_min_ago_dt.timestamp()
self._1_h_ago = self._1_h_ago_dt.timestamp()
self._2_h_ago = self._2_h_ago_dt.timestamp()
self._3_h_ago = self._3_h_ago_dt.timestamp()
def test_no_sessions(self):
"""
Tests that when there are no sessions the function behaves and returns 0
"""
actual = self.backend.get_project_sessions_count(
project_id=self.project.id,
environment_id=None,
rollup=60,
start=self._30_min_ago_dt,
end=self.now_dt,
)
assert 0 == actual
def test_sessions_in_environment(self):
"""
Tests that it correctly picks up the sessions for the selected environment
in the selected time, not counting other environments and other times
"""
dev = self.dev_env.name
prod = self.prod_env.name
self.bulk_store_sessions(
[
self.build_session(
environment=dev, received=self._5_min_ago, started=self._5_min_ago
),
self.build_session(
environment=prod, received=self._5_min_ago, started=self._5_min_ago
),
self.build_session(
environment=prod, received=self._5_min_ago, started=self._5_min_ago
),
self.build_session(environment=prod, received=self._2_h_ago, started=self._2_h_ago),
]
)
actual = self.backend.get_project_sessions_count(
project_id=self.project.id,
environment_id=self.prod_env.id,
rollup=60,
start=self._1_h_ago_dt,
end=self.now_dt,
)
assert actual == 2
def test_environment_without_sessions(self):
"""
We should get zero sessions, even if the environment name has not been indexed
by the metrics indexer.
"""
env_without_sessions = self.create_environment(
name="this_has_no_sessions", project=self.project
)
self.bulk_store_sessions(
[
self.build_session(
environment=self.prod_env.name,
received=self._5_min_ago,
started=self._5_min_ago,
),
self.build_session(
environment=None, received=self._5_min_ago, started=self._5_min_ago
),
]
)
count_env_all = self.backend.get_project_sessions_count(
project_id=self.project.id,
environment_id=None,
rollup=60,
start=self._1_h_ago_dt,
end=self.now_dt,
)
assert count_env_all == 2
count_env_new = self.backend.get_project_sessions_count(
project_id=self.project.id,
environment_id=env_without_sessions.id,
rollup=60,
start=self._1_h_ago_dt,
end=self.now_dt,
)
assert count_env_new == 0
def test_sessions_in_all_environments(self):
"""
When the environment is not specified sessions from all environments are counted
"""
dev = self.dev_env.name
prod = self.prod_env.name
self.bulk_store_sessions(
[
self.build_session(
environment=dev, received=self._5_min_ago, started=self._5_min_ago
),
self.build_session(
environment=prod, received=self._5_min_ago, started=self._5_min_ago
),
self.build_session(
environment=prod, received=self._5_min_ago, started=self._5_min_ago
),
self.build_session(environment=prod, received=self._2_h_ago, started=self._2_h_ago),
self.build_session(environment=dev, received=self._2_h_ago, started=self._2_h_ago),
]
)
actual = self.backend.get_project_sessions_count(
project_id=self.project.id,
environment_id=None,
rollup=60,
start=self._1_h_ago_dt,
end=self.now_dt,
)
assert actual == 3
def test_sessions_from_multiple_projects(self):
"""
Only sessions from the specified project are considered
"""
dev = self.dev_env.name
prod = self.prod_env.name
self.bulk_store_sessions(
[
self.build_session(
environment=dev, received=self._5_min_ago, started=self._5_min_ago
),
self.build_session(
environment=prod, received=self._5_min_ago, started=self._5_min_ago
),
self.build_session(
environment=prod,
received=self._5_min_ago,
project_id=self.another_project.id,
started=self._5_min_ago,
),
]
)
actual = self.backend.get_project_sessions_count(
project_id=self.project.id,
environment_id=None,
rollup=60,
start=self._1_h_ago_dt,
end=self.now_dt,
)
assert actual == 2
def test_sessions_per_project_no_sessions(self):
"""
Tests that no sessions are returned
"""
actual = self.backend.get_num_sessions_per_project(
project_ids=[self.project.id, self.another_project.id],
environment_ids=None,
rollup=60,
start=self._30_min_ago_dt,
end=self.now_dt,
)
assert [] == actual
def test_sesions_per_project_multiple_projects(self):
dev = self.dev_env.name
prod = self.prod_env.name
test = self.test_env.name
p1 = self.project
p2 = self.another_project
p3 = self.third_project
self.bulk_store_sessions(
[
# counted in p1
self.build_session(
environment=dev, received=self._5_min_ago, started=self._5_min_ago
),
self.build_session(
environment=prod, received=self._5_min_ago, started=self._5_min_ago
),
self.build_session(
environment=dev, received=self._30_min_ago, started=self._30_min_ago
),
# ignored in p1
# ignored env
self.build_session(
environment=test, received=self._30_min_ago, started=self._30_min_ago
),
# too old
self.build_session(environment=prod, received=self._3_h_ago, started=self._3_h_ago),
# counted in p2
self.build_session(
environment=dev,
received=self._5_min_ago,
project_id=p2.id,
started=self._5_min_ago,
),
# ignored in p2
# ignored env
self.build_session(
environment=test,
received=self._5_min_ago,
project_id=p2.id,
started=self._5_min_ago,
),
# too old
self.build_session(
environment=prod,
received=self._3_h_ago,
project_id=p2.id,
started=self._3_h_ago,
),
# ignored p3
self.build_session(
environment=dev,
received=self._5_min_ago,
project_id=p3.id,
started=self._5_min_ago,
),
]
)
actual = self.backend.get_num_sessions_per_project(
project_ids=[self.project.id, self.another_project.id],
environment_ids=[self.dev_env.id, self.prod_env.id],
rollup=60,
start=self._2_h_ago_dt,
end=self.now_dt,
)
assert set(actual) == {(p1.id, 3), (p2.id, 1)}
eids_tests: tuple[list[int] | None, ...] = ([], None)
for eids in eids_tests:
actual = self.backend.get_num_sessions_per_project(
project_ids=[self.project.id, self.another_project.id],
environment_ids=eids,
rollup=60,
start=self._2_h_ago_dt,
end=self.now_dt,
)
assert set(actual) == {(p1.id, 4), (p2.id, 2)}
class InitWithoutUserTestCase(TestCase, BaseMetricsTestCase):
backend = MetricsReleaseHealthBackend()
def setUp(self):
super().setUp()
self.received = time.time()
self.session_started = time.time() // 60 * 60
self.session_release = "foo@1.0.0"
session_1 = "5d52fd05-fcc9-4bf3-9dc9-267783670341"
session_2 = "5e910c1a-6941-460e-9843-24103fb6a63c"
session_3 = "a148c0c5-06a2-423b-8901-6b43b812cf82"
user_1 = "39887d89-13b2-4c84-8c23-5d13d2102666"
user_2 = "39887d89-13b2-4c84-8c23-5d13d2102667"
user_3 = "39887d89-13b2-4c84-8c23-5d13d2102668"
self.bulk_store_sessions(
[
self.build_session(
distinct_id=user_1,
session_id=session_1,
status="exited",
release=self.session_release,
environment="prod",
started=self.session_started,
received=self.received,
),
self.build_session(
distinct_id=user_2,
session_id=session_2,
status="crashed",
release=self.session_release,
environment="prod",
started=self.session_started,
received=self.received,
),
# session_3 initial update: no user ID
self.build_session(
distinct_id=None,
session_id=session_3,
status="ok",
seq=0,
release=self.session_release,
environment="prod",
started=self.session_started,
received=self.received,
),
# session_3 subsequent update: user ID is here!
self.build_session(
distinct_id=user_3,
session_id=session_3,
status="ok",
seq=123,
release=self.session_release,
environment="prod",
started=self.session_started,
received=self.received,
),
]
)
def test_get_release_adoption(self):
data = self.backend.get_release_adoption(
[
(self.project.id, self.session_release),
]
)
inner = data[(self.project.id, self.session_release)]
assert inner["users_24h"] == 3
def test_get_release_health_data_overview_users(self):
data = self.backend.get_release_health_data_overview(
[
(self.project.id, self.session_release),
],
summary_stats_period="24h",
health_stats_period="24h",
stat="users",
)
inner = data[(self.project.id, self.session_release)]
assert inner["total_users"] == 3
assert inner["crash_free_users"] == 66.66666666666667
def test_get_crash_free_breakdown(self):
start = timezone.now() - timedelta(days=4)
data = self.backend.get_crash_free_breakdown(
project_id=self.project.id,
release=self.session_release,
start=start,
environments=["prod"],
)
# Last returned date is generated within function, should be close to now:
last_date = data[-1]["date"]
assert timezone.now() - last_date < timedelta(seconds=1)
assert data == [
{
"crash_free_sessions": None,
"crash_free_users": None,
"date": start + timedelta(days=1),
"total_sessions": 0,
"total_users": 0,
},
{
"crash_free_sessions": None,
"crash_free_users": None,
"date": start + timedelta(days=2),
"total_sessions": 0,
"total_users": 0,
},
{
"crash_free_sessions": 66.66666666666667,
"crash_free_users": 66.66666666666667,
"total_sessions": 3,
"total_users": 3,
"date": mock.ANY, # tested above
},
]
def test_get_project_release_stats_users(self):
end = timezone.now()
start = end - timedelta(days=4)
stats, totals = self.backend.get_project_release_stats(
self.project.id,
release=self.session_release,
stat="users",
rollup=86400,
start=start,
end=end,
)
assert stats[3][1] == {
"duration_p50": 60.0,
"duration_p90": 60.0,
"users": 3,
"users_abnormal": 0,
"users_crashed": 1,
"users_errored": 0,
"users_healthy": 2,
}