test_snuba.py 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203
  1. import copy
  2. import time
  3. import uuid
  4. from datetime import datetime, timedelta
  5. from unittest import mock
  6. import pytest
  7. from django.utils import timezone
  8. from snuba_sdk.column import InvalidColumnError
  9. from sentry.testutils.cases import SnubaTestCase, TestCase
  10. from sentry.testutils.helpers.datetime import before_now, iso_format
  11. from sentry.utils import snuba
  12. class SnubaTest(TestCase, SnubaTestCase):
  13. def _insert_event_for_time(self, ts, hash="a" * 32, group_id=None):
  14. self.snuba_insert(
  15. (
  16. 2,
  17. "insert",
  18. {
  19. "event_id": uuid.uuid4().hex,
  20. "primary_hash": hash,
  21. "group_id": group_id if group_id else int(hash[:16], 16),
  22. "project_id": self.project.id,
  23. "message": "message",
  24. "platform": "python",
  25. "datetime": ts.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
  26. "data": {"received": time.mktime(ts.timetuple())},
  27. },
  28. )
  29. )
  30. def test(self) -> None:
  31. "This is just a simple 'hello, world' example test."
  32. now = datetime.now()
  33. events = [
  34. (
  35. 2,
  36. "insert",
  37. {
  38. "event_id": "a" * 32,
  39. "primary_hash": "1" * 32,
  40. "group_id": 1,
  41. "project_id": self.project.id,
  42. "message": "message",
  43. "platform": "python",
  44. "datetime": now.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
  45. "data": {"received": time.mktime(now.timetuple())},
  46. },
  47. )
  48. ]
  49. self.snuba_insert(events)
  50. assert snuba.query(
  51. start=now - timedelta(days=1),
  52. end=now + timedelta(days=1),
  53. groupby=["project_id"],
  54. filter_keys={"project_id": [self.project.id]},
  55. referrer="testing.test",
  56. tenant_ids={"referrer": "testing.test", "organization_id": 1},
  57. ) == {self.project.id: 1}
  58. def test_fail(self) -> None:
  59. now = datetime.now()
  60. with pytest.raises(InvalidColumnError):
  61. snuba.query(
  62. start=now - timedelta(days=1),
  63. end=now + timedelta(days=1),
  64. filter_keys={"project_id": [self.project.id]},
  65. groupby=[")("],
  66. referrer="testing.test",
  67. )
  68. def test_organization_retention_respected(self) -> None:
  69. base_time = datetime.utcnow()
  70. self._insert_event_for_time(base_time - timedelta(minutes=1))
  71. self._insert_event_for_time(base_time - timedelta(days=2))
  72. def _get_event_count():
  73. # attempt to query back 90 days
  74. return snuba.query(
  75. start=base_time - timedelta(days=90),
  76. end=base_time + timedelta(days=1),
  77. groupby=["project_id"],
  78. filter_keys={"project_id": [self.project.id]},
  79. referrer="testing.test",
  80. tenant_ids={"referrer": "testing.test", "organization_id": 1},
  81. )
  82. assert _get_event_count() == {self.project.id: 2}
  83. with self.options({"system.event-retention-days": 1}):
  84. assert _get_event_count() == {self.project.id: 1}
  85. def test_organization_retention_larger_than_end_date(self) -> None:
  86. base_time = datetime.utcnow()
  87. with self.options({"system.event-retention-days": 1}):
  88. assert (
  89. snuba.query(
  90. start=base_time - timedelta(days=90),
  91. end=base_time - timedelta(days=60),
  92. groupby=["project_id"],
  93. filter_keys={"project_id": [self.project.id]},
  94. referrer="testing.test",
  95. )
  96. == {}
  97. )
  98. class BulkRawQueryTest(TestCase, SnubaTestCase):
  99. def test_simple(self) -> None:
  100. one_min_ago = iso_format(before_now(minutes=1))
  101. event_1 = self.store_event(
  102. data={"fingerprint": ["group-1"], "message": "hello", "timestamp": one_min_ago},
  103. project_id=self.project.id,
  104. )
  105. event_2 = self.store_event(
  106. data={"fingerprint": ["group-2"], "message": "hello", "timestamp": one_min_ago},
  107. project_id=self.project.id,
  108. )
  109. results = snuba.bulk_raw_query(
  110. [
  111. snuba.SnubaQueryParams(
  112. start=timezone.now() - timedelta(days=1),
  113. end=timezone.now(),
  114. selected_columns=["event_id", "group_id", "timestamp"],
  115. filter_keys={"project_id": [self.project.id], "group_id": [event_1.group.id]},
  116. tenant_ids={"referrer": "testing.test", "organization_id": 1},
  117. ),
  118. snuba.SnubaQueryParams(
  119. start=timezone.now() - timedelta(days=1),
  120. end=timezone.now(),
  121. selected_columns=["event_id", "group_id", "timestamp"],
  122. filter_keys={"project_id": [self.project.id], "group_id": [event_2.group.id]},
  123. tenant_ids={"referrer": "testing.test", "organization_id": 1},
  124. ),
  125. ],
  126. )
  127. assert [{(item["group_id"], item["event_id"]) for item in r["data"]} for r in results] == [
  128. {(event_1.group.id, event_1.event_id)},
  129. {(event_2.group.id, event_2.event_id)},
  130. ]
  131. @mock.patch("sentry.utils.snuba._bulk_snuba_query", side_effect=snuba._bulk_snuba_query)
  132. def test_cache(self, _bulk_snuba_query):
  133. one_min_ago = iso_format(before_now(minutes=1))
  134. event_1 = self.store_event(
  135. data={"fingerprint": ["group-1"], "message": "hello", "timestamp": one_min_ago},
  136. project_id=self.project.id,
  137. )
  138. event_2 = self.store_event(
  139. data={"fingerprint": ["group-2"], "message": "hello", "timestamp": one_min_ago},
  140. project_id=self.project.id,
  141. )
  142. params = [
  143. snuba.SnubaQueryParams(
  144. start=timezone.now() - timedelta(days=1),
  145. end=timezone.now(),
  146. selected_columns=["event_id", "group_id", "timestamp"],
  147. filter_keys={"project_id": [self.project.id], "group_id": [event_1.group.id]},
  148. tenant_ids={"referrer": "testing.test", "organization_id": 1},
  149. ),
  150. snuba.SnubaQueryParams(
  151. start=timezone.now() - timedelta(days=1),
  152. end=timezone.now(),
  153. selected_columns=["event_id", "group_id", "timestamp"],
  154. filter_keys={"project_id": [self.project.id], "group_id": [event_2.group.id]},
  155. tenant_ids={"referrer": "testing.test", "organization_id": 1},
  156. ),
  157. ]
  158. results = snuba.bulk_raw_query(
  159. copy.deepcopy(params),
  160. use_cache=True,
  161. )
  162. assert [{(item["group_id"], item["event_id"]) for item in r["data"]} for r in results] == [
  163. {(event_1.group.id, event_1.event_id)},
  164. {(event_2.group.id, event_2.event_id)},
  165. ]
  166. assert _bulk_snuba_query.call_count == 1
  167. _bulk_snuba_query.reset_mock()
  168. # # Make sure this doesn't appear in the cached results
  169. self.store_event(
  170. data={"fingerprint": ["group-2"], "message": "hello there", "timestamp": one_min_ago},
  171. project_id=self.project.id,
  172. )
  173. results = snuba.bulk_raw_query(
  174. copy.deepcopy(params),
  175. use_cache=True,
  176. )
  177. assert [{(item["group_id"], item["event_id"]) for item in r["data"]} for r in results] == [
  178. {(event_1.group.id, event_1.event_id)},
  179. {(event_2.group.id, event_2.event_id)},
  180. ]
  181. assert _bulk_snuba_query.call_count == 0