load-mocks 43 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078
  1. #!/usr/bin/env python
  2. import time
  3. from sentry.runner import configure
  4. from sentry.types.activity import ActivityType
  5. configure()
  6. import itertools
  7. import random
  8. from datetime import datetime, timedelta
  9. from hashlib import sha1
  10. from random import randint
  11. from uuid import uuid4
  12. from django.conf import settings
  13. from django.db import IntegrityError, transaction
  14. from django.db.models import F
  15. from django.utils import timezone
  16. from pytz import utc
  17. from sentry import buffer, roles, tsdb
  18. from sentry.event_manager import HashDiscarded
  19. from sentry.incidents.logic import create_alert_rule, create_alert_rule_trigger, create_incident
  20. from sentry.incidents.models import AlertRuleThresholdType, IncidentType
  21. from sentry.models import (
  22. TOMBSTONE_FIELDS_FROM_GROUP,
  23. Activity,
  24. Broadcast,
  25. CheckInStatus,
  26. Commit,
  27. CommitAuthor,
  28. CommitFileChange,
  29. Deploy,
  30. Environment,
  31. EventAttachment,
  32. File,
  33. Group,
  34. GroupRelease,
  35. GroupTombstone,
  36. Monitor,
  37. MonitorCheckIn,
  38. MonitorStatus,
  39. MonitorType,
  40. Organization,
  41. OrganizationAccessRequest,
  42. OrganizationMember,
  43. Project,
  44. Release,
  45. ReleaseCommit,
  46. ReleaseEnvironment,
  47. ReleaseFile,
  48. ReleaseProjectEnvironment,
  49. Repository,
  50. Team,
  51. User,
  52. UserReport,
  53. )
  54. from sentry.signals import mocks_loaded
  55. from sentry.similarity import features
  56. from sentry.utils import loremipsum
  57. from sentry.utils.hashlib import md5_text
  58. from sentry.utils.samples import create_sample_event as _create_sample_event
  59. from sentry.utils.samples import create_trace, generate_user, random_normal
  60. PLATFORMS = itertools.cycle(["ruby", "php", "python", "java", "javascript"])
  61. LEVELS = itertools.cycle(["error", "error", "error", "fatal", "warning"])
  62. ENVIRONMENTS = itertools.cycle(["production", "production", "staging", "alpha", "beta", ""])
  63. MONITOR_NAMES = itertools.cycle(settings.CELERYBEAT_SCHEDULE.keys())
  64. MONITOR_SCHEDULES = itertools.cycle(["* * * * *", "0 * * * *", "0 0 * * *"])
  65. LONG_MESSAGE = """Code: 0.
  66. DB::Exception: String is too long for DateTime: 2018-10-26T19:14:18+00:00. Stack trace:
  67. 0. clickhouse-server(StackTrace::StackTrace()+0x16) [0x99e9626]
  68. 1. clickhouse-server(DB::Exception::Exception(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int)+0x22) [0x3087172]
  69. 2. clickhouse-server(DB::FunctionComparison<DB::EqualsOp, DB::NameEquals>::executeDateOrDateTimeOrEnumOrUUIDWithConstString(DB::Block&, unsigned long, DB::IColumn const*, DB::IColumn const*, std::shared_ptr<DB::IDataType const> const&, std::shared_ptr<DB::IDataType const> const&, bool, unsigned long)+0x13c8) [0x3b233d8]
  70. 3. clickhouse-server(DB::FunctionComparison<DB::EqualsOp, DB::NameEquals>::executeImpl(DB::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long, unsigned long)+0x576) [0x3bafc86]
  71. 4. clickhouse-server(DB::PreparedFunctionImpl::defaultImplementationForNulls(DB::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long, unsigned long)+0x174) [0x7953cd4]
  72. 5. clickhouse-server(DB::PreparedFunctionImpl::executeWithoutLowCardinalityColumns(DB::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long, unsigned long)+0x54) [0x7953b04]
  73. 6. clickhouse-server(DB::PreparedFunctionImpl::execute(DB::Block&, std::vector<unsigned long, std::allocator<unsigned long> > const&, unsigned long, unsigned long)+0x3e2) [0x7954222]
  74. 7. clickhouse-server(DB::ExpressionAction::execute(DB::Block&, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, unsigned long, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, unsigned long> > >&) const+0x69b) [0x7b021fb]
  75. 8. clickhouse-server(DB::ExpressionActions::execute(DB::Block&) const+0xe6) [0x7b03676]
  76. 9. clickhouse-server(DB::FilterBlockInputStream::FilterBlockInputStream(std::shared_ptr<DB::IBlockInputStream> const&, std::shared_ptr<DB::ExpressionActions> const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x711) [0x79b7e31]
  77. 10. clickhouse-server() [0x75e9443]
  78. 11. clickhouse-server(DB::InterpreterSelectQuery::executeImpl(DB::InterpreterSelectQuery::Pipeline&, std::shared_ptr<DB::IBlockInputStream> const&, bool)+0x118f) [0x75f212f]
  79. 12. clickhouse-server(DB::InterpreterSelectQuery::InterpreterSelectQuery(std::shared_ptr<DB::IAST> const&, DB::Context const&, std::shared_ptr<DB::IBlockInputStream> const&, std::shared_ptr<DB::IStorage> const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, DB::QueryProcessingStage::Enum, unsigned long, bool)+0x5e6) [0x75f2d46]
  80. 13. clickhouse-server(DB::InterpreterSelectQuery::InterpreterSelectQuery(std::shared_ptr<DB::IAST> const&, DB::Context const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, DB::QueryProcessingStage::Enum, unsigned long, bool)+0x56) [0x75f3aa6]
  81. 14. clickhouse-server(DB::InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery(std::shared_ptr<DB::IAST> const&, DB::Context const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, DB::QueryProcessingStage::Enum, unsigned long, bool)+0x7e7) [0x75ffab7]
  82. 15. clickhouse-server(DB::InterpreterFactory::get(std::shared_ptr<DB::IAST>&, DB::Context&, DB::QueryProcessingStage::Enum)+0x3a8) [0x75dc138]
  83. 16. clickhouse-server() [0x768fad9]
  84. 17. clickhouse-server(DB::executeQuery(std::__cxx11::basic..."""
  85. def make_sentence(words=None):
  86. if words is None:
  87. words = int(random.weibullvariate(8, 3))
  88. return " ".join(random.choice(loremipsum.words) for _ in range(words))
  89. def create_sample_event(*args, **kwargs):
  90. try:
  91. event = _create_sample_event(*args, **kwargs)
  92. except HashDiscarded as e:
  93. print(f"> Skipping Event: {e.message}") # NOQA
  94. else:
  95. if event is not None:
  96. features.record([event])
  97. return event
  98. def generate_commits(user):
  99. commits = []
  100. for i in range(random.randint(1, 20)):
  101. if i == 1:
  102. filename = "raven/base.py"
  103. else:
  104. filename = random.choice(loremipsum.words) + ".js"
  105. if random.randint(0, 5) == 1:
  106. author = (user.name, user.email)
  107. else:
  108. author = (
  109. f"{random.choice(loremipsum.words)} {random.choice(loremipsum.words)}",
  110. f"{random.choice(loremipsum.words)}@example.com",
  111. )
  112. commits.append(
  113. {
  114. "key": sha1(uuid4().bytes).hexdigest(),
  115. "message": f"feat: Do something to {filename}\n{make_sentence()}",
  116. "author": author,
  117. "files": [(filename, "M")],
  118. }
  119. )
  120. return commits
  121. def generate_tombstones(project, user):
  122. # attempt to create a high enough previous_group_id
  123. # that it won't conflict with any group ids
  124. prev_group_id = 100000
  125. try:
  126. prev_group_id = (
  127. max(
  128. GroupTombstone.objects.order_by("-previous_group_id")[0].previous_group_id,
  129. prev_group_id,
  130. )
  131. + 1
  132. )
  133. except IndexError:
  134. pass
  135. for group in Group.objects.filter(project=project)[:5]:
  136. GroupTombstone.objects.create(
  137. previous_group_id=prev_group_id,
  138. actor_id=user.id,
  139. **{name: getattr(group, name) for name in TOMBSTONE_FIELDS_FROM_GROUP},
  140. )
  141. prev_group_id += 1
  142. def create_system_time_series():
  143. now = datetime.utcnow().replace(tzinfo=utc)
  144. for _ in range(60):
  145. count = randint(1, 10)
  146. tsdb.incr_multi(
  147. (
  148. (tsdb.models.internal, "client-api.all-versions.responses.2xx"),
  149. (tsdb.models.internal, "client-api.all-versions.requests"),
  150. ),
  151. now,
  152. int(count * 0.9),
  153. )
  154. tsdb.incr_multi(
  155. ((tsdb.models.internal, "client-api.all-versions.responses.4xx"),),
  156. now,
  157. int(count * 0.05),
  158. )
  159. tsdb.incr_multi(
  160. ((tsdb.models.internal, "client-api.all-versions.responses.5xx"),),
  161. now,
  162. int(count * 0.1),
  163. )
  164. now = now - timedelta(seconds=1)
  165. for _ in range(24 * 30):
  166. count = randint(100, 1000)
  167. tsdb.incr_multi(
  168. (
  169. (tsdb.models.internal, "client-api.all-versions.responses.2xx"),
  170. (tsdb.models.internal, "client-api.all-versions.requests"),
  171. ),
  172. now,
  173. int(count * 4.9),
  174. )
  175. tsdb.incr_multi(
  176. ((tsdb.models.internal, "client-api.all-versions.responses.4xx"),),
  177. now,
  178. int(count * 0.05),
  179. )
  180. tsdb.incr_multi(
  181. ((tsdb.models.internal, "client-api.all-versions.responses.5xx"),),
  182. now,
  183. int(count * 0.1),
  184. )
  185. now = now - timedelta(hours=1)
  186. def create_sample_time_series(event, release=None):
  187. if event is None:
  188. return
  189. group = event.group
  190. project = group.project
  191. key = project.key_set.all()[0]
  192. now = datetime.utcnow().replace(tzinfo=utc)
  193. environment = Environment.get_or_create(
  194. project=project, name=Environment.get_name_or_default(event.get_tag("environment"))
  195. )
  196. if release:
  197. ReleaseEnvironment.get_or_create(
  198. project=project, release=release, environment=environment, datetime=now
  199. )
  200. grouprelease = GroupRelease.get_or_create(
  201. group=group, release=release, environment=environment, datetime=now
  202. )
  203. for _ in range(60):
  204. count = randint(1, 10)
  205. tsdb.incr_multi(
  206. ((tsdb.models.project, project.id), (tsdb.models.group, group.id)),
  207. now,
  208. count,
  209. environment_id=environment.id,
  210. )
  211. tsdb.incr_multi(
  212. (
  213. (tsdb.models.organization_total_received, project.organization_id),
  214. (tsdb.models.project_total_received, project.id),
  215. (tsdb.models.key_total_received, key.id),
  216. ),
  217. now,
  218. int(count * 1.1),
  219. )
  220. tsdb.incr(
  221. tsdb.models.project_total_forwarded,
  222. project.id,
  223. now,
  224. int(count * 1.1),
  225. )
  226. tsdb.incr_multi(
  227. (
  228. (tsdb.models.organization_total_rejected, project.organization_id),
  229. (tsdb.models.project_total_rejected, project.id),
  230. (tsdb.models.key_total_rejected, key.id),
  231. ),
  232. now,
  233. int(count * 0.1),
  234. )
  235. frequencies = [
  236. (tsdb.models.frequent_issues_by_project, {project.id: {group.id: count}}),
  237. (tsdb.models.frequent_environments_by_group, {group.id: {environment.id: count}}),
  238. ]
  239. if release:
  240. frequencies.append(
  241. (tsdb.models.frequent_releases_by_group, {group.id: {grouprelease.id: count}})
  242. )
  243. tsdb.record_frequency_multi(frequencies, now)
  244. now = now - timedelta(seconds=1)
  245. for _ in range(24 * 30):
  246. count = randint(100, 1000)
  247. tsdb.incr_multi(
  248. ((tsdb.models.project, group.project.id), (tsdb.models.group, group.id)),
  249. now,
  250. count,
  251. environment_id=environment.id,
  252. )
  253. tsdb.incr_multi(
  254. (
  255. (tsdb.models.organization_total_received, project.organization_id),
  256. (tsdb.models.project_total_received, project.id),
  257. (tsdb.models.key_total_received, key.id),
  258. ),
  259. now,
  260. int(count * 1.1),
  261. )
  262. tsdb.incr_multi(
  263. (
  264. (tsdb.models.organization_total_rejected, project.organization_id),
  265. (tsdb.models.project_total_rejected, project.id),
  266. (tsdb.models.key_total_rejected, key.id),
  267. ),
  268. now,
  269. int(count * 0.1),
  270. )
  271. frequencies = [
  272. (tsdb.models.frequent_issues_by_project, {project.id: {group.id: count}}),
  273. (tsdb.models.frequent_environments_by_group, {group.id: {environment.id: count}}),
  274. ]
  275. if release:
  276. frequencies.append(
  277. (tsdb.models.frequent_releases_by_group, {group.id: {grouprelease.id: count}})
  278. )
  279. tsdb.record_frequency_multi(frequencies, now)
  280. now = now - timedelta(hours=1)
  281. def main(
  282. skip_default_setup=False,
  283. num_events=1,
  284. extra_events=False,
  285. load_trends=False,
  286. load_performance_issues=False,
  287. slow=False,
  288. ):
  289. try:
  290. user = User.objects.filter(is_superuser=True)[0]
  291. except IndexError:
  292. raise Exception("No superuser exists (run `make bootstrap`)")
  293. dummy_user, _ = User.objects.get_or_create(
  294. username="dummy@example.com", defaults={"email": "dummy@example.com"}
  295. )
  296. dummy_user.set_password("dummy")
  297. dummy_user.save()
  298. mocks = (
  299. ("Massive Dynamic", ("Ludic Science",)),
  300. ("Captain Planet", ("Earth", "Fire", "Wind", "Water", "Heart")),
  301. )
  302. project_map = {}
  303. Broadcast.objects.create(
  304. title="Learn about Source Maps",
  305. message="Source maps are JSON files that contain information on how to map your transpiled source code back to their original source.",
  306. link="https://docs.sentry.io/platforms/javascript/#source-maps",
  307. )
  308. if settings.SENTRY_SINGLE_ORGANIZATION:
  309. org = Organization.get_default()
  310. print(f"Mocking org {org.name}") # NOQA
  311. else:
  312. print("Mocking org {}".format("Default")) # NOQA
  313. org, _ = Organization.objects.get_or_create(slug="default")
  314. OrganizationMember.objects.get_or_create(
  315. user=user, organization=org, role=roles.get_top_dog().id
  316. )
  317. dummy_member, _ = OrganizationMember.objects.get_or_create(
  318. user=dummy_user, organization=org, defaults={"role": roles.get_default().id}
  319. )
  320. # Allow for 0 events, if you only want transactions
  321. event1 = event2 = event3 = event4 = event5 = None
  322. if skip_default_setup:
  323. # Quickly fetch/create the teams and projects
  324. for team_name, project_names in mocks:
  325. print(f"> Mocking team {team_name}") # NOQA
  326. team, _ = Team.objects.get_or_create(name=team_name, defaults={"organization": org})
  327. for project_name in project_names:
  328. print(f" > Mocking project {project_name}") # NOQA
  329. project, _ = Project.objects.get_or_create(
  330. name=project_name,
  331. defaults={
  332. "organization": org,
  333. "first_event": timezone.now(),
  334. "flags": Project.flags.has_releases,
  335. },
  336. )
  337. project_map[project_name] = project
  338. project.add_team(team)
  339. else:
  340. for team_name, project_names in mocks:
  341. print(f"> Mocking team {team_name}") # NOQA
  342. team, _ = Team.objects.get_or_create(name=team_name, defaults={"organization": org})
  343. for project_name in project_names:
  344. print(f" > Mocking project {project_name}") # NOQA
  345. project, _ = Project.objects.get_or_create(
  346. name=project_name,
  347. defaults={
  348. "organization": org,
  349. "first_event": timezone.now(),
  350. "flags": Project.flags.has_releases,
  351. },
  352. )
  353. project_map[project_name] = project
  354. project.add_team(team)
  355. if not project.first_event:
  356. project.update(first_event=project.date_added)
  357. if not project.flags.has_releases:
  358. project.update(flags=F("flags").bitor(Project.flags.has_releases))
  359. monitor, created = Monitor.objects.get_or_create(
  360. name=next(MONITOR_NAMES),
  361. project_id=project.id,
  362. organization_id=org.id,
  363. type=MonitorType.CRON_JOB,
  364. defaults={
  365. "config": {"schedule": next(MONITOR_SCHEDULES)},
  366. "next_checkin": timezone.now() + timedelta(minutes=60),
  367. "last_checkin": timezone.now(),
  368. },
  369. )
  370. if not created:
  371. if not (monitor.config or {}).get("schedule"):
  372. monitor.config = {"schedule": next(MONITOR_SCHEDULES)}
  373. monitor.update(
  374. config=monitor.config,
  375. status=MonitorStatus.OK if randint(0, 10) < 7 else MonitorStatus.ERROR,
  376. last_checkin=timezone.now(),
  377. next_checkin=monitor.get_next_scheduled_checkin(timezone.now()),
  378. )
  379. MonitorCheckIn.objects.create(
  380. project_id=monitor.project_id,
  381. monitor=monitor,
  382. status=CheckInStatus.OK
  383. if monitor.status == MonitorStatus.OK
  384. else CheckInStatus.ERROR,
  385. )
  386. with transaction.atomic():
  387. has_release = Release.objects.filter(
  388. version=sha1(uuid4().bytes).hexdigest(),
  389. organization_id=project.organization_id,
  390. projects=project,
  391. ).exists()
  392. if not has_release:
  393. release = Release.objects.filter(
  394. version=sha1(uuid4().bytes).hexdigest(),
  395. organization_id=project.organization_id,
  396. ).first()
  397. if not release:
  398. release = Release.objects.create(
  399. version=sha1(uuid4().bytes).hexdigest(),
  400. organization_id=project.organization_id,
  401. )
  402. release.add_project(project)
  403. generate_tombstones(project, user)
  404. raw_commits = generate_commits(user)
  405. try:
  406. with transaction.atomic():
  407. repo, _ = Repository.objects.get_or_create(
  408. organization_id=org.id,
  409. provider="integrations:github",
  410. external_id="example/example",
  411. defaults={
  412. "name": "Example Repo",
  413. "url": "https://github.com/example/example",
  414. },
  415. )
  416. except IntegrityError:
  417. # for users with legacy github plugin
  418. # upgrade to the new integration
  419. repo = Repository.objects.get(
  420. organization_id=org.id,
  421. provider="github",
  422. external_id="example/example",
  423. name="Example Repo",
  424. )
  425. repo.provider = "integrations:github"
  426. repo.save()
  427. authors = set()
  428. for commit_index, raw_commit in enumerate(raw_commits):
  429. author = CommitAuthor.objects.get_or_create(
  430. organization_id=org.id,
  431. email=raw_commit["author"][1],
  432. defaults={"name": raw_commit["author"][0]},
  433. )[0]
  434. commit = Commit.objects.get_or_create(
  435. organization_id=org.id,
  436. repository_id=repo.id,
  437. key=raw_commit["key"],
  438. defaults={"author": author, "message": raw_commit["message"]},
  439. )[0]
  440. authors.add(author)
  441. for file in raw_commit["files"]:
  442. ReleaseFile.objects.get_or_create(
  443. organization_id=project.organization_id,
  444. release_id=release.id,
  445. name=file[0],
  446. file=File.objects.get_or_create(
  447. name=file[0], type="release.file", checksum="abcde" * 8, size=13043
  448. )[0],
  449. defaults={"organization_id": project.organization_id},
  450. )
  451. CommitFileChange.objects.get_or_create(
  452. organization_id=org.id, commit=commit, filename=file[0], type=file[1]
  453. )
  454. ReleaseCommit.objects.get_or_create(
  455. organization_id=org.id, release=release, commit=commit, order=commit_index
  456. )
  457. # create an unreleased commit
  458. Commit.objects.get_or_create(
  459. organization_id=org.id,
  460. repository_id=repo.id,
  461. key=sha1(uuid4().bytes).hexdigest(),
  462. defaults={
  463. "author": CommitAuthor.objects.get_or_create(
  464. organization_id=org.id, email=user.email, defaults={"name": user.name}
  465. )[0],
  466. "message": "feat: Do something to {}\n{}".format(
  467. random.choice(loremipsum.words) + ".js", make_sentence()
  468. ),
  469. },
  470. )[0]
  471. Activity.objects.create(
  472. type=ActivityType.RELEASE.value,
  473. project=project,
  474. ident=release.version,
  475. user=user,
  476. data={"version": release.version},
  477. )
  478. environment = Environment.get_or_create(project=project, name=next(ENVIRONMENTS))
  479. deploy = Deploy.objects.create(
  480. organization_id=project.organization_id,
  481. release=release,
  482. environment_id=environment.id,
  483. )
  484. release.update(
  485. commit_count=len(raw_commits),
  486. last_commit_id=commit.id,
  487. total_deploys=Deploy.objects.filter(release=release).count(),
  488. last_deploy_id=deploy.id,
  489. authors=[str(a.id) for a in authors],
  490. )
  491. ReleaseProjectEnvironment.objects.create_or_update(
  492. project=project,
  493. environment=environment,
  494. release=release,
  495. defaults={"last_deploy_id": deploy.id},
  496. )
  497. Activity.objects.create(
  498. type=ActivityType.DEPLOY.value,
  499. project=project,
  500. ident=release.version,
  501. data={
  502. "version": release.version,
  503. "deploy_id": deploy.id,
  504. "environment": environment.name,
  505. },
  506. datetime=deploy.date_finished,
  507. )
  508. # Add a bunch of additional dummy events to support pagination
  509. if extra_events:
  510. for _ in range(45):
  511. platform = next(PLATFORMS)
  512. create_sample_event(
  513. project=project,
  514. platform=platform,
  515. release=release.version,
  516. level=next(LEVELS),
  517. environment=next(ENVIRONMENTS),
  518. message="This is a mostly useless example %s exception" % platform,
  519. checksum=md5_text(platform + str(_)).hexdigest(),
  520. user=generate_user(),
  521. )
  522. for _ in range(num_events):
  523. event1 = create_sample_event(
  524. project=project,
  525. platform="python",
  526. release=release.version,
  527. environment=next(ENVIRONMENTS),
  528. user=generate_user(),
  529. )
  530. EventAttachment.objects.create(
  531. project_id=project.id,
  532. event_id=event1.event_id,
  533. name="example-logfile.txt",
  534. file_id=File.objects.get_or_create(
  535. name="example-logfile.txt",
  536. type="text/plain",
  537. checksum="abcde" * 8,
  538. size=13043,
  539. )[0].id,
  540. )
  541. event2 = create_sample_event(
  542. project=project,
  543. platform="javascript",
  544. release=release.version,
  545. environment=next(ENVIRONMENTS),
  546. sdk={"name": "raven-js", "version": "2.1.0"},
  547. user=generate_user(),
  548. )
  549. event3 = create_sample_event(project, "java")
  550. event4 = create_sample_event(
  551. project=project,
  552. platform="ruby",
  553. release=release.version,
  554. environment=next(ENVIRONMENTS),
  555. user=generate_user(),
  556. )
  557. event5 = create_sample_event(
  558. project=project,
  559. platform="cocoa",
  560. release=release.version,
  561. environment=next(ENVIRONMENTS),
  562. user=generate_user(),
  563. )
  564. create_sample_event(
  565. project=project,
  566. platform="php",
  567. release=release.version,
  568. environment=next(ENVIRONMENTS),
  569. message=LONG_MESSAGE,
  570. user=generate_user(),
  571. )
  572. create_sample_event(
  573. project=project,
  574. platform="cocoa",
  575. sample_name="react-native",
  576. release=release.version,
  577. environment=next(ENVIRONMENTS),
  578. user=generate_user(),
  579. )
  580. create_sample_event(
  581. project=project,
  582. platform="pii",
  583. release=release.version,
  584. environment=next(ENVIRONMENTS),
  585. user=generate_user(),
  586. )
  587. if event5:
  588. Commit.objects.get_or_create(
  589. organization_id=org.id,
  590. repository_id=repo.id,
  591. key=sha1(uuid4().bytes).hexdigest(),
  592. defaults={
  593. "author": CommitAuthor.objects.get_or_create(
  594. organization_id=org.id,
  595. email=user.email,
  596. defaults={"name": user.name},
  597. )[0],
  598. "message": f"Ooops!\nFixes {event5.group.qualified_short_id}",
  599. },
  600. )[0]
  601. create_sample_event(project=project, environment=next(ENVIRONMENTS), platform="csp")
  602. if event3:
  603. UserReport.objects.create(
  604. project_id=project.id,
  605. event_id=event3.event_id,
  606. group_id=event3.group.id,
  607. name="Jane Bloggs",
  608. email="jane@example.com",
  609. comments=make_sentence(),
  610. )
  611. # Metric alerts
  612. alert_rule = create_alert_rule(
  613. org,
  614. [project],
  615. "My Alert Rule",
  616. "level:error",
  617. "count()",
  618. 10,
  619. AlertRuleThresholdType.ABOVE,
  620. 1,
  621. )
  622. create_alert_rule_trigger(alert_rule, "critical", 10)
  623. create_incident(
  624. org,
  625. type_=IncidentType.DETECTED,
  626. title="My Incident",
  627. date_started=datetime.utcnow().replace(tzinfo=utc),
  628. alert_rule=alert_rule,
  629. projects=[project],
  630. )
  631. print(f" > Loading time series data") # NOQA
  632. if event1:
  633. create_sample_time_series(event1, release=release)
  634. if event2:
  635. create_sample_time_series(event2, release=release)
  636. if event3:
  637. create_sample_time_series(event3)
  638. if event4:
  639. create_sample_time_series(event4, release=release)
  640. if event5:
  641. create_sample_time_series(event5, release=release)
  642. if hasattr(buffer, "process_pending"):
  643. print(" > Processing pending buffers") # NOQA
  644. buffer.process_pending()
  645. mocks_loaded.send(project=project, sender=__name__)
  646. OrganizationAccessRequest.objects.create_or_update(member=dummy_member, team=team)
  647. create_mock_transactions(project_map, load_trends, load_performance_issues, slow)
  648. Activity.objects.create(
  649. type=ActivityType.RELEASE.value,
  650. project=project,
  651. ident="4f38b65c62c4565aa94bba391ff8946922a8eed4",
  652. user=user,
  653. data={"version": "4f38b65c62c4565aa94bba391ff8946922a8eed4"},
  654. )
  655. create_system_time_series()
  656. def create_mock_transactions(
  657. project_map, load_trends=False, load_performance_issues=False, slow=False
  658. ):
  659. backend_project = project_map["Earth"]
  660. frontend_project = project_map["Fire"]
  661. service_projects = [
  662. project_map["Wind"],
  663. project_map["Water"],
  664. project_map["Heart"],
  665. ]
  666. for project in project_map.values():
  667. if not project.flags.has_transactions:
  668. project.update(flags=F("flags").bitor(Project.flags.has_transactions))
  669. timestamp = timezone.now()
  670. print(f" > Loading a trace") # NOQA
  671. create_trace(
  672. slow,
  673. timestamp - timedelta(milliseconds=random_normal(4000, 250, 1000)),
  674. timestamp,
  675. generate_user(),
  676. uuid4().hex,
  677. None,
  678. {
  679. "project": frontend_project,
  680. "transaction": "/plants/:plantId/",
  681. "frontend": True,
  682. "errors": 1,
  683. "children": [
  684. {
  685. "project": backend_project,
  686. "transaction": "/api/plants/",
  687. "children": [
  688. {
  689. "project": service_projects[0],
  690. "transaction": "/products/all/",
  691. "children": [],
  692. },
  693. {
  694. "project": service_projects[1],
  695. "transaction": "/analytics/",
  696. "children": [],
  697. },
  698. {
  699. "project": service_projects[2],
  700. "transaction": "tasks.create_invoice",
  701. "children": [
  702. {
  703. "project": service_projects[2],
  704. "transaction": "tasks.process_invoice",
  705. "children": [
  706. {
  707. "project": service_projects[2],
  708. "transaction": "tasks.process_invoice",
  709. "children": [
  710. {
  711. "project": service_projects[2],
  712. "transaction": "tasks.process_invoice",
  713. "children": [
  714. {
  715. "project": service_projects[2],
  716. "transaction": "tasks.process_invoice",
  717. "children": [],
  718. },
  719. ],
  720. },
  721. ],
  722. },
  723. ],
  724. },
  725. ],
  726. },
  727. ],
  728. },
  729. ],
  730. },
  731. )
  732. if load_trends:
  733. print(f" > Loading trends data") # NOQA
  734. for day in range(14):
  735. for hour in range(24):
  736. timestamp = timezone.now() - timedelta(days=day, hours=hour)
  737. transaction_user = generate_user()
  738. trace_id = uuid4().hex
  739. frontend_span_id = uuid4().hex[:16]
  740. frontend_root_span_id = uuid4().hex[:16]
  741. frontend_duration = random_normal(2000 - 50 * day, 250, 1000)
  742. create_sample_event(
  743. project=frontend_project,
  744. platform="javascript-transaction",
  745. transaction="/trends/:frontend/",
  746. event_id=uuid4().hex,
  747. user=transaction_user,
  748. timestamp=timestamp,
  749. # start_timestamp decreases based on day so that there's a trend
  750. start_timestamp=timestamp - timedelta(milliseconds=frontend_duration),
  751. measurements={
  752. "fp": {"value": random_normal(1250 - 50 * day, 200, 500)},
  753. "fcp": {"value": random_normal(1250 - 50 * day, 200, 500)},
  754. "lcp": {"value": random_normal(2800 - 50 * day, 400, 2000)},
  755. "fid": {"value": random_normal(5 - 0.125 * day, 2, 1)},
  756. },
  757. # Root
  758. parent_span_id=None,
  759. span_id=frontend_root_span_id,
  760. trace=trace_id,
  761. spans=[
  762. {
  763. "same_process_as_parent": True,
  764. "op": "http",
  765. "description": "GET /api/plants/?all_plants=1",
  766. "data": {
  767. "duration": random_normal(
  768. 1 - 0.05 * day, 0.25, 0.01, frontend_duration / 1000
  769. ),
  770. "offset": 0.02,
  771. },
  772. "span_id": frontend_span_id,
  773. "trace_id": trace_id,
  774. }
  775. ],
  776. )
  777. # try to give clickhouse some breathing room
  778. if slow:
  779. time.sleep(0.05)
  780. backend_duration = random_normal(1500 + 50 * day, 250, 500)
  781. create_sample_event(
  782. project=backend_project,
  783. platform="transaction",
  784. transaction="/trends/backend/",
  785. event_id=uuid4().hex,
  786. user=transaction_user,
  787. timestamp=timestamp,
  788. start_timestamp=timestamp - timedelta(milliseconds=backend_duration),
  789. # match the trace from the javascript transaction
  790. trace=trace_id,
  791. parent_span_id=frontend_root_span_id,
  792. spans=[],
  793. )
  794. # try to give clickhouse some breathing room
  795. if slow:
  796. time.sleep(0.05)
  797. if load_performance_issues:
  798. print(f" > Loading performance issues data") # NOQA
  799. trace_id = uuid4().hex
  800. transaction_user = generate_user()
  801. frontend_root_span_id = uuid4().hex[:16]
  802. n_plus_one_db_current_offset = timestamp
  803. n_plus_one_db_duration = timedelta(milliseconds=100)
  804. parent_span_id = uuid4().hex[:16]
  805. source_span = {
  806. "timestamp": (timestamp + n_plus_one_db_duration).timestamp(),
  807. "start_timestamp": (timestamp + timedelta(milliseconds=10)).timestamp(),
  808. "description": "SELECT `books_book`.`id`, `books_book`.`title`, `books_book`.`author_id` FROM `books_book` ORDER BY `books_book`.`id` DESC LIMIT 10",
  809. "op": "db",
  810. "parent_span_id": parent_span_id,
  811. "span_id": uuid4().hex[:16],
  812. "hash": "858fea692d4d93e8",
  813. }
  814. def make_repeating_span(duration):
  815. nonlocal timestamp
  816. nonlocal n_plus_one_db_current_offset
  817. nonlocal n_plus_one_db_duration
  818. n_plus_one_db_duration += timedelta(milliseconds=duration) + timedelta(milliseconds=1)
  819. n_plus_one_db_current_offset = timestamp + n_plus_one_db_duration
  820. return {
  821. "timestamp": (
  822. n_plus_one_db_current_offset + timedelta(milliseconds=duration)
  823. ).timestamp(),
  824. "start_timestamp": (
  825. n_plus_one_db_current_offset + timedelta(milliseconds=1)
  826. ).timestamp(),
  827. "description": "SELECT `books_author`.`id`, `books_author`.`name` FROM `books_author` WHERE `books_author`.`id` = %s LIMIT 21",
  828. "op": "db",
  829. "span_id": uuid4().hex[:16],
  830. "parent_span_id": parent_span_id,
  831. "hash": "63f1e89e6a073441",
  832. }
  833. repeating_spans = [make_repeating_span(200) for _ in range(10)]
  834. parent_span = {
  835. "timestamp": (
  836. timestamp + n_plus_one_db_duration + timedelta(milliseconds=200)
  837. ).timestamp(),
  838. "start_timestamp": timestamp.timestamp(),
  839. "description": "new",
  840. "op": "django.view",
  841. "parent_span_id": uuid4().hex[:16],
  842. "span_id": parent_span_id,
  843. "hash": "0f43fb6f6e01ca52",
  844. }
  845. create_sample_event(
  846. project=backend_project,
  847. platform="transaction",
  848. transaction="/n_plus_one_db/backend/",
  849. event_id=uuid4().hex,
  850. user=transaction_user,
  851. timestamp=timestamp + n_plus_one_db_duration + timedelta(milliseconds=300),
  852. start_timestamp=timestamp,
  853. trace=trace_id,
  854. parent_span_id=frontend_root_span_id,
  855. spans=[
  856. parent_span,
  857. source_span,
  858. ]
  859. + repeating_spans,
  860. )
  861. time.sleep(1.0)
  862. create_sample_event(
  863. project=backend_project,
  864. platform="transaction",
  865. transaction="/file-io-main-thread/",
  866. event_id=uuid4().hex,
  867. user=transaction_user,
  868. timestamp=timestamp + timedelta(milliseconds=300),
  869. start_timestamp=timestamp,
  870. trace=trace_id,
  871. parent_span_id=frontend_root_span_id,
  872. spans=[
  873. parent_span,
  874. {
  875. "timestamp": (timestamp + timedelta(milliseconds=200)).timestamp(),
  876. "start_timestamp": timestamp.timestamp(),
  877. "description": "1669031858711_file.txt (4.0 kB)",
  878. "op": "file.write",
  879. "span_id": uuid4().hex[:16],
  880. "parent_span_id": parent_span_id,
  881. "status": "ok",
  882. "data": {
  883. "blocked_ui_thread": True,
  884. "call_stack": [
  885. {
  886. "function": "onClick",
  887. "in_app": True,
  888. "lineno": 2,
  889. "module": "io.sentry.samples.android.MainActivity$$ExternalSyntheticLambda6",
  890. "native": False,
  891. },
  892. {
  893. "filename": "MainActivity.java",
  894. "function": "lambda$onCreate$5$io-sentry-samples-android-MainActivity",
  895. "in_app": True,
  896. "lineno": 93,
  897. "module": "io.sentry.samples.android.MainActivity",
  898. "native": False,
  899. },
  900. ],
  901. "file.path": "/data/user/0/io.sentry.samples.android/files/1669031858711_file.txt",
  902. "file.size": 4010,
  903. },
  904. },
  905. ],
  906. )
  907. if __name__ == "__main__":
  908. settings.CELERY_ALWAYS_EAGER = True
  909. from optparse import OptionParser
  910. parser = OptionParser()
  911. parser.add_option("--events", default=1, type=int, help="number of events to generate")
  912. parser.add_option(
  913. "--skip-default-setup",
  914. default=False,
  915. action="store_true",
  916. help="Skips creating the default project, teams and timeseries, useful when only loading specific transactions",
  917. )
  918. parser.add_option(
  919. "--extra-events",
  920. default=False,
  921. action="store_true",
  922. help="add multiple events for each error group",
  923. )
  924. parser.add_option(
  925. "--load-trends",
  926. default=False,
  927. action="store_true",
  928. help="load multiple transactions for each id to show trends",
  929. )
  930. parser.add_option(
  931. "--load-performance-issues",
  932. default=False,
  933. action="store_true",
  934. help="load transactions with performance issues, still needs options/flags on for issues to appear.",
  935. )
  936. parser.add_option(
  937. "--slow",
  938. default=False,
  939. action="store_true",
  940. help="sleep between each transaction to let clickhouse rest",
  941. )
  942. (options, args) = parser.parse_args()
  943. try:
  944. main(
  945. skip_default_setup=options.skip_default_setup,
  946. num_events=options.events,
  947. extra_events=options.extra_events,
  948. load_trends=options.load_trends,
  949. load_performance_issues=options.load_performance_issues,
  950. slow=options.slow,
  951. )
  952. except Exception:
  953. # Avoid reporting any issues recursively back into Sentry
  954. import sys
  955. import traceback
  956. traceback.print_exc()
  957. sys.exit(1)