serializers.py 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553
  1. import uuid
  2. from typing import Dict, List, Tuple, Union
  3. from urllib.parse import urlparse
  4. from anonymizeip import anonymize_ip
  5. from django.db import transaction
  6. from django.db.models.expressions import OuterRef, RawSQL
  7. from django.db.utils import IntegrityError
  8. from ipware import get_client_ip
  9. from rest_framework import serializers
  10. from rest_framework.exceptions import PermissionDenied
  11. from environments.models import Environment
  12. from glitchtip.serializers import FlexibleDateTimeField
  13. from issues.models import EventType, Issue
  14. from issues.serializers import BaseBreadcrumbsSerializer
  15. from issues.tasks import update_search_index_issue
  16. from releases.models import Release
  17. from sentry.eventtypes.base import DefaultEvent
  18. from sentry.eventtypes.error import ErrorEvent
  19. from .event_context_processors import EVENT_CONTEXT_PROCESSORS
  20. from .event_processors import EVENT_PROCESSORS
  21. from .event_tag_processors import TAG_PROCESSORS
  22. from .fields import (
  23. ForgivingDisallowRegexField,
  24. ForgivingHStoreField,
  25. GenericField,
  26. QueryStringField,
  27. )
  28. from .models import Event, LogLevel
  29. def replace(data: Union[str, dict, list], match: str, repl: str):
  30. """A recursive replace function"""
  31. if isinstance(data, dict):
  32. return {k: replace(v, match, repl) for k, v in data.items()}
  33. elif isinstance(data, list):
  34. return [replace(i, match, repl) for i in data]
  35. elif isinstance(data, str):
  36. return data.replace(match, repl)
  37. return data
  38. def sanitize_bad_postgres_chars(data: Union[str, dict, list]):
  39. """
  40. Remove values which are not supported by the postgres string data types
  41. """
  42. known_bads = ["\x00"]
  43. for known_bad in known_bads:
  44. data = data.replace(known_bad, " ")
  45. return data
  46. def sanitize_bad_postgres_json(data: Union[str, dict, list]):
  47. """
  48. Remove values which are not supported by the postgres JSONB data type
  49. """
  50. known_bads = ["\u0000"]
  51. for known_bad in known_bads:
  52. data = replace(data, known_bad, " ")
  53. return data
  54. class RequestSerializer(serializers.Serializer):
  55. env = serializers.DictField(
  56. child=serializers.CharField(allow_blank=True, allow_null=True), required=False
  57. )
  58. # Dict values can be both str and List[str]
  59. headers = serializers.DictField(required=False)
  60. url = serializers.CharField(required=False, allow_blank=True)
  61. method = serializers.CharField(required=False, allow_blank=True)
  62. query_string = QueryStringField(required=False, allow_null=True)
  63. class BreadcrumbsSerializer(BaseBreadcrumbsSerializer):
  64. timestamp = GenericField(required=False)
  65. def validate_level(self, value):
  66. if value == "log":
  67. return "info"
  68. return value
  69. class BaseSerializer(serializers.Serializer):
  70. def process_user(self, project, data):
  71. """Fetch user data from SDK event and request"""
  72. user = data.get("user", {})
  73. if self.context and self.context.get("request"):
  74. client_ip, is_routable = get_client_ip(self.context["request"])
  75. if user or is_routable:
  76. if is_routable:
  77. if project.should_scrub_ip_addresses:
  78. client_ip = anonymize_ip(client_ip)
  79. user["ip_address"] = client_ip
  80. return user
  81. class SentrySDKEventSerializer(BaseSerializer):
  82. """Represents events coming from a OSS sentry SDK client"""
  83. breadcrumbs = serializers.JSONField(required=False)
  84. tags = ForgivingHStoreField(required=False)
  85. event_id = serializers.UUIDField(required=False, default=uuid.uuid4)
  86. extra = serializers.JSONField(required=False)
  87. request = RequestSerializer(required=False)
  88. server_name = serializers.CharField(required=False)
  89. sdk = serializers.JSONField(required=False)
  90. platform = serializers.CharField(required=False)
  91. release = serializers.CharField(required=False, allow_null=True, allow_blank=True)
  92. environment = ForgivingDisallowRegexField(
  93. required=False, allow_null=True, disallow_regex=r"^[^\n\r\f\/]*$"
  94. )
  95. _meta = serializers.JSONField(required=False)
  96. def set_environment(self, name: str, project) -> str:
  97. if not project.environment_id and name:
  98. environment, _ = Environment.objects.get_or_create(
  99. name=name[: Environment._meta.get_field("name").max_length],
  100. organization=project.organization,
  101. )
  102. environment.projects.add(project)
  103. project.environment_id = environment.id
  104. return environment.name
  105. return name
  106. def set_release(self, version: str, project) -> str:
  107. """
  108. Set project.release_id if not already so
  109. Create needed Release if necessary
  110. """
  111. if not project.release_id and version:
  112. release, _ = Release.objects.get_or_create(
  113. version=version, organization=project.organization
  114. )
  115. release.projects.add(project)
  116. project.release_id = release.id
  117. return release.version
  118. return version
  119. class FormattedMessageSerializer(serializers.Serializer):
  120. formatted = serializers.CharField(
  121. required=False
  122. ) # Documented as required, but some Sentry SDKs don't send it
  123. message = serializers.CharField(required=False)
  124. params = serializers.JSONField(required=False)
  125. def validate(self, attrs):
  126. data = super().validate(attrs)
  127. if not data.get("formatted") and data.get("params"):
  128. params = data["params"]
  129. if isinstance(params, list):
  130. data["formatted"] = data["message"] % tuple(params)
  131. elif isinstance(params, dict):
  132. data["formatted"] = data["message"].format(**params)
  133. return data
  134. # OSS Sentry only keeps unformatted "message" when it creates a formatted message
  135. return {key: data[key] for key in data if key != "message"}
  136. class MessageField(serializers.CharField):
  137. def to_internal_value(self, data):
  138. if isinstance(data, dict):
  139. serializer = FormattedMessageSerializer(data=data)
  140. serializer.is_valid(raise_exception=True)
  141. return serializer.validated_data
  142. return super().to_internal_value(data)
  143. class LogEntrySerializer(serializers.Serializer):
  144. formatted = serializers.CharField(required=False)
  145. message = serializers.CharField(required=False)
  146. params = serializers.JSONField(required=False)
  147. def validate(self, attrs):
  148. data = super().validate(attrs)
  149. if not data.get("formatted") and data.get("params"):
  150. params = data["params"]
  151. if isinstance(params, list):
  152. data["formatted"] = data["message"] % tuple(data["params"])
  153. elif isinstance(params, dict):
  154. data["formatted"] = data["message"].format(**params)
  155. return data
  156. class StoreDefaultSerializer(SentrySDKEventSerializer):
  157. """
  158. Default serializer. Used as both a base class and for default error types
  159. """
  160. type = EventType.DEFAULT
  161. contexts = serializers.JSONField(required=False)
  162. level = serializers.CharField(required=False)
  163. logentry = LogEntrySerializer(required=False)
  164. message = MessageField(required=False, allow_blank=True, allow_null=True)
  165. timestamp = FlexibleDateTimeField(required=False)
  166. transaction = serializers.CharField(
  167. required=False, allow_null=True, allow_blank=True
  168. )
  169. user = serializers.JSONField(required=False)
  170. modules = serializers.JSONField(required=False)
  171. def validate_breadcrumbs(self, value):
  172. """
  173. Normalize breadcrumbs, which may come in as dict or list
  174. """
  175. if isinstance(value, list):
  176. value = {"values": value}
  177. if value.get("values") == []:
  178. return None
  179. serializer = BreadcrumbsSerializer(data=value.get("values"), many=True)
  180. if serializer.is_valid():
  181. return {"values": serializer.validated_data}
  182. return value
  183. def get_eventtype(self):
  184. """Get event type class from self.type"""
  185. if self.type is EventType.DEFAULT:
  186. return DefaultEvent()
  187. if self.type is EventType.ERROR:
  188. return ErrorEvent()
  189. def modify_exception(self, exception):
  190. """OSS Sentry does this, I have no idea why"""
  191. if exception:
  192. for value in exception.get("values", []):
  193. value.pop("module", None)
  194. if value.get("stacktrace") and value["stacktrace"].get("frames"):
  195. frames = value["stacktrace"]["frames"]
  196. # If in_app is always true, make it false ¯\_(ツ)_/¯
  197. if all(x.get("in_app") for x in frames):
  198. for frame in frames:
  199. frame["in_app"] = False
  200. return exception
  201. def generate_tags(self, data: Dict, tags: List[Tuple[str, str]] = None):
  202. """
  203. Determine tag relational data
  204. Optionally pass tags array for existing known tags to generate
  205. """
  206. if tags is None:
  207. tags = []
  208. for Processor in TAG_PROCESSORS:
  209. processor = Processor()
  210. value = processor.get_tag_values(data)
  211. if value:
  212. tags.append((processor.tag, value))
  213. if data.get("tags"):
  214. tags += [(k, v) for k, v in data["tags"].items()]
  215. return tags
  216. def annotate_contexts(self, event):
  217. """
  218. SDK events may contain contexts. This function adds additional contexts data
  219. """
  220. contexts = event.get("contexts")
  221. for Processor in EVENT_CONTEXT_PROCESSORS:
  222. processor = Processor()
  223. if contexts is None or not contexts.get(processor.name):
  224. processor_contexts = processor.get_context(event)
  225. if processor_contexts:
  226. if contexts is None:
  227. contexts = {}
  228. contexts[processor.name] = processor_contexts
  229. return contexts
  230. def get_message(self, data):
  231. """Prefer message over logentry"""
  232. if "message" in data:
  233. if isinstance(data["message"], dict):
  234. return data["message"].get("formatted") or data["message"].get(
  235. "message", ""
  236. )
  237. return data["message"]
  238. return data.get("logentry", {}).get("message", "")
  239. def get_logentry(self, data):
  240. if "logentry" in data:
  241. return data.get("logentry")
  242. elif "message" in data:
  243. message = data["message"]
  244. if isinstance(message, dict):
  245. return message
  246. return {"formatted": message}
  247. def is_url(self, filename: str) -> bool:
  248. return filename.startswith(("file:", "http:", "https:", "applewebdata:"))
  249. def normalize_stacktrace(self, stacktrace):
  250. """
  251. Port of semaphore store/normalize/stacktrace.rs
  252. """
  253. if not stacktrace:
  254. return
  255. for frame in stacktrace.get("frames", []):
  256. if not frame.get("abs_path") and frame.get("filename"):
  257. frame["abs_path"] = frame["filename"]
  258. if frame.get("filename") and self.is_url(frame["filename"]):
  259. frame["filename"] = urlparse(frame["filename"]).path
  260. def create(self, validated_data):
  261. data = validated_data
  262. project = self.context.get("project")
  263. eventtype = self.get_eventtype()
  264. metadata = eventtype.get_metadata(data)
  265. exception = data.get("exception")
  266. if (
  267. data.get("stacktrace")
  268. and exception
  269. and len(exception.get("values", 0)) > 0
  270. and not exception["values"][0].get("stacktrace")
  271. ):
  272. # stacktrace is deprecated, but supported at this time
  273. # Assume it's for the first exception value
  274. exception["values"][0]["stacktrace"] = data.get("stacktrace")
  275. exception = self.modify_exception(exception)
  276. if isinstance(exception, dict):
  277. for value in exception.get("values", []):
  278. self.normalize_stacktrace(value.get("stacktrace"))
  279. tags = []
  280. release = self.set_release(data.get("release"), project)
  281. if project.release_id:
  282. tags.append(("release", release))
  283. environment = self.set_environment(data.get("environment"), project)
  284. if project.environment_id:
  285. tags.append(("environment", environment))
  286. if data.get("server_name"):
  287. tags.append(("server_name", data.get("server_name")))
  288. for Processor in EVENT_PROCESSORS:
  289. Processor(project, project.release_id, data).run()
  290. title = eventtype.get_title(metadata)
  291. culprit = eventtype.get_location(data)
  292. request = data.get("request")
  293. breadcrumbs = data.get("breadcrumbs")
  294. level = None
  295. if data.get("level"):
  296. level = LogLevel.from_string(data["level"])
  297. if request:
  298. headers = request.get("headers")
  299. if headers:
  300. request["inferred_content_type"] = headers.get("Content-Type")
  301. sorted_headers = sorted([pair for pair in headers.items()])
  302. for idx, header in enumerate(sorted_headers):
  303. if isinstance(header[1], list):
  304. sorted_headers[idx] = (header[0], header[1][0])
  305. request["headers"] = sorted_headers
  306. contexts = self.annotate_contexts(data)
  307. data["contexts"] = contexts
  308. with transaction.atomic():
  309. if not project.first_event:
  310. project.first_event = data.get("timestamp")
  311. project.save(update_fields=["first_event"])
  312. defaults = {
  313. "metadata": sanitize_bad_postgres_json(metadata),
  314. }
  315. if level:
  316. defaults["level"] = level
  317. tags = self.generate_tags(data, tags)
  318. defaults["tags"] = {tag[0]: [tag[1]] for tag in tags}
  319. issue, issue_created = Issue.objects.get_or_create(
  320. title=sanitize_bad_postgres_chars(title),
  321. culprit=sanitize_bad_postgres_chars(culprit),
  322. project_id=project.id,
  323. type=self.type,
  324. defaults=defaults,
  325. )
  326. json_data = {
  327. "breadcrumbs": breadcrumbs,
  328. "contexts": contexts,
  329. "culprit": culprit,
  330. "exception": exception,
  331. "logentry": self.get_logentry(data),
  332. "metadata": metadata,
  333. "message": self.get_message(data),
  334. "modules": data.get("modules"),
  335. "platform": data.get("platform", "other"),
  336. "request": request,
  337. "sdk": data.get("sdk"),
  338. "title": title,
  339. "type": self.type.label,
  340. }
  341. if project.environment_id:
  342. json_data["environment"] = data.get("environment")
  343. if data.get("logentry"):
  344. json_data["logentry"] = data.get("logentry")
  345. extra = data.get("extra")
  346. if extra:
  347. json_data["extra"] = extra
  348. user = self.process_user(project, data)
  349. if user:
  350. json_data["user"] = user
  351. errors = None
  352. handled_errors = self.context.get("handled_errors")
  353. if handled_errors:
  354. errors = []
  355. for field_name, field_errors in handled_errors.items():
  356. for error in field_errors:
  357. errors.append(
  358. {
  359. "reason": str(error),
  360. "type": error.code,
  361. "name": field_name,
  362. "value": error.value,
  363. }
  364. )
  365. params = {
  366. "event_id": data["event_id"],
  367. "issue": issue,
  368. "tags": {tag[0]: tag[1] for tag in tags},
  369. "errors": errors,
  370. "timestamp": data.get("timestamp"),
  371. "data": sanitize_bad_postgres_json(json_data),
  372. "release_id": project.release_id,
  373. }
  374. if level:
  375. params["level"] = level
  376. try:
  377. event = Event.objects.create(**params)
  378. except IntegrityError as err:
  379. # This except is more efficient than a query for exists().
  380. if err.args and "event_id" in err.args[0]:
  381. raise PermissionDenied(
  382. "An event with the same ID already exists (%s)"
  383. % params["event_id"]
  384. ) from err
  385. raise err
  386. if issue_created: # Do it right now, so that new issues look correct
  387. event_data = Event.objects.filter(issue_id=OuterRef("id")).values("data")[
  388. :1
  389. ]
  390. event_vector = event_data.annotate(
  391. search_vector=RawSQL("select generate_issue_tsvector(data)", [])
  392. ).values("search_vector")
  393. Issue.objects.filter(pk=issue.pk).update(
  394. search_vector=event_vector, last_seen=event.created
  395. )
  396. else: # Updates can be slower and debounced
  397. issue.check_for_status_update()
  398. # Expire after 1 hour - in case of major backup
  399. update_search_index_issue(args=[issue.pk])
  400. return event
  401. class StoreErrorSerializer(StoreDefaultSerializer):
  402. """Primary difference is the presense of exception attribute"""
  403. type = EventType.ERROR
  404. exception = serializers.JSONField(required=False)
  405. stacktrace = serializers.JSONField(
  406. required=False, help_text="Deprecated but supported at this time"
  407. )
  408. class StoreCSPReportSerializer(BaseSerializer):
  409. """
  410. CSP Report Serializer
  411. Very different format from others Store serializers.
  412. Does not extend base class due to differences.
  413. """
  414. type = EventType.CSP
  415. def __init__(self, *args, **kwargs):
  416. super().__init__(*args, **kwargs)
  417. # This is done to support the hyphen
  418. self.fields.update({"csp-report": serializers.JSONField()})
  419. def create(self, validated_data):
  420. project = self.context.get("project")
  421. csp = validated_data["csp-report"]
  422. title = self.get_title(csp)
  423. culprit = self.get_culprit(csp)
  424. uri = self.get_uri(csp)
  425. directive = self.get_effective_directive(csp)
  426. metadata = {
  427. "message": title,
  428. "uri": uri,
  429. "directive": directive,
  430. }
  431. issue, _ = Issue.objects.get_or_create(
  432. title=title,
  433. culprit=culprit,
  434. project_id=project.id,
  435. type=EventType.CSP,
  436. defaults={"metadata": metadata},
  437. )
  438. # Convert - to _
  439. normalized_csp = dict((k.replace("-", "_"), v) for k, v in csp.items())
  440. if "effective_directive" not in normalized_csp:
  441. normalized_csp["effective_directive"] = directive
  442. json_data = {
  443. "culprit": culprit,
  444. "csp": normalized_csp,
  445. "title": title,
  446. "metadata": metadata,
  447. "message": title,
  448. "type": EventType.CSP.label,
  449. }
  450. user = self.process_user(project, validated_data)
  451. if user:
  452. json_data["user"] = user
  453. params = {
  454. "issue": issue,
  455. "data": json_data,
  456. }
  457. return Event.objects.create(**params)
  458. def get_effective_directive(self, data):
  459. """
  460. Some browers return effective-directive and others don't.
  461. Infer missing ones from violated directive
  462. """
  463. if "effective-directive" in data:
  464. return data["effective-directive"]
  465. first_violation = data["violated-directive"].split()[0]
  466. return first_violation
  467. def get_uri(self, data):
  468. url = data["blocked-uri"]
  469. return urlparse(url).netloc
  470. def get_title(self, data):
  471. effective_directive = self.get_effective_directive(data)
  472. humanized_directive = effective_directive.replace("-src", "")
  473. uri = self.get_uri(data)
  474. return f"Blocked '{humanized_directive}' from '{uri}'"
  475. def get_culprit(self, data):
  476. # "style-src cdn.example.com"
  477. return data.get("violated-directive")
  478. class EnvelopeHeaderSerializer(serializers.Serializer):
  479. event_id = serializers.UUIDField(required=False)
  480. sent_at = FlexibleDateTimeField(required=False)