Add event breadcrumbs handling

event_store/serializers.py

@@ -1,9 +1,7 @@
 from typing import Dict, List, Tuple, Union
 from urllib.parse import urlparse
-from datetime import datetime
 from django.db import transaction, connection
 from django.db.utils import IntegrityError
-from django.utils.timezone import make_aware
 from ipware import get_client_ip
 from anonymizeip import anonymize_ip
 from rest_framework import serializers
@@ -11,7 +9,9 @@ from rest_framework.exceptions import PermissionDenied
 from sentry.eventtypes.error import ErrorEvent
 from sentry.eventtypes.base import DefaultEvent
 from issues.models import EventType, Event, Issue, EventTagKey
+from issues.serializers import BaseBreadcrumbsSerializer
 from environments.models import Environment
+from glitchtip.serializers import FlexibleDateTimeField
 from .event_tag_processors import TAG_PROCESSORS
 from .event_context_processors import EVENT_CONTEXT_PROCESSORS
 
@@ -47,16 +47,6 @@ def sanitize_bad_postgres_json(data: Union[str, dict, list]):
     return data
 
 
-class FlexibleDateTimeField(serializers.DateTimeField):
-    """ Supports both DateTime and unix epoch timestamp """
-
-    def to_internal_value(self, timestamp):
-        try:
-            return make_aware(datetime.fromtimestamp(float(timestamp)))
-        except ValueError:
-            return super().to_internal_value(timestamp)
-
-
 class RequestSerializer(serializers.Serializer):
     env = serializers.DictField(
         child=serializers.CharField(allow_blank=True), required=False
@@ -68,6 +58,20 @@ class RequestSerializer(serializers.Serializer):
     query_string = serializers.CharField(required=False, allow_blank=True)
 
 
+class GenericField(serializers.Field):
+    def to_internal_value(self, data):
+        return data
+
+
+class BreadcrumbsSerializer(BaseBreadcrumbsSerializer):
+    timestamp = GenericField(required=False)
+
+    def validate_level(self, value):
+        if value == "log":
+            return "info"
+        return value
+
+
 class BaseSerializer(serializers.Serializer):
     def process_user(self, project, data):
         """ Fetch user data from SDK event and request """
@@ -88,7 +92,7 @@ class StoreDefaultSerializer(BaseSerializer):
     """
 
     type = EventType.DEFAULT
-    breadcrumbs = serializers.JSONField(required=False)
+    breadcrumbs = BreadcrumbsSerializer(required=False, many=True)
     contexts = serializers.JSONField(required=False)
     environment = serializers.CharField(required=False)
     event_id = serializers.UUIDField()
@@ -189,6 +193,9 @@ class StoreDefaultSerializer(BaseSerializer):
         title = eventtype.get_title(metadata)
         culprit = eventtype.get_location(data)
         request = data.get("request")
+        breadcrumbs = data.get("breadcrumbs")
+        if breadcrumbs:
+            breadcrumbs = {"values": breadcrumbs}
         exception = self.modify_exception(data.get("exception"))
         if request:
             headers = request.get("headers")
@@ -219,6 +226,7 @@ class StoreDefaultSerializer(BaseSerializer):
                 environment = self.get_environment(data["environment"], project)
 
             json_data = {
+                "breadcrumbs": breadcrumbs,
                 "contexts": contexts,
                 "culprit": culprit,
                 "exception": exception,

event_store/views.py

@@ -102,6 +102,7 @@ class EventStoreAPIView(APIView):
         if serializer.is_valid():
             event = serializer.save()
             return Response({"id": event.event_id_hex})
+        print(serializer.errors)
         return Response()
 
     @classmethod

glitchtip/serializers.py

@@ -0,0 +1,14 @@
+from datetime import datetime
+from django.utils.timezone import make_aware
+from rest_framework import serializers
+
+
+class FlexibleDateTimeField(serializers.DateTimeField):
+    """ Supports both DateTime and unix epoch timestamp """
+
+    def to_internal_value(self, timestamp):
+        try:
+            return make_aware(datetime.fromtimestamp(float(timestamp)))
+        except ValueError:
+            return super().to_internal_value(timestamp)
+

issues/models.py

@@ -2,7 +2,6 @@ import uuid
 from django.contrib.postgres.search import SearchVectorField
 from django.contrib.postgres.indexes import GinIndex
 from django.db import models
-from sentry.interfaces.stacktrace import get_context
 from user_reports.models import UserReport
 from .utils import base32_encode
 
@@ -186,62 +185,6 @@ class Event(models.Model):
     def user_report(self):
         return UserReport.objects.filter(event_id=self.pk).first()
 
-    @property
-    def entries(self):
-        def get_has_system_frames(frames):
-            return any(frame.in_app for frame in frames)
-
-        entries = []
-
-        exception = self.data.get("exception")
-        # Some, but not all, keys are made more JS camel case like
-        if exception and exception.get("values"):
-            # https://gitlab.com/glitchtip/sentry-open-source/sentry/-/blob/master/src/sentry/interfaces/stacktrace.py#L487
-            # if any frame is "in_app" set this to True
-            exception["hasSystemFrames"] = False
-            for value in exception["values"]:
-                if "stacktrace" in value and "frames" in value["stacktrace"]:
-                    for frame in value["stacktrace"]["frames"]:
-                        if frame.get("in_app") == True:
-                            exception["hasSystemFrames"] = True
-                        if "in_app" in frame:
-                            frame["inApp"] = frame.pop("in_app")
-                        if "abs_path" in frame:
-                            frame["absPath"] = frame.pop("abs_path")
-                        if "colno" in frame:
-                            frame["colNo"] = frame.pop("colno")
-                        if "lineno" in frame:
-                            frame["lineNo"] = frame.pop("lineno")
-                            pre_context = frame.pop("pre_context", None)
-                            post_context = frame.pop("post_context", None)
-                            frame["context"] = get_context(
-                                frame["lineNo"],
-                                frame.get("context_line"),
-                                pre_context,
-                                post_context,
-                            )
-
-            entries.append({"type": "exception", "data": exception})
-
-        request = self.data.get("request")
-        if request:
-            request["inferredContentType"] = request.pop("inferred_content_type")
-            entries.append({"type": "request", "data": request})
-
-        breadcrumbs = self.data.get("breadcrumbs")
-        if breadcrumbs:
-            entries.append({"type": "breadcrumbs", "data": {"values": breadcrumbs}})
-
-        message = self.data.get("message")
-        if message:
-            entries.append({"type": "message", "data": {"formatted": message}})
-
-        csp = self.data.get("csp")
-        if csp:
-            entries.append({"type": EventType.CSP.label, "data": csp})
-
-        return entries
-
     def _build_context(self, context: list, base_line_no: int, is_pre: bool):
         context_length = len(context)
         result = []

issues/serializers.py

@@ -1,7 +1,9 @@
 from rest_framework import serializers
 from projects.serializers.base_serializers import ProjectReferenceSerializer
 from user_reports.serializers import UserReportSerializer
-from .models import Issue, Event, EventTag
+from sentry.interfaces.stacktrace import get_context
+from glitchtip.serializers import FlexibleDateTimeField
+from .models import Issue, Event, EventTag, EventType
 
 
 class EventTagSerializer(serializers.ModelSerializer):
@@ -21,11 +23,94 @@ class EventUserSerializer(serializers.Serializer):
     id = serializers.CharField(allow_null=True)
 
 
+class BaseBreadcrumbsSerializer(serializers.Serializer):
+    category = serializers.CharField()
+    level = serializers.CharField(default="info")
+    event_id = serializers.CharField(required=False)
+    data = serializers.JSONField(required=False)
+    message = serializers.CharField(required=False)
+    type = serializers.CharField(default="default")
+
+
+class BreadcrumbsSerializer(BaseBreadcrumbsSerializer):
+    timestamp = FlexibleDateTimeField()
+    message = serializers.CharField(default=None)
+    event_id = serializers.CharField(default=None)
+    data = serializers.JSONField(default=None)
+
+
+class EventEntriesSerializer(serializers.Serializer):
+    def to_representation(self, instance):
+        def get_has_system_frames(frames):
+            return any(frame.in_app for frame in frames)
+
+        entries = []
+
+        exception = instance.get("exception")
+        # Some, but not all, keys are made more JS camel case like
+        if exception and exception.get("values"):
+            # https://gitlab.com/glitchtip/sentry-open-source/sentry/-/blob/master/src/sentry/interfaces/stacktrace.py#L487
+            # if any frame is "in_app" set this to True
+            exception["hasSystemFrames"] = False
+            for value in exception["values"]:
+                if "stacktrace" in value and "frames" in value["stacktrace"]:
+                    for frame in value["stacktrace"]["frames"]:
+                        if frame.get("in_app") == True:
+                            exception["hasSystemFrames"] = True
+                        if "in_app" in frame:
+                            frame["inApp"] = frame.pop("in_app")
+                        if "abs_path" in frame:
+                            frame["absPath"] = frame.pop("abs_path")
+                        if "colno" in frame:
+                            frame["colNo"] = frame.pop("colno")
+                        if "lineno" in frame:
+                            frame["lineNo"] = frame.pop("lineno")
+                            pre_context = frame.pop("pre_context", None)
+                            post_context = frame.pop("post_context", None)
+                            frame["context"] = get_context(
+                                frame["lineNo"],
+                                frame.get("context_line"),
+                                pre_context,
+                                post_context,
+                            )
+
+            entries.append({"type": "exception", "data": exception})
+
+        breadcrumbs = instance.get("breadcrumbs")
+        if breadcrumbs:
+            breadcrumbs_serializer = BreadcrumbsSerializer(
+                data=breadcrumbs.get("values"), many=True
+            )
+            if breadcrumbs_serializer.is_valid():
+                entries.append(
+                    {
+                        "type": "breadcrumbs",
+                        "data": {"values": breadcrumbs_serializer.validated_data},
+                    }
+                )
+
+        request = instance.get("request")
+        if request:
+            request["inferredContentType"] = request.pop("inferred_content_type")
+            entries.append({"type": "request", "data": request})
+
+        message = instance.get("message")
+        if message:
+            entries.append({"type": "message", "data": {"formatted": message}})
+
+        csp = instance.get("csp")
+        if csp:
+            entries.append({"type": EventType.CSP.label, "data": csp})
+
+        return entries
+
+
 class EventSerializer(serializers.ModelSerializer):
     eventID = serializers.CharField(source="event_id_hex")
     id = serializers.CharField(source="event_id_hex")
     dateCreated = serializers.DateTimeField(source="timestamp")
     dateReceived = serializers.DateTimeField(source="created")
+    entries = EventEntriesSerializer(source="data")
     tags = EventTagSerializer(many=True)
     user = EventUserSerializer()
 

issues/tests/test_sentry_api_compat.py

@@ -1,5 +1,6 @@
 import json
-from typing import List, Dict
+from typing import List, Dict, Union
+from django.utils import dateparse
 from django.urls import reverse
 from event_store.test_data.django_error_factory import message
 from event_store.test_data.csp import mdn_sample_csp
@@ -44,6 +45,31 @@ class SentryAPICompatTestCase(GlitchTipTestCase):
         value["title"] = self.upgrade_title(value.get("title"))
         return value
 
+    def upgrade_datetime(self, value: str):
+        """
+        DRF DateTimeField sets precision while Sentry OSS does not
+        DRF DateTimeField 2020-07-23T02:54:37.823000Z
+        Sentry OSS: 2020-07-23T02:54:37.823Z
+        """
+        date_value = dateparse.parse_datetime(value)
+        if date_value:
+            value = date_value.isoformat()
+            if value.endswith("+00:00"):
+                value = value[:-6] + "Z"
+        return value
+
+    def upgrade_data(self, data: Union[str, dict, list]):
+        """ A recursive replace function """
+        if isinstance(data, dict):
+            return {k: self.upgrade_data(v) for k, v in data.items()}
+        elif isinstance(data, list):
+            return [self.upgrade_data(i) for i in data]
+        elif isinstance(data, str):
+            if data.endswith("Z"):
+                return self.upgrade_datetime(data)
+            return data
+        return data
+
     def assertCompareData(self, data1: Dict, data2: Dict, fields: List[str]):
         """ Compare data of two dict objects. Compare only provided fields list """
         for field in fields:
@@ -145,8 +171,18 @@ class SentryAPICompatTestCase(GlitchTipTestCase):
         event = Event.objects.first()
         self.assertIsNotNone(event.timestamp)
         self.assertNotEqual(event.timestamp, sdk_error["timestamp"])
+
         event_json = event.event_json()
-        self.assertEqual(event_json["datetime"], sentry_json["datetime"])
+        self.assertCompareData(event_json, sentry_json, ["datetime", "breadcrumbs"])
+
+        url = self.get_project_events_detail(event.pk)
+        res = self.client.get(url)
+        res_data = res.json()
+        self.assertCompareData(res_data, sentry_data, ["datetime"])
+        self.assertEqual(res_data["entries"][1].get("type"), "breadcrumbs")
+        self.assertEqual(
+            res_data["entries"][1], self.upgrade_data(sentry_data["entries"][1]),
+        )
 
     def test_dotnet_error(self):
         # Don't mimic this test, use self.get_jest_test_data instead