import logging import typing import uuid from datetime import datetime from typing import Annotated, Any, Literal, Optional, Union from urllib.parse import parse_qs, urlparse from django.utils.timezone import now from ninja import Field from pydantic import ( AliasChoices, BeforeValidator, RootModel, ValidationError, WrapValidator, field_validator, model_validator, ) from apps.issue_events.constants import IssueEventType from ..shared.schema.base import LaxIngestSchema from ..shared.schema.contexts import ContextsSchema from ..shared.schema.event import ( BaseIssueEvent, BaseRequest, EventBreadcrumb, ListKeyValue, ) from ..shared.schema.user import EventUser from ..shared.schema.utils import invalid_to_none logger = logging.getLogger(__name__) CoercedStr = Annotated[ str, BeforeValidator(lambda v: str(v) if isinstance(v, bool) else v) ] """ Coerced Str that will coerce bool to str when found """ def coerce_list(v: Any) -> Any: """Wrap non-list dict into list: {"a": 1} to [{"a": 1}]""" return v if not isinstance(v, dict) else [v] class Signal(LaxIngestSchema): number: int code: Optional[int] name: Optional[str] code_name: Optional[str] class MachException(LaxIngestSchema): number: int code: int subcode: int name: Optional[str] class NSError(LaxIngestSchema): code: int domain: str class Errno(LaxIngestSchema): number: int name: Optional[str] class MechanismMeta(LaxIngestSchema): signal: Optional[Signal] = None match_exception: Optional[MachException] = None ns_error: Optional[NSError] = None errno: Optional[Errno] = None class ExceptionMechanism(LaxIngestSchema): type: str description: Optional[str] = None help_link: Optional[str] = None handled: Optional[bool] = None synthetic: Optional[bool] = None meta: Optional[dict] = None data: Optional[dict] = None class StackTraceFrame(LaxIngestSchema): filename: Optional[str] = None function: Optional[str] = None raw_function: Optional[str] = None module: Optional[str] = None lineno: Optional[int] = None colno: Optional[int] = None abs_path: Optional[str] = None context_line: Optional[str] = None pre_context: Optional[list[Optional[str]]] = None post_context: Optional[list[Optional[str]]] = None source_link: Optional[str] = None in_app: Optional[bool] = None stack_start: Optional[bool] = None vars: Optional[dict[str, Union[str, dict, list]]] = None instruction_addr: Optional[str] = None addr_mode: Optional[str] = None symbol_addr: Optional[str] = None image_addr: Optional[str] = None package: Optional[str] = None platform: Optional[str] = None def is_url(self, filename: str) -> bool: return filename.startswith(("file:", "http:", "https:", "applewebdata:")) @model_validator(mode="after") def normalize_files(self): if not self.abs_path and self.filename: self.abs_path = self.filename if self.filename and self.is_url(self.filename): self.filename = urlparse(self.filename).path return self @field_validator("pre_context", "post_context") @classmethod def replace_null(cls, context: list[Optional[str]]) -> list[Optional[str]]: if context: return [line if line else "" for line in context] class StackTrace(LaxIngestSchema): frames: list[StackTraceFrame] registers: Optional[dict[str, str]] = None class EventException(LaxIngestSchema): type: Optional[str] = None value: Annotated[Optional[str], WrapValidator(invalid_to_none)] = None module: Optional[str] = None thread_id: Optional[str] = None mechanism: Optional[ExceptionMechanism] = None stacktrace: Annotated[Optional[StackTrace], WrapValidator(invalid_to_none)] = None @model_validator(mode="after") def check_type_value(self): if self.type is None and self.value is None: return None return self class ValueEventException(LaxIngestSchema): values: list[EventException] @field_validator("values") @classmethod def strip_null(cls, v: list[EventException]) -> list[EventException]: return [e for e in v if e is not None] class EventMessage(LaxIngestSchema): formatted: str = Field(max_length=8192, default="") message: Optional[str] = None params: Optional[Union[list[str], dict[str, str]]] = None @model_validator(mode="after") def set_formatted(self) -> "EventMessage": """ When the EventMessage formatted string is not set, attempt to set it based on message and params interpolation """ if not self.formatted and self.message: params = self.params if isinstance(params, list) and params: self.formatted = self.message % tuple(params) elif isinstance(params, dict): self.formatted = self.message.format(**params) return self class EventTemplate(LaxIngestSchema): lineno: int abs_path: Optional[str] = None filename: str context_line: str pre_context: Optional[list[str]] = None post_context: Optional[list[str]] = None class ValueEventBreadcrumb(LaxIngestSchema): values: list[EventBreadcrumb] class ClientSDKPackage(LaxIngestSchema): name: Optional[str] = None version: Optional[str] = None class ClientSDKInfo(LaxIngestSchema): integrations: Optional[list[Optional[str]]] = None name: Optional[str] packages: Optional[list[ClientSDKPackage]] = None version: Optional[str] @field_validator("packages", mode="before") def name_must_contain_space(cls, v: Any) -> Any: return coerce_list(v) class RequestHeaders(LaxIngestSchema): content_type: Optional[str] class RequestEnv(LaxIngestSchema): remote_addr: Optional[str] QueryString = Union[str, ListKeyValue, dict[str, Optional[str]]] """Raw URL querystring, list, or dict""" KeyValueFormat = Union[list[list[Optional[str]]], dict[str, Optional[CoercedStr]]] """ key-values in list or dict format. Example {browser: firefox} or [[browser, firefox]] """ class IngestRequest(BaseRequest): headers: Optional[KeyValueFormat] = None query_string: Optional[QueryString] = None @field_validator("headers", mode="before") @classmethod def fix_non_standard_headers(cls, v): """ Fix non-documented format used by PHP Sentry Client Convert {"Foo": ["bar"]} into {"Foo: "bar"} """ if isinstance(v, dict): return { key: value[0] if isinstance(value, list) else value for key, value in v.items() } return v @field_validator("query_string", "headers") @classmethod def prefer_list_key_value( cls, v: Optional[Union[QueryString, KeyValueFormat]] ) -> Optional[ListKeyValue]: """Store all querystring, header formats in a list format""" result: Optional[ListKeyValue] = None if isinstance(v, str) and v: # It must be a raw querystring, parse it qs = parse_qs(v) result = [[key, value] for key, values in qs.items() for value in values] elif isinstance(v, dict): # Convert dict to list result = [[key, value] for key, value in v.items()] elif isinstance(v, list): # Normalize list (throw out any weird data) result = [item[:2] for item in v if len(item) >= 2] if result: # Remove empty and any key called "Cookie" which could be sensitive data entry_to_remove = ["Cookie", ""] return sorted( [entry for entry in result if entry != entry_to_remove], key=lambda x: (x[0], x[1]), ) return result class IngestIssueEvent(BaseIssueEvent): timestamp: datetime = Field(default_factory=now) level: Optional[str] = "error" logentry: Optional[EventMessage] = None logger: Optional[str] = None transaction: Optional[str] = Field( validation_alias=AliasChoices("transaction", "culprit"), default=None ) server_name: Optional[str] = None release: Optional[str] = None dist: Optional[str] = None tags: Optional[KeyValueFormat] = None environment: Optional[str] = None modules: Optional[dict[str, Optional[str]]] = None extra: Optional[dict[str, Any]] = None fingerprint: Optional[list[str]] = None errors: Optional[list[Any]] = None exception: Optional[Union[list[EventException], ValueEventException]] = None message: Optional[Union[str, EventMessage]] = None template: Optional[EventTemplate] = None breadcrumbs: Optional[Union[list[EventBreadcrumb], ValueEventBreadcrumb]] = None sdk: Optional[ClientSDKInfo] = None request: Optional[IngestRequest] = None contexts: Optional[ContextsSchema] = None user: Optional[EventUser] = None @field_validator("tags") @classmethod def prefer_dict( cls, v: Optional[KeyValueFormat] ) -> Optional[dict[str, Optional[str]]]: if isinstance(v, list): return {key: value for key, value in v if key is not None} return v class EventIngestSchema(IngestIssueEvent): event_id: uuid.UUID class EnvelopeHeaderSchema(LaxIngestSchema): event_id: Optional[uuid.UUID] = None dsn: Optional[str] = None sdk: Optional[ClientSDKInfo] = None sent_at: datetime = Field(default_factory=now) SupportedItemType = Literal["transaction", "event"] IgnoredItemType = Literal[ "session", "sessions", "client_report", "attachment", "user_report", "check_in" ] SUPPORTED_ITEMS = typing.get_args(SupportedItemType) class ItemHeaderSchema(LaxIngestSchema): content_type: Optional[str] = None type: Union[SupportedItemType, IgnoredItemType] length: Optional[int] = None class EnvelopeSchema(RootModel[list[dict[str, Any]]]): root: list[dict[str, Any]] _header: EnvelopeHeaderSchema _items: list[tuple[ItemHeaderSchema, Union[IngestIssueEvent, dict[str, Any]]]] = [] @model_validator(mode="after") def validate_envelope(self) -> "EnvelopeSchema": data = self.root try: header = data.pop(0) except IndexError: raise ValidationError([{"message": "Envelope is empty"}]) self._header = EnvelopeHeaderSchema(**header) while len(data) >= 2: item_header_data = data.pop(0) if item_header_data.get("type", None) not in SUPPORTED_ITEMS: continue item_header = ItemHeaderSchema(**item_header_data) if item_header.type == "event": try: item = IngestIssueEvent(**data.pop(0)) except ValidationError as err: logger.warning("Envelope Event item invalid", exc_info=True) raise err self._items.append((item_header, item)) elif item_header.type == "transaction": item = data.pop(0) self._items.append((item_header, item)) return self class CSPReportSchema(LaxIngestSchema): """ https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Security-Policy-Report-Only#violation_report_syntax """ blocked_uri: str = Field(alias="blocked-uri") disposition: Literal["enforce", "report"] = Field(alias="disposition") document_uri: str = Field(alias="document-uri") effective_directive: str = Field(alias="effective-directive") original_policy: Optional[str] = Field(alias="original-policy") script_sample: Optional[str] = Field(alias="script-sample", default=None) status_code: Optional[int] = Field(alias="status-code") line_number: Optional[int] = None column_number: Optional[int] = None class SecuritySchema(LaxIngestSchema): csp_report: CSPReportSchema = Field(alias="csp-report") ## Normalized Interchange Issue Events class IssueEventSchema(IngestIssueEvent): """ Event storage and interchange format Used in json view and celery interchange Don't use this for api intake """ type: Literal[IssueEventType.DEFAULT] = IssueEventType.DEFAULT class ErrorIssueEventSchema(IngestIssueEvent): type: Literal[IssueEventType.ERROR] = IssueEventType.ERROR class CSPIssueEventSchema(IngestIssueEvent): type: Literal[IssueEventType.CSP] = IssueEventType.CSP csp: CSPReportSchema class InterchangeIssueEvent(LaxIngestSchema): """Normalized wrapper around issue event. Event should not contain repeat information.""" event_id: uuid.UUID = Field(default_factory=uuid.uuid4) project_id: int organization_id: int received: datetime = Field(default_factory=now) payload: Union[IssueEventSchema, ErrorIssueEventSchema, CSPIssueEventSchema] = ( Field(discriminator="type") )