ref: Remove Python store view (#19135)

Delete the endpoint and all associated tests. I have verified that the extra tests I removed have similar tests in either the Relay integration test suite or the ingest consumer test suite (event attachments)

getsentry PR: getsentry/getsentry#4158

.eslintignore

@@ -1,6 +1,6 @@
 **/dist/**/*
 **/vendor/**/*
-**/tests/sentry/lang/javascript/fixtures/**/*
-**/tests/sentry/lang/javascript/example-project/**/*
+**/tests/**/lang/javascript/fixtures/**/*
+**/tests/**/lang/javascript/example-project/**/*
 /examples/
 /scripts/

conftest.py

@@ -22,14 +22,6 @@ def pytest_configure(config):
     # always install plugins for the tests
     install_sentry_plugins()
 
-    # add custom test markers
-    config.addinivalue_line(
-        "markers",
-        "sentry_store_integration: mark test as using the sentry store endpoint and therefore using legacy code",
-    )
-    config.addinivalue_line(
-        "markers", "relay_store_integration: mark test as using the relay store endpoint"
-    )
     config.addinivalue_line("markers", "obsolete: mark test as obsolete and soon to be removed")
 
 

requirements-base.txt

@@ -43,7 +43,6 @@ python3-saml>=1.4.0,<1.5
 python-u2flib-server>=5.0.0,<6.0.0
 PyYAML>=5.3,<5.4
 qrcode>=6.1.0,<6.2.0
-querystring_parser>=1.2.3,<2.0.0
 rb>=1.7.0,<2.0.0
 redis-py-cluster==1.3.6
 redis==2.10.6

src/sentry/api/base.py

@@ -9,6 +9,7 @@ import sentry_sdk
 from datetime import datetime, timedelta
 from django.conf import settings
 from django.utils.http import urlquote
+from django.http import HttpResponse
 from django.views.decorators.csrf import csrf_exempt
 from enum import Enum
 from pytz import utc
@@ -23,11 +24,10 @@ from sentry.auth import access
 from sentry.models import Environment
 from sentry.utils.cursors import Cursor
 from sentry.utils.dates import to_datetime
-from sentry.utils.http import absolute_uri, is_valid_origin
+from sentry.utils.http import absolute_uri, is_valid_origin, origin_from_request
 from sentry.utils.audit import create_audit_entry
 from sentry.utils.sdk import capture_exception
 from sentry.utils import json
-from sentry.web.api import allow_cors_options
 
 
 from .authentication import ApiKeyAuthentication, TokenAuthentication
@@ -49,6 +49,51 @@ logger = logging.getLogger(__name__)
 audit_logger = logging.getLogger("sentry.audit.api")
 
 
+def allow_cors_options(func):
+    """
+    Decorator that adds automatic handling of OPTIONS requests for CORS
+
+    If the request is OPTIONS (i.e. pre flight CORS) construct a OK (200) response
+    in which we explicitly enable the caller and add the custom headers that we support
+    For other requests just add the appropriate CORS headers
+
+    :param func: the original request handler
+    :return: a request handler that shortcuts OPTIONS requests and just returns an OK (CORS allowed)
+    """
+
+    @functools.wraps(func)
+    def allow_cors_options_wrapper(self, request, *args, **kwargs):
+
+        if request.method == "OPTIONS":
+            response = HttpResponse(status=200)
+            response["Access-Control-Max-Age"] = "3600"  # don't ask for options again for 1 hour
+        else:
+            response = func(self, request, *args, **kwargs)
+
+        allow = ", ".join(self._allowed_methods())
+        response["Allow"] = allow
+        response["Access-Control-Allow-Methods"] = allow
+        response["Access-Control-Allow-Headers"] = (
+            "X-Sentry-Auth, X-Requested-With, Origin, Accept, "
+            "Content-Type, Authentication, Authorization, Content-Encoding"
+        )
+        response["Access-Control-Expose-Headers"] = "X-Sentry-Error, Retry-After"
+
+        if request.META.get("HTTP_ORIGIN") == "null":
+            origin = "null"  # if ORIGIN header is explicitly specified as 'null' leave it alone
+        else:
+            origin = origin_from_request(request)
+
+        if origin is None or origin == "null":
+            response["Access-Control-Allow-Origin"] = "*"
+        else:
+            response["Access-Control-Allow-Origin"] = origin
+
+        return response
+
+    return allow_cors_options_wrapper
+
+
 class DocSection(Enum):
     ACCOUNTS = "Accounts"
     EVENTS = "Events"

src/sentry/api/endpoints/project_filter_details.py

@@ -19,14 +19,14 @@ class ProjectFilterDetailsEndpoint(ProjectEndpoint):
 
         """
         current_filter = None
-        for flt in message_filters.get_all_filters():
-            if flt.spec.id == filter_id:
+        for flt in message_filters.get_all_filter_specs():
+            if flt.id == filter_id:
                 current_filter = flt
                 break
         else:
             raise ResourceDoesNotExist  # could not find filter with the requested id
 
-        serializer = current_filter.spec.serializer_cls(data=request.data, partial=True)
+        serializer = current_filter.serializer_cls(data=request.data, partial=True)
 
         if not serializer.is_valid():
             return Response(serializer.errors, status=400)

src/sentry/api/endpoints/project_filters.py

@@ -17,17 +17,16 @@ class ProjectFiltersEndpoint(ProjectEndpoint):
 
         """
         results = []
-        for flt in message_filters.get_all_filters():
-            filter_spec = flt.spec
+        for flt in message_filters.get_all_filter_specs():
             results.append(
                 {
-                    "id": filter_spec.id,
+                    "id": flt.id,
                     # 'active' will be either a boolean or list for the legacy browser filters
                     # all other filters will be boolean
-                    "active": message_filters.get_filter_state(filter_spec.id, project),
-                    "description": filter_spec.description,
-                    "name": filter_spec.name,
-                    "hello": filter_spec.id + " - " + filter_spec.name,
+                    "active": message_filters.get_filter_state(flt.id, project),
+                    "description": flt.description,
+                    "name": flt.name,
+                    "hello": flt.id + " - " + flt.name,
                 }
             )
         results.sort(key=lambda x: x["name"])

src/sentry/coreapi.py

@@ -3,29 +3,15 @@
 #       metadata (rather than generic log messages which aren't useful).
 from __future__ import absolute_import, print_function
 
-import abc
-import base64
 import logging
 import re
-import six
-import zlib
 
-from django.core.exceptions import SuspiciousOperation
-from django.utils.crypto import constant_time_compare
-from gzip import GzipFile
-from six import BytesIO
 from time import time
 
 from sentry.attachments import attachment_cache
 from sentry.cache import default_cache
-from sentry.models import ProjectKey
 from sentry.tasks.store import preprocess_event, preprocess_event_from_reprocessing
-from sentry.utils import json
-from sentry.utils.auth import parse_auth_header
 from sentry.utils.cache import cache_key_for_event
-from sentry.utils.http import origin_from_request
-from sentry.utils.strings import decompress
-from sentry.utils.sdk import configure_scope, set_current_project
 from sentry.utils.canonical import CANONICAL_TYPES
 
 
@@ -57,259 +43,33 @@ class APIForbidden(APIError):
     http_status = 403
 
 
-class APIRateLimited(APIError):
-    http_status = 429
-    msg = "Creation of this event was denied due to rate limiting"
-    name = "rate_limit"
+def insert_data_to_database_legacy(
+    data, start_time=None, from_reprocessing=False, attachments=None
+):
+    """
+    Yet another "fast path" to ingest an event without making it go
+    through Relay. Please consider using functions from the ingest consumer
+    instead, or, if you're within tests, to use `TestCase.store_event`.
+    """
 
-    def __init__(self, retry_after=None):
-        self.retry_after = retry_after
+    # XXX(markus): Delete this function and merge with ingest consumer logic.
 
+    if start_time is None:
+        start_time = time()
 
-class Auth(object):
-    def __init__(
-        self, client=None, version=None, secret_key=None, public_key=None, is_public=False
-    ):
-        self.client = client
-        self.version = version
-        self.secret_key = secret_key
-        self.public_key = public_key
-        self.is_public = is_public
+    # we might be passed some subclasses of dict that fail dumping
+    if isinstance(data, CANONICAL_TYPES):
+        data = dict(data.items())
 
+    cache_timeout = 3600
+    cache_key = cache_key_for_event(data)
+    default_cache.set(cache_key, data, cache_timeout)
 
-class ClientContext(object):
-    def __init__(self, agent=None, version=None, project_id=None, ip_address=None):
-        # user-agent (i.e. raven-python)
-        self.agent = agent
-        # protocol version
-        self.version = version
-        # project instance
-        self.project_id = project_id
-        self.project = None
-        self.ip_address = ip_address
+    # Attachments will be empty or None if the "event-attachments" feature
+    # is turned off. For native crash reports it will still contain the
+    # crash dump (e.g. minidump) so we can load it during processing.
+    if attachments is not None:
+        attachment_cache.set(cache_key, attachments, cache_timeout)
 
-    def bind_project(self, project):
-        self.project = project
-        self.project_id = project.id
-        set_current_project(project.id)
-
-    def bind_auth(self, auth):
-        self.agent = auth.client
-        self.version = auth.version
-
-        with configure_scope() as scope:
-            scope.set_tag("agent", self.agent)
-            scope.set_tag("protocol", self.version)
-
-
-class ClientApiHelper(object):
-    def __init__(self, agent=None, version=None, project_id=None, ip_address=None):
-        self.context = ClientContext(
-            agent=agent, version=version, project_id=project_id, ip_address=ip_address
-        )
-
-    def project_key_from_auth(self, auth):
-        if not auth.public_key:
-            raise APIUnauthorized("Invalid api key")
-
-        # Make sure the key even looks valid first, since it's
-        # possible to get some garbage input here causing further
-        # issues trying to query it from cache or the database.
-        if not ProjectKey.looks_like_api_key(auth.public_key):
-            raise APIUnauthorized("Invalid api key")
-
-        try:
-            pk = ProjectKey.objects.get_from_cache(public_key=auth.public_key)
-        except ProjectKey.DoesNotExist:
-            raise APIUnauthorized("Invalid api key")
-
-        # a secret key may not be present which will be validated elsewhere
-        if not constant_time_compare(pk.secret_key, auth.secret_key or pk.secret_key):
-            raise APIUnauthorized("Invalid api key")
-
-        if not pk.is_active:
-            raise APIUnauthorized("API key is disabled")
-
-        if not pk.roles.store:
-            raise APIUnauthorized("Key does not allow event storage access")
-
-        return pk
-
-    def project_id_from_auth(self, auth):
-        return self.project_key_from_auth(auth).project_id
-
-    def insert_data_to_database(
-        self, data, start_time=None, from_reprocessing=False, attachments=None
-    ):
-        if start_time is None:
-            start_time = time()
-
-        # we might be passed some subclasses of dict that fail dumping
-        if isinstance(data, CANONICAL_TYPES):
-            data = dict(data.items())
-
-        cache_timeout = 3600
-        cache_key = cache_key_for_event(data)
-        default_cache.set(cache_key, data, cache_timeout)
-
-        # Attachments will be empty or None if the "event-attachments" feature
-        # is turned off. For native crash reports it will still contain the
-        # crash dump (e.g. minidump) so we can load it during processing.
-        if attachments is not None:
-            attachment_cache.set(cache_key, attachments, cache_timeout)
-
-        task = from_reprocessing and preprocess_event_from_reprocessing or preprocess_event
-        task.delay(cache_key=cache_key, start_time=start_time, event_id=data["event_id"])
-
-
-@six.add_metaclass(abc.ABCMeta)
-class AbstractAuthHelper(object):
-    @abc.abstractmethod
-    def auth_from_request(cls, request):
-        pass
-
-    @abc.abstractmethod
-    def origin_from_request(cls, request):
-        pass
-
-
-class ClientAuthHelper(AbstractAuthHelper):
-    @classmethod
-    def auth_from_request(cls, request):
-        result = {k: request.GET[k] for k in six.iterkeys(request.GET) if k[:7] == "sentry_"}
-
-        if request.META.get("HTTP_X_SENTRY_AUTH", "")[:7].lower() == "sentry ":
-            if result:
-                raise SuspiciousOperation("Multiple authentication payloads were detected.")
-            result = parse_auth_header(request.META["HTTP_X_SENTRY_AUTH"])
-        elif request.META.get("HTTP_AUTHORIZATION", "")[:7].lower() == "sentry ":
-            if result:
-                raise SuspiciousOperation("Multiple authentication payloads were detected.")
-            result = parse_auth_header(request.META["HTTP_AUTHORIZATION"])
-
-        if not result:
-            raise APIUnauthorized("Unable to find authentication information")
-
-        origin = cls.origin_from_request(request)
-        auth = Auth(
-            client=result.get("sentry_client"),
-            version=six.text_type(result.get("sentry_version")),
-            secret_key=result.get("sentry_secret"),
-            public_key=result.get("sentry_key"),
-            is_public=bool(origin),
-        )
-        # default client to user agent
-        if not auth.client:
-            auth.client = request.META.get("HTTP_USER_AGENT")
-            if isinstance(auth.client, bytes):
-                auth.client = auth.client.decode("latin1")
-        return auth
-
-    @classmethod
-    def origin_from_request(cls, request):
-        """
-        Returns either the Origin or Referer value from the request headers.
-        """
-        if request.META.get("HTTP_ORIGIN") == "null":
-            return "null"
-        return origin_from_request(request)
-
-
-class MinidumpAuthHelper(AbstractAuthHelper):
-    @classmethod
-    def origin_from_request(cls, request):
-        # We don't use an origin here
-        return None
-
-    @classmethod
-    def auth_from_request(cls, request):
-        key = request.GET.get("sentry_key")
-        if not key:
-            raise APIUnauthorized("Unable to find authentication information")
-
-        # Minidump requests are always "trusted".  We at this point only
-        # use is_public to identify requests that have an origin set (via
-        # CORS)
-        auth = Auth(public_key=key, client="sentry-minidump", is_public=False)
-        return auth
-
-
-class SecurityAuthHelper(AbstractAuthHelper):
-    @classmethod
-    def origin_from_request(cls, request):
-        # In the case of security reports, the origin is not available at the
-        # dispatch() stage, as we need to parse it out of the request body, so
-        # we do our own CORS check once we have parsed it.
-        return None
-
-    @classmethod
-    def auth_from_request(cls, request):
-        key = request.GET.get("sentry_key")
-        if not key:
-            raise APIUnauthorized("Unable to find authentication information")
-
-        auth = Auth(public_key=key, is_public=True)
-        auth.client = request.META.get("HTTP_USER_AGENT")
-        return auth
-
-
-def decompress_deflate(encoded_data):
-    try:
-        return zlib.decompress(encoded_data).decode("utf-8")
-    except Exception as e:
-        # This error should be caught as it suggests that there's a
-        # bug somewhere in the client's code.
-        logger.debug(six.text_type(e), exc_info=True)
-        raise APIError("Bad data decoding request (%s, %s)" % (type(e).__name__, e))
-
-
-def decompress_gzip(encoded_data):
-    try:
-        fp = BytesIO(encoded_data)
-        try:
-            f = GzipFile(fileobj=fp)
-            return f.read().decode("utf-8")
-        finally:
-            f.close()
-    except Exception as e:
-        # This error should be caught as it suggests that there's a
-        # bug somewhere in the client's code.
-        logger.debug(six.text_type(e), exc_info=True)
-        raise APIError("Bad data decoding request (%s, %s)" % (type(e).__name__, e))
-
-
-def decode_and_decompress_data(encoded_data):
-    try:
-        try:
-            return decompress(encoded_data).decode("utf-8")
-        except zlib.error:
-            return base64.b64decode(encoded_data).decode("utf-8")
-    except Exception as e:
-        # This error should be caught as it suggests that there's a
-        # bug somewhere in the client's code.
-        logger.debug(six.text_type(e), exc_info=True)
-        raise APIError("Bad data decoding request (%s, %s)" % (type(e).__name__, e))
-
-
-def decode_data(encoded_data):
-    try:
-        return encoded_data.decode("utf-8")
-    except UnicodeDecodeError as e:
-        # This error should be caught as it suggests that there's a
-        # bug somewhere in the client's code.
-        logger.debug(six.text_type(e), exc_info=True)
-        raise APIError("Bad data decoding request (%s, %s)" % (type(e).__name__, e))
-
-
-def safely_load_json_string(json_string):
-    try:
-        if isinstance(json_string, six.binary_type):
-            json_string = json_string.decode("utf-8")
-        obj = json.loads(json_string)
-        assert isinstance(obj, dict)
-    except Exception as e:
-        # This error should be caught as it suggests that there's a
-        # bug somewhere in the client's code.
-        logger.debug(six.text_type(e), exc_info=True)
-        raise APIError("Bad data reconstructing object (%s, %s)" % (type(e).__name__, e))
-    return obj
+    task = from_reprocessing and preprocess_event_from_reprocessing or preprocess_event
+    task.delay(cache_key=cache_key, start_time=start_time, event_id=data["event_id"])

src/sentry/event_manager.py

@@ -4,7 +4,6 @@ import logging
 import time
 
 import ipaddress
-import jsonschema
 import six
 
 from datetime import timedelta
@@ -23,7 +22,6 @@ from sentry.constants import (
     MAX_SECS_IN_FUTURE,
     MAX_SECS_IN_PAST,
 )
-from sentry.message_filters import should_filter_event
 from sentry.grouping.api import (
     get_grouping_config_dict_for_project,
     get_grouping_config_dict_for_event_data,
@@ -32,16 +30,6 @@ from sentry.grouping.api import (
     get_fingerprinting_config_for_project,
     GroupingConfigNotFound,
 )
-from sentry.coreapi import (
-    APIError,
-    APIForbidden,
-    decompress_gzip,
-    decompress_deflate,
-    decode_and_decompress_data,
-    decode_data,
-    safely_load_json_string,
-)
-from sentry.interfaces.base import get_interface
 from sentry.lang.native.utils import STORE_CRASH_REPORTS_ALL, convert_crashreport_count
 from sentry.models import (
     Activity,
@@ -75,12 +63,7 @@ from sentry.signals import first_event_received
 from sentry.tasks.integrations import kick_off_status_syncs
 from sentry.utils import json, metrics
 from sentry.utils.canonical import CanonicalKeyDict
-from sentry.utils.data_filters import (
-    is_valid_ip,
-    is_valid_release,
-    is_valid_error_message,
-    FilterStatKeys,
-)
+from sentry.utils.data_filters import FilterStatKeys
 from sentry.utils.dates import to_timestamp, to_datetime
 from sentry.utils.outcomes import Outcome, track_outcome
 from sentry.utils.safe import safe_execute, trim, get_path, setdefault_path
@@ -137,19 +120,6 @@ def validate_and_set_timestamp(data, timestamp):
             data["timestamp"] = float(timestamp)
 
 
-def parse_client_as_sdk(value):
-    if not value:
-        return {}
-    try:
-        name, version = value.split("/", 1)
-    except ValueError:
-        try:
-            name, version = value.split(" ", 1)
-        except ValueError:
-            return {}
-    return {"name": name, "version": version}
-
-
 def plugin_is_regression(group, event):
     project = event.project
     for plugin in plugins.for_project(project):
@@ -237,36 +207,6 @@ class ScoreClause(Func):
         return (sql, [])
 
 
-def add_meta_errors(errors, meta):
-    for field_meta in meta:
-        original_value = field_meta.get().get("val")
-
-        for i, (err_type, err_data) in enumerate(field_meta.iter_errors()):
-            error = dict(err_data)
-            error["type"] = err_type
-            if field_meta.path:
-                error["name"] = field_meta.path
-            if i == 0 and original_value is not None:
-                error["value"] = original_value
-            errors.append(error)
-
-
-def _decode_event(data, content_encoding):
-    if isinstance(data, six.binary_type):
-        if content_encoding == "gzip":
-            data = decompress_gzip(data)
-        elif content_encoding == "deflate":
-            data = decompress_deflate(data)
-        elif data[0] != b"{":
-            data = decode_and_decompress_data(data)
-        else:
-            data = decode_data(data)
-    if isinstance(data, six.text_type):
-        data = safely_load_json_string(data)
-
-    return CanonicalKeyDict(data)
-
-
 class EventManager(object):
     """
     Handles normalization in both the store endpoint and the save task. The
@@ -289,7 +229,7 @@ class EventManager(object):
         project_config=None,
         sent_at=None,
     ):
-        self._data = _decode_event(data, content_encoding=content_encoding)
+        self._data = CanonicalKeyDict(data)
         self.version = version
         self._project = project
         # if not explicitly specified try to get the grouping from project_config
@@ -310,51 +250,6 @@ class EventManager(object):
         self.project_config = project_config
         self.sent_at = sent_at
 
-    def process_csp_report(self):
-        """Only called from the CSP report endpoint."""
-        data = self._data
-
-        try:
-            interface = get_interface(data.pop("interface"))
-            report = data.pop("report")
-        except KeyError:
-            raise APIForbidden("No report or interface data")
-
-        # To support testing, we can either accept a built interface instance, or the raw data in
-        # which case we build the instance ourselves
-        try:
-            instance = report if isinstance(report, interface) else interface.from_raw(report)
-        except jsonschema.ValidationError as e:
-            raise APIError("Invalid security report: %s" % str(e).splitlines()[0])
-
-        def clean(d):
-            return dict([x for x in d.items() if x[1]])
-
-        data.update(
-            {
-                "logger": "csp",
-                "message": instance.get_message(),
-                "culprit": instance.get_culprit(),
-                instance.path: instance.to_json(),
-                "tags": instance.get_tags(),
-                "errors": [],
-                "user": {"ip_address": self._client_ip},
-                # Construct a faux Http interface based on the little information we have
-                # This is a bit weird, since we don't have nearly enough
-                # information to create an Http interface, but
-                # this automatically will pick up tags for the User-Agent
-                # which is actually important here for CSP
-                "request": {
-                    "url": instance.get_origin(),
-                    "headers": clean(
-                        {"User-Agent": self._user_agent, "Referer": instance.get_referrer()}
-                    ),
-                },
-            }
-        )
-
-        self._data = data
-
     def normalize(self, project_id=None):
         with metrics.timer("events.store.normalize.duration"):
             self._normalize_impl(project_id=project_id)
@@ -388,41 +283,6 @@ class EventManager(object):
 
         self._data = CanonicalKeyDict(rust_normalizer.normalize_event(dict(self._data)))
 
-    def should_filter(self):
-        """
-        returns (result: bool, reason: string or None)
-        Result is True if an event should be filtered
-        The reason for filtering is passed along as a string
-        so that we can store it in metrics
-        """
-        for name in SECURITY_REPORT_INTERFACES:
-            if name in self._data:
-                interface = get_interface(name)
-                if interface.to_python(self._data[name]).should_filter(self._project):
-                    return (True, FilterStatKeys.INVALID_CSP)
-
-        if self._client_ip and not is_valid_ip(self.project_config, self._client_ip):
-            return (True, FilterStatKeys.IP_ADDRESS)
-
-        release = self._data.get("release")
-        if release and not is_valid_release(self.project_config, release):
-            return (True, FilterStatKeys.RELEASE_VERSION)
-
-        error_message = (
-            get_path(self._data, "logentry", "formatted")
-            or get_path(self._data, "logentry", "message")
-            or ""
-        )
-        if error_message and not is_valid_error_message(self.project_config, error_message):
-            return (True, FilterStatKeys.ERROR_MESSAGE)
-
-        for exc in get_path(self._data, "exception", "values", filter=True, default=[]):
-            message = u": ".join([_f for _f in map(exc.get, ["type", "value"]) if _f])
-            if message and not is_valid_error_message(self.project_config, message):
-                return (True, FilterStatKeys.ERROR_MESSAGE)
-
-        return should_filter_event(self.project_config, self._data)
-
     def get_data(self):
         return self._data
 

src/sentry/integrations/vercel/uihook.py

@@ -6,11 +6,10 @@ import logging
 from django.http import HttpResponse
 from django.views.decorators.csrf import csrf_exempt
 
-from sentry.api.base import Endpoint
+from sentry.api.base import Endpoint, allow_cors_options
 from sentry.constants import ObjectStatus
 from sentry.models import Integration, Organization, OrganizationIntegration, OrganizationStatus
 from sentry.utils.http import absolute_uri
-from sentry.web.api import allow_cors_options
 from sentry.web.helpers import render_to_response
 
 logger = logging.getLogger("sentry.integrations.vercel")

src/sentry/interfaces/schemas.py

@@ -266,6 +266,7 @@ TAGS_TUPLES_SCHEMA = {
 
 TAGS_SCHEMA = {"anyOf": [TAGS_DICT_SCHEMA, TAGS_TUPLES_SCHEMA]}
 
+# XXX(markus): Remove in favor of Relay's schema definition
 EVENT_SCHEMA = {
     "type": "object",
     "properties": {