perf: move interfaces and integrations to orjson (#71055)

Part of `orjson` move to get rid of `sentry.utils.json`

src/sentry/shared_integrations/client/base.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from collections.abc import Callable, Mapping, Sequence
 from typing import Any, Literal, Self, Union, overload
 
+import orjson
 import sentry_sdk
 from django.core.cache import cache
 from requests import PreparedRequest, Request, Response
@@ -19,7 +20,7 @@ from sentry.models.integrations.utils import is_response_error, is_response_succ
 from sentry.net.http import SafeSession
 from sentry.services.hybrid_cloud.integration import integration_service
 from sentry.services.hybrid_cloud.organization import organization_service
-from sentry.utils import json, metrics
+from sentry.utils import metrics
 from sentry.utils.audit import create_system_audit_entry
 from sentry.utils.hashlib import md5_text
 
@@ -332,7 +333,7 @@ class BaseApiClient(TrackResponseMixin):
         data = kwargs.get("data", None)
         query = ""
         if kwargs.get("params", None):
-            query = json.dumps(kwargs.get("params"))
+            query = orjson.dumps(kwargs.get("params")).decode()
 
         key = self.get_cache_key(path, query, data)
         result: BaseApiResponseX | None = self.check_cache(key)

src/sentry/shared_integrations/exceptions/__init__.py

@@ -5,13 +5,12 @@ from collections.abc import Mapping
 from typing import Any, Protocol
 from urllib.parse import urlparse
 
+import orjson
 from bs4 import BeautifulSoup
 from requests import Response
 from requests.adapters import RetryError
 from requests.exceptions import RequestException
 
-from sentry.utils import json
-
 __all__ = (
     "ApiConnectionResetError",
     "ApiError",
@@ -53,8 +52,8 @@ class ApiError(Exception):
         # TODO(dcramer): pull in XML support from Jira
         if text:
             try:
-                self.json = json.loads(text)
-            except (json.JSONDecodeError, ValueError):
+                self.json = orjson.loads(text)
+            except orjson.JSONDecodeError:
                 if self.text[:5] == "<?xml":
                     # perhaps it's XML?
                     self.xml = BeautifulSoup(self.text, "xml")

src/sentry/shared_integrations/response/base.py

@@ -3,12 +3,12 @@ from __future__ import annotations
 from collections.abc import Mapping
 from typing import Any
 
+import orjson
 import requests
 from django.utils.functional import cached_property
 from requests import Response
 
 from sentry.shared_integrations.exceptions import UnsupportedResponseType
-from sentry.utils import json
 
 
 class BaseApiResponse:
@@ -77,8 +77,8 @@ class BaseApiResponse:
         # to decode it anyways
         if "application/json" not in response.headers.get("Content-Type", ""):
             try:
-                data = json.loads(response.text)
-            except (TypeError, ValueError):
+                data = orjson.loads(response.text)
+            except orjson.JSONDecodeError:
                 if allow_text:
                     return TextApiResponse(response.text, response.headers, response.status_code)
                 raise UnsupportedResponseType(
@@ -87,7 +87,7 @@ class BaseApiResponse:
         elif response.text == "":
             return TextApiResponse(response.text, response.headers, response.status_code)
         else:
-            data = json.loads(response.text)
+            data = orjson.loads(response.text)
 
         if isinstance(data, dict):
             return MappingApiResponse(data, response.headers, response.status_code)

src/sentry/testutils/cases.py

@@ -17,6 +17,7 @@ from urllib.parse import urlencode
 from uuid import uuid4
 from zlib import compress
 
+import orjson
 import pytest
 import requests
 import responses
@@ -135,7 +136,6 @@ from sentry.testutils.helpers.notifications import TEST_ISSUE_OCCURRENCE
 from sentry.testutils.helpers.slack import install_slack
 from sentry.testutils.pytest.selenium import Browser
 from sentry.types.condition_activity import ConditionActivity, ConditionActivityType
-from sentry.utils import json
 from sentry.utils.auth import SsoSession
 from sentry.utils.json import dumps_htmlsafe
 from sentry.utils.performance_issues.performance_detection import detect_performance_problems
@@ -726,7 +726,7 @@ class APITestCase(BaseTestCase, BaseAPITestCase):
         if raw_data and isinstance(raw_data, bytes):
             raw_data = raw_data.decode("utf-8")
         if raw_data and isinstance(raw_data, str):
-            raw_data = json.loads(raw_data)
+            raw_data = orjson.loads(raw_data)
         data = raw_data or params
         method = params.pop("method", self.method).lower()
 
@@ -1167,7 +1167,7 @@ class AcceptanceTestCase(TransactionTestCase):
             res = self.client.put(
                 "/api/0/assistant/",
                 content_type="application/json",
-                data=json.dumps({"guide": item, "status": "viewed", "useful": True}),
+                data=orjson.dumps({"guide": item, "status": "viewed", "useful": True}).decode(),
             )
             assert res.status_code == 201, res.content
 
@@ -1245,7 +1245,7 @@ class SnubaTestCase(BaseTestCase):
 
     @classmethod
     def snuba_update_config(cls, config_vals):
-        return _snuba_pool.request("POST", "/config.json", body=json.dumps(config_vals))
+        return _snuba_pool.request("POST", "/config.json", body=orjson.dumps(config_vals).decode())
 
     def init_snuba(self):
         self.snuba_eventstream = SnubaEventStream()
@@ -1335,7 +1335,7 @@ class SnubaTestCase(BaseTestCase):
             _snuba_pool.urlopen(
                 "POST",
                 "/tests/entities/groupedmessage/insert",
-                body=json.dumps(data),
+                body=orjson.dumps(data).decode(),
                 headers={},
             ).status
             == 200
@@ -1345,7 +1345,8 @@ class SnubaTestCase(BaseTestCase):
         data = [self.__wrap_group(group)]
         assert (
             requests.post(
-                settings.SENTRY_SNUBA + "/tests/entities/outcomes/insert", data=json.dumps(data)
+                settings.SENTRY_SNUBA + "/tests/entities/outcomes/insert",
+                data=orjson.dumps(data),
             ).status_code
             == 200
         )
@@ -1353,7 +1354,8 @@ class SnubaTestCase(BaseTestCase):
     def store_span(self, span):
         assert (
             requests.post(
-                settings.SENTRY_SNUBA + "/tests/entities/spans/insert", data=json.dumps([span])
+                settings.SENTRY_SNUBA + "/tests/entities/spans/insert",
+                data=orjson.dumps([span]),
             ).status_code
             == 200
         )
@@ -1361,7 +1363,8 @@ class SnubaTestCase(BaseTestCase):
     def store_spans(self, spans):
         assert (
             requests.post(
-                settings.SENTRY_SNUBA + "/tests/entities/spans/insert", data=json.dumps(spans)
+                settings.SENTRY_SNUBA + "/tests/entities/spans/insert",
+                data=orjson.dumps(spans),
             ).status_code
             == 200
         )
@@ -1398,7 +1401,7 @@ class SnubaTestCase(BaseTestCase):
         assert (
             requests.post(
                 settings.SENTRY_SNUBA + "/tests/entities/metrics_summaries/insert",
-                data=json.dumps(rows),
+                data=orjson.dumps(rows),
             ).status_code
             == 200
         )
@@ -1466,7 +1469,8 @@ class SnubaTestCase(BaseTestCase):
 
         assert (
             requests.post(
-                settings.SENTRY_SNUBA + "/tests/entities/events/insert", data=json.dumps(events)
+                settings.SENTRY_SNUBA + "/tests/entities/events/insert",
+                data=orjson.dumps(events),
             ).status_code
             == 200
         )
@@ -1788,7 +1792,7 @@ class BaseMetricsTestCase(SnubaTestCase):
         assert (
             requests.post(
                 settings.SENTRY_SNUBA + cls.snuba_endpoint.format(entity=entity),
-                data=json.dumps(buckets),
+                data=orjson.dumps(buckets),
             ).status_code
             == 200
         )
@@ -2309,7 +2313,8 @@ class OutcomesSnubaTest(TestCase):
 
         assert (
             requests.post(
-                settings.SENTRY_SNUBA + "/tests/entities/outcomes/insert", data=json.dumps(outcomes)
+                settings.SENTRY_SNUBA + "/tests/entities/outcomes/insert",
+                data=orjson.dumps(outcomes),
             ).status_code
             == 200
         )

src/sentry/testutils/helpers/apigateway.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from urllib.parse import parse_qs
 
+import orjson
 import responses
 from django.conf import settings
 from django.http import HttpResponseRedirect
@@ -15,7 +16,6 @@ from sentry.api.base import Endpoint, control_silo_endpoint, region_silo_endpoin
 from sentry.api.bases.organization import ControlSiloOrganizationEndpoint, OrganizationEndpoint
 from sentry.testutils.cases import APITestCase
 from sentry.types.region import Region, RegionCategory
-from sentry.utils import json
 
 
 @control_silo_endpoint
@@ -79,11 +79,11 @@ def verify_request_body(body, headers):
 
     def request_callback(request):
         if request.headers.get("content-type") == "application/json":
-            assert json.load(request.body) == body
+            assert orjson.loads(request.body.read()) == body
         else:
             assert request.body.read() == body
         assert (request.headers[key] == headers[key] for key in headers)
-        return 200, {}, json.dumps({"proxy": True})
+        return 200, {}, orjson.dumps({"proxy": True}).decode()
 
     return request_callback
 
@@ -93,7 +93,7 @@ def verify_request_headers(headers):
 
     def request_callback(request):
         assert (request.headers[key] == headers[key] for key in headers)
-        return 200, {}, json.dumps({"proxy": True})
+        return 200, {}, orjson.dumps({"proxy": True}).decode()
 
     return request_callback
 
@@ -110,7 +110,7 @@ def verify_request_params(params, headers):
                 assert request_params[key] == params[key]
             else:
                 assert request_params[key][0] == params[key]
-        return 200, {}, json.dumps({"proxy": True})
+        return 200, {}, orjson.dumps({"proxy": True}).decode()
 
     return request_callback
 
@@ -121,7 +121,7 @@ def verify_file_body(file_body, headers):
     def request_callback(request):
         assert file_body in request.body.read()
         assert (request.headers[key] == headers[key] for key in headers)
-        return 200, {}, json.dumps({"proxy": True})
+        return 200, {}, orjson.dumps({"proxy": True}).decode()
 
     return request_callback
 
@@ -150,14 +150,14 @@ class ApiGatewayTestCase(APITestCase):
         responses.add(
             responses.GET,
             f"{self.REGION.address}/get",
-            body=json.dumps({"proxy": True}),
+            body=orjson.dumps({"proxy": True}).decode(),
             content_type="application/json",
             adding_headers={"test": "header"},
         )
         responses.add(
             responses.GET,
             f"{self.REGION.address}/error",
-            body=json.dumps({"proxy": True}),
+            body=orjson.dumps({"proxy": True}).decode(),
             status=400,
             content_type="application/json",
             adding_headers={"test": "header"},
@@ -167,12 +167,12 @@ class ApiGatewayTestCase(APITestCase):
 
         # Echos the request body and header back for verification
         def return_request_body(request):
-            return (200, request.headers, request.body)
+            return 200, request.headers, request.body
 
         # Echos the query params and header back for verification
         def return_request_params(request):
             params = parse_qs(request.url.split("?")[1])
-            return (200, request.headers, json.dumps(params).encode())
+            return 200, request.headers, orjson.dumps(params)
 
         responses.add_callback(responses.GET, f"{self.REGION.address}/echo", return_request_params)
         responses.add_callback(responses.POST, f"{self.REGION.address}/echo", return_request_body)

src/sentry/testutils/helpers/backups.py

@@ -10,6 +10,7 @@ from typing import Any
 from unittest.mock import MagicMock
 from uuid import uuid4
 
+import orjson
 from cryptography.hazmat.backends import default_backend
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric import rsa
@@ -102,7 +103,6 @@ from sentry.testutils.cases import TransactionTestCase
 from sentry.testutils.factories import get_fixture_path
 from sentry.testutils.silo import assume_test_silo_mode
 from sentry.types.token import AuthTokenType
-from sentry.utils import json
 
 __all__ = [
     "export_to_file",
@@ -153,8 +153,8 @@ def export_to_file(path: Path, scope: ExportScope, filter_by: set[str] | None =
         else:
             raise AssertionError(f"Unknown `ExportScope`: `{scope.name}`")
 
-    with open(json_file_path) as tmp_file:
-        output = json.load(tmp_file)
+    with open(json_file_path, "rb") as tmp_file:
+        output = orjson.loads(tmp_file.read())
     return output
 
 
@@ -224,7 +224,7 @@ def export_to_encrypted_tarball(
     # part of the encrypt/decrypt tar-ing API, so we need to ensure that these exact names are
     # present and contain the data we expect.
     with open(tar_file_path, "rb") as f:
-        return json.loads(
+        return orjson.loads(
             decrypt_encrypted_tarball(f, LocalFileDecryptor.from_bytes(private_key_pem))
         )
 
@@ -682,32 +682,40 @@ class BackupTestCase(TransactionTestCase):
 
     @cached_property
     def _json_of_exhaustive_user_with_maximum_privileges(self) -> Any:
-        with open(get_fixture_path("backup", "user-with-maximum-privileges.json")) as backup_file:
-            return json.load(backup_file)
+        with open(
+            get_fixture_path("backup", "user-with-maximum-privileges.json"), "rb"
+        ) as backup_file:
+            return orjson.loads(backup_file.read())
 
     def json_of_exhaustive_user_with_maximum_privileges(self) -> Any:
         return deepcopy(self._json_of_exhaustive_user_with_maximum_privileges)
 
     @cached_property
     def _json_of_exhaustive_user_with_minimum_privileges(self) -> Any:
-        with open(get_fixture_path("backup", "user-with-minimum-privileges.json")) as backup_file:
-            return json.load(backup_file)
+        with open(
+            get_fixture_path("backup", "user-with-minimum-privileges.json"), "rb"
+        ) as backup_file:
+            return orjson.loads(backup_file.read())
 
     def json_of_exhaustive_user_with_minimum_privileges(self) -> Any:
         return deepcopy(self._json_of_exhaustive_user_with_minimum_privileges)
 
     @cached_property
     def _json_of_exhaustive_user_with_roles_no_superadmin(self) -> Any:
-        with open(get_fixture_path("backup", "user-with-roles-no-superadmin.json")) as backup_file:
-            return json.load(backup_file)
+        with open(
+            get_fixture_path("backup", "user-with-roles-no-superadmin.json"), "rb"
+        ) as backup_file:
+            return orjson.loads(backup_file.read())
 
     def json_of_exhaustive_user_with_roles_no_superadmin(self) -> Any:
         return deepcopy(self._json_of_exhaustive_user_with_roles_no_superadmin)
 
     @cached_property
     def _json_of_exhaustive_user_with_superadmin_no_roles(self) -> Any:
-        with open(get_fixture_path("backup", "user-with-superadmin-no-roles.json")) as backup_file:
-            return json.load(backup_file)
+        with open(
+            get_fixture_path("backup", "user-with-superadmin-no-roles.json"), "rb"
+        ) as backup_file:
+            return orjson.loads(backup_file.read())
 
     def json_of_exhaustive_user_with_superadmin_no_roles(self) -> Any:
         return deepcopy(self._json_of_exhaustive_user_with_superadmin_no_roles)
@@ -759,5 +767,5 @@ class BackupTestCase(TransactionTestCase):
         """
 
         data = self.generate_tmp_users_json()
-        with open(tmp_path, "w+") as tmp_file:
-            json.dump(data, tmp_file)
+        with open(tmp_path, "wb+") as tmp_file:
+            tmp_file.write(orjson.dumps(data))

src/sentry/testutils/helpers/slack.py

@@ -1,5 +1,6 @@
 from urllib.parse import parse_qs, quote
 
+import orjson
 import responses
 
 from sentry.integrations.slack.message_builder import SlackBody
@@ -13,7 +14,6 @@ from sentry.models.user import User
 from sentry.silo.base import SiloMode
 from sentry.testutils.silo import assume_test_silo_mode
 from sentry.types.integrations import EXTERNAL_PROVIDERS, ExternalProviders
-from sentry.utils import json
 
 
 def get_response_text(data: SlackBody) -> str:
@@ -95,7 +95,7 @@ def get_channel(index=0):
     assert len(responses.calls) >= 1
     data = parse_qs(responses.calls[index].request.body)
     assert "channel" in data
-    channel = json.loads(data["channel"][0])
+    channel = orjson.loads(data["channel"][0])
 
     return channel
 
@@ -105,7 +105,7 @@ def get_attachment(index=0):
     data = parse_qs(responses.calls[index].request.body)
     assert "text" in data
     assert "attachments" in data
-    attachments = json.loads(data["attachments"][0])
+    attachments = orjson.loads(data["attachments"][0])
 
     assert len(attachments) == 1
     return attachments[0], data["text"][0]
@@ -115,7 +115,7 @@ def get_attachment_no_text():
     assert len(responses.calls) >= 1
     data = parse_qs(responses.calls[0].request.body)
     assert "attachments" in data
-    attachments = json.loads(data["attachments"][0])
+    attachments = orjson.loads(data["attachments"][0])
     assert len(attachments) == 1
     return attachments[0]
 
@@ -125,7 +125,7 @@ def get_blocks_and_fallback_text(index=0):
     data = parse_qs(responses.calls[index].request.body)
     assert "blocks" in data
     assert "text" in data
-    blocks = json.loads(data["blocks"][0])
+    blocks = orjson.loads(data["blocks"][0])
     fallback_text = data["text"][0]
     return blocks, fallback_text
 
@@ -137,8 +137,8 @@ def get_block_kit_preview_url(index=0) -> str:
     assert data["blocks"][0]
 
     stringified_blocks = data["blocks"][0]
-    blocks = json.loads(data["blocks"][0])
-    stringified_blocks = json.dumps({"blocks": blocks})
+    blocks = orjson.loads(data["blocks"][0])
+    stringified_blocks = orjson.dumps({"blocks": blocks}).decode()
 
     encoded_blocks = quote(stringified_blocks)
     base_url = "https://app.slack.com/block-kit-builder/#"

src/sentry/testutils/performance_issues/event_generators.py

@@ -4,8 +4,9 @@ import os
 from copy import deepcopy
 from typing import Any
 
+import orjson
+
 from sentry.testutils.factories import get_fixture_path
-from sentry.utils import json
 
 from .span_builder import SpanBuilder
 
@@ -27,8 +28,8 @@ for (dirpath, dirnames, filenames) in os.walk(_fixture_path):
 
         [full_event_name, _] = relative_path.split(".")
 
-        with open(filepath) as f:
-            event = json.load(f)
+        with open(filepath, "rb") as f:
+            event = orjson.loads(f.read())
             event["project"] = PROJECT_ID
 
         EVENTS[full_event_name] = event

src/sentry/utils/assets.py

@@ -2,19 +2,18 @@ from __future__ import annotations
 
 import os.path
 
+import orjson
 from cachetools.func import ttl_cache
 from django.conf import settings
 
-from sentry.utils import json
-
 
 @ttl_cache(ttl=60)
 def _frontend_versions() -> dict[str, str]:
     try:
         with open(
-            os.path.join(settings.CONF_DIR, "settings", "frontend", "frontend-versions.json")
+            os.path.join(settings.CONF_DIR, "settings", "frontend", "frontend-versions.json"), "rb"
         ) as f:
-            return json.load(f)  # getsentry path
+            return orjson.loads(f.read())  # getsentry path
     except OSError:
         return {}  # common case for self-hosted
 

src/sentry/utils/codecs.py

@@ -2,10 +2,9 @@ import zlib
 from abc import ABC, abstractmethod
 from typing import Any, Generic, TypeVar
 
+import orjson
 import zstandard
 
-from sentry.utils import json
-
 T = TypeVar("T")
 
 TDecoded = TypeVar("TDecoded")
@@ -71,10 +70,10 @@ class JSONCodec(Codec[Any, str]):
     """
 
     def encode(self, value: Any) -> str:
-        return str(json.dumps(value))
+        return orjson.dumps(value).decode()
 
     def decode(self, value: str) -> Any:
-        return json.loads(value)
+        return orjson.loads(value)
 
 
 class ZlibCodec(Codec[bytes, bytes]):