ref: improve types a bit for some 3rd party code (#39687)

- also remove `object_pairs_hook=OrderedDict` as python3.6+ dicts are
ordered




<!-- Describe your PR here. -->

mypy.ini

@@ -165,8 +165,6 @@ no_implicit_reexport=True
 [mypy-sentry.*]
 follow_imports = skip
 
-[mypy-bs4]
-ignore_missing_imports = True
 [mypy-celery.*]
 ignore_missing_imports = True
 [mypy-confluent_kafka.*]
@@ -179,12 +177,8 @@ ignore_missing_imports = True
 ignore_missing_imports = True
 [mypy-honcho.*]
 ignore_missing_imports = True
-[mypy-jsonschema]
-ignore_missing_imports = True
 [mypy-kombu]
 ignore_missing_imports = True
-[mypy-lxml]
-ignore_missing_imports = True
 [mypy-mistune.*]
 ignore_missing_imports = True
 [mypy-parsimonious.*]

requirements-dev-frozen.txt

@@ -72,6 +72,7 @@ jsonschema==3.2.0
 kombu==4.6.11
 lazy-object-proxy==1.7.1
 lxml==4.6.5
+lxml-stubs==0.4.0
 maxminddb==2.0.3
 mccabe==0.7.0
 milksnake==0.1.5
@@ -165,7 +166,9 @@ tomli==2.0.1
 toronado==0.1.0
 trio==0.21.0
 trio-websocket==0.9.2
+types-beautifulsoup4==4.11.6
 types-freezegun==1.1.10
+types-jsonschema==4.16.1
 types-psycopg2==2.9.21
 types-python-dateutil==2.8.19
 types-pytz==2022.1.2

requirements-dev-only-frozen.txt

@@ -29,6 +29,7 @@ isodate==0.6.1
 isort==5.10.1
 jsonschema==4.5.1
 lazy-object-proxy==1.7.1
+lxml-stubs==0.4.0
 mccabe==0.7.0
 mock==4.0.3
 more-itertools==8.13.0
@@ -69,7 +70,9 @@ six==1.16.0
 tokenize-rt==4.2.1
 toml==0.10.2
 tomli==2.0.1
+types-beautifulsoup4==4.11.6
 types-freezegun==1.1.10
+types-jsonschema==4.16.1
 types-psycopg2==2.9.21
 types-python-dateutil==2.8.19
 types-pytz==2022.1.2

requirements-dev.txt

@@ -27,12 +27,15 @@ pip-tools>=6.7.0
 packaging>=21.3
 
 # for type checking
+lxml-stubs
 msgpack-types>=0.2.0
 mypy>=0.971
+types-beautifulsoup4
 types-freezegun
-types-pyyaml
+types-jsonschema
+types-psycopg2
 types-python-dateutil
 types-pytz
+types-pyyaml
 types-redis
 types-requests
-types-psycopg2

src/sentry/shared_integrations/exceptions/base.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
-from collections import OrderedDict
-from typing import Any, Mapping
+from typing import Any
 
 from bs4 import BeautifulSoup
 from requests import Response
@@ -11,27 +10,22 @@ from sentry.utils import json
 
 class ApiError(Exception):
     code: int | None = None
-    json: Mapping[str, Any] | None = None
-    xml: str | None = None
 
     def __init__(self, text: str, code: int | None = None, url: str | None = None) -> None:
         if code is not None:
             self.code = code
         self.text = text
         self.url = url
-        self.xml = None
+        self.json: dict[str, Any] | None = None
+        self.xml: BeautifulSoup | None = None
         # TODO(dcramer): pull in XML support from Jira
         if text:
             try:
-                self.json = json.loads(text, object_pairs_hook=OrderedDict)
+                self.json = json.loads(text)
             except (json.JSONDecodeError, ValueError):
                 if self.text[:5] == "<?xml":
                     # perhaps it's XML?
                     self.xml = BeautifulSoup(self.text, "xml")
-                # must be an awful code.
-                self.json = None
-        else:
-            self.json = None
         super().__init__(text[:1024])
 
     @classmethod

src/sentry/shared_integrations/response/base.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-from collections import OrderedDict
 from typing import Any, Mapping
 
 import requests
@@ -73,7 +72,7 @@ class BaseApiResponse:
         # to decode it anyways
         if "application/json" not in response.headers.get("Content-Type", ""):
             try:
-                data = json.loads(response.text, object_pairs_hook=OrderedDict)
+                data = json.loads(response.text)
             except (TypeError, ValueError):
                 if allow_text:
                     return TextApiResponse(response.text, response.headers, response.status_code)
@@ -83,7 +82,7 @@ class BaseApiResponse:
         elif response.text == "":
             return TextApiResponse(response.text, response.headers, response.status_code)
         else:
-            data = json.loads(response.text, object_pairs_hook=OrderedDict)
+            data = json.loads(response.text)
 
         if isinstance(data, dict):
             return MappingApiResponse(data, response.headers, response.status_code)

src/sentry/utils/email/message_builder.py

@@ -8,7 +8,7 @@ from operator import attrgetter
 from random import randrange
 from typing import Any, Callable, Iterable, Mapping, MutableMapping, Sequence
 
-import lxml
+import lxml.html
 import toronado
 from django.core.mail import EmailMultiAlternatives
 from django.utils.encoding import force_text