helpers.py 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029
  1. """Various helper functions"""
  2. import asyncio
  3. import base64
  4. import binascii
  5. import contextlib
  6. import datetime
  7. import enum
  8. import functools
  9. import inspect
  10. import netrc
  11. import os
  12. import platform
  13. import re
  14. import sys
  15. import time
  16. import warnings
  17. import weakref
  18. from collections import namedtuple
  19. from contextlib import suppress
  20. from email.parser import HeaderParser
  21. from email.utils import parsedate
  22. from math import ceil
  23. from pathlib import Path
  24. from types import TracebackType
  25. from typing import (
  26. Any,
  27. Callable,
  28. ContextManager,
  29. Dict,
  30. Generator,
  31. Generic,
  32. Iterable,
  33. Iterator,
  34. List,
  35. Mapping,
  36. Optional,
  37. Pattern,
  38. Protocol,
  39. Tuple,
  40. Type,
  41. TypeVar,
  42. Union,
  43. get_args,
  44. overload,
  45. )
  46. from urllib.parse import quote
  47. from urllib.request import getproxies, proxy_bypass
  48. import attr
  49. from multidict import MultiDict, MultiDictProxy, MultiMapping
  50. from yarl import URL
  51. from . import hdrs
  52. from .log import client_logger, internal_logger
  53. if sys.version_info >= (3, 11):
  54. import asyncio as async_timeout
  55. else:
  56. import async_timeout
  57. __all__ = ("BasicAuth", "ChainMapProxy", "ETag")
  58. IS_MACOS = platform.system() == "Darwin"
  59. IS_WINDOWS = platform.system() == "Windows"
  60. PY_310 = sys.version_info >= (3, 10)
  61. PY_311 = sys.version_info >= (3, 11)
  62. _T = TypeVar("_T")
  63. _S = TypeVar("_S")
  64. _SENTINEL = enum.Enum("_SENTINEL", "sentinel")
  65. sentinel = _SENTINEL.sentinel
  66. NO_EXTENSIONS = bool(os.environ.get("AIOHTTP_NO_EXTENSIONS"))
  67. DEBUG = sys.flags.dev_mode or (
  68. not sys.flags.ignore_environment and bool(os.environ.get("PYTHONASYNCIODEBUG"))
  69. )
  70. CHAR = {chr(i) for i in range(0, 128)}
  71. CTL = {chr(i) for i in range(0, 32)} | {
  72. chr(127),
  73. }
  74. SEPARATORS = {
  75. "(",
  76. ")",
  77. "<",
  78. ">",
  79. "@",
  80. ",",
  81. ";",
  82. ":",
  83. "\\",
  84. '"',
  85. "/",
  86. "[",
  87. "]",
  88. "?",
  89. "=",
  90. "{",
  91. "}",
  92. " ",
  93. chr(9),
  94. }
  95. TOKEN = CHAR ^ CTL ^ SEPARATORS
  96. class noop:
  97. def __await__(self) -> Generator[None, None, None]:
  98. yield
  99. class BasicAuth(namedtuple("BasicAuth", ["login", "password", "encoding"])):
  100. """Http basic authentication helper."""
  101. def __new__(
  102. cls, login: str, password: str = "", encoding: str = "latin1"
  103. ) -> "BasicAuth":
  104. if login is None:
  105. raise ValueError("None is not allowed as login value")
  106. if password is None:
  107. raise ValueError("None is not allowed as password value")
  108. if ":" in login:
  109. raise ValueError('A ":" is not allowed in login (RFC 1945#section-11.1)')
  110. return super().__new__(cls, login, password, encoding)
  111. @classmethod
  112. def decode(cls, auth_header: str, encoding: str = "latin1") -> "BasicAuth":
  113. """Create a BasicAuth object from an Authorization HTTP header."""
  114. try:
  115. auth_type, encoded_credentials = auth_header.split(" ", 1)
  116. except ValueError:
  117. raise ValueError("Could not parse authorization header.")
  118. if auth_type.lower() != "basic":
  119. raise ValueError("Unknown authorization method %s" % auth_type)
  120. try:
  121. decoded = base64.b64decode(
  122. encoded_credentials.encode("ascii"), validate=True
  123. ).decode(encoding)
  124. except binascii.Error:
  125. raise ValueError("Invalid base64 encoding.")
  126. try:
  127. # RFC 2617 HTTP Authentication
  128. # https://www.ietf.org/rfc/rfc2617.txt
  129. # the colon must be present, but the username and password may be
  130. # otherwise blank.
  131. username, password = decoded.split(":", 1)
  132. except ValueError:
  133. raise ValueError("Invalid credentials.")
  134. return cls(username, password, encoding=encoding)
  135. @classmethod
  136. def from_url(cls, url: URL, *, encoding: str = "latin1") -> Optional["BasicAuth"]:
  137. """Create BasicAuth from url."""
  138. if not isinstance(url, URL):
  139. raise TypeError("url should be yarl.URL instance")
  140. if url.user is None:
  141. return None
  142. return cls(url.user, url.password or "", encoding=encoding)
  143. def encode(self) -> str:
  144. """Encode credentials."""
  145. creds = (f"{self.login}:{self.password}").encode(self.encoding)
  146. return "Basic %s" % base64.b64encode(creds).decode(self.encoding)
  147. def strip_auth_from_url(url: URL) -> Tuple[URL, Optional[BasicAuth]]:
  148. auth = BasicAuth.from_url(url)
  149. if auth is None:
  150. return url, None
  151. else:
  152. return url.with_user(None), auth
  153. def netrc_from_env() -> Optional[netrc.netrc]:
  154. """Load netrc from file.
  155. Attempt to load it from the path specified by the env-var
  156. NETRC or in the default location in the user's home directory.
  157. Returns None if it couldn't be found or fails to parse.
  158. """
  159. netrc_env = os.environ.get("NETRC")
  160. if netrc_env is not None:
  161. netrc_path = Path(netrc_env)
  162. else:
  163. try:
  164. home_dir = Path.home()
  165. except RuntimeError as e: # pragma: no cover
  166. # if pathlib can't resolve home, it may raise a RuntimeError
  167. client_logger.debug(
  168. "Could not resolve home directory when "
  169. "trying to look for .netrc file: %s",
  170. e,
  171. )
  172. return None
  173. netrc_path = home_dir / ("_netrc" if IS_WINDOWS else ".netrc")
  174. try:
  175. return netrc.netrc(str(netrc_path))
  176. except netrc.NetrcParseError as e:
  177. client_logger.warning("Could not parse .netrc file: %s", e)
  178. except OSError as e:
  179. netrc_exists = False
  180. with contextlib.suppress(OSError):
  181. netrc_exists = netrc_path.is_file()
  182. # we couldn't read the file (doesn't exist, permissions, etc.)
  183. if netrc_env or netrc_exists:
  184. # only warn if the environment wanted us to load it,
  185. # or it appears like the default file does actually exist
  186. client_logger.warning("Could not read .netrc file: %s", e)
  187. return None
  188. @attr.s(auto_attribs=True, frozen=True, slots=True)
  189. class ProxyInfo:
  190. proxy: URL
  191. proxy_auth: Optional[BasicAuth]
  192. def basicauth_from_netrc(netrc_obj: Optional[netrc.netrc], host: str) -> BasicAuth:
  193. """
  194. Return :py:class:`~aiohttp.BasicAuth` credentials for ``host`` from ``netrc_obj``.
  195. :raises LookupError: if ``netrc_obj`` is :py:data:`None` or if no
  196. entry is found for the ``host``.
  197. """
  198. if netrc_obj is None:
  199. raise LookupError("No .netrc file found")
  200. auth_from_netrc = netrc_obj.authenticators(host)
  201. if auth_from_netrc is None:
  202. raise LookupError(f"No entry for {host!s} found in the `.netrc` file.")
  203. login, account, password = auth_from_netrc
  204. # TODO(PY311): username = login or account
  205. # Up to python 3.10, account could be None if not specified,
  206. # and login will be empty string if not specified. From 3.11,
  207. # login and account will be empty string if not specified.
  208. username = login if (login or account is None) else account
  209. # TODO(PY311): Remove this, as password will be empty string
  210. # if not specified
  211. if password is None:
  212. password = ""
  213. return BasicAuth(username, password)
  214. def proxies_from_env() -> Dict[str, ProxyInfo]:
  215. proxy_urls = {
  216. k: URL(v)
  217. for k, v in getproxies().items()
  218. if k in ("http", "https", "ws", "wss")
  219. }
  220. netrc_obj = netrc_from_env()
  221. stripped = {k: strip_auth_from_url(v) for k, v in proxy_urls.items()}
  222. ret = {}
  223. for proto, val in stripped.items():
  224. proxy, auth = val
  225. if proxy.scheme in ("https", "wss"):
  226. client_logger.warning(
  227. "%s proxies %s are not supported, ignoring", proxy.scheme.upper(), proxy
  228. )
  229. continue
  230. if netrc_obj and auth is None:
  231. if proxy.host is not None:
  232. try:
  233. auth = basicauth_from_netrc(netrc_obj, proxy.host)
  234. except LookupError:
  235. auth = None
  236. ret[proto] = ProxyInfo(proxy, auth)
  237. return ret
  238. def current_task(
  239. loop: Optional[asyncio.AbstractEventLoop] = None,
  240. ) -> "Optional[asyncio.Task[Any]]":
  241. return asyncio.current_task(loop=loop)
  242. def get_running_loop(
  243. loop: Optional[asyncio.AbstractEventLoop] = None,
  244. ) -> asyncio.AbstractEventLoop:
  245. if loop is None:
  246. loop = asyncio.get_event_loop()
  247. if not loop.is_running():
  248. warnings.warn(
  249. "The object should be created within an async function",
  250. DeprecationWarning,
  251. stacklevel=3,
  252. )
  253. if loop.get_debug():
  254. internal_logger.warning(
  255. "The object should be created within an async function", stack_info=True
  256. )
  257. return loop
  258. def isasyncgenfunction(obj: Any) -> bool:
  259. func = getattr(inspect, "isasyncgenfunction", None)
  260. if func is not None:
  261. return func(obj) # type: ignore[no-any-return]
  262. else:
  263. return False
  264. def get_env_proxy_for_url(url: URL) -> Tuple[URL, Optional[BasicAuth]]:
  265. """Get a permitted proxy for the given URL from the env."""
  266. if url.host is not None and proxy_bypass(url.host):
  267. raise LookupError(f"Proxying is disallowed for `{url.host!r}`")
  268. proxies_in_env = proxies_from_env()
  269. try:
  270. proxy_info = proxies_in_env[url.scheme]
  271. except KeyError:
  272. raise LookupError(f"No proxies found for `{url!s}` in the env")
  273. else:
  274. return proxy_info.proxy, proxy_info.proxy_auth
  275. @attr.s(auto_attribs=True, frozen=True, slots=True)
  276. class MimeType:
  277. type: str
  278. subtype: str
  279. suffix: str
  280. parameters: "MultiDictProxy[str]"
  281. @functools.lru_cache(maxsize=56)
  282. def parse_mimetype(mimetype: str) -> MimeType:
  283. """Parses a MIME type into its components.
  284. mimetype is a MIME type string.
  285. Returns a MimeType object.
  286. Example:
  287. >>> parse_mimetype('text/html; charset=utf-8')
  288. MimeType(type='text', subtype='html', suffix='',
  289. parameters={'charset': 'utf-8'})
  290. """
  291. if not mimetype:
  292. return MimeType(
  293. type="", subtype="", suffix="", parameters=MultiDictProxy(MultiDict())
  294. )
  295. parts = mimetype.split(";")
  296. params: MultiDict[str] = MultiDict()
  297. for item in parts[1:]:
  298. if not item:
  299. continue
  300. key, _, value = item.partition("=")
  301. params.add(key.lower().strip(), value.strip(' "'))
  302. fulltype = parts[0].strip().lower()
  303. if fulltype == "*":
  304. fulltype = "*/*"
  305. mtype, _, stype = fulltype.partition("/")
  306. stype, _, suffix = stype.partition("+")
  307. return MimeType(
  308. type=mtype, subtype=stype, suffix=suffix, parameters=MultiDictProxy(params)
  309. )
  310. def guess_filename(obj: Any, default: Optional[str] = None) -> Optional[str]:
  311. name = getattr(obj, "name", None)
  312. if name and isinstance(name, str) and name[0] != "<" and name[-1] != ">":
  313. return Path(name).name
  314. return default
  315. not_qtext_re = re.compile(r"[^\041\043-\133\135-\176]")
  316. QCONTENT = {chr(i) for i in range(0x20, 0x7F)} | {"\t"}
  317. def quoted_string(content: str) -> str:
  318. """Return 7-bit content as quoted-string.
  319. Format content into a quoted-string as defined in RFC5322 for
  320. Internet Message Format. Notice that this is not the 8-bit HTTP
  321. format, but the 7-bit email format. Content must be in usascii or
  322. a ValueError is raised.
  323. """
  324. if not (QCONTENT > set(content)):
  325. raise ValueError(f"bad content for quoted-string {content!r}")
  326. return not_qtext_re.sub(lambda x: "\\" + x.group(0), content)
  327. def content_disposition_header(
  328. disptype: str, quote_fields: bool = True, _charset: str = "utf-8", **params: str
  329. ) -> str:
  330. """Sets ``Content-Disposition`` header for MIME.
  331. This is the MIME payload Content-Disposition header from RFC 2183
  332. and RFC 7579 section 4.2, not the HTTP Content-Disposition from
  333. RFC 6266.
  334. disptype is a disposition type: inline, attachment, form-data.
  335. Should be valid extension token (see RFC 2183)
  336. quote_fields performs value quoting to 7-bit MIME headers
  337. according to RFC 7578. Set to quote_fields to False if recipient
  338. can take 8-bit file names and field values.
  339. _charset specifies the charset to use when quote_fields is True.
  340. params is a dict with disposition params.
  341. """
  342. if not disptype or not (TOKEN > set(disptype)):
  343. raise ValueError("bad content disposition type {!r}" "".format(disptype))
  344. value = disptype
  345. if params:
  346. lparams = []
  347. for key, val in params.items():
  348. if not key or not (TOKEN > set(key)):
  349. raise ValueError(
  350. "bad content disposition parameter" " {!r}={!r}".format(key, val)
  351. )
  352. if quote_fields:
  353. if key.lower() == "filename":
  354. qval = quote(val, "", encoding=_charset)
  355. lparams.append((key, '"%s"' % qval))
  356. else:
  357. try:
  358. qval = quoted_string(val)
  359. except ValueError:
  360. qval = "".join(
  361. (_charset, "''", quote(val, "", encoding=_charset))
  362. )
  363. lparams.append((key + "*", qval))
  364. else:
  365. lparams.append((key, '"%s"' % qval))
  366. else:
  367. qval = val.replace("\\", "\\\\").replace('"', '\\"')
  368. lparams.append((key, '"%s"' % qval))
  369. sparams = "; ".join("=".join(pair) for pair in lparams)
  370. value = "; ".join((value, sparams))
  371. return value
  372. class _TSelf(Protocol, Generic[_T]):
  373. _cache: Dict[str, _T]
  374. class reify(Generic[_T]):
  375. """Use as a class method decorator.
  376. It operates almost exactly like
  377. the Python `@property` decorator, but it puts the result of the
  378. method it decorates into the instance dict after the first call,
  379. effectively replacing the function it decorates with an instance
  380. variable. It is, in Python parlance, a data descriptor.
  381. """
  382. def __init__(self, wrapped: Callable[..., _T]) -> None:
  383. self.wrapped = wrapped
  384. self.__doc__ = wrapped.__doc__
  385. self.name = wrapped.__name__
  386. def __get__(self, inst: _TSelf[_T], owner: Optional[Type[Any]] = None) -> _T:
  387. try:
  388. try:
  389. return inst._cache[self.name]
  390. except KeyError:
  391. val = self.wrapped(inst)
  392. inst._cache[self.name] = val
  393. return val
  394. except AttributeError:
  395. if inst is None:
  396. return self
  397. raise
  398. def __set__(self, inst: _TSelf[_T], value: _T) -> None:
  399. raise AttributeError("reified property is read-only")
  400. reify_py = reify
  401. try:
  402. from ._helpers import reify as reify_c
  403. if not NO_EXTENSIONS:
  404. reify = reify_c # type: ignore[misc,assignment]
  405. except ImportError:
  406. pass
  407. _ipv4_pattern = (
  408. r"^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}"
  409. r"(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$"
  410. )
  411. _ipv6_pattern = (
  412. r"^(?:(?:(?:[A-F0-9]{1,4}:){6}|(?=(?:[A-F0-9]{0,4}:){0,6}"
  413. r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}$)(([0-9A-F]{1,4}:){0,5}|:)"
  414. r"((:[0-9A-F]{1,4}){1,5}:|:)|::(?:[A-F0-9]{1,4}:){5})"
  415. r"(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\.){3}"
  416. r"(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])|(?:[A-F0-9]{1,4}:){7}"
  417. r"[A-F0-9]{1,4}|(?=(?:[A-F0-9]{0,4}:){0,7}[A-F0-9]{0,4}$)"
  418. r"(([0-9A-F]{1,4}:){1,7}|:)((:[0-9A-F]{1,4}){1,7}|:)|(?:[A-F0-9]{1,4}:){7}"
  419. r":|:(:[A-F0-9]{1,4}){7})$"
  420. )
  421. _ipv4_regex = re.compile(_ipv4_pattern)
  422. _ipv6_regex = re.compile(_ipv6_pattern, flags=re.IGNORECASE)
  423. _ipv4_regexb = re.compile(_ipv4_pattern.encode("ascii"))
  424. _ipv6_regexb = re.compile(_ipv6_pattern.encode("ascii"), flags=re.IGNORECASE)
  425. def _is_ip_address(
  426. regex: Pattern[str], regexb: Pattern[bytes], host: Optional[Union[str, bytes]]
  427. ) -> bool:
  428. if host is None:
  429. return False
  430. if isinstance(host, str):
  431. return bool(regex.match(host))
  432. elif isinstance(host, (bytes, bytearray, memoryview)):
  433. return bool(regexb.match(host))
  434. else:
  435. raise TypeError(f"{host} [{type(host)}] is not a str or bytes")
  436. is_ipv4_address = functools.partial(_is_ip_address, _ipv4_regex, _ipv4_regexb)
  437. is_ipv6_address = functools.partial(_is_ip_address, _ipv6_regex, _ipv6_regexb)
  438. def is_ip_address(host: Optional[Union[str, bytes, bytearray, memoryview]]) -> bool:
  439. return is_ipv4_address(host) or is_ipv6_address(host)
  440. _cached_current_datetime: Optional[int] = None
  441. _cached_formatted_datetime = ""
  442. def rfc822_formatted_time() -> str:
  443. global _cached_current_datetime
  444. global _cached_formatted_datetime
  445. now = int(time.time())
  446. if now != _cached_current_datetime:
  447. # Weekday and month names for HTTP date/time formatting;
  448. # always English!
  449. # Tuples are constants stored in codeobject!
  450. _weekdayname = ("Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun")
  451. _monthname = (
  452. "", # Dummy so we can use 1-based month numbers
  453. "Jan",
  454. "Feb",
  455. "Mar",
  456. "Apr",
  457. "May",
  458. "Jun",
  459. "Jul",
  460. "Aug",
  461. "Sep",
  462. "Oct",
  463. "Nov",
  464. "Dec",
  465. )
  466. year, month, day, hh, mm, ss, wd, *tail = time.gmtime(now)
  467. _cached_formatted_datetime = "%s, %02d %3s %4d %02d:%02d:%02d GMT" % (
  468. _weekdayname[wd],
  469. day,
  470. _monthname[month],
  471. year,
  472. hh,
  473. mm,
  474. ss,
  475. )
  476. _cached_current_datetime = now
  477. return _cached_formatted_datetime
  478. def _weakref_handle(info: "Tuple[weakref.ref[object], str]") -> None:
  479. ref, name = info
  480. ob = ref()
  481. if ob is not None:
  482. with suppress(Exception):
  483. getattr(ob, name)()
  484. def weakref_handle(
  485. ob: object,
  486. name: str,
  487. timeout: float,
  488. loop: asyncio.AbstractEventLoop,
  489. timeout_ceil_threshold: float = 5,
  490. ) -> Optional[asyncio.TimerHandle]:
  491. if timeout is not None and timeout > 0:
  492. when = loop.time() + timeout
  493. if timeout >= timeout_ceil_threshold:
  494. when = ceil(when)
  495. return loop.call_at(when, _weakref_handle, (weakref.ref(ob), name))
  496. return None
  497. def call_later(
  498. cb: Callable[[], Any],
  499. timeout: float,
  500. loop: asyncio.AbstractEventLoop,
  501. timeout_ceil_threshold: float = 5,
  502. ) -> Optional[asyncio.TimerHandle]:
  503. if timeout is not None and timeout > 0:
  504. when = loop.time() + timeout
  505. if timeout > timeout_ceil_threshold:
  506. when = ceil(when)
  507. return loop.call_at(when, cb)
  508. return None
  509. class TimeoutHandle:
  510. """Timeout handle"""
  511. def __init__(
  512. self,
  513. loop: asyncio.AbstractEventLoop,
  514. timeout: Optional[float],
  515. ceil_threshold: float = 5,
  516. ) -> None:
  517. self._timeout = timeout
  518. self._loop = loop
  519. self._ceil_threshold = ceil_threshold
  520. self._callbacks: List[
  521. Tuple[Callable[..., None], Tuple[Any, ...], Dict[str, Any]]
  522. ] = []
  523. def register(
  524. self, callback: Callable[..., None], *args: Any, **kwargs: Any
  525. ) -> None:
  526. self._callbacks.append((callback, args, kwargs))
  527. def close(self) -> None:
  528. self._callbacks.clear()
  529. def start(self) -> Optional[asyncio.Handle]:
  530. timeout = self._timeout
  531. if timeout is not None and timeout > 0:
  532. when = self._loop.time() + timeout
  533. if timeout >= self._ceil_threshold:
  534. when = ceil(when)
  535. return self._loop.call_at(when, self.__call__)
  536. else:
  537. return None
  538. def timer(self) -> "BaseTimerContext":
  539. if self._timeout is not None and self._timeout > 0:
  540. timer = TimerContext(self._loop)
  541. self.register(timer.timeout)
  542. return timer
  543. else:
  544. return TimerNoop()
  545. def __call__(self) -> None:
  546. for cb, args, kwargs in self._callbacks:
  547. with suppress(Exception):
  548. cb(*args, **kwargs)
  549. self._callbacks.clear()
  550. class BaseTimerContext(ContextManager["BaseTimerContext"]):
  551. def assert_timeout(self) -> None:
  552. """Raise TimeoutError if timeout has been exceeded."""
  553. class TimerNoop(BaseTimerContext):
  554. def __enter__(self) -> BaseTimerContext:
  555. return self
  556. def __exit__(
  557. self,
  558. exc_type: Optional[Type[BaseException]],
  559. exc_val: Optional[BaseException],
  560. exc_tb: Optional[TracebackType],
  561. ) -> None:
  562. return
  563. class TimerContext(BaseTimerContext):
  564. """Low resolution timeout context manager"""
  565. def __init__(self, loop: asyncio.AbstractEventLoop) -> None:
  566. self._loop = loop
  567. self._tasks: List[asyncio.Task[Any]] = []
  568. self._cancelled = False
  569. def assert_timeout(self) -> None:
  570. """Raise TimeoutError if timer has already been cancelled."""
  571. if self._cancelled:
  572. raise asyncio.TimeoutError from None
  573. def __enter__(self) -> BaseTimerContext:
  574. task = current_task(loop=self._loop)
  575. if task is None:
  576. raise RuntimeError(
  577. "Timeout context manager should be used " "inside a task"
  578. )
  579. if self._cancelled:
  580. raise asyncio.TimeoutError from None
  581. self._tasks.append(task)
  582. return self
  583. def __exit__(
  584. self,
  585. exc_type: Optional[Type[BaseException]],
  586. exc_val: Optional[BaseException],
  587. exc_tb: Optional[TracebackType],
  588. ) -> Optional[bool]:
  589. if self._tasks:
  590. self._tasks.pop()
  591. if exc_type is asyncio.CancelledError and self._cancelled:
  592. raise asyncio.TimeoutError from None
  593. return None
  594. def timeout(self) -> None:
  595. if not self._cancelled:
  596. for task in set(self._tasks):
  597. task.cancel()
  598. self._cancelled = True
  599. def ceil_timeout(
  600. delay: Optional[float], ceil_threshold: float = 5
  601. ) -> async_timeout.Timeout:
  602. if delay is None or delay <= 0:
  603. return async_timeout.timeout(None)
  604. loop = get_running_loop()
  605. now = loop.time()
  606. when = now + delay
  607. if delay > ceil_threshold:
  608. when = ceil(when)
  609. return async_timeout.timeout_at(when)
  610. class HeadersMixin:
  611. ATTRS = frozenset(["_content_type", "_content_dict", "_stored_content_type"])
  612. _headers: MultiMapping[str]
  613. _content_type: Optional[str] = None
  614. _content_dict: Optional[Dict[str, str]] = None
  615. _stored_content_type: Union[str, None, _SENTINEL] = sentinel
  616. def _parse_content_type(self, raw: Optional[str]) -> None:
  617. self._stored_content_type = raw
  618. if raw is None:
  619. # default value according to RFC 2616
  620. self._content_type = "application/octet-stream"
  621. self._content_dict = {}
  622. else:
  623. msg = HeaderParser().parsestr("Content-Type: " + raw)
  624. self._content_type = msg.get_content_type()
  625. params = msg.get_params(())
  626. self._content_dict = dict(params[1:]) # First element is content type again
  627. @property
  628. def content_type(self) -> str:
  629. """The value of content part for Content-Type HTTP header."""
  630. raw = self._headers.get(hdrs.CONTENT_TYPE)
  631. if self._stored_content_type != raw:
  632. self._parse_content_type(raw)
  633. return self._content_type # type: ignore[return-value]
  634. @property
  635. def charset(self) -> Optional[str]:
  636. """The value of charset part for Content-Type HTTP header."""
  637. raw = self._headers.get(hdrs.CONTENT_TYPE)
  638. if self._stored_content_type != raw:
  639. self._parse_content_type(raw)
  640. return self._content_dict.get("charset") # type: ignore[union-attr]
  641. @property
  642. def content_length(self) -> Optional[int]:
  643. """The value of Content-Length HTTP header."""
  644. content_length = self._headers.get(hdrs.CONTENT_LENGTH)
  645. if content_length is not None:
  646. return int(content_length)
  647. else:
  648. return None
  649. def set_result(fut: "asyncio.Future[_T]", result: _T) -> None:
  650. if not fut.done():
  651. fut.set_result(result)
  652. _EXC_SENTINEL = BaseException()
  653. class ErrorableProtocol(Protocol):
  654. def set_exception(
  655. self,
  656. exc: BaseException,
  657. exc_cause: BaseException = ...,
  658. ) -> None:
  659. ... # pragma: no cover
  660. def set_exception(
  661. fut: "asyncio.Future[_T] | ErrorableProtocol",
  662. exc: BaseException,
  663. exc_cause: BaseException = _EXC_SENTINEL,
  664. ) -> None:
  665. """Set future exception.
  666. If the future is marked as complete, this function is a no-op.
  667. :param exc_cause: An exception that is a direct cause of ``exc``.
  668. Only set if provided.
  669. """
  670. if asyncio.isfuture(fut) and fut.done():
  671. return
  672. exc_is_sentinel = exc_cause is _EXC_SENTINEL
  673. exc_causes_itself = exc is exc_cause
  674. if not exc_is_sentinel and not exc_causes_itself:
  675. exc.__cause__ = exc_cause
  676. fut.set_exception(exc)
  677. @functools.total_ordering
  678. class AppKey(Generic[_T]):
  679. """Keys for static typing support in Application."""
  680. __slots__ = ("_name", "_t", "__orig_class__")
  681. # This may be set by Python when instantiating with a generic type. We need to
  682. # support this, in order to support types that are not concrete classes,
  683. # like Iterable, which can't be passed as the second parameter to __init__.
  684. __orig_class__: Type[object]
  685. def __init__(self, name: str, t: Optional[Type[_T]] = None):
  686. # Prefix with module name to help deduplicate key names.
  687. frame = inspect.currentframe()
  688. while frame:
  689. if frame.f_code.co_name == "<module>":
  690. module: str = frame.f_globals["__name__"]
  691. break
  692. frame = frame.f_back
  693. self._name = module + "." + name
  694. self._t = t
  695. def __lt__(self, other: object) -> bool:
  696. if isinstance(other, AppKey):
  697. return self._name < other._name
  698. return True # Order AppKey above other types.
  699. def __repr__(self) -> str:
  700. t = self._t
  701. if t is None:
  702. with suppress(AttributeError):
  703. # Set to type arg.
  704. t = get_args(self.__orig_class__)[0]
  705. if t is None:
  706. t_repr = "<<Unknown>>"
  707. elif isinstance(t, type):
  708. if t.__module__ == "builtins":
  709. t_repr = t.__qualname__
  710. else:
  711. t_repr = f"{t.__module__}.{t.__qualname__}"
  712. else:
  713. t_repr = repr(t)
  714. return f"<AppKey({self._name}, type={t_repr})>"
  715. class ChainMapProxy(Mapping[Union[str, AppKey[Any]], Any]):
  716. __slots__ = ("_maps",)
  717. def __init__(self, maps: Iterable[Mapping[Union[str, AppKey[Any]], Any]]) -> None:
  718. self._maps = tuple(maps)
  719. def __init_subclass__(cls) -> None:
  720. raise TypeError(
  721. "Inheritance class {} from ChainMapProxy "
  722. "is forbidden".format(cls.__name__)
  723. )
  724. @overload # type: ignore[override]
  725. def __getitem__(self, key: AppKey[_T]) -> _T:
  726. ...
  727. @overload
  728. def __getitem__(self, key: str) -> Any:
  729. ...
  730. def __getitem__(self, key: Union[str, AppKey[_T]]) -> Any:
  731. for mapping in self._maps:
  732. try:
  733. return mapping[key]
  734. except KeyError:
  735. pass
  736. raise KeyError(key)
  737. @overload # type: ignore[override]
  738. def get(self, key: AppKey[_T], default: _S) -> Union[_T, _S]:
  739. ...
  740. @overload
  741. def get(self, key: AppKey[_T], default: None = ...) -> Optional[_T]:
  742. ...
  743. @overload
  744. def get(self, key: str, default: Any = ...) -> Any:
  745. ...
  746. def get(self, key: Union[str, AppKey[_T]], default: Any = None) -> Any:
  747. try:
  748. return self[key]
  749. except KeyError:
  750. return default
  751. def __len__(self) -> int:
  752. # reuses stored hash values if possible
  753. return len(set().union(*self._maps))
  754. def __iter__(self) -> Iterator[Union[str, AppKey[Any]]]:
  755. d: Dict[Union[str, AppKey[Any]], Any] = {}
  756. for mapping in reversed(self._maps):
  757. # reuses stored hash values if possible
  758. d.update(mapping)
  759. return iter(d)
  760. def __contains__(self, key: object) -> bool:
  761. return any(key in m for m in self._maps)
  762. def __bool__(self) -> bool:
  763. return any(self._maps)
  764. def __repr__(self) -> str:
  765. content = ", ".join(map(repr, self._maps))
  766. return f"ChainMapProxy({content})"
  767. # https://tools.ietf.org/html/rfc7232#section-2.3
  768. _ETAGC = r"[!\x23-\x7E\x80-\xff]+"
  769. _ETAGC_RE = re.compile(_ETAGC)
  770. _QUOTED_ETAG = rf'(W/)?"({_ETAGC})"'
  771. QUOTED_ETAG_RE = re.compile(_QUOTED_ETAG)
  772. LIST_QUOTED_ETAG_RE = re.compile(rf"({_QUOTED_ETAG})(?:\s*,\s*|$)|(.)")
  773. ETAG_ANY = "*"
  774. @attr.s(auto_attribs=True, frozen=True, slots=True)
  775. class ETag:
  776. value: str
  777. is_weak: bool = False
  778. def validate_etag_value(value: str) -> None:
  779. if value != ETAG_ANY and not _ETAGC_RE.fullmatch(value):
  780. raise ValueError(
  781. f"Value {value!r} is not a valid etag. Maybe it contains '\"'?"
  782. )
  783. def parse_http_date(date_str: Optional[str]) -> Optional[datetime.datetime]:
  784. """Process a date string, return a datetime object"""
  785. if date_str is not None:
  786. timetuple = parsedate(date_str)
  787. if timetuple is not None:
  788. with suppress(ValueError):
  789. return datetime.datetime(*timetuple[:6], tzinfo=datetime.timezone.utc)
  790. return None
  791. def must_be_empty_body(method: str, code: int) -> bool:
  792. """Check if a request must return an empty body."""
  793. return (
  794. status_code_must_be_empty_body(code)
  795. or method_must_be_empty_body(method)
  796. or (200 <= code < 300 and method.upper() == hdrs.METH_CONNECT)
  797. )
  798. def method_must_be_empty_body(method: str) -> bool:
  799. """Check if a method must return an empty body."""
  800. # https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.1
  801. # https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.2
  802. return method.upper() == hdrs.METH_HEAD
  803. def status_code_must_be_empty_body(code: int) -> bool:
  804. """Check if a status code must return an empty body."""
  805. # https://datatracker.ietf.org/doc/html/rfc9112#section-6.3-2.1
  806. return code in {204, 304} or 100 <= code < 200
  807. def should_remove_content_length(method: str, code: int) -> bool:
  808. """Check if a Content-Length header should be removed.
  809. This should always be a subset of must_be_empty_body
  810. """
  811. # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.6-8
  812. # https://www.rfc-editor.org/rfc/rfc9110.html#section-15.4.5-4
  813. return (
  814. code in {204, 304}
  815. or 100 <= code < 200
  816. or (200 <= code < 300 and method.upper() == hdrs.METH_CONNECT)
  817. )