From 41eab91a864834ec170a4947c8d548de2016252c Mon Sep 17 00:00:00 2001 From: TAHRI Ahmed R Date: Thu, 16 May 2024 07:51:35 +0200 Subject: [PATCH] :zap: overall performance improvement by reducing redundant iterations from requests era (#121) also avoid parsing x509 certificate many times for the same remote peer --- HISTORY.md | 7 +++++++ pyproject.toml | 2 +- src/niquests/__version__.py | 4 ++-- src/niquests/extensions/_async_ocsp.py | 8 ++++++-- src/niquests/extensions/_ocsp.py | 8 +++++++- src/niquests/sessions.py | 15 ++++++++++++--- src/niquests/structures.py | 6 +++++- 7 files changed, 40 insertions(+), 10 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 49049706f5..5f1f27690f 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,13 @@ Release History =============== +3.6.4 (2024-05-16) +------------------ + +**Changed** +- Avoid parsing X509 peer certificate in the certificate revocation check process over and over again. +- Avoid iterating over header items redundantly or needlessly. + 3.6.3 (2024-05-06) ------------------ diff --git a/pyproject.toml b/pyproject.toml index b2556726ab..67bc9e5ffd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ description = "Niquests is a simple, yet elegant, HTTP library. It is a drop-in readme = "README.md" license-files = { paths = ["LICENSE"] } license = "Apache-2.0" -keywords = ["requests", "http/2", "http/3", "QUIC", "http", "https", "http client", "http/1.1", "ocsp", "revocation", "tls", "multiplexed", "dns-over-quic", "doq", "dns-over-tls", "dot", "dns-over-https", "doh", "dnssec"] +keywords = ["requests", "http2", "http3", "QUIC", "http", "https", "http client", "http/1.1", "ocsp", "revocation", "tls", "multiplexed", "dns-over-quic", "doq", "dns-over-tls", "dot", "dns-over-https", "doh", "dnssec"] authors = [ {name = "Kenneth Reitz", email = "me@kennethreitz.org"} ] diff --git a/src/niquests/__version__.py b/src/niquests/__version__.py index b73b75b459..12078d1119 100644 --- a/src/niquests/__version__.py +++ b/src/niquests/__version__.py @@ -9,9 +9,9 @@ __url__: str = "https://niquests.readthedocs.io" __version__: str -__version__ = "3.6.3" +__version__ = "3.6.4" -__build__: int = 0x030603 +__build__: int = 0x030604 __author__: str = "Kenneth Reitz" __author_email__: str = "me@kennethreitz.org" __license__: str = "Apache-2.0" diff --git a/src/niquests/extensions/_async_ocsp.py b/src/niquests/extensions/_async_ocsp.py index 6c47125490..810f588c29 100644 --- a/src/niquests/extensions/_async_ocsp.py +++ b/src/niquests/extensions/_async_ocsp.py @@ -53,7 +53,11 @@ async_recv_tls_and_decrypt, async_send_tls, ) -from ._ocsp import _str_fingerprint_of, readable_revocation_reason +from ._ocsp import ( + _str_fingerprint_of, + readable_revocation_reason, + _parse_x509_der_cached, +) async def _ask_nicely_for_issuer( @@ -320,7 +324,7 @@ async def verify( if not endpoints: return - peer_certificate = Certificate(conn_info.certificate_der) + peer_certificate = _parse_x509_der_cached(conn_info.certificate_der) async with _SharedRevocationStatusCache.lock(peer_certificate): # this feature, by default, is reserved for a reasonable usage. diff --git a/src/niquests/extensions/_ocsp.py b/src/niquests/extensions/_ocsp.py index e78ff48633..a59a76b49d 100644 --- a/src/niquests/extensions/_ocsp.py +++ b/src/niquests/extensions/_ocsp.py @@ -9,6 +9,7 @@ from hashlib import sha256 from random import randint from statistics import mean +from functools import lru_cache from qh3._hazmat import ( OCSPRequest, @@ -52,6 +53,11 @@ ) +@lru_cache(maxsize=64) +def _parse_x509_der_cached(der: bytes) -> Certificate: + return Certificate(der) + + def _str_fingerprint_of(certificate: Certificate) -> str: return ":".join( [format(i, "02x") for i in sha256(certificate.public_bytes()).digest()] @@ -327,7 +333,7 @@ def verify( if _SharedRevocationStatusCache.hold: return - peer_certificate = Certificate(conn_info.certificate_der) + peer_certificate = _parse_x509_der_cached(conn_info.certificate_der) cached_response = _SharedRevocationStatusCache.check(peer_certificate) if cached_response is not None: diff --git a/src/niquests/sessions.py b/src/niquests/sessions.py index 3263a9d8df..7beb5d58e5 100644 --- a/src/niquests/sessions.py +++ b/src/niquests/sessions.py @@ -128,17 +128,26 @@ def merge_setting( return session_setting # Bypass if not a dictionary (e.g. verify) - if not ( + if isinstance(session_setting, bool) or not ( isinstance(session_setting, Mapping) and isinstance(request_setting, Mapping) ): return request_setting - merged_setting = dict_class(to_key_val_list(session_setting)) + if hasattr(session_setting, "copy"): + merged_setting = ( + session_setting.copy() + if session_setting.__class__ is dict_class + else dict_class(session_setting.copy()) + ) + else: + merged_setting = dict_class(to_key_val_list(session_setting)) + merged_setting.update(to_key_val_list(request_setting)) # Remove keys that are set to None. Extract keys first to avoid altering # the dictionary during iteration. - none_keys = [k for (k, v) in merged_setting.items() if v is None] + none_keys = [k for k in merged_setting if merged_setting[k] is None] + for key in none_keys: del merged_setting[key] diff --git a/src/niquests/structures.py b/src/niquests/structures.py index 949149bd5b..3c48f207e1 100644 --- a/src/niquests/structures.py +++ b/src/niquests/structures.py @@ -126,6 +126,10 @@ def lower_items(self) -> typing.Iterator[tuple[bytes | str, bytes | str]]: """Like iteritems(), but with all lowercase keys.""" return ((lowerkey, keyval[1]) for (lowerkey, keyval) in self._store.items()) + def items(self): + for k in self._store: + yield self._store[k] + def __eq__(self, other) -> bool: if isinstance(other, Mapping): other = CaseInsensitiveDict(other) @@ -136,7 +140,7 @@ def __eq__(self, other) -> bool: # Copy is required def copy(self) -> CaseInsensitiveDict: - return CaseInsensitiveDict(self._store.values()) + return CaseInsensitiveDict(self) def __repr__(self) -> str: return str(dict(self.items()))