From 43c2a12cb1761eae209ede18d0561d040e6fe1b0 Mon Sep 17 00:00:00 2001 From: TAHRI Ahmed R Date: Mon, 25 Mar 2024 06:48:38 +0100 Subject: [PATCH] :sparkle: Support for Happy Eyeballs (#101) 3.5.5 (2024-03-25) ------------------ **Added** - Support for Happy Eyeballs. This feature is disabled by default, you must pass `happy_eyeballs=True` within your session constructor or http adapter in order to leverage this. **Fixed** - Missed close implementation in AsyncSession causing the underlying poolmanager to remain open. - Additional OCSP requests (following a redirect) did not use specified custom DNS resolver. **Changed** - urllib3.future lower bound constraint has been raised to version 2.7.900 for the newly added happy eyeballs feature. --- HISTORY.md | 14 ++++++++++ README.md | 3 ++ docs/index.rst | 1 + docs/user/quickstart.rst | 38 ++++++++++++++++++++++++++ pyproject.toml | 2 +- src/niquests/__version__.py | 4 +-- src/niquests/_async.py | 13 ++++++++- src/niquests/_compat.py | 2 +- src/niquests/adapters.py | 26 ++++++++++++++++++ src/niquests/extensions/_async_ocsp.py | 5 +++- src/niquests/extensions/_ocsp.py | 3 +- src/niquests/models.py | 13 +++++---- src/niquests/sessions.py | 11 ++++++++ tests/test_async.py | 12 ++++++++ tests/test_live.py | 11 ++++++++ 15 files changed, 145 insertions(+), 13 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index f38f2c39f5..5311838332 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,20 @@ Release History =============== +3.5.5 (2024-03-25) +------------------ + +**Added** +- Support for Happy Eyeballs. This feature is disabled by default, you must pass `happy_eyeballs=True` within your session + constructor or http adapter in order to leverage this. + +**Fixed** +- Missed close implementation in AsyncSession causing the underlying poolmanager to remain open. +- Additional OCSP requests (following a redirect) did not use specified custom DNS resolver. + +**Changed** +- urllib3.future lower bound constraint has been raised to version 2.7.900 for the newly added happy eyeballs feature. + 3.5.4 (2024-03-17) ------------------ diff --git a/README.md b/README.md index a7dfb7bb0e..9fa524a86c 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Niquests, is the “**Safest**, **Fastest[^10]**, **Easiest**, and **Most advanc | `DNS over HTTPS` | ✅ | ❌ | ❌ | ❌ | | `DNS over QUIC` | ✅ | ❌ | ❌ | ❌ | | `DNS over TLS` | ✅ | ❌ | ❌ | ❌ | +| `Multiple DNS Resolver` | ✅ | ❌ | ❌ | ❌ | | `Network Fine Tuning & Inspect` | ✅ | ❌ | _Limited_[^6] | _Limited_[^6] | | `Certificate Revocation Protection` | ✅ | ❌ | ❌ | ❌ | | `Session Persistence` | ✅ | ✅ | ✅ | ✅ | @@ -36,6 +37,7 @@ Niquests, is the “**Safest**, **Fastest[^10]**, **Easiest**, and **Most advanc | `HTTP/HTTPS Proxies` | ✅ | ✅ | ✅ | ✅ | | `TLS-in-TLS Support` | ✅ | ✅ | ✅ | ✅ | | `Direct HTTP/3 Negotiation` | ✅[^9] | N/A[^8] | N/A[^8] | N/A[^8] | +| `Happy Eyeballs` | ✅ | ❌ | ❌ | ✅ | | `Package / SLSA Signed` | ✅ | ❌ | ❌ | ✅ | @@ -149,6 +151,7 @@ Niquests is ready for the demands of building scalable, robust and reliable HTTP - Streaming Downloads - HTTP/2 by default - HTTP/3 over QUIC +- Happy Eyeballs - Multiplexed! - Thread-safe! - DNSSEC! diff --git a/docs/index.rst b/docs/index.rst index 30c2e9b9fc..1cd545c51b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -85,6 +85,7 @@ Niquests is ready for today's web. - Streaming Downloads - HTTP/2 by default - HTTP/3 over QUIC +- Happy Eyeballs - Multiplexed! - Thread-safe! - DNSSEC! diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 294a87628e..346a1bb9ce 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -974,6 +974,44 @@ install Niquests with:: $ python -m pip install niquests[speedups] + +Happy Eyeballs +-------------- + +.. note:: Available since version 3.5.5+ + +Thanks to the underlying library (urllib3.future) we are able to serve the Happy Eyeballs feature, one toggle away. + +Happy Eyeballs (also called Fast Fallback) is an algorithm published by the IETF that makes dual-stack applications +(those that understand both IPv4 and IPv6) more responsive to users by attempting to connect using both IPv4 and IPv6 +at the same time (preferring IPv6), thus minimizing common problems experienced by users with imperfect IPv6 connections or setups. + +The name “happy eyeballs” derives from the term “eyeball” to describe endpoints which represent human Internet end-users, as opposed to servers. + +To enable Happy Eyeballs in Niquests, do as follow:: + + import niquests + + with niquests.Session(happy_eyeballs=True) as s: + ... + +Or.. in async:: + + import niquests + + async with niquests.AsyncSession(happy_eyeballs=True) as s: + ... + +A mere ``happy_eyeballs=True`` is sufficient to leverage its potential. + +.. note:: In case a server yield multiple IPv4 addresses but no IPv6, this still applies. Meaning that Niquests will connect concurrently to presented addresses and determine what is the fastest endpoint. + +.. note:: Like urllib3.future, you can pass an integer to increase the default number of concurrent connection to be tested. See https://urllib3future.readthedocs.io/en/latest/advanced-usage.html#happy-eyeballs to learn more. + +OCSP requests (certificate revocation checks) will follow given ``happy_eyeballs=True`` parameter. + +.. warning:: This feature is disabled by default and we are actually planning to make it enabled as the default in a future major. + ----------------------- Ready for more? Check out the :ref:`advanced ` section. diff --git a/pyproject.toml b/pyproject.toml index 596a263ed7..b1506f1c76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,7 @@ dynamic = ["version"] dependencies = [ "charset_normalizer>=2,<4", "idna>=2.5,<4", - "urllib3.future>=2.6.900,<3", + "urllib3.future>=2.7.900,<3", "wassima>=1.0.1,<2", "kiss_headers>=2,<4", ] diff --git a/src/niquests/__version__.py b/src/niquests/__version__.py index cbb0a8b421..70df28aa0c 100644 --- a/src/niquests/__version__.py +++ b/src/niquests/__version__.py @@ -9,9 +9,9 @@ __url__: str = "https://niquests.readthedocs.io" __version__: str -__version__ = "3.5.4" +__version__ = "3.5.5" -__build__: int = 0x030504 +__build__: int = 0x030505 __author__: str = "Kenneth Reitz" __author_email__: str = "me@kennethreitz.org" __license__: str = "Apache-2.0" diff --git a/src/niquests/_async.py b/src/niquests/_async.py index 609a58afea..b8ad7bf598 100644 --- a/src/niquests/_async.py +++ b/src/niquests/_async.py @@ -129,6 +129,7 @@ def __init__( disable_ipv4: bool = False, pool_connections: int = DEFAULT_POOLSIZE, pool_maxsize: int = DEFAULT_POOLSIZE, + happy_eyeballs: bool | int = False, ): if [disable_ipv4, disable_ipv6].count(True) == 2: raise RuntimeError("Cannot disable both IPv4 and IPv6") @@ -191,6 +192,8 @@ def __init__( self._pool_connections = pool_connections self._pool_maxsize = pool_maxsize + self._happy_eyeballs = happy_eyeballs + #: SSL Verification default. #: Defaults to `True`, requiring requests to verify the TLS certificate at the #: remote end. @@ -245,6 +248,7 @@ def __init__( disable_ipv6=disable_ipv6, pool_connections=pool_connections, pool_maxsize=pool_maxsize, + happy_eyeballs=happy_eyeballs, ), ) self.mount( @@ -257,6 +261,7 @@ def __init__( disable_ipv6=disable_ipv6, pool_connections=pool_connections, pool_maxsize=pool_maxsize, + happy_eyeballs=happy_eyeballs, ), ) @@ -333,6 +338,7 @@ async def on_post_connection(conn_info: ConnectionInfo) -> None: 0.2 if not strict_ocsp_enabled else 1.0, kwargs["proxies"], resolver=self.resolver, + happy_eyeballs=self._happy_eyeballs, ) # don't trigger pre_send for redirects @@ -387,6 +393,7 @@ async def handle_upload_progress( disable_ipv6=self._disable_ipv6, pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, + happy_eyeballs=self._happy_eyeballs, ), ) self.mount( @@ -399,6 +406,7 @@ async def handle_upload_progress( disable_ipv6=self._disable_ipv6, pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, + happy_eyeballs=self._happy_eyeballs, ), ) @@ -1291,4 +1299,7 @@ async def gather(self, *responses: Response, max_fetch: int | None = None) -> No await adapter.gather(*responses, max_fetch=max_fetch) async def close(self) -> None: # type: ignore[override] - ... + for v in self.adapters.values(): + await v.close() + if self._own_resolver: + await self.resolver.close() diff --git a/src/niquests/_compat.py b/src/niquests/_compat.py index 856363f497..cf24df9900 100644 --- a/src/niquests/_compat.py +++ b/src/niquests/_compat.py @@ -3,7 +3,7 @@ import typing try: - from urllib3._version import __version__ + from urllib3 import __version__ HAS_LEGACY_URLLIB3: bool = int(__version__.split(".")[-1]) < 900 except (ValueError, ImportError): diff --git a/src/niquests/adapters.py b/src/niquests/adapters.py index 397e6307b5..ea09ccfa70 100644 --- a/src/niquests/adapters.py +++ b/src/niquests/adapters.py @@ -67,6 +67,8 @@ parse_url, Retry, ) + from urllib3.contrib.resolver import BaseResolver + from urllib3.contrib.resolver._async import AsyncBaseResolver else: from urllib3_future import ( # type: ignore[assignment] ConnectionInfo, @@ -106,6 +108,8 @@ parse_url, Retry, ) + from urllib3_future.contrib.resolver import BaseResolver # type: ignore[assignment] + from urllib3_future.contrib.resolver._async import AsyncBaseResolver # type: ignore[assignment] from ._constant import DEFAULT_POOLBLOCK, DEFAULT_POOLSIZE, DEFAULT_RETRIES from ._typing import ( @@ -319,6 +323,7 @@ class HTTPAdapter(BaseAdapter): "_source_address", "_disable_ipv4", "_disable_ipv6", + "_happy_eyeballs", ] def __init__( @@ -335,6 +340,7 @@ def __init__( source_address: tuple[str, int] | None = None, disable_ipv4: bool = False, disable_ipv6: bool = False, + happy_eyeballs: bool | int = False, ): if isinstance(max_retries, bool): self.max_retries: RetryType = False @@ -364,6 +370,7 @@ def __init__( self._source_address = source_address self._disable_ipv4 = disable_ipv4 self._disable_ipv6 = disable_ipv6 + self._happy_eyeballs = happy_eyeballs #: we keep a list of pending (lazy) response self._promises: dict[str, Response] = {} @@ -391,6 +398,7 @@ def __init__( resolver=resolver, source_address=source_address, socket_family=resolve_socket_family(disable_ipv4, disable_ipv6), + happy_eyeballs=happy_eyeballs, ) def __getstate__(self) -> dict[str, typing.Any | None]: @@ -422,6 +430,7 @@ def __setstate__(self, state): disabled_svn=disabled_svn, source_address=self._source_address, socket_family=resolve_socket_family(self._disable_ipv4, self._disable_ipv6), + happy_eyeballs=self._happy_eyeballs, ) def init_poolmanager( @@ -490,6 +499,7 @@ def proxy_manager_for(self, proxy: str, **proxy_kwargs: typing.Any) -> ProxyMana block=self._pool_block, disabled_svn=disabled_svn, resolver=self._resolver, + happy_eyeballs=self._happy_eyeballs, **proxy_kwargs, ) else: @@ -502,6 +512,7 @@ def proxy_manager_for(self, proxy: str, **proxy_kwargs: typing.Any) -> ProxyMana block=self._pool_block, disabled_svn=disabled_svn, resolver=self._resolver, + happy_eyeballs=self._happy_eyeballs, **proxy_kwargs, ) @@ -1050,6 +1061,10 @@ def on_post_connection(conn_info: ConnectionInfo) -> None: strict_ocsp_enabled, 0.2 if not strict_ocsp_enabled else 1.0, kwargs["proxies"], + self._resolver + if isinstance(self._resolver, BaseResolver) + else None, + self._happy_eyeballs, ) kwargs["on_post_connection"] = on_post_connection @@ -1282,6 +1297,7 @@ class AsyncHTTPAdapter(AsyncBaseAdapter): "_source_address", "_disable_ipv4", "_disable_ipv6", + "_happy_eyeballs", ] def __init__( @@ -1298,6 +1314,7 @@ def __init__( source_address: tuple[str, int] | None = None, disable_ipv4: bool = False, disable_ipv6: bool = False, + happy_eyeballs: bool | int = False, ): if isinstance(max_retries, bool): self.max_retries: RetryType = False @@ -1328,6 +1345,7 @@ def __init__( self._source_address = source_address self._disable_ipv4 = disable_ipv4 self._disable_ipv6 = disable_ipv6 + self._happy_eyeballs = happy_eyeballs #: we keep a list of pending (lazy) response self._promises: dict[str, Response | AsyncResponse] = {} @@ -1354,6 +1372,7 @@ def __init__( resolver=resolver, source_address=source_address, socket_family=resolve_socket_family(disable_ipv4, disable_ipv6), + happy_eyeballs=happy_eyeballs, ) def __getstate__(self) -> dict[str, typing.Any | None]: @@ -1385,6 +1404,7 @@ def __setstate__(self, state): disabled_svn=disabled_svn, source_address=self._source_address, socket_family=resolve_socket_family(self._disable_ipv4, self._disable_ipv6), + happy_eyeballs=self._happy_eyeballs, ) def init_poolmanager( @@ -1455,6 +1475,7 @@ def proxy_manager_for( block=self._pool_block, disabled_svn=disabled_svn, resolver=self._resolver, + happy_eyeballs=self._happy_eyeballs, **proxy_kwargs, ) else: @@ -1467,6 +1488,7 @@ def proxy_manager_for( block=self._pool_block, disabled_svn=disabled_svn, resolver=self._resolver, + happy_eyeballs=self._happy_eyeballs, **proxy_kwargs, ) @@ -2022,6 +2044,10 @@ async def on_post_connection(conn_info: ConnectionInfo) -> None: strict_ocsp_enabled, 0.2 if not strict_ocsp_enabled else 1.0, kwargs["proxies"], + self._resolver + if isinstance(self._resolver, AsyncBaseResolver) + else None, + self._happy_eyeballs, ) kwargs["on_post_connection"] = on_post_connection diff --git a/src/niquests/extensions/_async_ocsp.py b/src/niquests/extensions/_async_ocsp.py index 70ef2805da..63a9e2efbe 100644 --- a/src/niquests/extensions/_async_ocsp.py +++ b/src/niquests/extensions/_async_ocsp.py @@ -296,6 +296,7 @@ async def verify( timeout: float | int = 0.2, proxies: ProxyType | None = None, resolver: AsyncBaseResolver | None = None, + happy_eyeballs: bool | int = False, ) -> None: conn_info: ConnectionInfo | None = r.conn_info @@ -366,7 +367,9 @@ async def verify( from .._async import AsyncSession - async with AsyncSession(resolver=resolver) as session: + async with AsyncSession( + resolver=resolver, happy_eyeballs=happy_eyeballs + ) as session: session.trust_env = False session.proxies = proxies diff --git a/src/niquests/extensions/_ocsp.py b/src/niquests/extensions/_ocsp.py index b832ea88a7..9838514686 100644 --- a/src/niquests/extensions/_ocsp.py +++ b/src/niquests/extensions/_ocsp.py @@ -321,6 +321,7 @@ def verify( timeout: float | int = 0.2, proxies: ProxyType | None = None, resolver: BaseResolver | None = None, + happy_eyeballs: bool | int = False, ) -> None: conn_info: ConnectionInfo | None = r.conn_info @@ -389,7 +390,7 @@ def verify( from ..sessions import Session - with Session(resolver=resolver) as session: + with Session(resolver=resolver, happy_eyeballs=happy_eyeballs) as session: session.trust_env = False session.proxies = proxies diff --git a/src/niquests/models.py b/src/niquests/models.py index 96d0bdaec5..348affafae 100644 --- a/src/niquests/models.py +++ b/src/niquests/models.py @@ -636,14 +636,15 @@ def prepare_auth(self, auth: HttpAuthenticationType | None, url: str = "") -> No "Unexpected non-callable authentication. Did you pass unsupported tuple to auth argument?" ) - # Allow auth to make its changes. - r = auth(self) + if not asyncio.iscoroutinefunction(auth.__call__): + # Allow auth to make its changes. + r = auth(self) - # Update self to reflect the auth changes. - self.__dict__.update(r.__dict__) + # Update self to reflect the auth changes. + self.__dict__.update(r.__dict__) - # Recompute Content-Length - self.prepare_content_length(self.body) + # Recompute Content-Length + self.prepare_content_length(self.body) def prepare_cookies(self, cookies: CookiesType | None) -> None: """Prepares the given HTTP cookie data. diff --git a/src/niquests/sessions.py b/src/niquests/sessions.py index 234a491538..c6e05b2aae 100644 --- a/src/niquests/sessions.py +++ b/src/niquests/sessions.py @@ -221,6 +221,7 @@ class Session: "_disable_http3", "_pool_connections", "_pool_maxsize", + "_happy_eyeballs", ] def __init__( @@ -237,6 +238,7 @@ def __init__( disable_ipv4: bool = False, pool_connections: int = DEFAULT_POOLSIZE, pool_maxsize: int = DEFAULT_POOLSIZE, + happy_eyeballs: bool | int = False, ): """ :param resolver: Specify a DNS resolver that should be used within this Session. @@ -310,6 +312,8 @@ def __init__( self._pool_connections = pool_connections self._pool_maxsize = pool_maxsize + self._happy_eyeballs = happy_eyeballs + #: SSL Verification default. #: Defaults to `True`, requiring requests to verify the TLS certificate at the #: remote end. @@ -364,6 +368,7 @@ def __init__( disable_ipv6=disable_ipv6, pool_connections=pool_connections, pool_maxsize=pool_maxsize, + happy_eyeballs=happy_eyeballs, ), ) self.mount( @@ -376,6 +381,7 @@ def __init__( disable_ipv6=disable_ipv6, pool_connections=pool_connections, pool_maxsize=pool_maxsize, + happy_eyeballs=happy_eyeballs, ), ) @@ -1085,6 +1091,7 @@ def on_post_connection(conn_info: ConnectionInfo) -> None: 0.2 if not strict_ocsp_enabled else 1.0, kwargs["proxies"], resolver=self.resolver, + happy_eyeballs=self._happy_eyeballs, ) # don't trigger pre_send for redirects @@ -1139,6 +1146,7 @@ def handle_upload_progress( disable_ipv6=self._disable_ipv6, pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, + happy_eyeballs=self._happy_eyeballs, ), ) self.mount( @@ -1151,6 +1159,7 @@ def handle_upload_progress( disable_ipv6=self._disable_ipv6, pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, + happy_eyeballs=self._happy_eyeballs, ), ) @@ -1352,6 +1361,7 @@ def __setstate__(self, state): resolver=self.resolver, pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, + happy_eyeballs=self._happy_eyeballs, ), ) self.mount( @@ -1364,6 +1374,7 @@ def __setstate__(self, state): resolver=self.resolver, pool_connections=self._pool_connections, pool_maxsize=self._pool_maxsize, + happy_eyeballs=self._happy_eyeballs, ), ) diff --git a/tests/test_async.py b/tests/test_async.py index cf573789f6..b3ec3cd016 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -2,6 +2,7 @@ import asyncio import json +import os import pytest @@ -216,3 +217,14 @@ async def emit(): assert len(responses_bar) == 2 assert all(r.status_code == 200 for r in responses_foo + responses_bar) + + @pytest.mark.skipif(os.environ.get("CI") is None, reason="Worth nothing locally") + async def test_happy_eyeballs(self) -> None: + """A bit of context, this test, running it locally does not get us + any confidence about Happy Eyeballs. This test is valuable in Github CI where IPv6 addresses are unreachable. + We're using a custom DNS resolver that will yield the IPv6 addresses and IPv4 ones. + If this hang in CI, then you did something wrong...!""" + async with AsyncSession(resolver="doh+cloudflare://", happy_eyeballs=True) as s: + r = await s.get("https://pie.dev/get") + + assert r.ok diff --git a/tests/test_live.py b/tests/test_live.py index f365771e1e..5aa7b72d23 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -99,3 +99,14 @@ def test_owned_resolver_must_recycle(self) -> None: s.get("https://pie.dev/get") assert s.resolver.is_available() + + @pytest.mark.skipif(os.environ.get("CI") is None, reason="Worth nothing locally") + def test_happy_eyeballs(self) -> None: + """A bit of context, this test, running it locally does not get us + any confidence about Happy Eyeballs. This test is valuable in Github CI where IPv6 addresses are unreachable. + We're using a custom DNS resolver that will yield the IPv6 addresses and IPv4 ones. + If this hang in CI, then you did something wrong...!""" + with Session(resolver="doh+cloudflare://", happy_eyeballs=True) as s: + r = s.get("https://pie.dev/get") + + assert r.ok