From 654b5f415ef60c2d198166676465273c1c489821 Mon Sep 17 00:00:00 2001 From: "TAHRI Ahmed R." Date: Tue, 8 Oct 2024 08:53:03 +0200 Subject: [PATCH] :bookmark: Release 3.9.0 (#148) **Added** - Support for WebSocket over HTTP/1, HTTP/2 and HTTP/3. It brings a unified API that makes you leverage our powerful features like Happy Eyeballs, SOCKS/HTTP/HTTPS proxies, thread/task safety etc... - Hook for catching early responses like "103 Early Hints". **Fixed** - Informational responses are fully supported over HTTP/1, HTTP/2 and HTTP/3. **Changed** - urllib3-future lower bound version is raised to 2.10.900. --- .github/workflows/run-tests.yml | 2 +- .pre-commit-config.yaml | 4 +- HISTORY.md | 14 +++++ README.md | 15 +++-- docs/community/faq.rst | 2 +- docs/index.rst | 2 + docs/user/advanced.rst | 33 +++++++++- docs/user/quickstart.rst | 108 ++++++++++++++++++++++++++++++++ noxfile.py | 2 +- pyproject.toml | 5 +- requirements-dev.txt | 2 +- src/niquests/__version__.py | 4 +- src/niquests/_async.py | 34 +++++++++- src/niquests/adapters.py | 53 +++++++++++++++- src/niquests/cookies.py | 2 +- src/niquests/hooks.py | 1 + src/niquests/models.py | 22 +++++++ src/niquests/sessions.py | 26 ++++++++ src/niquests/utils.py | 27 ++++++++ tests/test_async.py | 17 +++++ tests/test_hooks.py | 1 + tests/test_live.py | 21 ++++++- tests/test_lowlevel.py | 21 +++---- tests/test_multiplexed.py | 19 ++++++ tests/test_ocsp.py | 14 +++-- tests/test_websocket.py | 59 +++++++++++++++++ 26 files changed, 473 insertions(+), 37 deletions(-) create mode 100644 tests/test_websocket.py diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index b7e8c6e4d2..0b7dc997be 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "pypy-3.9", "pypy-3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "pypy-3.9", "pypy-3.10"] os: [ubuntu-latest, macOS-13, windows-latest] include: # pypy-3.7, pypy-3.8 may fail due to missing cryptography wheels. Adapting. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6a234e1add..5a676282e5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,9 +23,9 @@ repos: # Run the formatter. - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.10.0 + rev: v1.11.2 hooks: - id: mypy args: [--check-untyped-defs] exclude: 'tests/|noxfile.py' - additional_dependencies: ['charset_normalizer', 'urllib3.future>=2.9.900', 'wassima>=1.0.1', 'idna', 'kiss_headers'] + additional_dependencies: ['charset_normalizer', 'urllib3.future>=2.10.900', 'wassima>=1.0.1', 'idna', 'kiss_headers'] diff --git a/HISTORY.md b/HISTORY.md index 95bb336000..ce9b0cafd4 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,20 @@ Release History =============== +3.9.0 (2024-10-08) +------------------ + +**Added** +- Support for WebSocket over HTTP/1, HTTP/2 and HTTP/3. It brings a unified API that makes you leverage + our powerful features like Happy Eyeballs, SOCKS/HTTP/HTTPS proxies, thread/task safety etc... +- Hook for catching early responses like "103 Early Hints". + +**Fixed** +- Informational responses are fully supported over HTTP/1, HTTP/2 and HTTP/3. + +**Changed** +- urllib3-future lower bound version is raised to 2.10.900. + 3.8.0 (2024-09-24) ------------------ diff --git a/README.md b/README.md index 68ff24632f..178b5fd533 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ Niquests, is the “**Safest**, **Fastest[^10]**, **Easiest**, and **Most advanc | `HTTP/1.1` | ✅ | ✅ | ✅ | ✅ | | `HTTP/2` | ✅ | ❌ | ✅[^7] | ❌ | | `HTTP/3 over QUIC` | ✅ | ❌ | ❌ | ❌ | -| `Synchronous` | ✅ | ✅ | ✅ | ❌ | +| `Synchronous` | ✅ | ✅ | ✅ | _N/A_[^1] | | `Asynchronous` | ✅ | ❌ | ✅ | ✅ | | `Thread Safe` | ✅ | ✅ | ❌[^5] | _N/A_[^1] | | `Task Safe` | ✅ | _N/A_[^2] | ✅ | ✅ | @@ -42,6 +42,9 @@ Niquests, is the “**Safest**, **Fastest[^10]**, **Easiest**, and **Most advanc | `HTTP/2 with prior knowledge (h2c)` | ✅ | ❌ | ✅ | ❌ | | `Post-Quantum Security` | _Limited_[^12] | ❌ | ❌ | ❌ | | `HTTP Trailers` | ✅ | ❌ | ❌ | ❌ | +| `Early Responses` | ✅ | ❌ | ❌ | ❌ | +| `WebSocket over HTTP/1` | ✅ | ❌[^14] | ❌[^14] | ❌[^14] | +| `WebSocket over HTTP/2 and HTTP/3` | ✅[^13] | ❌ | ❌ | ❌ |
@@ -158,9 +161,11 @@ Niquests is ready for the demands of building scalable, robust and reliable HTTP - Streaming Downloads - HTTP/2 by default - HTTP/3 over QUIC +- Early Responses - Happy Eyeballs - Multiplexed! - Thread-safe! +- WebSocket! - Trailers! - DNSSEC! - Async! @@ -191,15 +196,17 @@ You may also be interested in unlocking specific advantages _(like access to a p Niquests is a highly improved HTTP client that is based (forked) on Requests. The previous project original author is Kenneth Reitz and actually left the maintenance of Requests years ago. -[^1]: aiohttp has no support for synchronous request. +[^1]: aiohttp was conceived solely for an asynchronous context. [^2]: requests has no support for asynchronous request. [^3]: while the HTTP/2 connection object can handle concurrent requests, you cannot leverage its true potential. [^4]: loading client certificate without file can't be done. [^5]: httpx officially claim to be thread safe but recent tests demonstrate otherwise as of february 2024. https://github.com/jawah/niquests/issues/83#issuecomment-1956065258 https://github.com/encode/httpx/issues/3072 https://github.com/encode/httpx/issues/3002 [^6]: they do not expose anything to control network aspects such as IPv4/IPv6 toggles, and timings (e.g. DNS response time, established delay, TLS handshake delay, etc...) and such. -[^7]: while advertised as possible, they refuse to make it the default due to performance issues. as of february 2024 an extra is required to enable it manually. +[^7]: while advertised as possible, they refuse to make it the default due to performance issues. as of October 2024 an extra is required to enable it manually. [^8]: they don't support HTTP/3 at all. [^9]: you must use a custom DNS resolver so that it can preemptively connect using HTTP/3 over QUIC when remote is compatible. -[^10]: performance measured when leveraging a multiplexed connection with or without uses of any form of concurrency as of July 2024. The research compared `httpx`, `requests`, `aiohttp` against `niquests`. See https://github.com/Ousret/niquests-stats +[^10]: performance measured when leveraging a multiplexed connection with or without uses of any form of concurrency as of October 2024. The research compared `httpx`, `requests`, `aiohttp` against `niquests`. See https://github.com/Ousret/niquests-stats [^11]: enabled when using a custom DNS resolver. [^12]: available only when using HTTP/3 over QUIC and that the remote server support also the same post-quantum key-exchange algorithm. Also, the `qh3` installed version must be >= 1.1. +[^13]: most servers out there are not ready for this feature, but Niquests is already compliant and future-proof! Modern server like Caddy are still working on it, see https://github.com/caddyserver/caddy/pull/6567 for more. +[^14]: they don't offer any built-in to speak with a WebSocket server. diff --git a/docs/community/faq.rst b/docs/community/faq.rst index 664f33978a..4a0932aed7 100644 --- a/docs/community/faq.rst +++ b/docs/community/faq.rst @@ -140,7 +140,7 @@ Ever encountered something along:: Yes? Usually it means that you tried to load a certificate (CA or client cert) that is malformed. What does malformed means? -~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~ Could be just a missing newline character *RC*, or wrong format like passing a DER file instead of a PEM encoded certificate. diff --git a/docs/index.rst b/docs/index.rst index 6495bbd6ff..6328bd39f1 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -86,9 +86,11 @@ Niquests is ready for today's web. - Streaming Downloads - HTTP/2 by default - HTTP/3 over QUIC +- Early Responses - Happy Eyeballs - Multiplexed! - Thread-safe! +- WebSocket! - Trailers! - DNSSEC! - Async! diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 4a9bd9c0c5..d8efe743f3 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -453,6 +453,8 @@ the request process, or signal event handling. Available hooks: +``early_response``: + An early response caught before receiving the final Response for a given Request. Like but not limited to 103 Early Hints. ``response``: The response generated from a Request. ``pre_send``: @@ -706,6 +708,10 @@ Alternatively you can configure it once for an entire See `#2018 `_ for details. +.. note:: WebSocket are too concerned by that section. By default ``wss://...`` will pick the ``https`` proxy + and the ``ws://...`` the ``http`` entry. You are free to add a ``wss`` key in your proxies + to route them on another proxy. + When the proxies configuration is not overridden per request as shown above, Niquests relies on the proxy configuration defined by standard environment variables ``http_proxy``, ``https_proxy``, ``no_proxy``, @@ -1315,7 +1321,7 @@ by passing a custom ``QuicSharedCache`` instance like so:: When the cache is full, the oldest entry is removed. Disable HTTP/1.1, HTTP/2, and/or HTTP/3 ------------------------------ +--------------------------------------- You can at your own discretion disable a protocol by passing ``disable_http2=True`` or ``disable_http3=True`` within your ``Session`` constructor. @@ -1507,3 +1513,28 @@ For example, we retrieve our trailers this way:: .. warning:: The ``trailers`` property is only filled when the response has been consumed entirely. The server only send them after finishing sending the body. By default, ``trailers`` is an empty CaseInsensibleDict. + +Early Response +-------------- + +A server may send one or several (informational) response before the final response. Before this, those responses were +silently ignored or worst, misinterpreted. + +Most notably, the status https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/103 is one of the most useful use case out there. + +To catch response like those:: + + from niquests import Session + + def early_response_hook(early_response): + print(early_response) # + print(early_response.headers) # {'origin-trial': ..., 'link': '; rel=preload; as=image'} + + with Session() as s: + resp = s.get("https://early-hints.fastlylabs.com/", hooks={"early_response": early_response_hook}) + + print(resp) # + +Isn't it easy and pleasant to write ? + +.. warning:: Some servers choose to enable it in HTTP/2, and HTTP/3 but not in HTTP/1.1 for security concerns. But rest assured that Niquests support this no matter the protocol. diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index 2d0d0ad84b..07cb25ea61 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -1012,6 +1012,114 @@ OCSP requests (certificate revocation checks) will follow given ``happy_eyeballs .. warning:: This feature is disabled by default and we are actually planning to make it enabled as the default in a future major. +WebSockets +---------- + +.. note:: Available since version 3.9+ and requires to install an extra. ``pip install niquests[ws]``. + +It is undeniable that WebSockets are a vital part of the web ecosystem along with HTTP. We noticed that +most users met frictions when trying to deal with a WebSocket server for the first time, that is why +we decided to expand Niquests capabilities to automatically handle WebSockets for you. + +Synchronous +~~~~~~~~~~~ + +In the following example, we will explore how to interact with a basic, but well known echo server:: + + from niquests import Session + + with Session() as s: + resp = s.get( + "wss://echo.websocket.org", + ) + + print(resp.status_code) # it says "101", for "Switching Protocol" + + print(resp.extension.next_payload()) # unpack the next message from server + + resp.extension.send_payload("Hello World") # automatically sends a text message to the server + + print(resp.extension.next_payload() == "Hello World") # output True! + + resp.extension.close() # don't forget this call to release the connection! + +.. warning:: Without the extra installed, you will get an exception that indicate that the scheme is unsupported. + +.. note:: Historically, Requests only accepted http:// and https:// as schemes. But now, you may use wss:// for WebSocket Secure or ws:// for WebSocket over PlainText. + +.. warning:: Be careful when accessing ``resp.extension``, if anything goes wrong in the "establishment" phase, meaning the server denies us the WebSocket upgrade, it will be worth ``None``. + +WebSocket and HTTP/2+ +~~~~~~~~~~~~~~~~~~~~~ + +By default, Niquests negotiate WebSocket over HTTP/1.1 but it is well capable of doing so over HTTP/2 and HTTP/3 following RFC8441. +But rare are the servers capable of bootstrapping WebSocket over a multiplexed connection. There's a little tweak to the URL +so that it can infer your desire to use a modern protocol, like so ``wss+rfc8441://echo.websocket.org``. + +Asynchronous +~~~~~~~~~~~~ + +Of course, as per our feature coverage, this is doable both in synchronous and asynchronous contexts. +Like so:: + + from niquests import AsyncSession + import asyncio + + async def main() -> None: + async with AsyncSession() as s: + resp = await s.get("wss://echo.websocket.org") + + # ... + + print(await resp.extension.next_payload()) # unpack the next message from server + + await resp.extension.send_payload("Hello World") # automatically sends a text message to the server + + print((await resp.extension.next_payload()) == "Hello World") # output True! + + await resp.extension.close() + + +Ping and Pong +~~~~~~~~~~~~~ + +Ping sent by a server are automatically handled/answered by Niquests each time to read from the socket with `next_payload()`. +However, we do not send automatically Ping TO the server. + +In order to do so:: + + from niquests import Session + + with Session() as s: + resp = s.get( + "wss://echo.websocket.org", + ) + + print(resp.extension.ping()) # send a ping to the websocket server, notify it that you're still there! + +You can use the elementary methods provided by Niquests to construct your own logic. + +Binary and Text Messages +~~~~~~~~~~~~~~~~~~~~~~~~ + +You may use ``next_payload()`` and ``send_payload(...)`` with str or bytes. + +If ``next_payload()`` output bytes, then it is a BinaryMessage. +If ``next_payload()`` output str, then it is a TextMessage. + +The same apply to ``send_payload(...)``, if passed item is str, then we send a TextMessage. +Otherwise, it will be a BinaryMessage. + +.. warning:: Niquests does not buffer "incomplete" message (e.g. end marker for a message). It returns every chunk received as is. + +.. note:: If ``next_payload()`` returns ``None``, that means that the remote choose to close the connection. + +Others +~~~~~~ + +Every other features still applies with WebSocket, like proxies, happy eyeballs, thread/task safety, etc... +See relevant docs for more. + ----------------------- Ready for more? Check out the :ref:`advanced ` section. diff --git a/noxfile.py b/noxfile.py index 18e197c9d2..55bfb1f41c 100644 --- a/noxfile.py +++ b/noxfile.py @@ -8,7 +8,7 @@ def tests_impl( session: nox.Session, - extras: str = "socks", + extras: str = "socks,ws", cohabitation: bool | None = False, ) -> None: # Install deps and the package itself. diff --git a/pyproject.toml b/pyproject.toml index ffeb9bc294..9baf880044 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dynamic = ["version"] dependencies = [ "charset_normalizer>=2,<4", "idna>=2.5,<4", - "urllib3.future>=2.9.900,<3", + "urllib3.future>=2.10.900,<3", "wassima>=1.0.1,<2", "kiss_headers>=2,<4", ] @@ -57,6 +57,9 @@ http3 = [ ocsp = [ "urllib3.future[qh3]", ] +ws = [ + "urllib3.future[ws]", +] speedups = [ "orjson>=3,<4", "urllib3.future[zstd,brotli]", diff --git a/requirements-dev.txt b/requirements-dev.txt index cce6d71024..bf529793aa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ -e .[socks] pytest>=2.8.0,<=7.4.4 pytest-cov -pytest-httpbin==2.0.0 +pytest-httpbin>=2,<3 pytest-asyncio>=0.21.1,<1.0 httpbin==0.10.2 trustme diff --git a/src/niquests/__version__.py b/src/niquests/__version__.py index 41217d0dc3..6d9274892a 100644 --- a/src/niquests/__version__.py +++ b/src/niquests/__version__.py @@ -9,9 +9,9 @@ __url__: str = "https://niquests.readthedocs.io" __version__: str -__version__ = "3.8.0" +__version__ = "3.9.0" -__build__: int = 0x030800 +__build__: int = 0x030900 __author__: str = "Kenneth Reitz" __author_email__: str = "me@kennethreitz.org" __license__: str = "Apache-2.0" diff --git a/src/niquests/_async.py b/src/niquests/_async.py index 8dfc9b7df8..316420266c 100644 --- a/src/niquests/_async.py +++ b/src/niquests/_async.py @@ -19,9 +19,11 @@ if HAS_LEGACY_URLLIB3 is False: from urllib3 import ConnectionInfo from urllib3.contrib.resolver._async import AsyncBaseResolver + from urllib3.contrib.webextensions._async import load_extension else: from urllib3_future import ConnectionInfo # type: ignore[assignment] from urllib3_future.contrib.resolver._async import AsyncBaseResolver # type: ignore[assignment] + from urllib3_future.contrib.webextensions._async import load_extension # type: ignore[assignment] from ._constant import ( READ_DEFAULT_TIMEOUT, @@ -54,6 +56,7 @@ ChunkedEncodingError, ContentDecodingError, TooManyRedirects, + InvalidSchema, ) from .hooks import async_dispatch_hook, default_hooks from .models import ( @@ -280,7 +283,32 @@ def mount(self, prefix: str, adapter: AsyncBaseAdapter) -> None: # type: ignore super().mount(prefix, adapter) # type: ignore[arg-type] def get_adapter(self, url: str) -> AsyncBaseAdapter: # type: ignore[override] - return super().get_adapter(url) # type: ignore[return-value] + for prefix, adapter in self.adapters.items(): + if url.lower().startswith(prefix.lower()): + return adapter + + # If no adapter matches our prefix, that usually means we want + # an HTTP extension like wss (e.g. WebSocket). + scheme = parse_scheme(url) + + if "+" in scheme: + scheme, implementation = tuple(scheme.split("+", maxsplit=1)) + else: + implementation = None + + try: + extension = load_extension(scheme, implementation=implementation) + for prefix, adapter in self.adapters.items(): + if ( + scheme in extension.supported_schemes() + and extension.scheme_to_http_scheme(scheme) == parse_scheme(prefix) + ): + return adapter + except ImportError: + pass + + # Nothing matches :-/ + raise InvalidSchema(f"No connection adapters were found for {url!r}") async def send( # type: ignore[override] self, request: PreparedRequest, **kwargs: typing.Any @@ -369,9 +397,13 @@ async def handle_upload_progress( await async_dispatch_hook("on_upload", hooks, request) # type: ignore[arg-type] + async def on_early_response(early_response: Response) -> None: + await async_dispatch_hook("early_response", hooks, early_response) # type: ignore[arg-type] + kwargs.setdefault("on_post_connection", on_post_connection) kwargs.setdefault("on_upload_body", handle_upload_progress) kwargs.setdefault("multiplexed", self.multiplexed) + kwargs.setdefault("on_early_response", on_early_response) assert request.url is not None diff --git a/src/niquests/adapters.py b/src/niquests/adapters.py index e6e3990a83..a92e64c8fd 100644 --- a/src/niquests/adapters.py +++ b/src/niquests/adapters.py @@ -69,6 +69,10 @@ ) from urllib3.contrib.resolver import BaseResolver from urllib3.contrib.resolver._async import AsyncBaseResolver + from urllib3.contrib.webextensions import load_extension + from urllib3.contrib.webextensions._async import ( + load_extension as async_load_extension, + ) else: from urllib3_future import ( # type: ignore[assignment] ConnectionInfo, @@ -110,6 +114,10 @@ ) from urllib3_future.contrib.resolver import BaseResolver # type: ignore[assignment] from urllib3_future.contrib.resolver._async import AsyncBaseResolver # type: ignore[assignment] + from urllib3_future.contrib.webextensions import load_extension # type: ignore[assignment] + from urllib3_future.contrib.webextensions._async import ( # type: ignore[assignment] + load_extension as async_load_extension, + ) from ._constant import DEFAULT_POOLBLOCK, DEFAULT_POOLSIZE, DEFAULT_RETRIES from ._typing import ( @@ -189,6 +197,7 @@ def send( on_post_connection: typing.Callable[[typing.Any], None] | None = None, on_upload_body: typing.Callable[[int, int | None, bool, bool], None] | None = None, + on_early_response: typing.Callable[[Response], None] | None = None, multiplexed: bool = False, ) -> Response: """Sends PreparedRequest object. Returns Response object. @@ -244,6 +253,8 @@ async def send( [int, int | None, bool, bool], typing.Awaitable[None] ] | None = None, + on_early_response: typing.Callable[[Response], typing.Awaitable[None]] + | None = None, multiplexed: bool = False, ) -> AsyncResponse: """Sends PreparedRequest object. Returns Response object. @@ -819,6 +830,7 @@ def send( on_post_connection: typing.Callable[[typing.Any], None] | None = None, on_upload_body: typing.Callable[[int, int | None, bool, bool], None] | None = None, + on_early_response: typing.Callable[[Response], None] | None = None, multiplexed: bool = False, ) -> Response: """Sends PreparedRequest object. Returns Response object. @@ -895,8 +907,19 @@ def send( if isinstance(request.body, (list, dict)): raise ValueError("Body contains unprepared native list or dict.") + scheme = parse_scheme(request.url) + extension = None + + if scheme is not None and scheme not in ("http", "https"): + extension = load_extension(scheme)() + + def early_response_hook(early_response: BaseHTTPResponse) -> None: + nonlocal on_early_response + assert on_early_response is not None + on_early_response(self.build_response(request, early_response)) + try: - resp_or_promise = conn.urlopen( # type: ignore[call-overload] + resp_or_promise = conn.urlopen( # type: ignore[call-overload,misc] method=request.method, url=url, body=request.body, @@ -910,6 +933,10 @@ def send( chunked=chunked, on_post_connection=on_post_connection, on_upload_body=on_upload_body, + on_early_response=early_response_hook + if on_early_response is not None + else None, + extension=extension, multiplexed=multiplexed, ) @@ -1794,6 +1821,8 @@ async def send( [int, int | None, bool, bool], typing.Awaitable[None] ] | None = None, + on_early_response: typing.Callable[[Response], typing.Awaitable[None]] + | None = None, multiplexed: bool = False, ) -> AsyncResponse: """Sends PreparedRequest object. Returns Response object. @@ -1878,8 +1907,24 @@ async def send( if isinstance(request.body, (list, dict)): raise ValueError("Body contains unprepared native list or dict.") + scheme = parse_scheme(request.url) + extension = None + + if scheme is not None and scheme not in ("http", "https"): + if "+" in scheme: + scheme, implementation = tuple(scheme.split("+", maxsplit=1)) + else: + implementation = None + + extension = async_load_extension(scheme, implementation=implementation)() + + async def early_response_hook(early_response: BaseAsyncHTTPResponse) -> None: + nonlocal on_early_response + assert on_early_response is not None + await on_early_response(self.build_response(request, early_response)) + try: - resp_or_promise = await conn.urlopen( # type: ignore[call-overload] + resp_or_promise = await conn.urlopen( # type: ignore[call-overload,misc] method=request.method, url=url, body=request.body, @@ -1893,6 +1938,10 @@ async def send( chunked=chunked, on_post_connection=on_post_connection, on_upload_body=on_upload_body, + on_early_response=early_response_hook + if on_early_response is not None + else None, + extension=extension, multiplexed=multiplexed, ) diff --git a/src/niquests/cookies.py b/src/niquests/cookies.py index 8e3ecddf81..1cd2246b3a 100644 --- a/src/niquests/cookies.py +++ b/src/niquests/cookies.py @@ -405,7 +405,7 @@ def set_cookie(self, cookie, *args, **kwargs): cookie.value = cookie.value.replace('\\"', "") return super().set_cookie(cookie, *args, **kwargs) - def update(self, other): + def update(self, other): # type: ignore[override] """Updates this jar with cookies from another CookieJar or dict-like""" if isinstance(other, cookielib.CookieJar): for cookie in other: diff --git a/src/niquests/hooks.py b/src/niquests/hooks.py index 199ff0b735..b971cb48a8 100644 --- a/src/niquests/hooks.py +++ b/src/niquests/hooks.py @@ -34,6 +34,7 @@ "pre_request", "pre_send", "on_upload", + "early_response", "response", ] diff --git a/src/niquests/models.py b/src/niquests/models.py index 7e97e7c465..9e8074e427 100644 --- a/src/niquests/models.py +++ b/src/niquests/models.py @@ -54,6 +54,8 @@ from urllib3.fields import RequestField from urllib3.filepost import choose_boundary, encode_multipart_formdata from urllib3.util import parse_url + from urllib3.contrib.webextensions._async import AsyncExtensionFromHTTP + from urllib3.contrib.webextensions import ExtensionFromHTTP else: from urllib3_future import ( # type: ignore[assignment] BaseHTTPResponse, @@ -71,6 +73,8 @@ from urllib3_future.fields import RequestField # type: ignore[assignment] from urllib3_future.filepost import choose_boundary, encode_multipart_formdata # type: ignore[assignment] from urllib3_future.util import parse_url # type: ignore[assignment] + from urllib3_future.contrib.webextensions._async import AsyncExtensionFromHTTP # type: ignore[assignment] + from urllib3_future.contrib.webextensions import ExtensionFromHTTP # type: ignore[assignment] from ._typing import ( BodyFormType, @@ -1018,6 +1022,15 @@ def __init__(self) -> None: #: using stream=True + iter_content + content-length set! self.download_progress: TransferProgress | None = None + @property + def extension(self) -> ExtensionFromHTTP | None: + """Access the I/O after an Upgraded connection. E.g. for a WebSocket handler.""" + return ( + self.raw.extension + if self.raw is not None and hasattr(self.raw, "extension") + else None + ) + @property def lazy(self) -> bool: """ @@ -1598,6 +1611,15 @@ class AsyncResponse(Response): "elapsed", } + @property + def extension(self) -> AsyncExtensionFromHTTP | None: # type: ignore[override] + """Access the I/O after an Upgraded connection. E.g. for a WebSocket handler.""" + return ( + self.raw.extension + if self.raw is not None and hasattr(self.raw, "extension") + else None + ) + def __aenter__(self) -> AsyncResponse: return self diff --git a/src/niquests/sessions.py b/src/niquests/sessions.py index c2caff048b..73a9382a05 100644 --- a/src/niquests/sessions.py +++ b/src/niquests/sessions.py @@ -24,8 +24,10 @@ if HAS_LEGACY_URLLIB3 is False: from urllib3 import ConnectionInfo + from urllib3.contrib.webextensions import load_extension else: from urllib3_future import ConnectionInfo # type: ignore[assignment] + from urllib3_future.contrib.webextensions import load_extension # type: ignore[assignment] from ._constant import ( DEFAULT_RETRIES, @@ -1141,9 +1143,13 @@ def handle_upload_progress( dispatch_hook("on_upload", hooks, request) # type: ignore[arg-type] + def on_early_response(early_response) -> None: + dispatch_hook("early_response", hooks, early_response) + kwargs.setdefault("on_post_connection", on_post_connection) kwargs.setdefault("on_upload_body", handle_upload_progress) kwargs.setdefault("multiplexed", self.multiplexed) + kwargs.setdefault("on_early_response", on_early_response) assert request.url is not None @@ -1365,6 +1371,26 @@ def get_adapter(self, url: str) -> BaseAdapter: if url.lower().startswith(prefix.lower()): return adapter + # If no adapter matches our prefix, that usually means we want + # an HTTP extension like wss (e.g. WebSocket). + scheme = parse_scheme(url) + + if "+" in scheme: + scheme, implementation = tuple(scheme.split("+", maxsplit=1)) + else: + implementation = None + + try: + extension = load_extension(scheme, implementation=implementation) + for prefix, adapter in self.adapters.items(): + if ( + scheme in extension.supported_schemes() + and extension.scheme_to_http_scheme(scheme) == parse_scheme(prefix) + ): + return adapter + except ImportError: + pass + # Nothing matches :-/ raise InvalidSchema(f"No connection adapters were found for {url!r}") diff --git a/src/niquests/utils.py b/src/niquests/utils.py index 4e8fc63359..fe61aad5c4 100644 --- a/src/niquests/utils.py +++ b/src/niquests/utils.py @@ -50,6 +50,7 @@ AsyncManyResolver, ) from urllib3 import ConnectionInfo + from urllib3.contrib.webextensions import load_extension else: from urllib3_future.util import make_headers, parse_url # type: ignore[assignment] from urllib3_future.contrib.resolver import ( # type: ignore[assignment] @@ -64,6 +65,7 @@ AsyncManyResolver, ) from urllib3_future import ConnectionInfo # type: ignore[assignment] + from urllib3_future.contrib.webextensions import load_extension # type: ignore[assignment] from .__version__ import __version__ from .exceptions import InvalidURL, UnrewindableBodyError, MissingSchema @@ -826,6 +828,31 @@ def select_proxy( "all://" + urlparts.hostname, "all", ] + + if urlparts.scheme.lower() not in ( + "http", + "https", + ): + maybe_extension_scheme = urlparts.scheme + implementation = None + + if "+" in maybe_extension_scheme: + maybe_extension_scheme, implementation = tuple( + maybe_extension_scheme.split("+", maxsplit=1) + ) + + try: + extension_class = load_extension(maybe_extension_scheme, implementation) + except ImportError: + pass + else: + parent_scheme = extension_class.scheme_to_http_scheme( + maybe_extension_scheme + ) + + proxy_keys.append(parent_scheme) + proxy_keys.append(parent_scheme + "://" + urlparts.hostname) + proxy = None for proxy_key in proxy_keys: if proxy_key in proxies: diff --git a/tests/test_async.py b/tests/test_async.py index 79a8ddf682..a14878f1a2 100644 --- a/tests/test_async.py +++ b/tests/test_async.py @@ -89,6 +89,23 @@ async def fake_aauth(p): assert r.status_code == 200 assert "X-Async-Auth" in r.json()["headers"] + async def test_early_response(self) -> None: + received_early_response: bool = False + + async def callback_on_early(early_resp) -> None: + nonlocal received_early_response + if early_resp.status_code == 103: + received_early_response = True + + async with AsyncSession() as s: + resp = await s.get( + "https://early-hints.fastlylabs.com/", + hooks={"early_response": [callback_on_early]}, + ) + + assert resp.status_code == 200 + assert received_early_response is True + # async def test_http_trailer_preload(self) -> None: # async with AsyncSession() as s: # r = await s.get("https://httpbingo.org/trailers?foo=baz") diff --git a/tests/test_hooks.py b/tests/test_hooks.py index a5c884aef8..ce2c5057ae 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -41,5 +41,6 @@ def test_default_hooks(): "pre_request": [], "pre_send": [], "on_upload": [], + "early_response": [], "response": [], } diff --git a/tests/test_live.py b/tests/test_live.py index bdf45dd410..ee30f19c7b 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -113,7 +113,7 @@ def test_happy_eyeballs(self) -> None: # def test_http_trailer_preload(self) -> None: # with Session() as s: - # r = s.get("https://httpbingo.org/trailers?foo=baz") + # r = s.get("https://httpbingo.org/trailers?foo=baz", headers={"TE": "trailers"}) # # assert r.ok # assert r.trailers @@ -122,7 +122,7 @@ def test_happy_eyeballs(self) -> None: # # def test_http_trailer_no_preload(self) -> None: # with Session() as s: - # r = s.get("https://httpbingo.org/trailers?foo=baz", stream=True) + # r = s.get("https://httpbingo.org/trailers?foo=baz", headers={"TE": "trailers"}, stream=True) # # assert r.ok # assert not r.trailers @@ -133,3 +133,20 @@ def test_happy_eyeballs(self) -> None: # assert r.trailers # assert "foo" in r.trailers # assert r.trailers["foo"] == "baz" + + def test_early_response(self) -> None: + received_early_response: bool = False + + def callback_on_early(early_resp) -> None: + nonlocal received_early_response + if early_resp.status_code == 103: + received_early_response = True + + with Session() as s: + resp = s.get( + "https://early-hints.fastlylabs.com/", + hooks={"early_response": [callback_on_early]}, + ) + + assert resp.status_code == 200 + assert received_early_response is True diff --git a/tests/test_lowlevel.py b/tests/test_lowlevel.py index 45bd9078c1..600dd12eb2 100644 --- a/tests/test_lowlevel.py +++ b/tests/test_lowlevel.py @@ -1,6 +1,5 @@ from __future__ import annotations -import platform import sys import threading from json import JSONDecodeError @@ -125,13 +124,12 @@ def multiple_content_length_response_handler(sock): @pytest.mark.xfail( - platform.python_implementation() == "PyPy" - and sys.version_info + sys.version_info < ( 3, 8, ), - reason="PyPy 3.7 bug with socket unexpected close server side", + reason="Bug with socket unexpected close server side", ) def test_digestauth_401_count_reset_on_redirect(): """Ensure we correctly reset num_401_calls after a successful digest auth, @@ -200,13 +198,12 @@ def digest_response_handler(sock): @pytest.mark.xfail( - platform.python_implementation() == "PyPy" - and sys.version_info + sys.version_info < ( 3, 8, ), - reason="PyPy 3.7 bug with socket unexpected close server side", + reason="Bug with socket unexpected close server side", ) def test_digestauth_401_only_sent_once(): """Ensure we correctly respond to a 401 challenge once, and then @@ -331,13 +328,12 @@ def test_use_proxy_from_environment(httpbin, var, scheme): @pytest.mark.xfail( - platform.python_implementation() == "PyPy" - and sys.version_info + sys.version_info < ( 3, 8, ), - reason="PyPy 3.7 bug with socket unexpected close server side", + reason="Bug with socket unexpected close server side", ) def test_redirect_rfc1808_to_non_ascii_location(): path = "š" @@ -397,13 +393,12 @@ def test_fragment_not_sent_with_request(): @pytest.mark.xfail( - platform.python_implementation() == "PyPy" - and sys.version_info + sys.version_info < ( 3, 8, ), - reason="PyPy 3.7 bug with socket unexpected close server side", + reason="Bug with socket unexpected close server side", ) def test_fragment_update_on_redirect(): """Verify we only append previous fragment if one doesn't exist on new diff --git a/tests/test_multiplexed.py b/tests/test_multiplexed.py index 29aad597dc..121ad717bc 100644 --- a/tests/test_multiplexed.py +++ b/tests/test_multiplexed.py @@ -117,3 +117,22 @@ def test_early_close_no_error(self): # since urllib3.future 2.5, the scheduler ensure we kept track of ongoing request even if pool is # shutdown. assert all([r.json() for r in responses]) + + def test_early_response(self) -> None: + received_early_response: bool = False + + def callback_on_early(early_resp) -> None: + nonlocal received_early_response + if early_resp.status_code == 103: + received_early_response = True + + with Session(multiplexed=True) as s: + resp = s.get( + "https://early-hints.fastlylabs.com/", + hooks={"early_response": [callback_on_early]}, + ) + + assert received_early_response is False + + assert resp.status_code == 200 + assert received_early_response is True diff --git a/tests/test_ocsp.py b/tests/test_ocsp.py index e0dace819e..1ea84b3aa2 100644 --- a/tests/test_ocsp.py +++ b/tests/test_ocsp.py @@ -3,10 +3,16 @@ from niquests import Session, AsyncSession from niquests.exceptions import ConnectionError, Timeout +try: + import qh3 +except ImportError: + qh3 = None + OCSP_MAX_DELAY_WAIT = 5 @pytest.mark.usefixtures("requires_wan") +@pytest.mark.skipif(qh3 is None, reason="qh3 unavailable") class TestOnlineCertificateRevocationProtocol: """This test class hold the minimal amount of confidence we need to ensure revoked certificate are properly rejected. @@ -16,9 +22,9 @@ class TestOnlineCertificateRevocationProtocol: @pytest.mark.parametrize( "revoked_peer_url", [ - "https://revoked.badssl.com/", + # "https://revoked.badssl.com/", # "https://revoked-rsa-ev.ssl.com/", - # "https://revoked-ecc-dv.ssl.com/", + "https://revoked-ecc-dv.ssl.com/", ], ) def test_sync_revoked_certificate(self, revoked_peer_url: str) -> None: @@ -40,9 +46,9 @@ def test_sync_revoked_certificate(self, revoked_peer_url: str) -> None: @pytest.mark.parametrize( "revoked_peer_url", [ - "https://revoked.badssl.com/", + # "https://revoked.badssl.com/", # "https://revoked-rsa-ev.ssl.com/", - # "https://revoked-ecc-dv.ssl.com/", + "https://revoked-ecc-dv.ssl.com/", ], ) async def test_async_revoked_certificate(self, revoked_peer_url: str) -> None: diff --git a/tests/test_websocket.py b/tests/test_websocket.py new file mode 100644 index 0000000000..799548acff --- /dev/null +++ b/tests/test_websocket.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import pytest + +from niquests import Session, AsyncSession + +try: + import wsproto +except ImportError: + wsproto = None + + +@pytest.mark.usefixtures("requires_wan") +@pytest.mark.skipif(wsproto is None, reason="wsproto unavailable") +class TestLiveWebSocket: + def test_sync_websocket_basic_example(self) -> None: + with Session() as s: + resp = s.get("wss://echo.websocket.org") + + assert resp.status_code == 101 + assert resp.extension is not None + assert resp.extension.closed is False + + greeting_msg = resp.extension.next_payload() + + assert greeting_msg is not None + assert isinstance(greeting_msg, str) + + resp.extension.send_payload("Hello World") + resp.extension.send_payload(b"Foo Bar Baz!") + + assert resp.extension.next_payload() == "Hello World" + assert resp.extension.next_payload() == b"Foo Bar Baz!" + + resp.extension.close() + assert resp.extension.closed is True + + @pytest.mark.asyncio + async def test_async_websocket_basic_example(self) -> None: + async with AsyncSession() as s: + resp = await s.get("wss://echo.websocket.org") + + assert resp.status_code == 101 + assert resp.extension is not None + assert resp.extension.closed is False + + greeting_msg = await resp.extension.next_payload() + + assert greeting_msg is not None + assert isinstance(greeting_msg, str) + + await resp.extension.send_payload("Hello World") + await resp.extension.send_payload(b"Foo Bar Baz!") + + assert (await resp.extension.next_payload()) == "Hello World" + assert (await resp.extension.next_payload()) == b"Foo Bar Baz!" + + await resp.extension.close() + assert resp.extension.closed is True