From 1d8561f5adb5e32d5fb7709ea3c1d6255184b60b Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Mon, 16 Oct 2023 08:55:09 +0200 Subject: [PATCH] :bookmark: Release 3.1.2 **Fixed** - Static type checker not accepting **list\[str\]** in values for argument **data**. **Misc** - Changed the documentation theme by **furo**. **Added** - IPv6 support in the `NO_PROXY` environment variable or in the **proxies** (key no_proxy) argument. Patch taken from idle upstream PR https://github.com/psf/requests/pull/5953 - Preemptively register a website to be HTTP/3 capable prior to the first TLS over TCP handshake. You can do so by doing like: ```python from niquests import Session s = Session() s.quic_cache_layer.add_domain("cloudflare.com") ``` - Passed **data** will be converted to form-data if headers have a Content-Type header and is set to `multipart/form-data`. Otherwise, by default, it is still urlencoded. If you specified a boundary, it will be used, otherwise, a random one will be generated. --- HISTORY.md | 25 +++++++++ README.md | 2 + docs/_templates/hacks.html | 27 --------- docs/_templates/sidebarintro.html | 37 ------------- docs/_templates/sidebarlogo.html | 31 ----------- docs/_themes/.gitignore | 3 - docs/_themes/LICENSE | 45 --------------- docs/_themes/flask_theme_support.py | 86 ----------------------------- docs/conf.py | 51 +++++++++-------- docs/index.rst | 10 ++-- docs/requirements.txt | 6 +- docs/user/advanced.rst | 35 +++++++++++- docs/user/quickstart.rst | 16 ++++++ setup.cfg | 1 - src/niquests/__version__.py | 4 +- src/niquests/_typing.py | 2 +- src/niquests/api.py | 4 +- src/niquests/models.py | 52 +++++++++++++++-- src/niquests/sessions.py | 6 +- src/niquests/structures.py | 11 ++++ src/niquests/utils.py | 75 +++++++++++++++++++++---- tests/test_requests.py | 21 +++++++ tests/test_utils.py | 58 +++++++++++++++++-- 23 files changed, 315 insertions(+), 293 deletions(-) delete mode 100644 docs/_templates/hacks.html delete mode 100644 docs/_templates/sidebarintro.html delete mode 100644 docs/_templates/sidebarlogo.html delete mode 100644 docs/_themes/.gitignore delete mode 100644 docs/_themes/LICENSE delete mode 100644 docs/_themes/flask_theme_support.py diff --git a/HISTORY.md b/HISTORY.md index d08950b8f2..61feaad2af 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,31 @@ Release History =============== +3.1.2 (2023-10-16) +------------------ + +**Fixed** +- Static type checker not accepting **list\[str\]** in values for argument **data**. + +**Misc** +- Changed the documentation theme by **furo**. + +**Added** +- IPv6 support in the `NO_PROXY` environment variable or in the **proxies** (key no_proxy) argument. + Patch taken from idle upstream PR https://github.com/psf/requests/pull/5953 +- Preemptively register a website to be HTTP/3 capable prior to the first TLS over TCP handshake. + You can do so by doing like: + + ```python + from niquests import Session + + s = Session() + s.quic_cache_layer.add_domain("cloudflare.com") + ``` +- Passed **data** will be converted to form-data if headers have a Content-Type header and is set to `multipart/form-data`. + Otherwise, by default, it is still urlencoded. If you specified a boundary, it will be used, otherwise, a random one will + be generated. + 3.1.1 (2023-10-11) ------------------ diff --git a/README.md b/README.md index aa06c9b246..fc6722cb08 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ intend to keep it that way. {'authenticated': True, ...} >>> r +>>> r.ocsp_verified +True ``` Niquests allows you to send HTTP requests extremely easily. There’s no need to manually add query strings to your URLs, or to form-encode your `PUT` & `POST` data — but nowadays, just use the `json` method! diff --git a/docs/_templates/hacks.html b/docs/_templates/hacks.html deleted file mode 100644 index c43e19cac5..0000000000 --- a/docs/_templates/hacks.html +++ /dev/null @@ -1,27 +0,0 @@ - - - - - diff --git a/docs/_templates/sidebarintro.html b/docs/_templates/sidebarintro.html deleted file mode 100644 index f3b557c19b..0000000000 --- a/docs/_templates/sidebarintro.html +++ /dev/null @@ -1,37 +0,0 @@ - - -

- -

- -

- Niquests is an elegant and simple HTTP library for Python, built for - human beings. Drop-in replacement for Requests, no longer under feature freeze. -

- -

Useful Links

- - -
-
diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html deleted file mode 100644 index d6582c5597..0000000000 --- a/docs/_templates/sidebarlogo.html +++ /dev/null @@ -1,31 +0,0 @@ -

- -

- -

- Niquests is an elegant and simple HTTP library for Python, built for - human beings. Drop-in replacement for Requests. No longer under feature freeze. - You are currently looking at the documentation of the - development release. -

- -

Useful Links

- - diff --git a/docs/_themes/.gitignore b/docs/_themes/.gitignore deleted file mode 100644 index 66b6e4c2f3..0000000000 --- a/docs/_themes/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.pyc -*.pyo -.DS_Store diff --git a/docs/_themes/LICENSE b/docs/_themes/LICENSE deleted file mode 100644 index 3d1e04a257..0000000000 --- a/docs/_themes/LICENSE +++ /dev/null @@ -1,45 +0,0 @@ -Modifications: - -Copyright (c) 2011 Kenneth Reitz. - - -Original Project: - -Copyright (c) 2010 by Armin Ronacher. - - -Some rights reserved. - -Redistribution and use in source and binary forms of the theme, with or -without modification, are permitted provided that the following conditions -are met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -* The names of the contributors may not be used to endorse or - promote products derived from this software without specific - prior written permission. - -We kindly ask you to only use these themes in an unmodified manner just -for Flask and Flask-related products, not for unrelated projects. If you -like the visual style and want to use it for your own projects, please -consider making some larger changes to the themes (such as changing -font faces, sizes, colors or margins). - -THIS THEME IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS THEME, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. diff --git a/docs/_themes/flask_theme_support.py b/docs/_themes/flask_theme_support.py deleted file mode 100644 index 33f47449c1..0000000000 --- a/docs/_themes/flask_theme_support.py +++ /dev/null @@ -1,86 +0,0 @@ -# flasky extensions. flasky pygments style based on tango style -from pygments.style import Style -from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator, Generic, Whitespace, Punctuation, Other, Literal - - -class FlaskyStyle(Style): - background_color = "#f8f8f8" - default_style = "" - - styles = { - # No corresponding class for the following: - #Text: "", # class: '' - Whitespace: "underline #f8f8f8", # class: 'w' - Error: "#a40000 border:#ef2929", # class: 'err' - Other: "#000000", # class 'x' - - Comment: "italic #8f5902", # class: 'c' - Comment.Preproc: "noitalic", # class: 'cp' - - Keyword: "bold #004461", # class: 'k' - Keyword.Constant: "bold #004461", # class: 'kc' - Keyword.Declaration: "bold #004461", # class: 'kd' - Keyword.Namespace: "bold #004461", # class: 'kn' - Keyword.Pseudo: "bold #004461", # class: 'kp' - Keyword.Reserved: "bold #004461", # class: 'kr' - Keyword.Type: "bold #004461", # class: 'kt' - - Operator: "#582800", # class: 'o' - Operator.Word: "bold #004461", # class: 'ow' - like keywords - - Punctuation: "bold #000000", # class: 'p' - - # because special names such as Name.Class, Name.Function, etc. - # are not recognized as such later in the parsing, we choose them - # to look the same as ordinary variables. - Name: "#000000", # class: 'n' - Name.Attribute: "#c4a000", # class: 'na' - to be revised - Name.Builtin: "#004461", # class: 'nb' - Name.Builtin.Pseudo: "#3465a4", # class: 'bp' - Name.Class: "#000000", # class: 'nc' - to be revised - Name.Constant: "#000000", # class: 'no' - to be revised - Name.Decorator: "#888", # class: 'nd' - to be revised - Name.Entity: "#ce5c00", # class: 'ni' - Name.Exception: "bold #cc0000", # class: 'ne' - Name.Function: "#000000", # class: 'nf' - Name.Property: "#000000", # class: 'py' - Name.Label: "#f57900", # class: 'nl' - Name.Namespace: "#000000", # class: 'nn' - to be revised - Name.Other: "#000000", # class: 'nx' - Name.Tag: "bold #004461", # class: 'nt' - like a keyword - Name.Variable: "#000000", # class: 'nv' - to be revised - Name.Variable.Class: "#000000", # class: 'vc' - to be revised - Name.Variable.Global: "#000000", # class: 'vg' - to be revised - Name.Variable.Instance: "#000000", # class: 'vi' - to be revised - - Number: "#990000", # class: 'm' - - Literal: "#000000", # class: 'l' - Literal.Date: "#000000", # class: 'ld' - - String: "#4e9a06", # class: 's' - String.Backtick: "#4e9a06", # class: 'sb' - String.Char: "#4e9a06", # class: 'sc' - String.Doc: "italic #8f5902", # class: 'sd' - like a comment - String.Double: "#4e9a06", # class: 's2' - String.Escape: "#4e9a06", # class: 'se' - String.Heredoc: "#4e9a06", # class: 'sh' - String.Interpol: "#4e9a06", # class: 'si' - String.Other: "#4e9a06", # class: 'sx' - String.Regex: "#4e9a06", # class: 'sr' - String.Single: "#4e9a06", # class: 's1' - String.Symbol: "#4e9a06", # class: 'ss' - - Generic: "#000000", # class: 'g' - Generic.Deleted: "#a40000", # class: 'gd' - Generic.Emph: "italic #000000", # class: 'ge' - Generic.Error: "#ef2929", # class: 'gr' - Generic.Heading: "bold #000080", # class: 'gh' - Generic.Inserted: "#00A000", # class: 'gi' - Generic.Output: "#888", # class: 'go' - Generic.Prompt: "#745334", # class: 'gp' - Generic.Strong: "bold #000000", # class: 'gs' - Generic.Subheading: "bold #800080", # class: 'gu' - Generic.Traceback: "bold #a40000", # class: 'gt' - } diff --git a/docs/conf.py b/docs/conf.py index 7e7adbfc38..e6e005da81 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -22,10 +22,11 @@ # Insert Requests' path into the system. sys.path.insert(0, os.path.abspath("../src")) -sys.path.insert(0, os.path.abspath("_themes")) import niquests +sys.modules["requests"] = niquests + # -- General configuration ------------------------------------------------ @@ -37,9 +38,9 @@ # ones. extensions = [ "sphinx.ext.autodoc", - "sphinx.ext.intersphinx", "sphinx.ext.todo", "sphinx.ext.viewcode", + "sphinx_copybutton", ] # Add any paths that contain templates here, relative to this directory. @@ -58,7 +59,7 @@ # General information about the project. project = u"Niquests" -copyright = u'MMXVIX. A Kenneth Reitz Project' +copyright = u'MMXVIX. A Kenneth Reitz Project' author = u"Kenneth Reitz" # The version info for the project you're documenting, acts as replacement for @@ -75,7 +76,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -103,7 +104,8 @@ # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = "flask_theme_support.FlaskyStyle" +pygments_style = "sphinx" +pygments_dark_style = "monokai" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -119,18 +121,19 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "alabaster" +html_theme = "furo" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. html_theme_options = { - "show_powered_by": False, - "github_user": "requests", - "github_repo": "requests", - "github_banner": True, - "show_related": False, - "note_bg": "#FFF59C", + "source_repository": "https://github.com/jawah/niquests/", + "source_branch": "main", + "source_directory": "docs/", + "light_css_variables": { + "color-brand-primary": "#7C4DFF", + "color-brand-content": "#7C4DFF", + }, } # Add any paths that contain custom themes here, relative to this directory. @@ -171,17 +174,17 @@ html_use_smartypants = False # Custom sidebar templates, maps document names to template names. -html_sidebars = { - "index": ["sidebarintro.html", "sourcelink.html", "searchbox.html", "hacks.html"], - "**": [ - "sidebarlogo.html", - "localtoc.html", - "relations.html", - "sourcelink.html", - "searchbox.html", - "hacks.html", - ], -} +# html_sidebars = { +# "index": ["sidebarintro.html", "sourcelink.html", "searchbox.html", "hacks.html"], +# "**": [ +# "sidebarlogo.html", +# "localtoc.html", +# "relations.html", +# "sourcelink.html", +# "searchbox.html", +# "hacks.html", +# ], +# } # Additional templates that should be rendered to pages, maps page names to # template names. @@ -382,5 +385,5 @@ intersphinx_mapping = { "python": ("https://docs.python.org/3/", None), - "urllib3.future": ("https://urllib3-future.readthedocs.io/en/latest", None), + "urllib3.future": ("https://urllib3future.readthedocs.io/en/latest", None), } diff --git a/docs/index.rst b/docs/index.rst index 77b03ff951..94cff6f6ff 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,10 +17,6 @@ Release v\ |version|. (:ref:`Installation `) :target: https://pypi.org/project/niquests/ :alt: License Badge -.. image:: https://img.shields.io/pypi/wheel/niquests.svg - :target: https://pypi.org/project/niquests/ - :alt: Wheel Support Badge - .. image:: https://img.shields.io/pypi/pyversions/niquests.svg :target: https://pypi.org/project/niquests/ :alt: Python Version Support Badge @@ -43,6 +39,10 @@ is designed to be a drop-in replacement for **Requests** that is no longer under '{"type":"User"...' >>> r.json() {'private_gists': 419, 'total_private_repos': 77, ...} + >>> r + + >>> r.ocsp_verified + True See `similar code, sans Niquests `_. @@ -50,7 +50,7 @@ See `similar code, sans Niquests `_. **Niquests** allows you to send HTTP/1.1, HTTP/2 and HTTP/3 requests extremely easily. There's no need to manually add query strings to your URLs, or to form-encode your POST data. Keep-alive and HTTP connection pooling -are 100% automatic, thanks to `urllib3 `_. +are 100% automatic, thanks to `urllib3.future `_. Beloved Features ---------------- diff --git a/docs/requirements.txt b/docs/requirements.txt index 0c43e4acbc..5e2d0ec541 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,8 @@ # Pinning to avoid unexpected breakages. # Used by RTD to generate docs. -Sphinx==4.2.0 -urllib3.future>=2.0.934 +Sphinx==7.2.6 +sphinx-copybutton==0.5.2 +urllib3.future>=2.1.900 wassima>=1,<2 kiss_headers>=2,<4 +furo>=2023.9.10 diff --git a/docs/user/advanced.rst b/docs/user/advanced.rst index 1f9bc7bee3..ee2772454c 100644 --- a/docs/user/advanced.rst +++ b/docs/user/advanced.rst @@ -12,7 +12,7 @@ Session Objects The Session object allows you to persist certain parameters across niquests. It also persists cookies across all requests made from the -Session instance, and will use ``urllib3``'s `connection pooling`_. So if +Session instance, and will use ``urllib3.future``'s `connection pooling`_. So if you're making several requests to the same host, the underlying TCP connection will be reused, which can result in a significant performance increase (see `HTTP persistent connection`_). @@ -640,7 +640,14 @@ Alternatively you can configure it once for an entire When the proxies configuration is not overridden per request as shown above, Niquests relies on the proxy configuration defined by standard environment variables ``http_proxy``, ``https_proxy``, ``no_proxy``, -and ``all_proxy``. Uppercase variants of these variables are also supported. +and ``all_proxy``. + +.. admonition:: IPv6 in NO_PROXY + :class: note + + Available since version 3.1.2 + +Uppercase variants of these variables are also supported. You can therefore set them to configure Niquests (only set the ones relevant to your needs):: @@ -1178,3 +1185,27 @@ This feature may not be available if the ``cryptography`` package is missing fro Verify the availability by running ``python -m niquests.help``. .. note:: Access property ``ocsp_verified`` in both ``PreparedRequest``, and ``Response`` to have information about this post handshake verification. + +Specify HTTP/3 capable endpoint preemptively +-------------------------------------------- + +Preemptively register a website to be HTTP/3 capable prior to the first TLS over TCP handshake. +You can do so by doing like:: + + from niquests import Session + + s = Session() + s.quic_cache_layer.add_domain("cloudflare.com") + +This will prevent the first request being made with HTTP/2 or HTTP/1.1. + +.. note:: You can also specify an alternate destination port if QUIC is being served on anything else than 443. + +Sample:: + + s.quic_cache_layer.add_domain("cloudflare.com", alt_port=8544) + +This would mean that attempting to request ``https://cloudflare.com/a/b`` will be routed through ``https://cloudflare.com:8544/a/b`` +over QUIC. + +.. warning:: You cannot specify another hostname for security reasons. diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index d13ae7cec2..8fb4353b26 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -299,6 +299,22 @@ and it will be encoded automatically: Note, the ``json`` parameter is ignored if either ``data`` or ``files`` is passed. +POST a Multipart Form-Data without File +--------------------------------------- + +Since Niquests 3.1.2 it is possible to overrule the default conversion to ``application/x-www-form-urlencoded`` type. +You can submit a form-data by helping Niquests understand what you meant. + + >>> url = 'https://httpbin.org/post' + >>> payload = {'some': 'data'} + + >>> r = niquests.post(url, data=payload, headers={"Content-Type": "multipart/form-data"}) + +Now, instead of submitting a urlencoded body, as per the default, Niquests will send instead a proper +form-data. + +.. note:: You can also specify manually a boundary in the header value. Niquests will reuse it. Otherwise it will assign a random one. + POST a Multipart-Encoded File ----------------------------- diff --git a/setup.cfg b/setup.cfg index f820ae2bc5..0875a6d193 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,5 +2,4 @@ ignore = E203, E501, W503 per-file-ignores = src/niquests/__init__.py:E402, F401 - src/niquests/compat.py:E402, F401 tests/compat.py:F401 diff --git a/src/niquests/__version__.py b/src/niquests/__version__.py index 78fa938297..9b6ae52486 100644 --- a/src/niquests/__version__.py +++ b/src/niquests/__version__.py @@ -9,9 +9,9 @@ __url__: str = "https://niquests.readthedocs.io" __version__: str -__version__ = "3.1.1" +__version__ = "3.1.2" -__build__: int = 0x030101 +__build__: int = 0x030102 __author__: str = "Kenneth Reitz" __author_email__: str = "me@kennethreitz.org" __license__: str = "Apache-2.0" diff --git a/src/niquests/_typing.py b/src/niquests/_typing.py index a1e637533e..c4c6b4c1ba 100644 --- a/src/niquests/_typing.py +++ b/src/niquests/_typing.py @@ -23,7 +23,7 @@ ] BodyFormType: typing.TypeAlias = typing.Union[ typing.List[typing.Tuple[str, str]], - typing.Dict[str, str], + typing.Dict[str, typing.Union[typing.List[str], str]], ] #: Accepted types for the payload in POST, PUT, and PATCH requests. BodyType: typing.TypeAlias = typing.Union[ diff --git a/src/niquests/api.py b/src/niquests/api.py index 049f96753d..9a42aa4f1c 100644 --- a/src/niquests/api.py +++ b/src/niquests/api.py @@ -31,9 +31,9 @@ TLSVerifyType, ) from .models import PreparedRequest, Response -from .structures import SharableLimitedDict +from .structures import QuicSharedCache -_SHARED_QUIC_CACHE: CacheLayerAltSvcType = SharableLimitedDict(max_size=12_288) +_SHARED_QUIC_CACHE: CacheLayerAltSvcType = QuicSharedCache(max_size=12_288) def request( diff --git a/src/niquests/models.py b/src/niquests/models.py index 3b5fb5ddae..fb1946859e 100644 --- a/src/niquests/models.py +++ b/src/niquests/models.py @@ -32,7 +32,7 @@ SSLError, ) from urllib3.fields import RequestField -from urllib3.filepost import encode_multipart_formdata +from urllib3.filepost import choose_boundary, encode_multipart_formdata from urllib3.util import parse_url from ._internal_utils import to_native_string @@ -257,6 +257,10 @@ def __init__(self) -> None: #: marker about if OCSP post-handshake verification took place. self.ocsp_verified: bool | None = None + @property + def oheaders(self) -> Headers: + return parse_it(self.headers) + def prepare( self, method: HttpMethodType | None = None, @@ -423,6 +427,7 @@ def prepare_body( # Nottin' on you. body: BodyType | None = None content_type: str | None = None + enforce_form_data = False if not data and json is not None: # urllib3 requires a bytes-like body. Python 2's json.dumps @@ -492,11 +497,35 @@ def prepare_body( (body, content_type) = self._encode_files(files, data) # type: ignore[arg-type] else: if data: - body = self._encode_params(data) + enforce_form_data = ( + "content-type" in self.oheaders + and isinstance(self.oheaders.content_type, list) is False + and "multipart/form-data" in self.oheaders.content_type + ) + + if enforce_form_data: + form_data_boundary = ( + self.oheaders.content_type.boundary # type: ignore[union-attr] + if enforce_form_data + and self.oheaders.content_type.get("boundary") # type: ignore[union-attr] + else choose_boundary() + ) + else: + form_data_boundary = None + + body = self._encode_params( + data, + boundary_for_multipart=form_data_boundary, + ) if isinstance(data, str) or hasattr(data, "read"): content_type = None else: - content_type = "application/x-www-form-urlencoded" + if not enforce_form_data: + content_type = "application/x-www-form-urlencoded" + else: + content_type = ( + f"multipart/form-data; boundary={form_data_boundary}" + ) assert isinstance(body, (list, dict)) is False self.prepare_content_length(body) @@ -504,6 +533,8 @@ def prepare_body( # Add content-type if it wasn't explicitly provided. if content_type and ("content-type" not in self.headers): self.headers["Content-Type"] = content_type + elif content_type and enforce_form_data: + self.headers["Content-Type"] = content_type self.body = body @@ -635,6 +666,7 @@ def path_url(self) -> str: @staticmethod def _encode_params( data: QueryParameterType | BodyFormType | typing.IO, + boundary_for_multipart: str | None = None, ) -> str | bytes | bytearray: """Encode parameters in a piece of data. @@ -650,9 +682,13 @@ def _encode_params( elif hasattr(data, "__iter__"): result = [] for k, vs in to_key_val_list(data): - iterable_vs: list[str | bytes] - if isinstance(vs, str) or not hasattr(vs, "__iter__"): - iterable_vs = [vs] + iterable_vs: typing.Iterable[str | bytes] + if isinstance(vs, (str, bytes, int, float)): + # not officially supported, but some people maybe passing ints or float. + if isinstance(vs, (str, bytes)) is False: + iterable_vs = [str(vs)] + else: + iterable_vs = [vs] else: iterable_vs = vs for v in iterable_vs: @@ -663,6 +699,10 @@ def _encode_params( v.encode("utf-8") if isinstance(v, str) else v, ) ) + if boundary_for_multipart: + return encode_multipart_formdata( + result, boundary=boundary_for_multipart # type: ignore[arg-type] + )[0] return urlencode(result, doseq=True) else: raise ValueError( diff --git a/src/niquests/sessions.py b/src/niquests/sessions.py index 0a154718d6..8ef082065d 100644 --- a/src/niquests/sessions.py +++ b/src/niquests/sessions.py @@ -70,7 +70,7 @@ Response, ) from .status_codes import codes -from .structures import CaseInsensitiveDict, SharableLimitedDict +from .structures import CaseInsensitiveDict, QuicSharedCache from .utils import ( # noqa: F401 DEFAULT_PORTS, default_headers, @@ -269,7 +269,7 @@ def __init__( self.quic_cache_layer = ( quic_cache_layer if quic_cache_layer is not None - else SharableLimitedDict(max_size=12_288) + else QuicSharedCache(max_size=12_288) ) # Default connection adapters. @@ -1116,7 +1116,7 @@ def __setstate__(self, state): for attr, value in state.items(): setattr(self, attr, value) - self.quic_cache_layer = SharableLimitedDict(max_size=12_288) + self.quic_cache_layer = QuicSharedCache(max_size=12_288) self.adapters = OrderedDict() self.mount( diff --git a/src/niquests/structures.py b/src/niquests/structures.py index 896bbe8aec..df38e4f638 100644 --- a/src/niquests/structures.py +++ b/src/niquests/structures.py @@ -132,3 +132,14 @@ def __setitem__(self, key, value): def __getitem__(self, item): with self._lock: return self._store[item] + + +class QuicSharedCache(SharableLimitedDict): + def add_domain( + self, host: str, port: int | None = None, alt_port: int | None = None + ) -> None: + if port is None: + port = 443 + if alt_port is None: + alt_port = port + self[(host, port)] = (host, alt_port) diff --git a/src/niquests/utils.py b/src/niquests/utils.py index 04a969c917..e5ab3bdf40 100644 --- a/src/niquests/utils.py +++ b/src/niquests/utils.py @@ -557,16 +557,42 @@ def requote_uri(uri: str) -> str: return quote(uri, safe=safe_without_percent) +def _get_mask_bits(mask: int, totalbits: int = 32) -> int: + """Converts a mask from /xx format to an integer + to be used as a mask for IP's in int format + Example: if mask is 24 function returns 0xFFFFFF00 + if mask is 24 and totalbits=128 function + returns 0xFFFFFF00000000000000000000000000 + """ + bits = ((1 << mask) - 1) << (totalbits - mask) + + return bits + + def address_in_network(ip: str, net: str) -> bool: """This function allows you to check if an IP belongs to a network subnet Example: returns True if ip = 192.168.1.1 and net = 192.168.1.0/24 returns False if ip = 192.168.1.1 and net = 192.168.100.0/24 + returns True if ip = 1:2:3:4::1 and net = 1:2:3:4::/64 """ - ipaddr = struct.unpack("=L", socket.inet_aton(ip))[0] netaddr, bits = net.split("/") - netmask = struct.unpack("=L", socket.inet_aton(dotted_netmask(int(bits))))[0] - network = struct.unpack("=L", socket.inet_aton(netaddr))[0] & netmask + if is_ipv4_address(ip) and is_ipv4_address(netaddr): + ipaddr = struct.unpack(">L", socket.inet_aton(ip))[0] + netmask = _get_mask_bits(int(bits)) + network = struct.unpack(">L", socket.inet_aton(netaddr))[0] + elif is_ipv6_address(ip) and is_ipv6_address(netaddr): + ipaddr_msb, ipaddr_lsb = struct.unpack( + ">QQ", socket.inet_pton(socket.AF_INET6, ip) + ) + ipaddr = (ipaddr_msb << 64) ^ ipaddr_lsb + netmask = _get_mask_bits(int(bits), 128) + network_msb, network_lsb = struct.unpack( + ">QQ", socket.inet_pton(socket.AF_INET6, netaddr) + ) + network = (network_msb << 64) ^ network_lsb + else: + return False return (ipaddr & netmask) == (network & netmask) @@ -587,22 +613,45 @@ def is_ipv4_address(string_ip: str) -> bool: return True +def is_ipv6_address(string_ip: str) -> bool: + try: + socket.inet_pton(socket.AF_INET6, string_ip) + except OSError: + return False + return True + + +def compare_ipv6(a: str, b: str): + """ + Compare 2 IPs, uses socket.inet_pton to normalize IPv6 IPs + """ + try: + return socket.inet_pton(socket.AF_INET6, a) == socket.inet_pton( + socket.AF_INET6, b + ) + except OSError: + return False + + def is_valid_cidr(string_network: str) -> bool: """ Very simple check of the cidr format in no_proxy variable. """ if string_network.count("/") == 1: - try: - mask = int(string_network.split("/")[1]) - except ValueError: - return False + address, mask = string_network.split("/") - if mask < 1 or mask > 32: + if len(mask) >= 4 or mask.isdigit() is False: return False - try: - socket.inet_aton(string_network.split("/")[0]) - except OSError: + mask_int = int(mask) + + if is_ipv4_address(address): + if mask_int < 1 or mask_int > 32: + return False + elif is_ipv6_address(address): + if mask_int < 1 or mask_int > 128: + return False + else: return False else: return False @@ -659,7 +708,7 @@ def get_proxy(key: str) -> str | None: host for host in no_proxy.replace(" ", "").split(",") if host ) - if is_ipv4_address(parsed.hostname): + if is_ipv4_address(parsed.hostname) or is_ipv6_address(parsed.hostname): for proxy_ip in no_proxy_list: if is_valid_cidr(proxy_ip): if address_in_network(parsed.hostname, proxy_ip): @@ -668,6 +717,8 @@ def get_proxy(key: str) -> str | None: # If no_proxy ip was defined in plain IP notation instead of cidr notation & # matches the IP of the index return True + elif compare_ipv6(parsed.hostname, proxy_ip): + return True else: host_with_port = parsed.hostname if parsed.port: diff --git a/tests/test_requests.py b/tests/test_requests.py index b85055ce3b..38ac744036 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -2163,6 +2163,27 @@ def test_chunked_upload_does_not_set_content_length_header(self, httpbin): assert "Transfer-Encoding" in prepared_request.headers assert "Content-Length" not in prepared_request.headers + def test_data_multipart_without_files(self, httpbin): + payload = {"hello": "world", "Niquests": "Welcome you!"} + url = httpbin("post") + r = niquests.post( url, data=payload, headers={"Content-Type": "multipart/form-data"}) + + assert r.status_code == 200 + assert b"Content-Disposition: form-data" in r.request.body + assert r.json()["form"]["hello"] == "world" + assert r.json()["form"]["Niquests"] == "Welcome you!" + + def test_data_multipart_pre_assigned_boundary_without_files(self, httpbin): + payload = {"hello": "world", "Niquests": "Welcome you!"} + url = httpbin("post") + r = niquests.post( url, data=payload, headers={"Content-Type": "multipart/form-data; boundary=c5c0e7f46481be260be8e9f68fcabf12"}) + + assert r.status_code == 200 + assert b"Content-Disposition: form-data" in r.request.body + assert b"--c5c0e7f46481be260be8e9f68fcabf12" + assert r.json()["form"]["hello"] == "world" + assert r.json()["form"]["Niquests"] == "Welcome you!" + def test_custom_redirect_mixin(self, httpbin): """Tests a custom mixin to overwrite ``get_redirect_target``. diff --git a/tests/test_utils.py b/tests/test_utils.py index d2072a8371..2134139098 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -14,9 +14,11 @@ from niquests.cookies import RequestsCookieJar from niquests.structures import CaseInsensitiveDict from niquests.utils import ( + _get_mask_bits, _parse_content_type_header, add_dict_to_cookiejar, address_in_network, + compare_ipv6, dotted_netmask, get_auth_from_url, get_encoding_from_headers, @@ -258,8 +260,15 @@ def test_invalid(self, value): class TestIsValidCIDR: - def test_valid(self): - assert is_valid_cidr("192.168.1.0/24") + @pytest.mark.parametrize( + "value", + ( + "192.168.1.0/24", + "1:2:3:4::/64", + ), + ) + def test_valid(self, value): + assert is_valid_cidr(value) @pytest.mark.parametrize( "value", @@ -269,6 +278,11 @@ def test_valid(self): "192.168.1.0/128", "192.168.1.0/-1", "192.168.1.999/24", + "1:2:3:4::1", + "1:2:3:4::/a", + "1:2:3:4::0/321", + "1:2:3:4::/-1", + "1:2:3:4::12211/64", ), ) def test_invalid(self, value): @@ -616,6 +630,11 @@ def test_urldefragauth(url, expected): ("http://172.16.1.12:5000/", False), ("http://google.com:5000/v1.0/", False), ("file:///some/path/on/disk", True), + ("http://[1:2:3:4:5:6:7:8]:5000/", True), + ("http://[1:2:3:4::1]/", True), + ("http://[1:2:3:9::1]/", True), + ("http://[1:2:3:9:0:0:0:1]/", True), + ("http://[1:2:3:9::2]/", False), ), ) def test_should_bypass_proxies(url, expected, monkeypatch): @@ -624,11 +643,11 @@ def test_should_bypass_proxies(url, expected, monkeypatch): """ monkeypatch.setenv( "no_proxy", - "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000", + "192.168.0.0/24,127.0.0.1,localhost.localdomain,1:2:3:4::/64,1:2:3:9::1,172.16.1.1, google.com:6000", ) monkeypatch.setenv( "NO_PROXY", - "192.168.0.0/24,127.0.0.1,localhost.localdomain,172.16.1.1, google.com:6000", + "192.168.0.0/24,127.0.0.1,localhost.localdomain,1:2:3:4::/64,1:2:3:9::1,172.16.1.1, google.com:6000", ) assert should_bypass_proxies(url, no_proxy=None) == expected @@ -806,3 +825,34 @@ def test_set_environ_raises_exception(): raise Exception("Expected exception") assert "Expected exception" in str(exception.value) + + +@pytest.mark.parametrize( + "mask, totalbits, maskbits", + ( + (24, None, 0xFFFFFF00), + (31, None, 0xFFFFFFFE), + (0, None, 0x0), + (4, 4, 0xF), + (24, 128, 0xFFFFFF00000000000000000000000000), + ), +) +def test__get_mask_bits(mask, totalbits, maskbits): + args = {"mask": mask} + if totalbits: + args["totalbits"] = totalbits + assert _get_mask_bits(**args) == maskbits + + +@pytest.mark.parametrize( + "a, b, expected", + ( + ("1::4", "1.2.3.4", False), + ("1::4", "1::4", True), + ("1::4", "1:0:0:0:0:0:0:4", True), + ("1::4", "1:0:0:0:0:0::4", True), + ("1::4", "1:0:0:0:0:0:1:4", False), + ), +) +def test_compare_ipv6(a, b, expected): + assert compare_ipv6(a, b) == expected