diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index b6d544640b..a504eccc5b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -45,7 +45,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@df5a14dc28094dc936e103b37d749c6628682b60 # v3.25.0 + uses: github/codeql-action/init@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 with: languages: "python" # If you wish to specify custom queries, you can do so here or in a config file. @@ -56,7 +56,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@df5a14dc28094dc936e103b37d749c6628682b60 # v3.25.0 + uses: github/codeql-action/autobuild@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -70,4 +70,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@df5a14dc28094dc936e103b37d749c6628682b60 # v3.25.0 + uses: github/codeql-action/analyze@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 46a7862eac..cbea3ae665 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,7 +13,7 @@ jobs: steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: Set up Python - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: "3.x" - name: Run pre-commit diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 674df73120..0a8d73af6a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -27,7 +27,7 @@ jobs: steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0 with: python-version: ${{ matrix.python-version }} cache: 'pip' @@ -47,7 +47,7 @@ jobs: steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 - name: 'Set up Python 3.8' - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 with: python-version: '3.8' - name: Install dependencies @@ -57,3 +57,23 @@ jobs: - name: Run tests run: | make ci + + urllib3: + name: 'urllib3 1.x' + runs-on: 'ubuntu-latest' + strategy: + fail-fast: true + + steps: + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + - name: 'Set up Python 3.8' + uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 + with: + python-version: '3.8' + - name: Install dependencies + run: | + make + python -m pip install "urllib3<2" + - name: Run tests + run: | + make ci diff --git a/HISTORY.md b/HISTORY.md index 4fca5894d7..e51a7ee2c2 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -6,6 +6,15 @@ dev - \[Short description of non-trivial change.\] +2.32.3 (2024-05-29) +------------------- + +**Bugfixes** +- Fixed bug breaking the ability to specify custom SSLContexts in sub-classes of + HTTPAdapter. (#6716) +- Fixed issue where Requests started failing to run on Python versions compiled + without the `ssl` module. (#6724) + 2.32.2 (2024-05-21) ------------------- diff --git a/setup.py b/setup.py index 1b0eb377b4..e18aa18bb6 100755 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ from codecs import open from setuptools import setup -from setuptools.command.test import test as TestCommand CURRENT_PYTHON = sys.version_info[:2] REQUIRED_PYTHON = (3, 8) @@ -28,30 +27,6 @@ sys.exit(1) -class PyTest(TestCommand): - user_options = [("pytest-args=", "a", "Arguments to pass into py.test")] - - def initialize_options(self): - TestCommand.initialize_options(self) - try: - from multiprocessing import cpu_count - - self.pytest_args = ["-n", str(cpu_count()), "--boxed"] - except (ImportError, NotImplementedError): - self.pytest_args = ["-n", "1", "--boxed"] - - def finalize_options(self): - TestCommand.finalize_options(self) - self.test_args = [] - self.test_suite = True - - def run_tests(self): - import pytest - - errno = pytest.main(self.pytest_args) - sys.exit(errno) - - # 'setup.py publish' shortcut. if sys.argv[-1] == "publish": os.system("python setup.py sdist bdist_wheel") @@ -118,7 +93,6 @@ def run_tests(self): "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries", ], - cmdclass={"test": PyTest}, tests_require=test_requirements, extras_require={ "security": [], diff --git a/src/requests/__version__.py b/src/requests/__version__.py index 24829c4b5f..2c105aca7d 100644 --- a/src/requests/__version__.py +++ b/src/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = "requests" __description__ = "Python HTTP for Humans." __url__ = "https://requests.readthedocs.io" -__version__ = "2.32.2" -__build__ = 0x023202 +__version__ = "2.32.3" +__build__ = 0x023203 __author__ = "Kenneth Reitz" __author_email__ = "me@kennethreitz.org" __license__ = "Apache-2.0" diff --git a/src/requests/adapters.py b/src/requests/adapters.py index 42fabe527c..9a58b16025 100644 --- a/src/requests/adapters.py +++ b/src/requests/adapters.py @@ -73,26 +73,44 @@ def SOCKSProxyManager(*args, **kwargs): DEFAULT_RETRIES = 0 DEFAULT_POOL_TIMEOUT = None -_preloaded_ssl_context = create_urllib3_context() -_preloaded_ssl_context.load_verify_locations( - extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) -) + +try: + import ssl # noqa: F401 + + _preloaded_ssl_context = create_urllib3_context() + _preloaded_ssl_context.load_verify_locations( + extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) + ) +except ImportError: + # Bypass default SSLContext creation when Python + # interpreter isn't built with the ssl module. + _preloaded_ssl_context = None def _urllib3_request_context( request: "PreparedRequest", verify: "bool | str | None", client_cert: "typing.Tuple[str, str] | str | None", + poolmanager: "PoolManager", ) -> "(typing.Dict[str, typing.Any], typing.Dict[str, typing.Any])": host_params = {} pool_kwargs = {} parsed_request_url = urlparse(request.url) scheme = parsed_request_url.scheme.lower() port = parsed_request_url.port + + # Determine if we have and should use our default SSLContext + # to optimize performance on standard requests. + poolmanager_kwargs = getattr(poolmanager, "connection_pool_kw", {}) + has_poolmanager_ssl_context = poolmanager_kwargs.get("ssl_context") + should_use_default_ssl_context = ( + _preloaded_ssl_context is not None and not has_poolmanager_ssl_context + ) + cert_reqs = "CERT_REQUIRED" if verify is False: cert_reqs = "CERT_NONE" - elif verify is True: + elif verify is True and should_use_default_ssl_context: pool_kwargs["ssl_context"] = _preloaded_ssl_context elif isinstance(verify, str): if not os.path.isdir(verify): @@ -375,23 +393,83 @@ def build_response(self, req, resp): return response + def build_connection_pool_key_attributes(self, request, verify, cert=None): + """Build the PoolKey attributes used by urllib3 to return a connection. + + This looks at the PreparedRequest, the user-specified verify value, + and the value of the cert parameter to determine what PoolKey values + to use to select a connection from a given urllib3 Connection Pool. + + The SSL related pool key arguments are not consistently set. As of + this writing, use the following to determine what keys may be in that + dictionary: + + * If ``verify`` is ``True``, ``"ssl_context"`` will be set and will be the + default Requests SSL Context + * If ``verify`` is ``False``, ``"ssl_context"`` will not be set but + ``"cert_reqs"`` will be set + * If ``verify`` is a string, (i.e., it is a user-specified trust bundle) + ``"ca_certs"`` will be set if the string is not a directory recognized + by :py:func:`os.path.isdir`, otherwise ``"ca_certs_dir"`` will be + set. + * If ``"cert"`` is specified, ``"cert_file"`` will always be set. If + ``"cert"`` is a tuple with a second item, ``"key_file"`` will also + be present + + To override these settings, one may subclass this class, call this + method and use the above logic to change parameters as desired. For + example, if one wishes to use a custom :py:class:`ssl.SSLContext` one + must both set ``"ssl_context"`` and based on what else they require, + alter the other keys to ensure the desired behaviour. + + :param request: + The PreparedReqest being sent over the connection. + :type request: + :class:`~requests.models.PreparedRequest` + :param verify: + Either a boolean, in which case it controls whether + we verify the server's TLS certificate, or a string, in which case it + must be a path to a CA bundle to use. + :param cert: + (optional) Any user-provided SSL certificate for client + authentication (a.k.a., mTLS). This may be a string (i.e., just + the path to a file which holds both certificate and key) or a + tuple of length 2 with the certificate file path and key file + path. + :returns: + A tuple of two dictionaries. The first is the "host parameters" + portion of the Pool Key including scheme, hostname, and port. The + second is a dictionary of SSLContext related parameters. + """ + return _urllib3_request_context(request, verify, cert, self.poolmanager) + def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None): """Returns a urllib3 connection for the given request and TLS settings. This should not be called from user code, and is only exposed for use when subclassing the :class:`HTTPAdapter `. - :param request: The :class:`PreparedRequest ` object - to be sent over the connection. - :param verify: Either a boolean, in which case it controls whether - we verify the server's TLS certificate, or a string, in which case it - must be a path to a CA bundle to use. - :param proxies: (optional) The proxies dictionary to apply to the request. - :param cert: (optional) Any user-provided SSL certificate to be trusted. - :rtype: urllib3.ConnectionPool + :param request: + The :class:`PreparedRequest ` object to be sent + over the connection. + :param verify: + Either a boolean, in which case it controls whether we verify the + server's TLS certificate, or a string, in which case it must be a + path to a CA bundle to use. + :param proxies: + (optional) The proxies dictionary to apply to the request. + :param cert: + (optional) Any user-provided SSL certificate to be used for client + authentication (a.k.a., mTLS). + :rtype: + urllib3.ConnectionPool """ proxy = select_proxy(request.url, proxies) try: - host_params, pool_kwargs = _urllib3_request_context(request, verify, cert) + host_params, pool_kwargs = self.build_connection_pool_key_attributes( + request, + verify, + cert, + ) except ValueError as e: raise InvalidURL(e, request=request) if proxy: diff --git a/src/requests/compat.py b/src/requests/compat.py index 095de1b6ca..7f9d754350 100644 --- a/src/requests/compat.py +++ b/src/requests/compat.py @@ -10,6 +10,18 @@ import importlib import sys +# ------- +# urllib3 +# ------- +from urllib3 import __version__ as urllib3_version + +# Detect which major version of urllib3 is being used. +try: + is_urllib3_1 = int(urllib3_version.split(".")[0]) == 1 +except (TypeError, AttributeError): + # If we can't discern a version, prefer old functionality. + is_urllib3_1 = True + # ------------------- # Character Detection # ------------------- diff --git a/src/requests/utils.py b/src/requests/utils.py index ae6c42f6cb..699683e5d9 100644 --- a/src/requests/utils.py +++ b/src/requests/utils.py @@ -38,6 +38,7 @@ getproxies, getproxies_environment, integer_types, + is_urllib3_1, ) from .compat import parse_http_list as _parse_list_header from .compat import ( @@ -136,7 +137,9 @@ def super_len(o): total_length = None current_position = 0 - if isinstance(o, str): + if not is_urllib3_1 and isinstance(o, str): + # urllib3 2.x+ treats all strings as utf-8 instead + # of latin-1 (iso-8859-1) like http.client. o = o.encode("utf-8") if hasattr(o, "__len__"): diff --git a/tests/test_requests.py b/tests/test_requests.py index b4e9fe92ae..d8fbb23688 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -25,6 +25,7 @@ builtin_str, cookielib, getproxies, + is_urllib3_1, urlparse, ) from requests.cookies import cookiejar_from_dict, morsel_to_cookie @@ -1810,23 +1811,6 @@ def test_autoset_header_values_are_native(self, httpbin): assert p.headers["Content-Length"] == length - def test_content_length_for_bytes_data(self, httpbin): - data = "This is a string containing multi-byte UTF-8 ☃ī¸" - encoded_data = data.encode("utf-8") - length = str(len(encoded_data)) - req = requests.Request("POST", httpbin("post"), data=encoded_data) - p = req.prepare() - - assert p.headers["Content-Length"] == length - - def test_content_length_for_string_data_counts_bytes(self, httpbin): - data = "This is a string containing multi-byte UTF-8 ☃ī¸" - length = str(len(data.encode("utf-8"))) - req = requests.Request("POST", httpbin("post"), data=data) - p = req.prepare() - - assert p.headers["Content-Length"] == length - def test_nonhttp_schemes_dont_check_URLs(self): test_urls = ( "data:image/gif;base64,R0lGODlhAQABAHAAACH5BAUAAAAALAAAAAABAAEAAAICRAEAOw==", @@ -2966,6 +2950,29 @@ def response_handler(sock): assert client_cert is not None +def test_content_length_for_bytes_data(httpbin): + data = "This is a string containing multi-byte UTF-8 ☃ī¸" + encoded_data = data.encode("utf-8") + length = str(len(encoded_data)) + req = requests.Request("POST", httpbin("post"), data=encoded_data) + p = req.prepare() + + assert p.headers["Content-Length"] == length + + +@pytest.mark.skipif( + is_urllib3_1, + reason="urllib3 2.x encodes all strings to utf-8, urllib3 1.x uses latin-1", +) +def test_content_length_for_string_data_counts_bytes(httpbin): + data = "This is a string containing multi-byte UTF-8 ☃ī¸" + length = str(len(data.encode("utf-8"))) + req = requests.Request("POST", httpbin("post"), data=data) + p = req.prepare() + + assert p.headers["Content-Length"] == length + + def test_json_decode_errors_are_serializable_deserializable(): json_decode_error = requests.exceptions.JSONDecodeError( "Extra data",