From e94346e13375d81b35644e95ea4340e957ee2a7d Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 17 Dec 2024 14:39:48 +0100 Subject: [PATCH 1/8] Use TLS settings in selecting connection pool Upstream commit: https://github.com/psf/requests/commit/c0813a2d910ea6b4f8438b91d315b8d181302356 --- requests/adapters.py | 53 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/requests/adapters.py b/requests/adapters.py index fa4d9b3..b768460 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -10,6 +10,7 @@ and maintain connections. import os.path import socket +import typing from urllib3.poolmanager import PoolManager, proxy_from_url from urllib3.response import HTTPResponse @@ -52,6 +53,28 @@ DEFAULT_RETRIES = 0 DEFAULT_POOL_TIMEOUT = None +def _urllib3_request_context( + request: "PreparedRequest", verify: "bool | str | None" +) -> "(typing.Dict[str, typing.Any], typing.Dict[str, typing.Any])": + host_params = {} + pool_kwargs = {} + parsed_request_url = urlparse(request.url) + scheme = parsed_request_url.scheme.lower() + port = parsed_request_url.port + cert_reqs = "CERT_REQUIRED" + if verify is False: + cert_reqs = "CERT_NONE" + if isinstance(verify, str): + pool_kwargs["ca_certs"] = verify + pool_kwargs["cert_reqs"] = cert_reqs + host_params = { + "scheme": scheme, + "host": parsed_request_url.hostname, + "port": port, + } + return host_params, pool_kwargs + + class BaseAdapter(object): """The Base Transport Adapter""" @@ -289,6 +312,34 @@ class HTTPAdapter(BaseAdapter): return response + def _get_connection(self, request, verify, proxies=None): + # Replace the existing get_connection without breaking things and + # ensure that TLS settings are considered when we interact with + # urllib3 HTTP Pools + proxy = select_proxy(request.url, proxies) + try: + host_params, pool_kwargs = _urllib3_request_context(request, verify) + except ValueError as e: + raise InvalidURL(e, request=request) + if proxy: + proxy = prepend_scheme_if_needed(proxy, "http") + proxy_url = parse_url(proxy) + if not proxy_url.host: + raise InvalidProxyURL( + "Please check proxy URL. It is malformed " + "and could be missing the host." + ) + proxy_manager = self.proxy_manager_for(proxy) + conn = proxy_manager.connection_from_host( + **host_params, pool_kwargs=pool_kwargs + ) + else: + # Only scheme should be lower case + conn = self.poolmanager.connection_from_host( + **host_params, pool_kwargs=pool_kwargs + ) + return conn + def get_connection(self, url, proxies=None): """Returns a urllib3 connection for the given URL. This should not be called from user code, and is only exposed for use when subclassing the @@ -409,7 +460,7 @@ class HTTPAdapter(BaseAdapter): """ try: - conn = self.get_connection(request.url, proxies) + conn = self._get_connection(request, verify, proxies) except LocationValueError as e: raise InvalidURL(e, request=request) -- 2.47.1 From d3c30b0c69d8efe9a8ebce1f05d72dc0ac47ed67 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 17 Dec 2024 14:45:08 +0100 Subject: [PATCH 2/8] Add additional context parameters for our pool manager Upstream commit: https://github.com/psf/requests/commit/a94e9b5308ffcc3d2913ab873e9810a6601a67da --- requests/adapters.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index b768460..65ad876 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -54,7 +54,9 @@ DEFAULT_POOL_TIMEOUT = None def _urllib3_request_context( - request: "PreparedRequest", verify: "bool | str | None" + request: "PreparedRequest", + verify: "bool | str | None", + client_cert: "typing.Tuple[str, str] | str | None", ) -> "(typing.Dict[str, typing.Any], typing.Dict[str, typing.Any])": host_params = {} pool_kwargs = {} @@ -67,6 +69,14 @@ def _urllib3_request_context( if isinstance(verify, str): pool_kwargs["ca_certs"] = verify pool_kwargs["cert_reqs"] = cert_reqs + if client_cert is not None: + if isinstance(client_cert, tuple) and len(client_cert) == 2: + pool_kwargs["cert_file"] = client_cert[0] + pool_kwargs["key_file"] = client_cert[1] + else: + # According to our docs, we allow users to specify just the client + # cert path + pool_kwargs["cert_file"] = client_cert host_params = { "scheme": scheme, "host": parsed_request_url.hostname, @@ -312,13 +322,13 @@ class HTTPAdapter(BaseAdapter): return response - def _get_connection(self, request, verify, proxies=None): + def _get_connection(self, request, verify, proxies=None, cert=None): # Replace the existing get_connection without breaking things and # ensure that TLS settings are considered when we interact with # urllib3 HTTP Pools proxy = select_proxy(request.url, proxies) try: - host_params, pool_kwargs = _urllib3_request_context(request, verify) + host_params, pool_kwargs = _urllib3_request_context(request, verify, cert) except ValueError as e: raise InvalidURL(e, request=request) if proxy: @@ -460,7 +470,7 @@ class HTTPAdapter(BaseAdapter): """ try: - conn = self._get_connection(request, verify, proxies) + conn = self._get_connection(request, verify, proxies=proxies, cert=cert) except LocationValueError as e: raise InvalidURL(e, request=request) -- 2.47.1 From 5dbe98fe21871f315cc68473165cbbed5eb5f048 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 17 Dec 2024 14:51:34 +0100 Subject: [PATCH 3/8] Avoid reloading root certificates to improve concurrent performance Upstream commit: https://github.com/psf/requests/commit/9a40d1277807f0a4f26c9a37eea8ec90faa8aadc --- requests/adapters.py | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 65ad876..7502059 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -17,6 +17,7 @@ from urllib3.response import HTTPResponse from urllib3.util import parse_url from urllib3.util import Timeout as TimeoutSauce from urllib3.util.retry import Retry +from urllib3.util.ssl_ import create_urllib3_context from urllib3.exceptions import ClosedPoolError from urllib3.exceptions import ConnectTimeoutError from urllib3.exceptions import HTTPError as _HTTPError @@ -52,6 +53,11 @@ DEFAULT_POOLSIZE = 10 DEFAULT_RETRIES = 0 DEFAULT_POOL_TIMEOUT = None +_preloaded_ssl_context = create_urllib3_context() +_preloaded_ssl_context.load_verify_locations( + extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) +) + def _urllib3_request_context( request: "PreparedRequest", @@ -66,8 +72,13 @@ def _urllib3_request_context( cert_reqs = "CERT_REQUIRED" if verify is False: cert_reqs = "CERT_NONE" - if isinstance(verify, str): - pool_kwargs["ca_certs"] = verify + elif verify is True: + pool_kwargs["ssl_context"] = _preloaded_ssl_context + elif isinstance(verify, str): + if not os.path.isdir(verify): + pool_kwargs["ca_certs"] = verify + else: + pool_kwargs["ca_cert_dir"] = verify pool_kwargs["cert_reqs"] = cert_reqs if client_cert is not None: if isinstance(client_cert, tuple) and len(client_cert) == 2: @@ -247,25 +258,26 @@ class HTTPAdapter(BaseAdapter): """ if url.lower().startswith('https') and verify: - cert_loc = None + conn.cert_reqs = "CERT_REQUIRED" - # Allow self-specified cert location. + # Only load the CA certificates if 'verify' is a string indicating the CA bundle to use. + # Otherwise, if verify is a boolean, we don't load anything since + # the connection will be using a context with the default certificates already loaded, + # and this avoids a call to the slow load_verify_locations() if verify is not True: + # `verify` must be a str with a path then cert_loc = verify - if not cert_loc: - cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) - - if not cert_loc or not os.path.exists(cert_loc): - raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {}".format(cert_loc)) - - conn.cert_reqs = 'CERT_REQUIRED' + if not os.path.exists(cert_loc): + raise OSError( + f"Could not find a suitable TLS CA certificate bundle, " + f"invalid path: {cert_loc}" + ) - if not os.path.isdir(cert_loc): - conn.ca_certs = cert_loc - else: - conn.ca_cert_dir = cert_loc + if not os.path.isdir(cert_loc): + conn.ca_certs = cert_loc + else: + conn.ca_cert_dir = cert_loc else: conn.cert_reqs = 'CERT_NONE' conn.ca_certs = None -- 2.47.1 From 232d96f2662eefbb3ebcfde94532ae38a6fe6f6f Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 17 Dec 2024 14:53:47 +0100 Subject: [PATCH 4/8] Move _get_connection to get_connection_with_tls_context Upstream commit: https://github.com/psf/requests/commit/aa1461b68aa73e2f6ec0e78c8853b635c76fd099 --- requests/adapters.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 7502059..823efcd 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -334,10 +334,19 @@ class HTTPAdapter(BaseAdapter): return response - def _get_connection(self, request, verify, proxies=None, cert=None): - # Replace the existing get_connection without breaking things and - # ensure that TLS settings are considered when we interact with - # urllib3 HTTP Pools + def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None): + """Returns a urllib3 connection for the given request and TLS settings. + This should not be called from user code, and is only exposed for use + when subclassing the :class:`HTTPAdapter `. + :param request: The :class:`PreparedRequest ` object + to be sent over the connection. + :param verify: Either a boolean, in which case it controls whether + we verify the server's TLS certificate, or a string, in which case it + must be a path to a CA bundle to use. + :param proxies: (optional) The proxies dictionary to apply to the request. + :param cert: (optional) Any user-provided SSL certificate to be trusted. + :rtype: urllib3.ConnectionPool + """ proxy = select_proxy(request.url, proxies) try: host_params, pool_kwargs = _urllib3_request_context(request, verify, cert) @@ -363,7 +372,9 @@ class HTTPAdapter(BaseAdapter): return conn def get_connection(self, url, proxies=None): - """Returns a urllib3 connection for the given URL. This should not be + """DEPRECATED: Users should move to `get_connection_with_tls_context` + for all subclasses of HTTPAdapter using Requests>=2.32.2. + Returns a urllib3 connection for the given URL. This should not be called from user code, and is only exposed for use when subclassing the :class:`HTTPAdapter `. @@ -482,7 +493,9 @@ class HTTPAdapter(BaseAdapter): """ try: - conn = self._get_connection(request, verify, proxies=proxies, cert=cert) + conn = self.get_connection_with_tls_context( + request, verify, proxies=proxies, cert=cert + ) except LocationValueError as e: raise InvalidURL(e, request=request) -- 2.47.1 From c380f08f4ba26e8658f20347cf82b3c2c4b797ea Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 17 Dec 2024 14:57:09 +0100 Subject: [PATCH 5/8] Allow for overriding of specific pool key params Upstream commit: https://github.com/psf/requests/commit/a62a2d35d918baa8e793f7aa4fb41527644dfca5 --- requests/adapters.py | 73 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 9 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 823efcd..1ee302c 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -334,22 +334,77 @@ class HTTPAdapter(BaseAdapter): return response + def build_connection_pool_key_attributes(self, request, verify, cert=None): + """Build the PoolKey attributes used by urllib3 to return a connection. + This looks at the PreparedRequest, the user-specified verify value, + and the value of the cert parameter to determine what PoolKey values + to use to select a connection from a given urllib3 Connection Pool. + The SSL related pool key arguments are not consistently set. As of + this writing, use the following to determine what keys may be in that + dictionary: + * If ``verify`` is ``True``, ``"ssl_context"`` will be set and will be the + default Requests SSL Context + * If ``verify`` is ``False``, ``"ssl_context"`` will not be set but + ``"cert_reqs"`` will be set + * If ``verify`` is a string, (i.e., it is a user-specified trust bundle) + ``"ca_certs"`` will be set if the string is not a directory recognized + by :py:func:`os.path.isdir`, otherwise ``"ca_certs_dir"`` will be + set. + * If ``"cert"`` is specified, ``"cert_file"`` will always be set. If + ``"cert"`` is a tuple with a second item, ``"key_file"`` will also + be present + To override these settings, one may subclass this class, call this + method and use the above logic to change parameters as desired. For + example, if one wishes to use a custom :py:class:`ssl.SSLContext` one + must both set ``"ssl_context"`` and based on what else they require, + alter the other keys to ensure the desired behaviour. + :param request: + The PreparedReqest being sent over the connection. + :type request: + :class:`~requests.models.PreparedRequest` + :param verify: + Either a boolean, in which case it controls whether + we verify the server's TLS certificate, or a string, in which case it + must be a path to a CA bundle to use. + :param cert: + (optional) Any user-provided SSL certificate for client + authentication (a.k.a., mTLS). This may be a string (i.e., just + the path to a file which holds both certificate and key) or a + tuple of length 2 with the certificate file path and key file + path. + :returns: + A tuple of two dictionaries. The first is the "host parameters" + portion of the Pool Key including scheme, hostname, and port. The + second is a dictionary of SSLContext related parameters. + """ + return _urllib3_request_context(request, verify, cert) + def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None): """Returns a urllib3 connection for the given request and TLS settings. This should not be called from user code, and is only exposed for use when subclassing the :class:`HTTPAdapter `. - :param request: The :class:`PreparedRequest ` object - to be sent over the connection. - :param verify: Either a boolean, in which case it controls whether - we verify the server's TLS certificate, or a string, in which case it - must be a path to a CA bundle to use. - :param proxies: (optional) The proxies dictionary to apply to the request. - :param cert: (optional) Any user-provided SSL certificate to be trusted. - :rtype: urllib3.ConnectionPool + :param request: + The :class:`PreparedRequest ` object to be sent + over the connection. + :param verify: + Either a boolean, in which case it controls whether we verify the + server's TLS certificate, or a string, in which case it must be a + path to a CA bundle to use. + :param proxies: + (optional) The proxies dictionary to apply to the request. + :param cert: + (optional) Any user-provided SSL certificate to be used for client + authentication (a.k.a., mTLS). + :rtype: + urllib3.ConnectionPool """ proxy = select_proxy(request.url, proxies) try: - host_params, pool_kwargs = _urllib3_request_context(request, verify, cert) + host_params, pool_kwargs = self.build_connection_pool_key_attributes( + request, + verify, + cert, + ) except ValueError as e: raise InvalidURL(e, request=request) if proxy: -- 2.47.1 From 1f7a7a4748fec114fdb042649e8b2685fb2af464 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 17 Dec 2024 14:59:19 +0100 Subject: [PATCH 6/8] Don't use default SSLContext with custom poolmanager kwargs Upstream commit: https://github.com/psf/requests/commit/b1d73ddb509a3a2d3e10744e85f9cdebdbde90f0 --- requests/adapters.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 1ee302c..359bd22 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -63,16 +63,20 @@ def _urllib3_request_context( request: "PreparedRequest", verify: "bool | str | None", client_cert: "typing.Tuple[str, str] | str | None", + poolmanager: "PoolManager", ) -> "(typing.Dict[str, typing.Any], typing.Dict[str, typing.Any])": host_params = {} pool_kwargs = {} parsed_request_url = urlparse(request.url) scheme = parsed_request_url.scheme.lower() port = parsed_request_url.port + poolmanager_kwargs = getattr(poolmanager, "connection_pool_kw", {}) + has_poolmanager_ssl_context = poolmanager_kwargs.get("ssl_context") + cert_reqs = "CERT_REQUIRED" if verify is False: cert_reqs = "CERT_NONE" - elif verify is True: + elif verify is True and not has_poolmanager_ssl_context: pool_kwargs["ssl_context"] = _preloaded_ssl_context elif isinstance(verify, str): if not os.path.isdir(verify): @@ -377,7 +381,7 @@ class HTTPAdapter(BaseAdapter): portion of the Pool Key including scheme, hostname, and port. The second is a dictionary of SSLContext related parameters. """ - return _urllib3_request_context(request, verify, cert) + return _urllib3_request_context(request, verify, cert, self.poolmanager) def get_connection_with_tls_context(self, request, verify, proxies=None, cert=None): """Returns a urllib3 connection for the given request and TLS settings. -- 2.47.1 From f9e9a8b2a392b771d5ab644192246379667bbf08 Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 17 Dec 2024 15:01:24 +0100 Subject: [PATCH 7/8] Don't create default SSLContext if ssl module isn't present Upstream commit: https://github.com/psf/requests/commit/e18879932287c2bf4bcee4ddf6ccb8a69b6fc656 --- requests/adapters.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 359bd22..4062137 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -53,10 +53,17 @@ DEFAULT_POOLSIZE = 10 DEFAULT_RETRIES = 0 DEFAULT_POOL_TIMEOUT = None -_preloaded_ssl_context = create_urllib3_context() -_preloaded_ssl_context.load_verify_locations( - extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) -) + +try: + import ssl # noqa: F401 + _preloaded_ssl_context = create_urllib3_context() + _preloaded_ssl_context.load_verify_locations( + extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) + ) +except ImportError: + # Bypass default SSLContext creation when Python + # interpreter isn't built with the ssl module. + _preloaded_ssl_context = None def _urllib3_request_context( @@ -70,13 +77,19 @@ def _urllib3_request_context( parsed_request_url = urlparse(request.url) scheme = parsed_request_url.scheme.lower() port = parsed_request_url.port + + # Determine if we have and should use our default SSLContext + # to optimize performance on standard requests. poolmanager_kwargs = getattr(poolmanager, "connection_pool_kw", {}) has_poolmanager_ssl_context = poolmanager_kwargs.get("ssl_context") + should_use_default_ssl_context = ( + _preloaded_ssl_context is not None and not has_poolmanager_ssl_context + ) cert_reqs = "CERT_REQUIRED" if verify is False: cert_reqs = "CERT_NONE" - elif verify is True and not has_poolmanager_ssl_context: + elif verify is True and should_use_default_ssl_context: pool_kwargs["ssl_context"] = _preloaded_ssl_context elif isinstance(verify, str): if not os.path.isdir(verify): -- 2.47.1 From fd57339bb1f7f0e1726d52f4b45d54ae1262d09f Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Tue, 17 Dec 2024 15:08:02 +0100 Subject: [PATCH 8/8] Address certificate loading regression Upstream source: https://github.com/psf/requests/pull/6731 --- requests/adapters.py | 45 +++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/requests/adapters.py b/requests/adapters.py index 4062137..6dac45e 100644 --- a/requests/adapters.py +++ b/requests/adapters.py @@ -66,6 +66,23 @@ except ImportError: _preloaded_ssl_context = None +def _should_use_default_context( + verify: "bool | str | None", + client_cert: "typing.Tuple[str, str] | str | None", + poolmanager_kwargs: typing.Dict[str, typing.Any], +) -> bool: + # Determine if we have and should use our default SSLContext + # to optimize performance on standard requests. + has_poolmanager_ssl_context = poolmanager_kwargs.get("ssl_context") + should_use_default_ssl_context = ( + verify is True + and _preloaded_ssl_context is not None + and not has_poolmanager_ssl_context + and client_cert is None + ) + return should_use_default_ssl_context + + def _urllib3_request_context( request: "PreparedRequest", verify: "bool | str | None", @@ -77,25 +94,25 @@ def _urllib3_request_context( parsed_request_url = urlparse(request.url) scheme = parsed_request_url.scheme.lower() port = parsed_request_url.port - - # Determine if we have and should use our default SSLContext - # to optimize performance on standard requests. poolmanager_kwargs = getattr(poolmanager, "connection_pool_kw", {}) - has_poolmanager_ssl_context = poolmanager_kwargs.get("ssl_context") - should_use_default_ssl_context = ( - _preloaded_ssl_context is not None and not has_poolmanager_ssl_context - ) cert_reqs = "CERT_REQUIRED" + cert_loc = None if verify is False: cert_reqs = "CERT_NONE" - elif verify is True and should_use_default_ssl_context: + elif _should_use_default_context(verify, client_cert, poolmanager_kwargs): pool_kwargs["ssl_context"] = _preloaded_ssl_context + elif verify is True: + # Set default ca cert location if none provided + cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) elif isinstance(verify, str): - if not os.path.isdir(verify): - pool_kwargs["ca_certs"] = verify + cert_loc = verify + + if cert_loc is not None: + if not os.path.isdir(cert_loc): + pool_kwargs["ca_certs"] = cert_loc else: - pool_kwargs["ca_cert_dir"] = verify + pool_kwargs["ca_cert_dir"] = cert_loc pool_kwargs["cert_reqs"] = cert_reqs if client_cert is not None: if isinstance(client_cert, tuple) and len(client_cert) == 2: @@ -277,10 +294,8 @@ class HTTPAdapter(BaseAdapter): conn.cert_reqs = "CERT_REQUIRED" - # Only load the CA certificates if 'verify' is a string indicating the CA bundle to use. - # Otherwise, if verify is a boolean, we don't load anything since - # the connection will be using a context with the default certificates already loaded, - # and this avoids a call to the slow load_verify_locations() + # Only load the CA certificates if `verify` is a + # string indicating the CA bundle to use. if verify is not True: # `verify` must be a str with a path then cert_loc = verify -- 2.47.1