Wrap proxy TLS, connection, and timeout errors with ProxyError
MODIFY
docs/advanced-usage.rst
docs/advanced-usage.rst
@@ -212,10 +212,10 @@ full visibility of your requests.
.. _https_proxy_error_http_proxy:
-How to fix HTTPSProxyError?
-~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Your proxy appears to only use HTTP and not HTTPS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-If you're receiving the :class:`~urllib3.exceptions.HTTPSProxyError` and it mentions
+If you're receiving the :class:`~urllib3.exceptions.ProxyError` and it mentions
your proxy only speaks HTTP and not HTTPS here's what to do to solve your issue:
If you're using ``urllib3`` directly, make sure the URL you're passing into :class:`urllib3.ProxyManager`

MODIFY
src/urllib3/connection.py
src/urllib3/connection.py
@@ -23,7 +23,7 @@ from typing import (
)
if TYPE_CHECKING:
- from typing_extensions import Literal, NoReturn
+ from typing_extensions import Literal
from .util.proxy import create_proxy_ssl_context
from .util.timeout import _DEFAULT_TIMEOUT, _TYPE_TIMEOUT, Timeout
@@ -43,9 +43,9 @@ except (ImportError, AttributeError): # Platform-specific: No SSL.
from ._version import __version__
from .exceptions import (
ConnectTimeoutError,
- HTTPSProxyError,
NameResolutionError,
NewConnectionError,
+ ProxyError,
SystemTimeWarning,
)
from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection, ssl_
@@ -123,6 +123,7 @@ class HTTPConnection(_HTTPConnection):
socket_options: Optional[connection._TYPE_SOCKET_OPTIONS]
_tunnel_host: Optional[str]
_tunnel: Callable[["HTTPConnection"], None]
+ _connecting_to_proxy: bool
def __init__(
self,
@@ -154,6 +155,8 @@ class HTTPConnection(_HTTPConnection):
blocksize=blocksize,
)
+ self._connecting_to_proxy = False
+
# https://github.com/python/mypy/issues/4125
# Mypy treats this as LSP violation, which is considered a bug.
# If `host` is made a property it violates LSP, because a writeable attribute is overridden with a read-only one.
@@ -224,12 +227,19 @@ class HTTPConnection(_HTTPConnection):
if self._is_using_tunnel():
# TODO: Fix tunnel so it doesn't depend on self.sock state.
self._tunnel()
+ self._connecting_to_proxy = False
# Mark this connection as not reusable
self.auto_open = 0
def connect(self) -> None:
+ self._connecting_to_proxy = bool(self.proxy)
conn = self._new_conn()
self._prepare_conn(conn)
+ self._connecting_to_proxy = False
+
+ def close(self) -> None:
+ self._connecting_to_proxy = False
+ super().close()
def putrequest(
self,
@@ -428,6 +438,7 @@ class HTTPSConnection(HTTPConnection):
self.ca_cert_data = ca_cert_data
def connect(self) -> None:
+ self._connecting_to_proxy = bool(self.proxy)
# Add certificate verification
conn = self._new_conn()
hostname: str = self.host
@@ -438,6 +449,7 @@ class HTTPSConnection(HTTPConnection):
conn = self._connect_tls_proxy(hostname, conn)
tls_in_tls = True
+ self._connecting_to_proxy = False
self.sock = conn
# Calls self._set_hostport(), so self.host is
@@ -514,6 +526,7 @@ class HTTPSConnection(HTTPConnection):
ssl_context=context,
tls_in_tls=tls_in_tls,
)
+ self._connecting_to_proxy = False
if self.assert_fingerprint:
assert_fingerprint(
@@ -541,7 +554,6 @@ class HTTPSConnection(HTTPConnection):
) # `_connect_tls_proxy` is called when self._is_using_tunnel() is truthy.
ssl_context = proxy_config.ssl_context
- try:
if ssl_context:
# If the user provided a proxy context, we assume CA and client
# certificates have already been set
@@ -569,10 +581,6 @@ class HTTPSConnection(HTTPConnection):
server_hostname=hostname,
ssl_context=ssl_context,
)
- except Exception as e:
- if not _should_wrap_https_proxy_error(e):
- raise
- _wrap_https_proxy_error(e, hostname)
def _match_hostname(cert: _TYPE_PEER_CERT_RET, asserted_hostname: str) -> None:
@@ -590,12 +598,7 @@ def _match_hostname(cert: _TYPE_PEER_CERT_RET, asserted_hostname: str) -> None:
raise
-def _should_wrap_https_proxy_error(err: Exception) -> bool:
- """Detects if the error should be wrapped into :class:`urllib3.exceptions.HTTPSProxyError`"""
- return isinstance(err, (CertificateError, BaseSSLError, ssl_.SSLError)) # type: ignore[attr-defined]
-
-
-def _wrap_https_proxy_error(err: Exception, hostname: str) -> "NoReturn":
+def _wrap_proxy_error(err: Exception) -> ProxyError:
# Look for the phrase 'wrong version number', if found
# then we should warn the user that we're very sure that
# this proxy is HTTP-only and they have a configuration issue.
@@ -605,15 +608,13 @@ def _wrap_https_proxy_error(err: Exception, hostname: str) -> "NoReturn":
". Your proxy appears to only use HTTP and not HTTPS, "
"did you intend to set a proxy URL using HTTPS instead of HTTP?"
)
-
- # Wrap into an HTTPSProxyError for easier diagnosis.
- # Original exception is available on original_error and
- # as __cause__.
- raise HTTPSProxyError(
- f"Unable to establish a TLS connection to {hostname}"
+ new_err = ProxyError(
+ f"Unable to connect to proxy"
f"{http_proxy_warning if is_likely_http_proxy else ''}",
err,
- ) from err
+ )
+ new_err.__cause__ = err
+ return new_err
def _get_default_user_agent() -> str:

MODIFY
src/urllib3/connectionpool.py
src/urllib3/connectionpool.py
@@ -6,17 +6,7 @@ import warnings
from http.client import HTTPResponse as _HttplibHTTPResponse
from socket import timeout as SocketTimeout
from types import TracebackType
-from typing import (
- TYPE_CHECKING,
- Any,
- Mapping,
- Optional,
- Type,
- TypeVar,
- Union,
- cast,
- overload,
-)
+from typing import TYPE_CHECKING, Any, Mapping, Optional, Type, TypeVar, Union, overload
from ._request_methods import RequestMethods
from .connection import (
@@ -29,8 +19,7 @@ from .connection import (
HTTPSConnection,
ProxyConfig,
VerifiedHTTPSConnection,
- _should_wrap_https_proxy_error,
- _wrap_https_proxy_error,
+ _wrap_proxy_error,
)
from .connection import port_by_scheme as port_by_scheme
from .exceptions import (
@@ -39,7 +28,6 @@ from .exceptions import (
FullPoolError,
HeaderParsingError,
HostChangedError,
- HTTPSProxyError,
InsecureRequestWarning,
LocationValueError,
MaxRetryError,
@@ -407,6 +395,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
timeout_obj.start_connect()
conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
+ try:
# Trigger any extra validation we need to do.
try:
self._validate_conn(conn)
@@ -414,6 +403,25 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
raise
+ # _validate_conn() starts the connection to an HTTPS proxy
+ # so we need to wrap errors with 'ProxyError' here too.
+ except (
+ OSError,
+ NewConnectionError,
+ TimeoutError,
+ BaseSSLError,
+ CertificateError,
+ SSLError,
+ ) as e:
+ new_e: Exception = e
+ if isinstance(e, (BaseSSLError, CertificateError)):
+ new_e = SSLError(e)
+ if isinstance(
+ new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
+ ) and (conn and conn._connecting_to_proxy):
+ new_e = _wrap_proxy_error(new_e)
+ raise new_e
+
# conn.request() calls http.client.*.request, not the method in
# urllib3.request. It also calls makefile (recv) on the socket.
try:
@@ -701,8 +709,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
is_new_proxy_conn = self.proxy is not None and not getattr(
conn, "sock", None
)
- if is_new_proxy_conn and http_tunnel_required:
+ if is_new_proxy_conn:
+ assert isinstance(self.proxy, Url)
+ conn._connecting_to_proxy = True
+ if http_tunnel_required:
+ try:
self._prepare_proxy(conn)
+ except (BaseSSLError, OSError, SocketTimeout) as e:
+ self._raise_timeout(
+ err=e, url=self.proxy.url, timeout_value=conn.timeout
+ )
+ raise
# Make the request on the httplib connection object.
httplib_response = self._make_request(
@@ -750,7 +767,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
BaseSSLError,
SSLError,
CertificateError,
- HTTPSProxyError,
+ ProxyError,
) as e:
# Discard the connection for these exceptions. It will be
# replaced during the next _get_conn() call.
@@ -758,10 +775,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
new_e: Exception = e
if isinstance(e, (BaseSSLError, CertificateError)):
new_e = SSLError(e)
- elif isinstance(e, (OSError, NewConnectionError)) and self.proxy:
- new_e = ProxyError("Cannot connect to proxy.", e)
- elif isinstance(e, (OSError, HTTPException)):
- new_e = ProtocolError("Connection aborted.", e)
+ if (
+ isinstance(
+ new_e,
+ (
+ OSError,
+ NewConnectionError,
+ TimeoutError,
+ SSLError,
+ HTTPException,
+ ),
+ )
+ and (conn and conn._connecting_to_proxy)
+ ):
+ new_e = _wrap_proxy_error(new_e)
+ elif isinstance(new_e, (OSError, HTTPException)):
+ new_e = ProtocolError("Connection aborted.", new_e)
retries = retries.increment(
method, url, error=new_e, _pool=self, _stacktrace=sys.exc_info()[2]
@@ -972,7 +1001,6 @@ class HTTPSConnectionPool(HTTPConnectionPool):
Tunnel connection is established early because otherwise httplib would
improperly set Host: header to proxy's IP:port.
"""
-
conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers)
if self.proxy and self.proxy.scheme == "https":
@@ -1022,18 +1050,8 @@ class HTTPSConnectionPool(HTTPConnectionPool):
super()._validate_conn(conn)
# Force connect early to allow us to validate the connection.
- try:
if not conn.sock:
conn.connect()
- except Exception as e:
- # We only want to wrap errors for HTTPS proxies
- if (
- not self.proxy
- or self.proxy.scheme != "https"
- or not _should_wrap_https_proxy_error(e)
- ):
- raise
- _wrap_https_proxy_error(e, cast(str, self.proxy.host))
if not conn.is_verified:
warnings.warn(

MODIFY
src/urllib3/exceptions.py
src/urllib3/exceptions.py
@@ -69,12 +69,6 @@ class ProxyError(HTTPError):
self.original_error = error
-class HTTPSProxyError(ProxyError):
- """Used only when establishing a TLS connection to a proxy"""
-
- pass
-
-
class DecodeError(HTTPError):
"""Raised when automatic decoding based on Content-Type fails."""

MODIFY
test/with_dummyserver/test_proxy_poolmanager.py
test/with_dummyserver/test_proxy_poolmanager.py
import os.path
import shutil
import socket
+import ssl
import tempfile
from test import LONG_TIMEOUT, SHORT_TIMEOUT, onlySecureTransport, withPyOpenSSL
@@ -25,6 +26,7 @@ from urllib3.exceptions import (
)
from urllib3.poolmanager import ProxyManager, proxy_from_url
from urllib3.util.ssl_ import create_urllib3_context
+from urllib3.util.timeout import Timeout
from .. import TARPIT_HOST, requires_network
@@ -158,22 +160,6 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
== urllib3.exceptions.NameResolutionError
)
- @pytest.mark.parametrize("target_scheme", ["http", "https"])
- def test_https_proxy_error_doesnt_wrap_timeouts(self, target_scheme: str) -> None:
- bad_proxy_url = f"https://{self.proxy_host}:{int(self.proxy_port)}"
- with proxy_from_url(
- bad_proxy_url, ca_certs=DEFAULT_CA, timeout=LONG_TIMEOUT
- ) as http:
- with pytest.raises(MaxRetryError) as e:
- http.request(
- "GET", f"{target_scheme}://{self.http_host}:{self.http_port}"
- )
-
- if type(e.value.reason) == ProxyError:
- assert type(e.value.reason.original_error) == socket.timeout
- else:
- assert type(e.value.reason) == ReadTimeoutError
-
def test_oldapi(self) -> None:
with ProxyManager(
connection_from_url(self.proxy_url), ca_certs=DEFAULT_CA # type: ignore[arg-type]
@@ -460,21 +446,143 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
assert sc2 != sc3
assert sc3 == sc4
- @pytest.mark.timeout(0.5)
@requires_network()
- def test_https_proxy_timeout(self) -> None:
- with proxy_from_url(f"https://{TARPIT_HOST}") as https:
+ @pytest.mark.parametrize(
+ ["proxy_scheme", "target_scheme", "use_forwarding_for_https"],
+ [
+ ("http", "http", False),
+ ("https", "http", False),
+ # 'use_forwarding_for_https' is only valid for HTTPS+HTTPS.
+ ("https", "https", True),
+ ],
+ )
+ def test_forwarding_proxy_request_timeout(
+ self, proxy_scheme: str, target_scheme: str, use_forwarding_for_https: bool
+ ) -> None:
+ proxy_url = self.https_proxy_url if proxy_scheme == "https" else self.proxy_url
+ target_url = f"{target_scheme}://{TARPIT_HOST}"
+
+ with proxy_from_url(
+ proxy_url,
+ ca_certs=DEFAULT_CA,
+ use_forwarding_for_https=use_forwarding_for_https,
+ ) as proxy:
+ with pytest.raises(MaxRetryError) as e:
+ timeout = Timeout(connect=LONG_TIMEOUT, read=SHORT_TIMEOUT)
+ proxy.request("GET", target_url, timeout=timeout)
+
+ # We sent the request to the proxy but didn't get any response
+ # so we're not sure if that's being caused by the proxy or the
+ # target so we put the blame on the target.
+ assert type(e.value.reason) == ReadTimeoutError
+
+ @requires_network()
+ @pytest.mark.parametrize(
+ ["proxy_scheme", "target_scheme"], [("http", "https"), ("https", "https")]
+ )
+ def test_tunneling_proxy_request_timeout(
+ self, proxy_scheme: str, target_scheme: str
+ ) -> None:
+ proxy_url = self.https_proxy_url if proxy_scheme == "https" else self.proxy_url
+ target_url = f"{target_scheme}://{TARPIT_HOST}"
+
+ with proxy_from_url(
+ proxy_url,
+ ca_certs=DEFAULT_CA,
+ ) as proxy:
with pytest.raises(MaxRetryError) as e:
- https.request("GET", self.http_url, timeout=SHORT_TIMEOUT)
- assert type(e.value.reason) == ConnectTimeoutError
+ timeout = Timeout(connect=LONG_TIMEOUT, read=SHORT_TIMEOUT)
+ proxy.request("GET", target_url, timeout=timeout)
+
+ assert type(e.value.reason) == ReadTimeoutError
- @pytest.mark.timeout(0.5)
@requires_network()
- def test_https_proxy_pool_timeout(self) -> None:
- with proxy_from_url(f"https://{TARPIT_HOST}", timeout=SHORT_TIMEOUT) as https:
+ @pytest.mark.parametrize(
+ ["proxy_scheme", "target_scheme", "use_forwarding_for_https"],
+ [
+ ("http", "http", False),
+ ("https", "http", False),
+ # 'use_forwarding_for_https' is only valid for HTTPS+HTTPS.
+ ("https", "https", True),
+ ],
+ )
+ def test_forwarding_proxy_connect_timeout(
+ self, proxy_scheme: str, target_scheme: str, use_forwarding_for_https: bool
+ ) -> None:
+ proxy_url = f"{proxy_scheme}://{TARPIT_HOST}"
+ target_url = self.https_url if target_scheme == "https" else self.http_url
+
+ with proxy_from_url(
+ proxy_url, ca_certs=DEFAULT_CA, timeout=SHORT_TIMEOUT
+ ) as proxy:
+ with pytest.raises(MaxRetryError) as e:
+ proxy.request("GET", target_url)
+
+ assert type(e.value.reason) == ProxyError
+ assert type(e.value.reason.original_error) == ConnectTimeoutError
+
+ @requires_network()
+ @pytest.mark.parametrize(
+ ["proxy_scheme", "target_scheme"], [("http", "https"), ("https", "https")]
+ )
+ def test_tunneling_proxy_connect_timeout(
+ self, proxy_scheme: str, target_scheme: str
+ ) -> None:
+ proxy_url = f"{proxy_scheme}://{TARPIT_HOST}"
+ target_url = self.https_url if target_scheme == "https" else self.http_url
+
+ with proxy_from_url(
+ proxy_url, ca_certs=DEFAULT_CA, timeout=SHORT_TIMEOUT
+ ) as proxy:
+ with pytest.raises(MaxRetryError) as e:
+ proxy.request("GET", target_url)
+
+ assert type(e.value.reason) == ProxyError
+ assert type(e.value.reason.original_error) == ConnectTimeoutError
+
+ @requires_network()
+ @pytest.mark.parametrize(
+ ["target_scheme", "use_forwarding_for_https"],
+ [
+ ("http", False),
+ ("https", False),
+ ("https", True),
+ ],
+ )
+ def test_https_proxy_tls_error(
+ self, target_scheme: str, use_forwarding_for_https: str
+ ) -> None:
+ target_url = self.https_url if target_scheme == "https" else self.http_url
+ proxy_ctx = ssl.create_default_context()
+ with proxy_from_url(self.https_proxy_url, proxy_ssl_context=proxy_ctx) as proxy:
+ with pytest.raises(MaxRetryError) as e:
+ proxy.request("GET", target_url)
+ assert type(e.value.reason) == ProxyError
+ assert type(e.value.reason.original_error) == SSLError
+
+ @requires_network()
+ @pytest.mark.parametrize(
+ ["proxy_scheme", "use_forwarding_for_https"],
+ [
+ ("http", False),
+ ("https", False),
+ ("https", True),
+ ],
+ )
+ def test_proxy_https_target_tls_error(
+ self, proxy_scheme: str, use_forwarding_for_https: str
+ ) -> None:
+ proxy_url = self.https_proxy_url if proxy_scheme == "https" else self.proxy_url
+ proxy_ctx = ssl.create_default_context()
+ proxy_ctx.load_verify_locations(DEFAULT_CA)
+ ctx = ssl.create_default_context()
+
+ with proxy_from_url(
+ proxy_url, proxy_ssl_context=proxy_ctx, ssl_context=ctx
+ ) as proxy:
with pytest.raises(MaxRetryError) as e:
- https.request("GET", self.http_url)
- assert type(e.value.reason) == ConnectTimeoutError
+ proxy.request("GET", self.https_url)
+ assert type(e.value.reason) == SSLError
def test_scheme_host_case_insensitive(self) -> None:
"""Assert that upper-case schemes and hosts are normalized."""

MODIFY
test/with_dummyserver/test_socketlevel.py
test/with_dummyserver/test_socketlevel.py
@@ -38,7 +38,6 @@ from urllib3 import HTTPConnectionPool, HTTPSConnectionPool, ProxyManager, util
from urllib3._collections import HTTPHeaderDict
from urllib3.connection import HTTPConnection, _get_default_user_agent
from urllib3.exceptions import (
- HTTPSProxyError,
MaxRetryError,
ProtocolError,
ProxyError,
@@ -1180,7 +1179,7 @@ class TestProxyManager(SocketDummyServerTestCase):
errored.set() # Avoid a ConnectionAbortedError on Windows.
- assert type(e.value.reason) == HTTPSProxyError
+ assert type(e.value.reason) == ProxyError
assert "Your proxy appears to only use HTTP and not HTTPS" in str(
e.value.reason
)

Wrap proxy TLS, connection, and timeout errors with ProxyError