Integrate TLS-in-TLS support into urllib3
MODIFY
src/urllib3/connection.py
src/urllib3/connection.py
@@ -9,6 +9,7 @@ import warnings
from .packages import six
from .packages.six.moves.http_client import HTTPConnection as _HTTPConnection
from .packages.six.moves.http_client import HTTPException # noqa: F401
+from .util.proxy import create_proxy_ssl_context
try: # Compiled with SSL?
import ssl
@@ -117,6 +118,11 @@ class HTTPConnection(_HTTPConnection, object):
#: The socket options provided by the user. If no options are
#: provided, we use the default options.
self.socket_options = kw.pop("socket_options", self.default_socket_options)
+
+ # Proxy options provided by the user.
+ self.proxy = kw.pop("proxy", None)
+ self.proxy_config = kw.pop("proxy_config", None)
+
_HTTPConnection.__init__(self, *args, **kw)
@property
@@ -271,6 +277,7 @@ class HTTPSConnection(HTTPConnection):
ca_cert_data = None
ssl_version = None
assert_fingerprint = None
+ tls_in_tls_required = False
def __init__(
self,
@@ -335,8 +342,13 @@ class HTTPSConnection(HTTPConnection):
# Add certificate verification
conn = self._new_conn()
hostname = self.host
+ tls_in_tls = False
if self._is_using_tunnel():
+ if self.tls_in_tls_required:
+ conn = self._connect_tls_proxy(hostname, conn)
+ tls_in_tls = True
+
self.sock = conn
# Calls self._set_hostport(), so self.host is
@@ -396,6 +408,7 @@ class HTTPSConnection(HTTPConnection):
ca_cert_data=self.ca_cert_data,
server_hostname=server_hostname,
ssl_context=context,
+ tls_in_tls=tls_in_tls,
)
if self.assert_fingerprint:
@@ -428,6 +441,40 @@ class HTTPSConnection(HTTPConnection):
or self.assert_fingerprint is not None
)
+ def _connect_tls_proxy(self, hostname, conn):
+ """
+ Establish a TLS connection to the proxy using the provided SSL context.
+ """
+ proxy_config = self.proxy_config
+ ssl_context = proxy_config.ssl_context
+ if ssl_context:
+ # If the user provided a proxy context, we assume CA and client
+ # certificates have already been set
+ return ssl_wrap_socket(
+ sock=conn,
+ server_hostname=hostname,
+ ssl_context=ssl_context,
+ )
+
+ ssl_context = create_proxy_ssl_context(
+ self.ssl_version,
+ self.cert_reqs,
+ self.ca_certs,
+ self.ca_cert_dir,
+ self.ca_cert_data,
+ )
+
+ # If no cert was provided, use only the default options for server
+ # certificate validation
+ return ssl_wrap_socket(
+ sock=conn,
+ ca_certs=self.ca_certs,
+ ca_cert_dir=self.ca_cert_dir,
+ ca_cert_data=self.ca_cert_data,
+ server_hostname=hostname,
+ ssl_context=ssl_context,
+ )
+
def _match_hostname(cert, asserted_hostname):
try:

MODIFY
src/urllib3/connectionpool.py
src/urllib3/connectionpool.py
@@ -40,6 +40,7 @@ from .request import RequestMethods
from .response import HTTPResponse
from .util.connection import is_connection_dropped
+from .util.proxy import connection_requires_http_tunnel
from .util.request import set_file_position
from .util.response import assert_header_parsing
from .util.retry import Retry
@@ -182,6 +183,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
retries=None,
_proxy=None,
_proxy_headers=None,
+ _proxy_config=None,
**conn_kw
):
ConnectionPool.__init__(self, host, port)
@@ -203,6 +205,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
self.proxy = _proxy
self.proxy_headers = _proxy_headers or {}
+ self.proxy_config = _proxy_config
# Fill the queue up so that doing get() on it will block properly
for _ in xrange(maxsize):
@@ -219,6 +222,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# list.
self.conn_kw.setdefault("socket_options", [])
+ self.conn_kw["proxy"] = self.proxy
+ self.conn_kw["proxy_config"] = self.proxy_config
+
def _new_conn(self):
"""
Return a fresh :class:`HTTPConnection`.
@@ -621,6 +627,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
Additional parameters are passed to
:meth:`urllib3.response.HTTPResponse.from_httplib`
"""
+
+ parsed_url = parse_url(url)
+ destination_scheme = parsed_url.scheme
+
if headers is None:
headers = self.headers
@@ -638,7 +648,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
if url.startswith("/"):
url = six.ensure_str(_encode_target(url))
else:
- url = six.ensure_str(parse_url(url).url)
+ url = six.ensure_str(parsed_url.url)
conn = None
@@ -653,10 +663,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# [1] <https://github.com/urllib3/urllib3/issues/651>
release_this_conn = release_conn
+ http_tunnel_required = connection_requires_http_tunnel(
+ self.proxy, self.proxy_config, destination_scheme
+ )
+
# Merge the proxy headers. Only done when not using HTTP CONNECT. We
# have to copy the headers dict so we can safely change it without those
# changes being reflected in anyone else's copy.
- if self.scheme == "http" or (self.proxy and self.proxy.scheme == "https"):
+ if not http_tunnel_required:
headers = headers.copy()
headers.update(self.proxy_headers)
@@ -682,7 +696,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
is_new_proxy_conn = self.proxy is not None and not getattr(
conn, "sock", None
)
- if is_new_proxy_conn:
+ if is_new_proxy_conn and http_tunnel_required:
self._prepare_proxy(conn)
# Make the request on the httplib connection object.
@@ -946,9 +960,11 @@ class HTTPSConnectionPool(HTTPConnectionPool):
improperly set Host: header to proxy's IP:port.
"""
- if self.proxy.scheme != "https":
conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers)
+ if self.proxy.scheme == "https":
+ conn.tls_in_tls_required = True
+
conn.connect()
def _new_conn(self):

MODIFY
src/urllib3/poolmanager.py
src/urllib3/poolmanager.py
@@ -2,14 +2,12 @@ from __future__ import absolute_import
import collections
import functools
import logging
-import warnings
from ._collections import RecentlyUsedContainer
from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool
from .connectionpool import port_by_scheme
from .exceptions import (
- HTTPWarning,
LocationValueError,
MaxRetryError,
ProxySchemeUnknown,
@@ -21,17 +19,13 @@ from .packages.six.moves.urllib.parse import urljoin
from .request import RequestMethods
from .util.url import parse_url
from .util.retry import Retry
+from .util.proxy import connection_requires_http_tunnel
+from .packages.six import PY3
__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"]
-class InvalidProxyConfigurationWarning(HTTPWarning):
- """Raised when a user has an HTTPS proxy without enabling HTTPS proxies."""
-
- pass
-
-
log = logging.getLogger(__name__)
SSL_KEYWORDS = (
@@ -68,6 +62,7 @@ _key_fields = (
"key_headers", # dict
"key__proxy", # parsed proxy url
"key__proxy_headers", # dict
+ "key__proxy_config", # class
"key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples
"key__socks_options", # dict
"key_assert_hostname", # bool or string
@@ -79,6 +74,9 @@ _key_fields = (
#: All custom key schemes should include the fields in this key at a minimum.
PoolKey = collections.namedtuple("PoolKey", _key_fields)
+_proxy_config_fields = ("ssl_context", "use_forwarding_for_https")
+ProxyConfig = collections.namedtuple("ProxyConfig", _proxy_config_fields)
+
def _default_key_normalizer(key_class, request_context):
"""
@@ -170,6 +168,7 @@ class PoolManager(RequestMethods):
"""
proxy = None
+ proxy_config = None
def __init__(self, num_pools=10, headers=None, **connection_pool_kw):
RequestMethods.__init__(self, headers)
@@ -326,14 +325,32 @@ class PoolManager(RequestMethods):
def _proxy_requires_url_absolute_form(self, parsed_url):
"""
Indicates if the proxy requires the complete destination URL in the
- request.
-
- Normally this is only needed when not using an HTTP CONNECT tunnel.
+ request. Normally this is only needed when not using an HTTP CONNECT
+ tunnel.
"""
if self.proxy is None:
return False
- return parsed_url.scheme == "http" or self.proxy.scheme == "https"
+ return not connection_requires_http_tunnel(
+ self.proxy, self.proxy_config, parsed_url.scheme
+ )
+
+ def _validate_proxy_scheme_url_selection(self, url_scheme):
+ """
+ Validates that were not attempting to do TLS in TLS connections on
+ Python2 or with unsupported SSL implementations.
+ """
+ if self.proxy is None or url_scheme != "https":
+ return
+
+ if self.proxy.scheme != "https":
+ return
+
+ if not PY3 and not self.proxy_config.use_forwarding_for_https:
+ raise ProxySchemeUnsupported(
+ "Contacting HTTPS destinations through HTTPS proxies "
+ "'via CONNECT tunnels' is not supported in Python 2"
+ )
def urlopen(self, method, url, redirect=True, **kw):
"""
@@ -345,6 +362,8 @@ class PoolManager(RequestMethods):
:class:`urllib3.connectionpool.ConnectionPool` can be chosen for it.
"""
u = parse_url(url)
+ self._validate_proxy_scheme_url_selection(u.scheme)
+
conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme)
kw["assert_same_host"] = False
@@ -415,11 +434,18 @@ class ProxyManager(PoolManager):
HTTPS/CONNECT case they are sent only once. Could be used for proxy
authentication.
- :param _allow_https_proxy_to_see_traffic:
- Allows forwarding of HTTPS requests to HTTPS proxies. The proxy will
- have visibility of all the traffic sent. ONLY USE IF YOU KNOW WHAT
- YOU'RE DOING. This flag might be removed at any time in any future
- update.
+ :param proxy_ssl_context:
+ The proxy SSL context is used to establish the TLS connection to the
+ proxy when using HTTPS proxies.
+
+ :param use_forwarding_for_https:
+ (Defaults to False) If set to True will forward requests to the HTTPS
+ proxy to be made on behalf of the client instead of creating a TLS
+ tunnel via the CONNECT method. **Enabling this flag means that request
+ and response headers and content will be visible from the HTTPS proxy**
+ whereas tunneling keeps request and response headers and content
+ private. IP address, target hostname, SNI, and port are always visible
+ to an HTTPS proxy even when this flag is disabled.
Example:
>>> proxy = urllib3.ProxyManager('http://localhost:3128/')
@@ -440,7 +466,8 @@ class ProxyManager(PoolManager):
num_pools=10,
headers=None,
proxy_headers=None,
- _allow_https_proxy_to_see_traffic=False,
+ proxy_ssl_context=None,
+ use_forwarding_for_https=False,
**connection_pool_kw
):
@@ -461,11 +488,12 @@ class ProxyManager(PoolManager):
self.proxy = proxy
self.proxy_headers = proxy_headers or {}
+ self.proxy_ssl_context = proxy_ssl_context
+ self.proxy_config = ProxyConfig(proxy_ssl_context, use_forwarding_for_https)
connection_pool_kw["_proxy"] = self.proxy
connection_pool_kw["_proxy_headers"] = self.proxy_headers
-
- self.allow_insecure_proxy = _allow_https_proxy_to_see_traffic
+ connection_pool_kw["_proxy_config"] = self.proxy_config
super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw)
@@ -494,35 +522,13 @@ class ProxyManager(PoolManager):
headers_.update(headers)
return headers_
- def _validate_proxy_scheme_url_selection(self, url_scheme):
- if (
- url_scheme == "https"
- and self.proxy.scheme == "https"
- and not self.allow_insecure_proxy
- ):
- warnings.warn(
- "Your proxy configuration specified an HTTPS scheme for the proxy. "
- "Are you sure you want to use HTTPS to contact the proxy? "
- "This most likely indicates an error in your configuration."
- "If you are sure you want use HTTPS to contact the proxy, enable "
- "the _allow_https_proxy_to_see_traffic.",
- InvalidProxyConfigurationWarning,
- )
-
- raise ProxySchemeUnsupported(
- "Contacting HTTPS destinations through HTTPS proxies is not supported."
- )
-
def urlopen(self, method, url, redirect=True, **kw):
"Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute."
u = parse_url(url)
- self._validate_proxy_scheme_url_selection(u.scheme)
-
- if u.scheme == "http" or self.proxy.scheme == "https":
+ if not connection_requires_http_tunnel(self.proxy, self.proxy_config, u.scheme):
# For connections using HTTP CONNECT, httplib sets the necessary
- # headers on the CONNECT to the proxy. For HTTP or when talking
- # HTTPS to the proxy, we'll definitely need to set 'Host' at the
- # very least.
+ # headers on the CONNECT to the proxy. If we're not using CONNECT,
+ # we'll definitely need to set 'Host' at the very least.
headers = kw.get("headers", self.headers)
kw["headers"] = self._set_proxy_headers(url, headers)

ADD
src/urllib3/util/proxy.py
+from .ssl_ import (
+ resolve_cert_reqs,
+ resolve_ssl_version,
+ create_urllib3_context,
+)
+
+
+def connection_requires_http_tunnel(
+ proxy_url=None, proxy_config=None, destination_scheme=None
+):
+ """
+ Returns True if the connection requires an HTTP CONNECT through the proxy.
+
+ :param URL proxy_url:
+ URL of the proxy.
+ :param ProxyConfig proxy_config:
+ Proxy configuration from poolmanager.py
+ :param str destination_scheme:
+ The scheme of the destination. (i.e https, http, etc)
+ """
+ # If we're not using a proxy, no way to use a tunnel.
+ if proxy_url is None:
+ return False
+
+ # HTTP destinations never require tunneling, we always forward.
+ if destination_scheme == "http":
+ return False
+
+ # Support for forwarding with HTTPS proxies and HTTPS destinations.
+ if (
+ proxy_url.scheme == "https"
+ and proxy_config
+ and proxy_config.use_forwarding_for_https
+ ):
+ return False
+
+ # Otherwise always use a tunnel.
+ return True
+
+
+def create_proxy_ssl_context(
+ ssl_version, cert_reqs, ca_certs=None, ca_cert_dir=None, ca_cert_data=None
+):
+ """
+ Generates a default proxy ssl context if one hasn't been provided by the
+ user.
+ """
+ ssl_context = create_urllib3_context(
+ ssl_version=resolve_ssl_version(ssl_version),
+ cert_reqs=resolve_cert_reqs(cert_reqs),
+ )
+ if (
+ not ca_certs
+ and not ca_cert_dir
+ and not ca_cert_data
+ and hasattr(ssl_context, "load_default_certs")
+ ):
+ ssl_context.load_default_certs()
+
+ return ssl_context

MODIFY
src/urllib3/util/ssl_.py
src/urllib3/util/ssl_.py
@@ -8,11 +8,17 @@ from binascii import hexlify, unhexlify
from hashlib import md5, sha1, sha256
from .url import IPV4_RE, BRACELESS_IPV6_ADDRZ_RE
-from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning
+from ..exceptions import (
+ SSLError,
+ InsecurePlatformWarning,
+ SNIMissingWarning,
+ ProxySchemeUnsupported,
+)
from ..packages import six
SSLContext = None
+SSLTransport = None
HAS_SNI = False
IS_PYOPENSSL = False
IS_SECURETRANSPORT = False
@@ -41,6 +47,7 @@ try: # Test for SSL features
import ssl
from ssl import wrap_socket, CERT_REQUIRED
from ssl import HAS_SNI # Has SNI?
+ from .ssltransport import SSLTransport
except ImportError:
pass
@@ -327,6 +334,7 @@ def ssl_wrap_socket(
ca_cert_dir=None,
key_password=None,
ca_cert_data=None,
+ tls_in_tls=False,
):
"""
All arguments except for server_hostname, ssl_context, and ca_cert_dir have
@@ -348,6 +356,8 @@ def ssl_wrap_socket(
:param ca_cert_data:
Optional string containing CA certificates in PEM format suitable for
passing as the cadata parameter to SSLContext.load_verify_locations()
+ :param tls_in_tls:
+ Use SSLTransport to wrap the existing socket.
"""
context = ssl_context
if context is None:
@@ -405,9 +415,11 @@ def ssl_wrap_socket(
)
if send_sni:
- ssl_sock = context.wrap_socket(sock, server_hostname=server_hostname)
+ ssl_sock = _ssl_wrap_socket_impl(
+ sock, context, tls_in_tls, server_hostname=server_hostname
+ )
else:
- ssl_sock = context.wrap_socket(sock)
+ ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls)
return ssl_sock
@@ -433,3 +445,20 @@ def _is_key_file_encrypted(key_file):
return True
return False
+
+
+def _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname=None):
+ if tls_in_tls:
+ if not SSLTransport:
+ # Import error, ssl is not available.
+ raise ProxySchemeUnsupported(
+ "TLS in TLS requires support for the 'ssl' module"
+ )
+
+ SSLTransport._validate_ssl_context_for_tls_in_tls(ssl_context)
+ return SSLTransport(sock, ssl_context, server_hostname)
+
+ if server_hostname:
+ return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
+ else:
+ return ssl_context.wrap_socket(sock)

RENAME
src/urllib3/contrib/ssl.py
src/urllib3/util/ssltransport.py
@@ -2,6 +2,9 @@ import ssl
import socket
import io
+from urllib3.exceptions import ProxySchemeUnsupported
+from urllib3.packages.six import PY3
+
SSL_BLOCKSIZE = 16384
@@ -16,6 +19,28 @@ class SSLTransport:
The class supports most of the socket API operations.
"""
+ @staticmethod
+ def _validate_ssl_context_for_tls_in_tls(ssl_context):
+ """
+ Raises a ProxySchemeUnsupported if the provided ssl_context can't be used
+ for TLS in TLS.
+
+ The only requirement is that the ssl_context provides the 'wrap_bio'
+ methods.
+ """
+
+ if not hasattr(ssl_context, "wrap_bio"):
+ if not PY3:
+ raise ProxySchemeUnsupported(
+ "TLS in TLS requires SSLContext.wrap_bio() which isn't "
+ "supported on Python 2"
+ )
+ else:
+ raise ProxySchemeUnsupported(
+ "TLS in TLS requires SSLContext.wrap_bio() which isn't "
+ "available on non-native SSLContext"
+ )
+
def __init__(
self, socket, ssl_context, suppress_ragged_eofs=True, server_hostname=None
):

MODIFY
test/__init__.py
test/__init__.py
@@ -19,6 +19,11 @@ from urllib3.packages import six
from urllib3.util import ssl_
from urllib3 import util
+try:
+ import urllib3.contrib.pyopenssl as pyopenssl
+except ImportError:
+ pyopenssl = None
+
# We need a host that will not immediately close the connection with a TCP
# Reset.
if platform.system() == "Windows":
@@ -166,6 +171,19 @@ def notBrotlipy():
)
+def onlySecureTransport(test):
+ """Runs this test when SecureTransport is in use."""
+
+ @six.wraps(test)
+ def wrapper(*args, **kwargs):
+ msg = "{name} only runs with SecureTransport".format(name=test.__name__)
+ if not ssl_.IS_SECURETRANSPORT:
+ pytest.skip(msg)
+ return test(*args, **kwargs)
+
+ return wrapper
+
+
def notSecureTransport(test):
"""Skips this test when SecureTransport is in use."""
@@ -290,6 +308,21 @@ def resolvesLocalhostFQDN(test):
return wrapper
+def withPyOpenSSL(test):
+ @six.wraps(test)
+ def wrapper(*args, **kwargs):
+ if not pyopenssl:
+ pytest.skip("pyopenssl not available, skipping test.")
+ return test(*args, **kwargs)
+
+ pyopenssl.inject_into_urllib3()
+ result = test(*args, **kwargs)
+ pyopenssl.extract_from_urllib3()
+ return result
+
+ return wrapper
+
+
class _ListHandler(logging.Handler):
def __init__(self):
super(_ListHandler, self).__init__()

MODIFY
test/test_poolmanager.py
test/test_poolmanager.py
@@ -366,3 +366,9 @@ class TestPoolManager(object):
p = PoolManager(strict=True)
merged = p._merge_pool_kwargs({"invalid_key": None})
assert p.connection_pool_kw == merged
+
+ def test_pool_manager_no_url_absolute_form(self):
+ """Valides we won't send a request with absolute form without a proxy"""
+ p = PoolManager(strict=True)
+ assert p._proxy_requires_url_absolute_form("http://example.com") is False
+ assert p._proxy_requires_url_absolute_form("https://example.com") is False

MODIFY
test/test_proxymanager.py
test/test_proxymanager.py
@@ -62,6 +62,10 @@ class TestProxyManager(object):
assert p._proxy_requires_url_absolute_form(https_url) is False
with ProxyManager("https://proxy:8080") as p:
+ assert p._proxy_requires_url_absolute_form(http_url)
+ assert p._proxy_requires_url_absolute_form(https_url) is False
+
+ with ProxyManager("https://proxy:8080", use_forwarding_for_https=True) as p:
assert p._proxy_requires_url_absolute_form(http_url)
assert p._proxy_requires_url_absolute_form(https_url)

RENAME
test/contrib/test_ssltransport.py
test/test_ssltransport.py
@@ -4,7 +4,7 @@ from dummyserver.server import (
DEFAULT_CA,
)
-from urllib3.contrib.ssl import SSLTransport
+from urllib3.util.ssltransport import SSLTransport
import select
import pytest

MODIFY
test/test_util.py
test/test_util.py
@@ -28,10 +28,16 @@ from urllib3.exceptions import (
SNIMissingWarning,
UnrewindableBodyError,
)
-from urllib3.util.connection import allowed_gai_family, _has_ipv6
+from urllib3.util.proxy import (
+ connection_requires_http_tunnel,
+ create_proxy_ssl_context,
+)
from urllib3.util import is_fp_closed
+from urllib3.util.connection import allowed_gai_family, _has_ipv6
from urllib3.packages import six
+from urllib3.poolmanager import ProxyConfig
+
from . import clear_warnings
from test import onlyPy3, onlyPy2, onlyBrotlipy, notBrotlipy
@@ -748,6 +754,34 @@ class TestUtil(object):
with pytest.raises(TypeError):
assert_header_parsing(headers)
+ def test_connection_requires_http_tunnel_no_proxy(self):
+ assert not connection_requires_http_tunnel(
+ proxy_url=None, proxy_config=None, destination_scheme=None
+ )
+
+ def test_connection_requires_http_tunnel_http_proxy(self):
+ proxy = parse_url("http://proxy:8080")
+ proxy_config = ProxyConfig(ssl_context=None, use_forwarding_for_https=False)
+ destination_scheme = "http"
+ assert not connection_requires_http_tunnel(
+ proxy, proxy_config, destination_scheme
+ )
+
+ destination_scheme = "https"
+ assert connection_requires_http_tunnel(proxy, proxy_config, destination_scheme)
+
+ def test_connection_requires_http_tunnel_https_proxy(self):
+ proxy = parse_url("https://proxy:8443")
+ proxy_config = ProxyConfig(ssl_context=None, use_forwarding_for_https=False)
+ destination_scheme = "http"
+ assert not connection_requires_http_tunnel(
+ proxy, proxy_config, destination_scheme
+ )
+
+ def test_create_proxy_ssl_context(self):
+ ssl_context = create_proxy_ssl_context(ssl_version=None, cert_reqs=None)
+ ssl_context.verify_mode = ssl.CERT_REQUIRED
+
@onlyPy3
def test_assert_header_parsing_no_error_on_multipart(self):
from http import client

MODIFY
test/with_dummyserver/test_proxy_poolmanager.py
test/with_dummyserver/test_proxy_poolmanager.py
@@ -22,8 +22,17 @@ from urllib3.exceptions import (
ProxySchemeUnsupported,
)
from urllib3.connectionpool import connection_from_url, VerifiedHTTPSConnection
+from urllib3.util.ssl_ import create_urllib3_context
+
+from test import (
+ SHORT_TIMEOUT,
+ LONG_TIMEOUT,
+ onlyPy3,
+ onlyPy2,
+ withPyOpenSSL,
+ onlySecureTransport,
+)
-from test import SHORT_TIMEOUT, LONG_TIMEOUT
# Retry failed tests
pytestmark = pytest.mark.flaky
@@ -63,21 +72,75 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
r = http.request("GET", "%s/" % self.https_url)
assert r.status == 200
+ @onlyPy3
def test_https_proxy(self):
+ with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https:
+ r = https.request("GET", "%s/" % self.https_url)
+ assert r.status == 200
+
+ r = https.request("GET", "%s/" % self.http_url)
+ assert r.status == 200
+
+ @onlyPy3
+ def test_https_proxy_with_proxy_ssl_context(self):
+ proxy_ssl_context = create_urllib3_context()
+ proxy_ssl_context.load_verify_locations(DEFAULT_CA)
+ with proxy_from_url(
+ self.https_proxy_url,
+ proxy_ssl_context=proxy_ssl_context,
+ ca_certs=DEFAULT_CA,
+ ) as https:
+ r = https.request("GET", "%s/" % self.https_url)
+ assert r.status == 200
+
+ r = https.request("GET", "%s/" % self.http_url)
+ assert r.status == 200
+
+ @onlyPy2
+ def test_https_proxy_not_supported(self):
+ with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https:
+ r = https.request("GET", "%s/" % self.http_url)
+ assert r.status == 200
+
+ with pytest.raises(ProxySchemeUnsupported) as excinfo:
+ https.request("GET", "%s/" % self.https_url)
+
+ assert "is not supported in Python 2" in str(excinfo.value)
+
+ @withPyOpenSSL
+ @onlyPy3
+ def test_https_proxy_pyopenssl_not_supported(self):
+ with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https:
+ r = https.request("GET", "%s/" % self.http_url)
+ assert r.status == 200
+
+ with pytest.raises(ProxySchemeUnsupported) as excinfo:
+ https.request("GET", "%s/" % self.https_url)
+
+ assert "isn't available on non-native SSLContext" in str(excinfo.value)
+
+ @onlySecureTransport
+ @onlyPy3
+ def test_https_proxy_securetransport_not_supported(self):
with proxy_from_url(self.https_proxy_url, ca_certs=DEFAULT_CA) as https:
r = https.request("GET", "%s/" % self.http_url)
assert r.status == 200
- with pytest.raises(ProxySchemeUnsupported):
+ with pytest.raises(ProxySchemeUnsupported) as excinfo:
https.request("GET", "%s/" % self.https_url)
+ assert "isn't available on non-native SSLContext" in str(excinfo.value)
+
+ def test_https_proxy_forwarding_for_https(self):
with proxy_from_url(
self.https_proxy_url,
ca_certs=DEFAULT_CA,
- _allow_https_proxy_to_see_traffic=True,
+ use_forwarding_for_https=True,
) as https:
r = https.request("GET", "%s/" % self.http_url)
- https.request("GET", "%s/" % self.https_url)
+ assert r.status == 200
+
+ r = https.request("GET", "%s/" % self.https_url)
assert r.status == 200
def test_nagle_proxy(self):
@@ -302,6 +365,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
self.https_port,
)
+ @onlyPy3
def test_https_headers(self):
with proxy_from_url(
self.https_proxy_url,
@@ -328,19 +392,34 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
self.http_port,
)
- with pytest.raises(ProxySchemeUnsupported):
- http.request_encode_url("GET", "%s/headers" % self.https_url)
-
- r = http.request_encode_url(
- "GET", "%s/headers" % self.http_url, headers={"Baz": "quux"}
+ r = http.request_encode_body(
+ "GET", "%s/headers" % self.https_url, headers={"Baz": "quux"}
)
returned_headers = json.loads(r.data.decode())
assert returned_headers.get("Foo") is None
assert returned_headers.get("Baz") == "quux"
+ assert returned_headers.get("Hickory") is None
+ assert returned_headers.get("Host") == "%s:%s" % (
+ self.https_host,
+ self.https_port,
+ )
+
+ def test_https_headers_forwarding_for_https(self):
+ with proxy_from_url(
+ self.https_proxy_url,
+ headers={"Foo": "bar"},
+ proxy_headers={"Hickory": "dickory"},
+ ca_certs=DEFAULT_CA,
+ use_forwarding_for_https=True,
+ ) as http:
+
+ r = http.request_encode_url("GET", "%s/headers" % self.https_url)
+ returned_headers = json.loads(r.data.decode())
+ assert returned_headers.get("Foo") == "bar"
assert returned_headers.get("Hickory") == "dickory"
assert returned_headers.get("Host") == "%s:%s" % (
- self.http_host,
- self.http_port,
+ self.https_host,
+ self.https_port,
)
def test_headerdict(self):

Integrate TLS-in-TLS support into urllib3 (#1923)