Add type hints to urllib3.connectionpool
MODIFY
noxfile.py
noxfile.py
@@ -10,6 +10,7 @@ import nox
TYPED_FILES = {
"src/urllib3/contrib/__init__.py",
"src/urllib3/connection.py",
+ "src/urllib3/connectionpool.py",
"src/urllib3/exceptions.py",
"src/urllib3/_collections.py",
"src/urllib3/fields.py",

MODIFY
src/urllib3/connection.py
src/urllib3/connection.py
@@ -336,11 +336,11 @@ class HTTPSConnection(HTTPConnection):
default_port = port_by_scheme["https"]
- cert_reqs: Optional[int] = None
+ cert_reqs: Optional[Union[int, str]] = None
ca_certs: Optional[str] = None
ca_cert_dir: Optional[str] = None
ca_cert_data: Union[None, str, bytes] = None
- ssl_version: Optional[int] = None
+ ssl_version: Optional[Union[int, str]] = None
assert_fingerprint: Optional[str] = None
tls_in_tls_required: bool = False
@@ -384,7 +384,7 @@ class HTTPSConnection(HTTPConnection):
self,
key_file: Optional[str] = None,
cert_file: Optional[str] = None,
- cert_reqs: Optional[int] = None,
+ cert_reqs: Optional[Union[int, str]] = None,
key_password: Optional[str] = None,
ca_certs: Optional[str] = None,
assert_hostname: Union[None, str, "Literal[False]"] = None,

MODIFY
src/urllib3/connectionpool.py
src/urllib3/connectionpool.py
@@ -4,15 +4,19 @@ import queue
import socket
import sys
import warnings
+from http.client import HTTPResponse as _HttplibHTTPResponse
from socket import timeout as SocketTimeout
+from typing import TYPE_CHECKING, Any, Mapping, Optional, Type, Union, overload
-from .connection import (
+from .connection import ( # type: ignore
BaseSSLError,
BrokenPipeError,
DummyConnection,
+ HTTPBody,
HTTPConnection,
HTTPException,
HTTPSConnection,
+ ProxyConfig,
VerifiedHTTPSConnection,
port_by_scheme,
)
@@ -47,11 +51,17 @@ from .util.url import _normalize_host as normalize_host
from .util.url import parse_url
from .util.util import to_str
+if TYPE_CHECKING:
+ from typing_extensions import Literal
+
log = logging.getLogger(__name__)
_Default = object()
+_TYPE_TIMEOUT = Union[Timeout, int, float, object]
+
+
# Pool objects
class ConnectionPool:
"""
@@ -64,10 +74,10 @@ class ConnectionPool:
target URIs.
"""
- scheme = None
+ scheme: Optional[str] = None
QueueCls = queue.LifoQueue
- def __init__(self, host, port=None):
+ def __init__(self, host: str, port: Optional[int] = None) -> None:
if not host:
raise LocationValueError("No host specified.")
@@ -75,18 +85,20 @@ class ConnectionPool:
self._proxy_host = host.lower()
self.port = port
- def __str__(self):
+ def __str__(self) -> str:
return f"{type(self).__name__}(host={self.host!r}, port={self.port!r})"
- def __enter__(self):
+ def __enter__(self) -> "ConnectionPool":
return self
- def __exit__(self, exc_type, exc_val, exc_tb):
+ def __exit__(
+ self, exc_type: object, exc_val: object, exc_tb: object
+ ) -> "Literal[False]":
self.close()
# Return False to re-raise any potential exceptions
return False
- def close(self):
+ def close(self) -> None:
"""
Close all pooled connections and disable the pool.
"""
@@ -150,22 +162,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
"""
scheme = "http"
- ConnectionCls = HTTPConnection
+ ConnectionCls: Type[Union[HTTPConnection, HTTPSConnection]] = HTTPConnection
ResponseCls = HTTPResponse
def __init__(
self,
- host,
- port=None,
- timeout=Timeout.DEFAULT_TIMEOUT,
- maxsize=1,
- block=False,
- headers=None,
- retries=None,
- _proxy=None,
- _proxy_headers=None,
- _proxy_config=None,
- **conn_kw,
+ host: str,
+ port: Optional[int] = None,
+ timeout: Optional[Union[Timeout, float, int, object]] = Timeout.DEFAULT_TIMEOUT,
+ maxsize: int = 1,
+ block: bool = False,
+ headers: Optional[Mapping[str, str]] = None,
+ retries: Optional[Union[Retry, bool, int]] = None,
+ _proxy: Optional[Url] = None,
+ _proxy_headers: Optional[Mapping[str, str]] = None,
+ _proxy_config: Optional[ProxyConfig] = None,
+ **conn_kw: Any,
):
ConnectionPool.__init__(self, host, port)
RequestMethods.__init__(self, headers)
@@ -174,12 +186,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
timeout = Timeout.from_float(timeout)
if retries is None:
- retries = Retry.DEFAULT
+ retries = Retry.DEFAULT # type: ignore
self.timeout = timeout
self.retries = retries
- self.pool = self.QueueCls(maxsize)
+ self.pool: Optional[queue.LifoQueue[Any]] = self.QueueCls(maxsize)
self.block = block
self.proxy = _proxy
@@ -204,7 +216,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
self.conn_kw["proxy"] = self.proxy
self.conn_kw["proxy_config"] = self.proxy_config
- def _new_conn(self):
+ def _new_conn(self) -> HTTPConnection:
"""
Return a fresh :class:`HTTPConnection`.
"""
@@ -219,12 +231,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
conn = self.ConnectionCls(
host=self.host,
port=self.port,
- timeout=self.timeout.connect_timeout,
+ timeout=self.timeout.connect_timeout, # type: ignore
**self.conn_kw,
)
return conn
- def _get_conn(self, timeout=None):
+ def _get_conn(self, timeout: Optional[float] = None) -> HTTPConnection:
"""
Get a connection. Will return a pooled connection if one is available.
@@ -237,6 +249,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
:prop:`.block` is ``True``.
"""
conn = None
+
+ if self.pool is None:
+ raise ClosedPoolError(self, "Pool is closed.")
+
try:
conn = self.pool.get(block=self.block, timeout=timeout)
@@ -263,7 +279,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
return conn or self._new_conn()
- def _put_conn(self, conn):
+ def _put_conn(self, conn: Optional[HTTPConnection]) -> None:
"""
Put a connection back into the pool.
@@ -277,6 +293,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
If the pool is closed, then the connection will be closed and discarded.
"""
+ if self.pool is not None:
try:
self.pool.put(conn, block=False)
return # Everything is dandy, done.
@@ -296,23 +313,25 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
"Pool reached maximum size and no more connections are allowed.",
)
- log.warning("Connection pool is full, discarding connection: %s", self.host)
+ log.warning(
+ "Connection pool is full, discarding connection: %s", self.host
+ )
# Connection never got put back into the pool, close it.
if conn:
conn.close()
- def _validate_conn(self, conn):
+ def _validate_conn(self, conn: HTTPConnection) -> None:
"""
Called right before a request is made, after the socket is created.
"""
pass
- def _prepare_proxy(self, conn):
+ def _prepare_proxy(self, conn: HTTPConnection) -> None:
# Nothing to do for HTTP connections.
pass
- def _get_timeout(self, timeout):
+ def _get_timeout(self, timeout: _TYPE_TIMEOUT) -> Timeout:
""" Helper that always returns a :class:`urllib3.util.Timeout` """
if timeout is _Default:
return self.timeout.clone()
@@ -324,7 +343,12 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# can be removed later
return Timeout.from_float(timeout)
- def _raise_timeout(self, err, url, timeout_value):
+ def _raise_timeout(
+ self,
+ err: Union[BaseSSLError, OSError, SocketTimeout],
+ url: str,
+ timeout_value: _TYPE_TIMEOUT,
+ ) -> None:
"""Is the error actually a timeout? Will raise a ReadTimeout or pass"""
if isinstance(err, SocketTimeout):
@@ -339,8 +363,14 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
)
def _make_request(
- self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw
- ):
+ self,
+ conn: HTTPConnection,
+ method: str,
+ url: str,
+ timeout: _TYPE_TIMEOUT = _Default,
+ chunked: bool = False,
+ **httplib_request_kw: Any,
+ ) -> _HttplibHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
@@ -359,7 +389,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
- conn.timeout = timeout_obj.connect_timeout
+ conn.timeout = timeout_obj.connect_timeout # type: ignore
# Trigger any extra validation we need to do.
try:
@@ -421,9 +451,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
method,
url,
# HTTP version
- conn._http_vsn_str,
+ conn._http_vsn_str, # type: ignore
httplib_response.status,
- httplib_response.length,
+ httplib_response.length, # type: ignore
)
try:
@@ -438,10 +468,10 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
return httplib_response
- def _absolute_url(self, path):
+ def _absolute_url(self, path: str) -> str:
return Url(scheme=self.scheme, host=self.host, port=self.port, path=path).url
- def close(self):
+ def close(self) -> None:
"""
Close all pooled connections and disable the pool.
"""
@@ -481,21 +511,21 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
return (scheme, host, port) == (self.scheme, self.host, self.port)
- def urlopen(
+ def urlopen( # type: ignore
self,
- method,
- url,
- body=None,
- headers=None,
- retries=None,
- redirect=True,
- assert_same_host=True,
- timeout=_Default,
- pool_timeout=None,
- release_conn=None,
- chunked=False,
- body_pos=None,
- **response_kw,
+ method: str,
+ url: str,
+ body: Optional[HTTPBody] = None,
+ headers: Optional[Mapping[str, str]] = None,
+ retries: Optional[Union[Retry, bool, int]] = None,
+ redirect: bool = True,
+ assert_same_host: bool = True,
+ timeout: _TYPE_TIMEOUT = _Default,
+ pool_timeout: Optional[int] = None,
+ release_conn: Optional[bool] = None,
+ chunked: bool = False,
+ body_pos: Optional[Union[int, object]] = None,
+ **response_kw: Any,
) -> BaseHTTPResponse:
"""
Get a connection from the pool and perform an HTTP request. This is the
@@ -633,8 +663,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# have to copy the headers dict so we can safely change it without those
# changes being reflected in anyone else's copy.
if not http_tunnel_required:
- headers = headers.copy()
- headers.update(self.proxy_headers)
+ headers = headers.copy() # type: ignore
+ headers.update(self.proxy_headers) # type: ignore
# Must keep the exception bound to a separate variable or else Python 3
# complains about UnboundLocalError.
@@ -653,7 +683,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
timeout_obj = self._get_timeout(timeout)
conn = self._get_conn(timeout=pool_timeout)
- conn.timeout = timeout_obj.connect_timeout
+ conn.timeout = timeout_obj.connect_timeout # type: ignore
is_new_proxy_conn = self.proxy is not None and not getattr(
conn, "sock", None
@@ -733,7 +763,9 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods):
# to throw the connection away unless explicitly told not to.
# Close the connection, set the variable to None, and make sure
# we put the None back in the pool to avoid leaking it.
- conn = conn and conn.close()
+ if conn:
+ conn.close()
+ conn = None
release_this_conn = True
if release_this_conn:
@@ -848,26 +880,26 @@ class HTTPSConnectionPool(HTTPConnectionPool):
def __init__(
self,
- host,
- port=None,
- timeout=Timeout.DEFAULT_TIMEOUT,
- maxsize=1,
- block=False,
- headers=None,
- retries=None,
- _proxy=None,
- _proxy_headers=None,
- key_file=None,
- cert_file=None,
- cert_reqs=None,
- key_password=None,
- ca_certs=None,
- ssl_version=None,
- assert_hostname=None,
- assert_fingerprint=None,
- ca_cert_dir=None,
- **conn_kw,
- ):
+ host: str,
+ port: Optional[int] = None,
+ timeout: _TYPE_TIMEOUT = Timeout.DEFAULT_TIMEOUT,
+ maxsize: int = 1,
+ block: bool = False,
+ headers: Optional[Mapping[str, str]] = None,
+ retries: Optional[Union[Retry, bool, int]] = None,
+ _proxy: Optional[Url] = None,
+ _proxy_headers: Optional[Mapping[str, str]] = None,
+ key_file: Optional[str] = None,
+ cert_file: Optional[str] = None,
+ cert_reqs: Optional[Union[int, str]] = None,
+ key_password: Optional[str] = None,
+ ca_certs: Optional[str] = None,
+ ssl_version: Optional[Union[int, str]] = None,
+ assert_hostname: Optional[Union[str, "Literal[False]"]] = None,
+ assert_fingerprint: Optional[str] = None,
+ ca_cert_dir: Optional[str] = None,
+ **conn_kw: Any,
+ ) -> None:
super().__init__(
host,
@@ -892,7 +924,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
self.assert_hostname = assert_hostname
self.assert_fingerprint = assert_fingerprint
- def _prepare_conn(self, conn):
+ def _prepare_conn(self, conn: HTTPSConnection) -> HTTPConnection:
"""
Prepare the ``connection`` for :meth:`urllib3.util.ssl_wrap_socket`
and establish the tunnel if proxy is used.
@@ -912,7 +944,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
conn.ssl_version = self.ssl_version
return conn
- def _prepare_proxy(self, conn):
+ def _prepare_proxy(self, conn: HTTPSConnection) -> None: # type: ignore
"""
Establishes a tunnel connection through HTTP CONNECT.
@@ -922,14 +954,14 @@ class HTTPSConnectionPool(HTTPConnectionPool):
conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers)
- if self.proxy.scheme == "https":
+ if self.proxy and self.proxy.scheme == "https":
conn.tls_in_tls_required = True
conn.connect()
- def _new_conn(self):
+ def _new_conn(self) -> HTTPConnection:
"""
- Return a fresh :class:`http.client.HTTPSConnection`.
+ Return a fresh :class:`urllib3.connection.HTTPConnection`.
"""
self.num_connections += 1
log.debug(
@@ -939,21 +971,21 @@ class HTTPSConnectionPool(HTTPConnectionPool):
self.port or "443",
)
- if not self.ConnectionCls or self.ConnectionCls is DummyConnection:
+ if not self.ConnectionCls or self.ConnectionCls is DummyConnection: # type: ignore
raise SSLError(
"Can't connect to HTTPS URL because the SSL module is not available."
)
- actual_host = self.host
+ actual_host: str = self.host
actual_port = self.port
- if self.proxy is not None:
+ if self.proxy is not None and self.proxy.host is not None:
actual_host = self.proxy.host
actual_port = self.proxy.port
conn = self.ConnectionCls(
host=actual_host,
port=actual_port,
- timeout=self.timeout.connect_timeout,
+ timeout=self.timeout.connect_timeout, # type: ignore
cert_file=self.cert_file,
key_file=self.key_file,
key_password=self.key_password,
@@ -962,7 +994,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
return self._prepare_conn(conn)
- def _validate_conn(self, conn):
+ def _validate_conn(self, conn: HTTPConnection) -> None:
"""
Called right before a request is made, after the socket is created.
"""
@@ -984,7 +1016,7 @@ class HTTPSConnectionPool(HTTPConnectionPool):
)
-def connection_from_url(url, **kw):
+def connection_from_url(url: str, **kw: Any) -> ConnectionPool:
"""
Given a url, return an :class:`.ConnectionPool` instance of its host.
@@ -1008,12 +1040,22 @@ def connection_from_url(url, **kw):
scheme = scheme or "http"
port = port or port_by_scheme.get(scheme, 80)
if scheme == "https":
- return HTTPSConnectionPool(host, port=port, **kw)
+ return HTTPSConnectionPool(host, port=port, **kw) # type: ignore
else:
- return HTTPConnectionPool(host, port=port, **kw)
+ return HTTPConnectionPool(host, port=port, **kw) # type: ignore
+
+
+@overload
+def _normalize_host(host: None, scheme: Optional[str]) -> None:
+ ...
+
+
+@overload
+def _normalize_host(host: str, scheme: Optional[str]) -> str:
+ ...
-def _normalize_host(host, scheme):
+def _normalize_host(host: Optional[str], scheme: Optional[str]) -> Optional[str]:
"""
Normalize hosts for comparisons and use with sockets.
"""
@@ -1026,6 +1068,6 @@ def _normalize_host(host, scheme):
# Instead, we need to make sure we never pass ``None`` as the port.
# However, for backward compatibility reasons we can't actually
# *assert* that. See http://bugs.python.org/issue28539
- if host.startswith("[") and host.endswith("]"):
+ if host and host.startswith("[") and host.endswith("]"):
host = host[1:-1]
return host

MODIFY
src/urllib3/exceptions.py
src/urllib3/exceptions.py
@@ -6,6 +6,7 @@ if TYPE_CHECKING:
from urllib3.connectionpool import ConnectionPool
from urllib3.response import HTTPResponse
+ from urllib3.util.retry import Retry
# Base Exceptions
@@ -119,9 +120,11 @@ class MaxRetryError(RequestError):
class HostChangedError(RequestError):
"""Raised when an existing pool gets a request for a foreign host."""
- retries: int
+ retries: Union["Retry", int]
- def __init__(self, pool: "ConnectionPool", url: str, retries: int = 3) -> None:
+ def __init__(
+ self, pool: "ConnectionPool", url: str, retries: Union["Retry", int] = 3
+ ) -> None:
message = f"Tried to open a foreign host with url: {url}"
super().__init__(pool, url, message)
self.retries = retries

MODIFY
src/urllib3/response.py
src/urllib3/response.py
@@ -3,7 +3,9 @@ import logging
import typing
import zlib
from contextlib import contextmanager
+from http.client import HTTPResponse as _HttplibHTTPResponse
from socket import timeout as SocketTimeout
+from typing import Any, Type
try:
try:
@@ -435,7 +437,7 @@ class HTTPResponse(BaseHTTPResponse):
self._pool._put_conn(self._connection)
self._connection = None
- def drain_conn(self):
+ def drain_conn(self) -> None:
"""
Read and discard any remaining HTTP response data in the response connection.
@@ -678,7 +680,9 @@ class HTTPResponse(BaseHTTPResponse):
yield data
@classmethod
- def from_httplib(ResponseCls, r, **response_kw):
+ def from_httplib(
+ ResponseCls: Type["HTTPResponse"], r: _HttplibHTTPResponse, **response_kw: Any
+ ) -> "HTTPResponse":
"""
Given an :class:`http.client.HTTPResponse` instance ``r``, return a
corresponding :class:`urllib3.response.HTTPResponse` object.

MODIFY
src/urllib3/util/proxy.py
src/urllib3/util/proxy.py
@@ -45,8 +45,8 @@ def connection_requires_http_tunnel(
def create_proxy_ssl_context(
- ssl_version: Optional[int] = None,
- cert_reqs: Optional[int] = None,
+ ssl_version: Optional[Union[int, str]] = None,
+ cert_reqs: Optional[Union[int, str]] = None,
ca_certs: Optional[str] = None,
ca_cert_dir: Optional[str] = None,
ca_cert_data: Union[None, str, bytes] = None,

MODIFY
src/urllib3/util/request.py
src/urllib3/util/request.py
from base64 import b64encode
-from typing import IO, AnyStr, Dict, List, Optional, Union
+from typing import IO, Any, AnyStr, Dict, List, Optional, Union
from ..exceptions import UnrewindableBodyError
@@ -104,7 +104,7 @@ def make_headers(
def set_file_position(
- body: IO[AnyStr], pos: Optional[Union[int, object]]
+ body: Any, pos: Optional[Union[int, object]]
) -> Optional[Union[int, object]]:
"""
If a position is provided, move file to that point.

MODIFY
src/urllib3/util/timeout.py
src/urllib3/util/timeout.py
@@ -169,7 +169,7 @@ class Timeout:
return value
@classmethod
- def from_float(cls, timeout: float) -> "Timeout":
+ def from_float(cls, timeout: Optional[Union[int, float, object]]) -> "Timeout":
"""Create a new Timeout from a legacy timeout value.
The timeout value used by httplib.py sets the same timeout on the

Add type hints to urllib3.connectionpool