Add SKIP_HEADER for skipping automatically added headers
MODIFY
src/urllib3/_collections.py
src/urllib3/_collections.py
@@ -155,7 +155,7 @@ class HTTPHeaderDict(MutableMapping):
def __getitem__(self, key):
val = self._container[key.lower()]
- return ", ".join([six.ensure_str(v, "ascii") for v in val[1:]])
+ return ", ".join(val[1:])
def __delitem__(self, key):
del self._container[key.lower()]

MODIFY
src/urllib3/connection.py
src/urllib3/connection.py
@@ -43,7 +43,6 @@ except NameError: # Python 2:
pass
-from ._collections import HTTPHeaderDict
from ._version import __version__
from .exceptions import (
ConnectTimeoutError,
@@ -52,7 +51,7 @@ from .exceptions import (
SystemTimeWarning,
)
from .packages.ssl_match_hostname import CertificateError, match_hostname
-from .util import SUPPRESS_USER_AGENT, connection
+from .util import SKIP_HEADER, SKIPPABLE_HEADERS, connection
from .util.ssl_ import (
assert_fingerprint,
create_urllib3_context,
@@ -213,12 +212,24 @@ class HTTPConnection(_HTTPConnection, object):
return _HTTPConnection.putrequest(self, method, url, *args, **kwargs)
+ def putheader(self, header, *values):
+ """"""
+ if SKIP_HEADER not in values:
+ _HTTPConnection.putheader(self, header, *values)
+ elif six.ensure_str(header.lower()) not in SKIPPABLE_HEADERS:
+ raise ValueError(
+ "urllib3.util.SKIP_HEADER only supports '%s'"
+ % ("', '".join(map(str.title, sorted(SKIPPABLE_HEADERS))),)
+ )
+
def request(self, method, url, body=None, headers=None):
- headers = HTTPHeaderDict(headers if headers is not None else {})
- if "user-agent" not in headers:
+ if headers is None:
+ headers = {}
+ else:
+ # Avoid modifying the headers passed into .request()
+ headers = headers.copy()
+ if "user-agent" not in (k.lower() for k in headers):
headers["User-Agent"] = _get_default_user_agent()
- elif headers["user-agent"] == SUPPRESS_USER_AGENT:
- del headers["user-agent"]
super(HTTPConnection, self).request(method, url, body=body, headers=headers)
def request_chunked(self, method, url, body=None, headers=None):
@@ -226,16 +237,15 @@ class HTTPConnection(_HTTPConnection, object):
Alternative to the common request method, which sends the
body with chunked encoding and not as one block
"""
- headers = HTTPHeaderDict(headers if headers is not None else {})
- skip_accept_encoding = "accept-encoding" in headers
- skip_host = "host" in headers
+ headers = headers or {}
+ header_keys = set([six.ensure_str(k.lower()) for k in headers])
+ skip_accept_encoding = "accept-encoding" in header_keys
+ skip_host = "host" in header_keys
self.putrequest(
method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host
)
- if "user-agent" not in headers:
- headers["User-Agent"] = _get_default_user_agent()
- elif headers["user-agent"] == SUPPRESS_USER_AGENT:
- del headers["user-agent"]
+ if "user-agent" not in header_keys:
+ self.putheader("User-Agent", _get_default_user_agent())
for header, value in headers.items():
self.putheader(header, value)
if "transfer-encoding" not in headers:

MODIFY
src/urllib3/util/__init__.py
src/urllib3/util/__init__.py
@@ -2,7 +2,7 @@ from __future__ import absolute_import
# For backwards compatibility, provide imports that used to be here.
from .connection import is_connection_dropped
-from .request import SUPPRESS_USER_AGENT, make_headers
+from .request import SKIP_HEADER, SKIPPABLE_HEADERS, make_headers
from .response import is_fp_closed
from .retry import Retry
from .ssl_ import (
@@ -44,5 +44,6 @@ __all__ = (
"ssl_wrap_socket",
"wait_for_read",
"wait_for_write",
- "SUPPRESS_USER_AGENT",
+ "SKIP_HEADER",
+ "SKIPPABLE_HEADERS",
)

MODIFY
src/urllib3/util/request.py
src/urllib3/util/request.py
@@ -5,10 +5,13 @@ from base64 import b64encode
from ..exceptions import UnrewindableBodyError
from ..packages.six import b, integer_types
-# Use an invalid User-Agent to represent suppressing of default user agent.
-# See https://tools.ietf.org/html/rfc7231#section-5.5.3 and
-# https://tools.ietf.org/html/rfc7230#section-3.2.6
-SUPPRESS_USER_AGENT = "@@@INVALID_USER_AGENT@@@"
+# Pass as a value within ``headers`` to skip
+# emitting some HTTP headers that are added automatically.
+# The only headers that are supported are ``Accept-Encoding``,
+# ``Host``, and ``User-Agent``.
+SKIP_HEADER = "@@@SKIP_HEADER@@@"
+SKIPPABLE_HEADERS = frozenset(["accept-encoding", "host", "user-agent"])
+
ACCEPT_ENCODING = "gzip,deflate"
try:
import brotli as _unused_module_brotli # noqa: F401

MODIFY
test/with_dummyserver/test_chunked_transfer.py
test/with_dummyserver/test_chunked_transfer.py
@@ -8,7 +8,7 @@ from dummyserver.testcase import (
consume_socket,
)
from urllib3 import HTTPConnectionPool
-from urllib3.util import SUPPRESS_USER_AGENT
+from urllib3.util import SKIP_HEADER
from urllib3.util.retry import Retry
# Retry failed tests
@@ -123,7 +123,7 @@ class TestChunkedTransfer(SocketDummyServerTestCase):
"GET",
"/",
chunks,
- headers={"User-Agent": SUPPRESS_USER_AGENT},
+ headers={"User-Agent": SKIP_HEADER},
chunked=True,
)

Add SKIP_HEADER for skipping automatically added headers