Add support for configuring header combining with HTTPHeaderDict
ADD
changelog/2242.feature.rst
+Added support for configuring header merging behavior with HTTPHeaderDict
+
+When using a ``HTTPHeaderDict`` to provide headers for a request, by default duplicate
+header values will be repeated. But if ``combine=True`` is passed into a call to
+``HTTPHeaderDict.add``, then the added header value will be merged in with an existing
+value into a comma-separated list (``X-My-Header: foo, bar``).

MODIFY
src/urllib3/_collections.py
src/urllib3/_collections.py
@@ -174,14 +174,17 @@ class HTTPHeaderDictItemView(Set[Tuple[str, str]]):
>>> d['X-Header-Name']
'Value1, Value2, Value3'
- However, if we iterate over an HTTPHeaderDict's items, we want to get a
- distinct item for every different value of a header:
+ However, if we iterate over an HTTPHeaderDict's items, we will optionally combine
+ these values based on whether combine=True was called when building up the dictionary
+ >>> d = HTTPHeaderDict({"A": "1", "B": "foo"})
+ >>> d.add("A", "2", combine=True)
+ >>> d.add("B", "bar")
>>> list(d.items())
[
- ('X-Header-Name', 'Value1')
- ('X-Header-Name', 'Value2')
- ('X-Header-Name', 'Value3')
+ ('A', '1, 2'),
+ ('B', 'foo'),
+ ('B', 'bar'),
]
This class conforms to the interface required by the MutableMapping ABC while
@@ -273,6 +276,9 @@ class HTTPHeaderDict(MutableMapping[str, str]):
return key.lower() in self._container
return False
+ def setdefault(self, key: str, default: str = "") -> str:
+ return super().setdefault(key, default)
+
def __eq__(self, other: object) -> bool:
maybe_constructable = ensure_can_construct_http_header_dict(other)
if maybe_constructable is None:
@@ -301,14 +307,24 @@ class HTTPHeaderDict(MutableMapping[str, str]):
except KeyError:
pass
- def add(self, key: str, val: str) -> None:
+ def add(self, key: str, val: str, *, combine: bool = False) -> None:
"""Adds a (name, value) pair, doesn't overwrite the value if it already
exists.
+ If this is called with combine=True, instead of adding a new header value
+ as a distinct item during iteration, this will instead append the value to
+ any existing header value with a comma. If no existing header value exists
+ for the key, then the value will simply be added, ignoring the combine parameter.
+
>>> headers = HTTPHeaderDict(foo='bar')
>>> headers.add('Foo', 'baz')
>>> headers['foo']
'bar, baz'
+ >>> list(headers.items())
+ [('foo', 'bar'), ('foo', 'baz')]
+ >>> headers.add('foo', 'quz', combine=True)
+ >>> list(headers.items())
+ [('foo', 'bar, baz, quz')]
"""
# avoid a bytes/str comparison by decoding before httplib
if isinstance(key, bytes):
@@ -318,6 +334,12 @@ class HTTPHeaderDict(MutableMapping[str, str]):
# Keep the common case aka no item present as fast as possible
vals = self._container.setdefault(key_lower, new_vals)
if new_vals is not vals:
+ # if there are values here, then there is at least the initial
+ # key/value pair
+ assert len(vals) >= 2
+ if combine:
+ vals[-1] = vals[-1] + ", " + val
+ else:
vals.append(val)
def extend(self, *args: ValidHTTPHeaderSource, **kwargs: str) -> None:

MODIFY
src/urllib3/_request_methods.py
src/urllib3/_request_methods.py
@@ -2,6 +2,7 @@ import json as _json
from typing import Any, Dict, Mapping, Optional, Sequence, Tuple, Union
from urllib.parse import urlencode
+from ._collections import HTTPHeaderDict
from .connection import _TYPE_BODY
from .filepost import _TYPE_FIELDS, encode_multipart_formdata
from .response import BaseHTTPResponse
@@ -188,7 +189,7 @@ class RequestMethods:
if headers is None:
headers = self.headers
- extra_kw: Dict[str, Any] = {"headers": {}}
+ extra_kw: Dict[str, Any] = {"headers": HTTPHeaderDict(headers)}
body: Union[bytes, str]
if fields:
@@ -208,9 +209,8 @@ class RequestMethods:
)
extra_kw["body"] = body
- extra_kw["headers"] = {"Content-Type": content_type}
+ extra_kw["headers"].setdefault("Content-Type", content_type)
- extra_kw["headers"].update(headers)
extra_kw.update(urlopen_kw)
return self.urlopen(method, url, **extra_kw)

MODIFY
test/test_collections.py
test/test_collections.py
@@ -155,6 +155,14 @@ class TestHTTPHeaderDict:
assert len(h) == 4
assert "ab" in h
+ def test_setdefault(self) -> None:
+ h = HTTPHeaderDict(a="1")
+ assert h.setdefault("A", "3") == "1"
+ assert h.setdefault("b", "2") == "2"
+ assert h.setdefault("c") == ""
+ assert h["c"] == ""
+ assert h["b"] == "2"
+
def test_create_from_dict(self) -> None:
h = HTTPHeaderDict(dict(ab="1", cd="2", ef="3", gh="4"))
assert len(h) == 4
@@ -252,6 +260,34 @@ class TestHTTPHeaderDict:
assert d["e"] == "foofoo"
assert len(d) == 2
+ def test_header_repeat(self, d: HTTPHeaderDict) -> None:
+ d["other-header"] = "hello"
+ d.add("other-header", "world")
+
+ assert list(d.items()) == [
+ ("Cookie", "foo"),
+ ("Cookie", "bar"),
+ ("other-header", "hello"),
+ ("other-header", "world"),
+ ]
+
+ d.add("other-header", "!", combine=True)
+ expected_results = [
+ ("Cookie", "foo"),
+ ("Cookie", "bar"),
+ ("other-header", "hello"),
+ ("other-header", "world, !"),
+ ]
+
+ assert list(d.items()) == expected_results
+ # make sure the values persist over copys
+ assert list(d.copy().items()) == expected_results
+
+ other_dict = HTTPHeaderDict()
+ # we also need for extensions to properly maintain results
+ other_dict.extend(d)
+ assert list(other_dict.items()) == expected_results
+
def test_extend_from_headerdict(self, d: HTTPHeaderDict) -> None:
h = HTTPHeaderDict(Cookie="foo", e="foofoo")
d.extend(h)

MODIFY
test/with_dummyserver/test_poolmanager.py
test/with_dummyserver/test_poolmanager.py
@@ -380,6 +380,46 @@ class TestPoolManager(HTTPDummyServerTestCase):
["Extra", "extra"],
]
+ def test_headers_http_multi_header_multipart(self) -> None:
+ headers = HTTPHeaderDict()
+ headers.add("Multi", "1")
+ headers.add("Multi", "2")
+ old_headers = headers.copy()
+
+ with PoolManager(headers=headers) as http:
+ r = http.request(
+ "POST",
+ f"{self.base_url}/multi_headers",
+ fields={"k": "v"},
+ multipart_boundary="b",
+ encode_multipart=True,
+ )
+ returned_headers = r.json()["headers"]
+ assert returned_headers[4:] == [
+ ["Multi", "1"],
+ ["Multi", "2"],
+ ["Content-Type", "multipart/form-data; boundary=b"],
+ ]
+ # Assert that the previous headers weren't modified.
+ assert headers == old_headers
+
+ # Set a default value for the Content-Type
+ headers["Content-Type"] = "multipart/form-data; boundary=b; field=value"
+ r = http.request(
+ "POST",
+ f"{self.base_url}/multi_headers",
+ fields={"k": "v"},
+ multipart_boundary="b",
+ encode_multipart=True,
+ )
+ returned_headers = r.json()["headers"]
+ assert returned_headers[4:] == [
+ ["Multi", "1"],
+ ["Multi", "2"],
+ # Uses the set value, not the one that would be generated.
+ ["Content-Type", "multipart/form-data; boundary=b; field=value"],
+ ]
+
def test_body(self) -> None:
with PoolManager() as http:
r = http.request("POST", f"{self.base_url}/echo", body=b"test")

MODIFY
test/with_dummyserver/test_socketlevel.py
test/with_dummyserver/test_socketlevel.py
@@ -1643,6 +1643,75 @@ class TestHeaders(SocketDummyServerTestCase):
]
assert expected_response_headers == actual_response_headers
+ @pytest.mark.parametrize(
+ "method_type, body_type",
+ [
+ ("GET", None),
+ ("POST", None),
+ ("POST", "bytes"),
+ ("POST", "bytes-io"),
+ ],
+ )
+ def test_headers_sent_with_add(
+ self, method_type: str, body_type: Optional[str]
+ ) -> None:
+ """
+ Confirm that when adding headers with combine=True that we simply append to the
+ most recent value, rather than create a new header line.
+ """
+ body: Union[None, bytes, io.BytesIO]
+ if body_type is None:
+ body = None
+ elif body_type == "bytes":
+ body = b"my-body"
+ elif body_type == "bytes-io":
+ body = io.BytesIO(b"bytes-io-body")
+ body.seek(0, 0)
+ else:
+ raise ValueError("Unknonw body type")
+
+ buffer: bytes = b""
+
+ def socket_handler(listener: socket.socket) -> None:
+ nonlocal buffer
+ sock = listener.accept()[0]
+ sock.settimeout(0)
+
+ start = time.time()
+ while time.time() - start < (LONG_TIMEOUT / 2):
+ try:
+ buffer += sock.recv(65536)
+ except OSError:
+ continue
+
+ sock.sendall(
+ b"HTTP/1.1 200 OK\r\n"
+ b"Server: example.com\r\n"
+ b"Content-Length: 0\r\n\r\n"
+ )
+ sock.close()
+
+ self._start_server(socket_handler)
+
+ headers = HTTPHeaderDict()
+ headers.add("A", "1")
+ headers.add("C", "3")
+ headers.add("B", "2")
+ headers.add("B", "3")
+ headers.add("A", "4", combine=False)
+ headers.add("C", "5", combine=True)
+ headers.add("C", "6")
+
+ with HTTPConnectionPool(self.host, self.port, retries=False) as pool:
+ r = pool.request(
+ method_type,
+ "/",
+ body=body,
+ headers=headers,
+ )
+ assert r.status == 200
+ assert b"A: 1\r\nA: 4\r\nC: 3, 5\r\nC: 6\r\nB: 2\r\nB: 3" in buffer
+
class TestBrokenHeaders(SocketDummyServerTestCase):
def _test_broken_header_parsing(

Add support for configuring header combining with HTTPHeaderDict

Co-authored-by: Seth Michael Larson <sethmichaellarson@gmail.com>