Add support for native JSON request and response bodies
ADD
changelog/2243.feature.rst
+Added the ``json`` parameter to ``urllib3.request()``, ``PoolManager.request()``, and ``ConnectionPool.request()`` methods to send JSON bodies in requests. Using this parameter will set the header ``Content-Type: application/json`` if ``Content-Type`` isn't already defined.
+
+Added support for parsing JSON response bodies with ``HTTPResponse.json()`` method.

MODIFY
docs/user-guide.rst
docs/user-guide.rst
@@ -101,20 +101,33 @@ The :class:`~response.HTTPResponse` object provides
JSON Content
~~~~~~~~~~~~
-
-JSON content can be loaded by decoding and deserializing the
-:attr:`~response.HTTPResponse.data` attribute of the request:
+JSON content can be loaded by :meth:`~response.HTTPResponse.json`
+method of the response:
.. code-block:: python
- import json
import urllib3
resp = urllib3.request("GET", "https://httpbin.org/ip")
- print(json.loads(resp.data.decode("utf-8")))
+ print(resp.json())
# {"origin": "127.0.0.1"}
+Alternatively, Custom JSON libraries such as `orjson` can be used to encode data,
+retrieve data by decoding and deserializing the :attr:`~response.HTTPResponse.data`
+attribute of the request:
+
+.. code-block:: python
+
+ import orjson
+ import urllib3
+
+ encoded_data = orjson.dumps({"attribute": "value"})
+ resp = urllib3.request(method="POST", url="http://httpbin.org/post", body=encoded_data)
+
+ print(orjson.loads(resp.data)["json"])
+ # {'attribute': 'value'}
+
Binary Content
~~~~~~~~~~~~~~
@@ -170,7 +183,6 @@ You can specify headers as a dictionary in the ``headers`` argument in :meth:`~p
.. code-block:: python
- import json
import urllib3
resp = urllib3.request(
@@ -181,14 +193,13 @@ You can specify headers as a dictionary in the ``headers`` argument in :meth:`~p
}
)
- print(json.loads(resp.data.decode("utf-8"))["headers"])
+ print(resp.json()["headers"])
# {"X-Something": "value", ...}
Or you can use the ``HTTPHeaderDict`` class to create multi-valued HTTP headers:
.. code-block:: python
- import json
import urllib3
# Create an HTTPHeaderDict and add headers
@@ -203,7 +214,7 @@ Or you can use the ``HTTPHeaderDict`` class to create multi-valued HTTP headers:
headers=headers
)
- print(json.loads(resp.data.decode("utf-8"))["headers"])
+ print(resp.json()["headers"])
# {"Accept": "application/json, text/plain", ...}
@@ -216,7 +227,6 @@ arguments as a dictionary in the ``fields`` argument to
.. code-block:: python
- import json
import urllib3
resp = urllib3.request(
@@ -225,7 +235,7 @@ arguments as a dictionary in the ``fields`` argument to
fields={"arg": "value"}
)
- print(json.loads(resp.data.decode("utf-8"))["args"])
+ print(resp.json()["args"])
# {"arg": "value"}
For ``POST`` and ``PUT`` requests, you need to manually encode query parameters
@@ -233,7 +243,6 @@ in the URL:
.. code-block:: python
- import json
from urllib.parse import urlencode
import urllib3
@@ -244,11 +253,10 @@ in the URL:
url = "https://httpbin.org/post?" + encoded_args
resp = urllib3.request("POST", url)
- print(json.loads(resp.data.decode("utf-8"))["args"])
+ print(resp.json()["args"])
# {"arg": "value"}
-
.. _form_data:
Form Data
@@ -260,7 +268,6 @@ dictionary in the ``fields`` argument provided to
.. code-block:: python
- import json
import urllib3
resp = urllib3.request(
@@ -269,34 +276,34 @@ dictionary in the ``fields`` argument provided to
fields={"field": "value"}
)
- print(json.loads(resp.data.decode("utf-8"))["form"])
+ print(resp.json()["form"])
# {"field": "value"}
+.. _json:
+
JSON
~~~~
-You can send a JSON request by specifying the encoded data as the ``body``
-argument and setting the ``Content-Type`` header when calling
+You can send a JSON request by specifying the data as ``json`` argument,
+urllib3 automatically encodes data using ``json`` module with ``UTF-8``
+encoding. Also by default ``"Content-Type"`` in headers is set to
+``"application/json"`` if not specified when calling
:meth:`~poolmanager.PoolManager.request`:
.. code-block:: python
- import json
import urllib3
data = {"attribute": "value"}
- # Encoding the data in JSON format.
- encoded_data = json.dumps(data).encode("utf-8")
-
resp = urllib3.request(
"POST",
"https://httpbin.org/post",
- body=encoded_data, # Embedding JSON data into request body.
+ body=data,
headers={"Content-Type": "application/json"}
)
- print(json.loads(resp.data.decode("utf-8"))["json"])
+ print(resp.json())
# {"attribute": "value"}
Files & Binary Data
@@ -308,7 +315,6 @@ approach as :ref:`form_data` and specify the file field as a tuple of
.. code-block:: python
- import json
import urllib3
# Reading the text file from local storage.
@@ -324,7 +330,7 @@ approach as :ref:`form_data` and specify the file field as a tuple of
}
)
- print(json.loads(resp.data.decode("utf-8"))["files"])
+ print(resp.json()["files"])
# {"filefield": "..."}
While specifying the filename is not strictly required, it's recommended in
@@ -346,7 +352,6 @@ recommended to set the ``Content-Type`` header:
.. code-block:: python
- import json
import urllib3
with open("/home/samad/example.jpg", "rb") as fp:
@@ -359,7 +364,7 @@ recommended to set the ``Content-Type`` header:
headers={"Content-Type": "image/jpeg"}
)
- print(json.loads(resp.data.decode("utf-8"))["data"])
+ print(resp.json()["data"])
# data:application/octet-stream;base64,...
.. _ssl:

MODIFY
dummyserver/handlers.py
dummyserver/handlers.py
@@ -8,7 +8,7 @@ import zlib
from datetime import datetime, timedelta
from http.client import responses
from io import BytesIO
-from typing import Dict, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, Optional, Sequence, Tuple, Union
from urllib.parse import urlsplit
from tornado import httputil
@@ -25,9 +25,14 @@ class Response:
body: Union[str, bytes, Sequence[Union[str, bytes]]] = "",
status: str = "200 OK",
headers: Optional[Sequence[Tuple[str, Union[str, bytes]]]] = None,
+ json: Optional[Any] = None,
) -> None:
self.body = body
self.status = status
+ if json is not None:
+ self.headers = headers or [("Content-type", "application/json")]
+ self.body = json
+ else:
self.headers = headers or [("Content-type", "text/plain")]
def __call__(self, request_handler: RequestHandler) -> None:
@@ -238,6 +243,10 @@ class TestingApp(RequestHandler):
return Response(request.body)
+ def echo_json(self, request: httputil.HTTPServerRequest) -> Response:
+ "Echo back the JSON"
+ return Response(json=request.body, headers=list(request.headers.items()))
+
def echo_uri(self, request: httputil.HTTPServerRequest) -> Response:
"Echo back the requested URI"
assert request.uri is not None

MODIFY
src/urllib3/__init__.py
src/urllib3/__init__.py
@@ -6,7 +6,7 @@ Python HTTP library with thread-safe connection pooling, file post support, user
import logging
import warnings
from logging import NullHandler
-from typing import Mapping, Optional, Type, Union
+from typing import Any, Mapping, Optional, Type, Union
from . import exceptions
from ._collections import HTTPHeaderDict
@@ -100,6 +100,7 @@ def request(
redirect: Optional[bool] = True,
retries: Optional[Union[Retry, bool, int]] = None,
timeout: Optional[Union[Timeout, float, int]] = 3,
+ json: Optional[Any] = None,
) -> BaseHTTPResponse:
"""
A convenience, top-level request method. It uses a module-global ``PoolManager`` instance.
@@ -119,4 +120,5 @@ def request(
redirect=redirect,
retries=retries,
timeout=timeout,
+ json=json,
)

MODIFY
src/urllib3/_request_methods.py
src/urllib3/_request_methods.py
+import json as _json
from typing import Any, Dict, Mapping, Optional, Sequence, Tuple, Union
from urllib.parse import urlencode
@@ -68,6 +69,7 @@ class RequestMethods:
body: Optional[_TYPE_BODY] = None,
fields: Optional[_TYPE_FIELDS] = None,
headers: Optional[Mapping[str, str]] = None,
+ json: Optional[Any] = None,
**urlopen_kw: Any,
) -> BaseHTTPResponse:
"""
@@ -84,6 +86,21 @@ class RequestMethods:
urlopen_kw["request_url"] = url
+ if json is not None and body is not None:
+ raise TypeError(
+ "request got values for both 'body' and 'json' parameters which are mutually exclusive"
+ )
+
+ if json is not None:
+ if headers is None:
+ headers = self.headers.copy() # type: ignore
+ if not ("content-type" in map(str.lower, headers.keys())):
+ headers["Content-Type"] = "application/json" # type: ignore
+
+ body = _json.dumps(json, separators=(",", ":"), ensure_ascii=False).encode(
+ "utf-8"
+ )
+
if body is not None:
urlopen_kw["body"] = body

MODIFY
src/urllib3/response.py
src/urllib3/response.py
import io
+import json as _json
import logging
import zlib
from contextlib import contextmanager
@@ -236,6 +237,19 @@ class BaseHTTPResponse(io.IOBase):
def data(self) -> bytes:
raise NotImplementedError()
+ def json(self) -> Any:
+ """
+ Parses the body of the HTTP response as JSON.
+
+ To use a custom JSON decoder pass the result of :attr:`HTTPResponse.data` to the decoder.
+
+ This method can raise either `UnicodeDecodeError` or `json.JSONDecodeError`.
+
+ Read more :ref:`here <json>`.
+ """
+ data = self.data.decode("utf-8")
+ return _json.loads(data)
+
@property
def url(self) -> Optional[str]:
raise NotImplementedError()

MODIFY
test/with_dummyserver/test_https.py
test/with_dummyserver/test_https.py
import datetime
-import json
import logging
import os.path
import shutil
@@ -178,7 +177,7 @@ class TestHTTPS(HTTPSDummyServerTestCase):
ssl_minimum_version=self.tls_version(),
) as https_pool:
r = https_pool.request("GET", "/certificate")
- subject = json.loads(r.data.decode("utf-8"))
+ subject = r.json()
assert subject["organizationalUnitName"].startswith("Testing cert")
def test_client_no_intermediate(self) -> None:
@@ -210,7 +209,7 @@ class TestHTTPS(HTTPSDummyServerTestCase):
ssl_minimum_version=self.tls_version(),
) as https_pool:
r = https_pool.request("GET", "/certificate")
- subject = json.loads(r.data.decode("utf-8"))
+ subject = r.json()
assert subject["organizationalUnitName"].startswith("Testing cert")
@requires_ssl_context_keyfile_password()

MODIFY
test/with_dummyserver/test_poolmanager.py
test/with_dummyserver/test_poolmanager.py
import gzip
-import json
from test import LONG_TIMEOUT
from unittest import mock
@@ -145,7 +144,7 @@ class TestPoolManager(HTTPDummyServerTestCase):
assert r.status == 200
- data = json.loads(r.data.decode("utf-8"))
+ data = r.json()
assert "Authorization" not in data
@@ -158,7 +157,7 @@ class TestPoolManager(HTTPDummyServerTestCase):
assert r.status == 200
- data = json.loads(r.data.decode("utf-8"))
+ data = r.json()
assert "authorization" not in data
assert "Authorization" not in data
@@ -175,7 +174,7 @@ class TestPoolManager(HTTPDummyServerTestCase):
assert r.status == 200
- data = json.loads(r.data.decode("utf-8"))
+ data = r.json()
assert data["Authorization"] == "foo"
@@ -191,7 +190,7 @@ class TestPoolManager(HTTPDummyServerTestCase):
assert r.status == 200
- data = json.loads(r.data.decode("utf-8"))
+ data = r.json()
assert "X-API-Secret" not in data
assert data["Authorization"] == "bar"
@@ -207,7 +206,7 @@ class TestPoolManager(HTTPDummyServerTestCase):
assert r.status == 200
- data = json.loads(r.data.decode("utf-8"))
+ data = r.json()
assert "x-api-secret" not in data
assert "X-API-Secret" not in data
@@ -311,32 +310,32 @@ class TestPoolManager(HTTPDummyServerTestCase):
def test_headers(self):
with PoolManager(headers={"Foo": "bar"}) as http:
r = http.request("GET", f"{self.base_url}/headers")
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") == "bar"
r = http.request("POST", f"{self.base_url}/headers")
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") == "bar"
r = http.request_encode_url("GET", f"{self.base_url}/headers")
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") == "bar"
r = http.request_encode_body("POST", f"{self.base_url}/headers")
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") == "bar"
r = http.request_encode_url(
"GET", f"{self.base_url}/headers", headers={"Baz": "quux"}
)
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") is None
assert returned_headers.get("Baz") == "quux"
r = http.request_encode_body(
"GET", f"{self.base_url}/headers", headers={"Baz": "quux"}
)
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") is None
assert returned_headers.get("Baz") == "quux"
@@ -349,7 +348,7 @@ class TestPoolManager(HTTPDummyServerTestCase):
with PoolManager(headers=headers) as http:
r = http.request("GET", f"{self.base_url}/headers")
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers["Foo"] == "bar"
assert returned_headers["Multi"] == "1, 2"
assert returned_headers["Baz"] == "quux"
@@ -363,7 +362,7 @@ class TestPoolManager(HTTPDummyServerTestCase):
"Foo": "new",
},
)
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers["Foo"] == "new"
assert returned_headers["Multi"] == "1, 2"
assert returned_headers["Baz"] == "quux"
@@ -490,8 +489,50 @@ class TestPoolManager(HTTPDummyServerTestCase):
redirect=True,
retries=None,
timeout=2.5,
+ json=None,
)
+ @pytest.mark.parametrize(
+ "headers",
+ [
+ None,
+ {"content-Type": "application/json"},
+ {"content-Type": "text/plain"},
+ {"attribute": "value", "CONTENT-TYPE": "application/json"},
+ HTTPHeaderDict(cookie="foo, bar"),
+ ],
+ )
+ def test_request_with_json(self, headers):
+ body = {"attribute": "value"}
+ r = request(
+ method="POST", url=f"{self.base_url}/echo_json", headers=headers, json=body
+ )
+ assert r.status == 200
+ assert r.json() == body
+ if headers is not None and "application/json" not in headers.values():
+ assert "text/plain" in r.headers["Content-Type"].replace(" ", "").split(",")
+ else:
+ assert "application/json" in r.headers["Content-Type"].replace(
+ " ", ""
+ ).split(",")
+
+ def test_top_level_request_with_json_with_httpheaderdict(self):
+ body = {"attribute": "value"}
+ header = HTTPHeaderDict(cookie="foo, bar")
+ with PoolManager(headers=header) as http:
+ r = http.request(method="POST", url=f"{self.base_url}/echo_json", json=body)
+ assert r.status == 200
+ assert r.json() == body
+ assert "application/json" in r.headers["Content-Type"].replace(
+ " ", ""
+ ).split(",")
+
+ def test_top_level_request_with_body_and_json(self):
+ match = "request got values for both 'body' and 'json' parameters which are mutually exclusive"
+ with pytest.raises(TypeError, match=match):
+ body = {"attribute": "value"}
+ request(method="POST", url=f"{self.base_url}/echo", body=body, json=body)
+
@pytest.mark.skipif(not HAS_IPV6, reason="IPv6 is not supported on this system")
class TestIPv6PoolManager(IPv6HTTPDummyServerTestCase):

MODIFY
test/with_dummyserver/test_proxy_poolmanager.py
test/with_dummyserver/test_proxy_poolmanager.py
-import json
import os.path
import shutil
import socket
@@ -272,13 +271,13 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
) as http:
r = http.request_encode_url("GET", f"{self.http_url}/headers")
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") == "bar"
assert returned_headers.get("Hickory") == "dickory"
assert returned_headers.get("Host") == f"{self.http_host}:{self.http_port}"
r = http.request_encode_url("GET", f"{self.http_url_alt}/headers")
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") == "bar"
assert returned_headers.get("Hickory") == "dickory"
assert (
@@ -286,7 +285,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
)
r = http.request_encode_url("GET", f"{self.https_url}/headers")
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") == "bar"
assert returned_headers.get("Hickory") is None
assert (
@@ -294,7 +293,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
)
r = http.request_encode_body("POST", f"{self.http_url}/headers")
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") == "bar"
assert returned_headers.get("Hickory") == "dickory"
assert returned_headers.get("Host") == f"{self.http_host}:{self.http_port}"
@@ -302,7 +301,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
r = http.request_encode_url(
"GET", f"{self.http_url}/headers", headers={"Baz": "quux"}
)
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") is None
assert returned_headers.get("Baz") == "quux"
assert returned_headers.get("Hickory") == "dickory"
@@ -311,7 +310,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
r = http.request_encode_url(
"GET", f"{self.https_url}/headers", headers={"Baz": "quux"}
)
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") is None
assert returned_headers.get("Baz") == "quux"
assert returned_headers.get("Hickory") is None
@@ -322,7 +321,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
r = http.request_encode_body(
"GET", f"{self.http_url}/headers", headers={"Baz": "quux"}
)
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") is None
assert returned_headers.get("Baz") == "quux"
assert returned_headers.get("Hickory") == "dickory"
@@ -331,7 +330,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
r = http.request_encode_body(
"GET", f"{self.https_url}/headers", headers={"Baz": "quux"}
)
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") is None
assert returned_headers.get("Baz") == "quux"
assert returned_headers.get("Hickory") is None
@@ -348,13 +347,13 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
) as http:
r = http.request_encode_url("GET", f"{self.http_url}/headers")
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") == "bar"
assert returned_headers.get("Hickory") == "dickory"
assert returned_headers.get("Host") == f"{self.http_host}:{self.http_port}"
r = http.request_encode_url("GET", f"{self.http_url_alt}/headers")
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") == "bar"
assert returned_headers.get("Hickory") == "dickory"
assert (
@@ -364,7 +363,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
r = http.request_encode_body(
"GET", f"{self.https_url}/headers", headers={"Baz": "quux"}
)
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") is None
assert returned_headers.get("Baz") == "quux"
assert returned_headers.get("Hickory") is None
@@ -382,7 +381,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
) as http:
r = http.request_encode_url("GET", f"{self.https_url}/headers")
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") == "bar"
assert returned_headers.get("Hickory") == "dickory"
assert (
@@ -399,7 +398,7 @@ class TestHTTPProxyManager(HTTPDummyProxyTestCase):
) as http:
request_headers = HTTPHeaderDict(baz="quux")
r = http.request("GET", f"{self.http_url}/headers", headers=request_headers)
- returned_headers = json.loads(r.data.decode())
+ returned_headers = r.json()
assert returned_headers.get("Foo") == "bar"
assert returned_headers.get("Baz") == "quux"

Add support for native JSON request and response bodies

Co-authored-by: Quentin Pradet <quentin@pradet.me>
Co-authored-by: Seth Michael Larson <sethmichaellarson@gmail.com>