Add support for decoding Zstandard
MODIFY
.readthedocs.yml
.readthedocs.yml
@@ -9,6 +9,7 @@ python:
- brotli
- secure
- socks
+ - zstd
sphinx:
fail_on_warning: true

MODIFY
README.md
README.md
@@ -23,7 +23,7 @@ standard libraries:
- Client-side SSL/TLS verification.
- File uploads with multipart encoding.
- Helpers for retrying requests and dealing with HTTP redirects.
-- Support for gzip, deflate, and brotli encoding.
+- Support for gzip, deflate, brotli, and zstd encoding.
- Proxy support for HTTP and SOCKS.
- 100% test coverage.

MODIFY
docs/advanced-usage.rst
docs/advanced-usage.rst
@@ -538,6 +538,37 @@ Here's an example using brotli encoding via the ``Accept-Encoding`` header:
headers={"Accept-Encoding": "br"}
)
+Zstandard Encoding
+------------------
+
+`Zstandard <https://datatracker.ietf.org/doc/html/rfc8878>`_
+is a compression algorithm created by Facebook with better compression
+than brotli, gzip and deflate (see `benchmarks <https://facebook.github.io/zstd/#benchmarks>`_)
+and is supported by urllib3 if the `zstandard package <https://pypi.org/project/zstandard/>`_ is installed.
+You may also request the package be installed via the ``urllib3[zstd]`` extra:
+
+.. code-block:: bash
+
+ $ python -m pip install urllib3[zstd]
+
+.. note::
+
+ Zstandard support in urllib3 requires using v0.18.0 or later of the ``zstandard`` package.
+ If the version installed is less than v0.18.0 then Zstandard support won't be enabled.
+
+Here's an example using zstd encoding via the ``Accept-Encoding`` header:
+
+.. code-block:: python
+
+ import urllib3
+
+ urllib3.request(
+ "GET",
+ "https://www.facebook.com/",
+ headers={"Accept-Encoding": "zstd"}
+ )
+
+
Decrypting Captured TLS Sessions with Wireshark
-----------------------------------------------
Python 3.8 and higher support logging of TLS pre-master secrets.

MODIFY
docs/index.rst
docs/index.rst
@@ -26,7 +26,7 @@ standard libraries:
- Client-side TLS/SSL verification.
- File uploads with multipart encoding.
- Helpers for retrying requests and dealing with HTTP redirects.
-- Support for gzip, deflate, and brotli encoding.
+- Support for gzip, deflate, brotli, and zstd encoding.
- Proxy support for HTTP and SOCKS.
- 100% test coverage.

MODIFY
docs/reference/urllib3.response.rst
docs/reference/urllib3.response.rst
@@ -29,4 +29,5 @@ representation.
.. autoclass:: urllib3.response.BrotliDecoder
.. autoclass:: urllib3.response.DeflateDecoder
.. autoclass:: urllib3.response.GzipDecoder
+.. autoclass:: urllib3.response.ZstdDecoder
.. autoclass:: urllib3.response.MultiDecoder

MODIFY
noxfile.py
noxfile.py
@@ -16,7 +16,7 @@ SOURCE_FILES = [
def tests_impl(
session: nox.Session,
- extras: str = "socks,secure,brotli",
+ extras: str = "socks,secure,brotli,zstd",
byte_string_comparisons: bool = True,
) -> None:
# Install deps and the package itself.
@@ -174,7 +174,7 @@ def mypy(session: nox.Session) -> None:
@nox.session
def docs(session: nox.Session) -> None:
session.install("-r", "docs/requirements.txt")
- session.install(".[socks,secure,brotli]")
+ session.install(".[socks,secure,brotli,zstd]")
session.chdir("docs")
if os.path.exists("_build"):

MODIFY
pyproject.toml
pyproject.toml
@@ -42,6 +42,9 @@ brotli = [
"brotli>=1.0.9; platform_python_implementation == 'CPython'",
"brotlicffi>=0.8.0; platform_python_implementation != 'CPython'"
]
+zstd = [
+ "zstandard>=0.18.0",
+]
secure = [
"pyOpenSSL>=0.14",
"cryptography>=1.3.4",

MODIFY
src/urllib3/response.py
src/urllib3/response.py
import io
import json as _json
import logging
+import re
import zlib
from contextlib import contextmanager
from http.client import HTTPMessage as _HttplibHTTPMessage
@@ -27,6 +28,22 @@ try:
except ImportError:
brotli = None
+try:
+ import zstandard as zstd # type: ignore[import]
+
+ # The package 'zstandard' added the 'eof' property starting
+ # in v0.18.0 which we require to ensure a complete and
+ # valid zstd stream was fed into the ZstdDecoder.
+ # See: https://github.com/urllib3/urllib3/pull/2624
+ _zstd_version = _zstd_version = tuple(
+ map(int, re.search(r"^([0-9]+)\.([0-9]+)", zstd.__version__).groups()) # type: ignore[union-attr]
+ )
+ if _zstd_version < (0, 18): # Defensive:
+ zstd = None
+
+except (AttributeError, ImportError, ValueError): # Defensive:
+ zstd = None
+
from ._collections import HTTPHeaderDict
from .connection import _TYPE_BODY, BaseSSLError, HTTPConnection, HTTPException
from .exceptions import (
@@ -148,6 +165,24 @@ if brotli is not None:
return b""
+if zstd is not None:
+
+ class ZstdDecoder(ContentDecoder):
+ def __init__(self) -> None:
+ self._obj = zstd.ZstdDecompressor().decompressobj()
+
+ def decompress(self, data: bytes) -> bytes:
+ if not data:
+ return b""
+ return self._obj.decompress(data) # type: ignore[no-any-return]
+
+ def flush(self) -> bytes:
+ ret = self._obj.flush()
+ if not self._obj.eof:
+ raise DecodeError("Zstandard data is incomplete")
+ return ret # type: ignore[no-any-return]
+
+
class MultiDecoder(ContentDecoder):
"""
From RFC7231:
@@ -179,6 +214,9 @@ def _get_decoder(mode: str) -> ContentDecoder:
if brotli is not None and mode == "br":
return BrotliDecoder()
+ if zstd is not None and mode == "zstd":
+ return ZstdDecoder()
+
return DeflateDecoder()
@@ -186,12 +224,17 @@ class BaseHTTPResponse(io.IOBase):
CONTENT_DECODERS = ["gzip", "deflate"]
if brotli is not None:
CONTENT_DECODERS += ["br"]
+ if zstd is not None:
+ CONTENT_DECODERS += ["zstd"]
REDIRECT_STATUSES = [301, 302, 303, 307, 308]
DECODER_ERROR_CLASSES: Tuple[Type[Exception], ...] = (IOError, zlib.error)
if brotli is not None:
DECODER_ERROR_CLASSES += (brotli.error,)
+ if zstd is not None:
+ DECODER_ERROR_CLASSES += (zstd.ZstdError,)
+
def __init__(
self,
*,

MODIFY
src/urllib3/util/request.py
src/urllib3/util/request.py
@@ -37,6 +37,12 @@ except ImportError:
pass
else:
ACCEPT_ENCODING += ",br"
+try:
+ import zstandard as _unused_module_zstd # type: ignore[import] # noqa: F401
+except ImportError:
+ pass
+else:
+ ACCEPT_ENCODING += ",zstd"
class _TYPE_FAILEDTELL(Enum):

MODIFY
test/__init__.py
test/__init__.py
@@ -31,6 +31,11 @@ try:
except ImportError:
brotli = None
+try:
+ import zstandard as zstd # type: ignore[import]
+except ImportError:
+ zstd = None
+
import functools
from urllib3 import util
@@ -146,6 +151,19 @@ def notBrotli() -> Callable[[_TestFuncT], _TestFuncT]:
)
+def onlyZstd() -> Callable[[_TestFuncT], _TestFuncT]:
+ return pytest.mark.skipif(
+ zstd is None, reason="only run if a python-zstandard library is installed"
+ )
+
+
+def notZstd() -> Callable[[_TestFuncT], _TestFuncT]:
+ return pytest.mark.skipif(
+ zstd is not None,
+ reason="only run if a python-zstandard library is not installed",
+ )
+
+
# Hack to make pytest evaluate a condition at test runtime instead of collection time.
def lazy_condition(condition: Callable[[], bool]) -> bool:
class LazyCondition:

MODIFY
test/test_response.py
test/test_response.py
@@ -6,7 +6,7 @@ import zlib
from base64 import b64decode
from http.client import IncompleteRead as httplib_IncompleteRead
from io import BufferedReader, BytesIO, TextIOWrapper
-from test import onlyBrotli
+from test import onlyBrotli, onlyZstd
from typing import Any, Generator, List, Optional
from unittest import mock
@@ -22,7 +22,7 @@ from urllib3.exceptions import (
ResponseNotChunked,
SSLError,
)
-from urllib3.response import HTTPResponse, brotli # type: ignore[attr-defined]
+from urllib3.response import HTTPResponse, brotli, zstd # type: ignore[attr-defined]
from urllib3.util.response import is_fp_closed
from urllib3.util.retry import RequestHistory, Retry
@@ -254,6 +254,39 @@ class TestResponse:
with pytest.raises(DecodeError):
HTTPResponse(fp, headers={"content-encoding": "br"})
+ @onlyZstd()
+ def test_decode_zstd(self) -> None:
+ data = zstd.compress(b"foo")
+
+ fp = BytesIO(data)
+ r = HTTPResponse(fp, headers={"content-encoding": "zstd"})
+ assert r.data == b"foo"
+
+ @onlyZstd()
+ def test_chunked_decoding_zstd(self) -> None:
+ data = zstd.compress(b"foobarbaz")
+
+ fp = BytesIO(data)
+ r = HTTPResponse(
+ fp, headers={"content-encoding": "zstd"}, preload_content=False
+ )
+
+ ret = b""
+
+ for _ in range(100):
+ ret += r.read(1)
+ if r.closed:
+ break
+ assert ret == b"foobarbaz"
+
+ @onlyZstd()
+ @pytest.mark.parametrize("data", [b"foo", b"x" * 100])
+ def test_decode_zstd_error(self, data: bytes) -> None:
+ fp = BytesIO(data)
+
+ with pytest.raises(DecodeError):
+ HTTPResponse(fp, headers={"content-encoding": "zstd"})
+
def test_multi_decoding_deflate_deflate(self) -> None:
data = zlib.compress(zlib.compress(b"foo"))

MODIFY
test/test_util.py
test/test_util.py
@@ -5,7 +5,7 @@ import ssl
import sys
import warnings
from itertools import chain
-from test import ImportBlocker, ModuleStash, notBrotli, onlyBrotli
+from test import ImportBlocker, ModuleStash, notBrotli, notZstd, onlyBrotli, onlyZstd
from typing import TYPE_CHECKING, Dict, List, NoReturn, Optional, Tuple, Union
from unittest import mock
from unittest.mock import MagicMock, Mock, patch
@@ -515,27 +515,47 @@ class TestUtil:
@pytest.mark.parametrize(
"kwargs, expected",
[
+ pytest.param(
+ {"accept_encoding": True},
+ {"accept-encoding": "gzip,deflate,br,zstd"},
+ marks=[onlyBrotli(), onlyZstd()], # type: ignore[list-item]
+ ),
pytest.param(
{"accept_encoding": True},
{"accept-encoding": "gzip,deflate,br"},
- marks=onlyBrotli(), # type: ignore[arg-type]
+ marks=[onlyBrotli(), notZstd()], # type: ignore[list-item]
+ ),
+ pytest.param(
+ {"accept_encoding": True},
+ {"accept-encoding": "gzip,deflate,zstd"},
+ marks=[notBrotli(), onlyZstd()], # type: ignore[list-item]
),
pytest.param(
{"accept_encoding": True},
{"accept-encoding": "gzip,deflate"},
- marks=notBrotli(), # type: ignore[arg-type]
+ marks=[notBrotli(), notZstd()], # type: ignore[list-item]
),
({"accept_encoding": "foo,bar"}, {"accept-encoding": "foo,bar"}),
({"accept_encoding": ["foo", "bar"]}, {"accept-encoding": "foo,bar"}),
+ pytest.param(
+ {"accept_encoding": True, "user_agent": "banana"},
+ {"accept-encoding": "gzip,deflate,br,zstd", "user-agent": "banana"},
+ marks=[onlyBrotli(), onlyZstd()], # type: ignore[list-item]
+ ),
pytest.param(
{"accept_encoding": True, "user_agent": "banana"},
{"accept-encoding": "gzip,deflate,br", "user-agent": "banana"},
- marks=onlyBrotli(), # type: ignore[arg-type]
+ marks=[onlyBrotli(), notZstd()], # type: ignore[list-item]
+ ),
+ pytest.param(
+ {"accept_encoding": True, "user_agent": "banana"},
+ {"accept-encoding": "gzip,deflate,zstd", "user-agent": "banana"},
+ marks=[notBrotli(), onlyZstd()], # type: ignore[list-item]
),
pytest.param(
{"accept_encoding": True, "user_agent": "banana"},
{"accept-encoding": "gzip,deflate", "user-agent": "banana"},
- marks=notBrotli(), # type: ignore[arg-type]
+ marks=[notBrotli(), notZstd()], # type: ignore[list-item]
),
({"user_agent": "banana"}, {"user-agent": "banana"}),
({"keep_alive": True}, {"connection": "keep-alive"}),

Add support for decoding Zstandard

Co-authored-by: Quentin Pradet <quentin.pradet@gmail.com>
Co-authored-by: Ma Lin <malincns@163.com>
Co-authored-by: Seth Michael Larson <sethmichaellarson@gmail.com>