Add support for brotli content encoding via brotlipy package
MODIFY
.travis.yml
.travis.yml
@@ -39,6 +39,8 @@ matrix:
include:
- python: 2.7
env: TOXENV=py27
+ - python: 2.7
+ env: TOXENV=py27-nobrotli
- python: 3.4
env: TOXENV=py34
- python: 3.5
@@ -49,6 +51,10 @@ matrix:
env: TOXENV=py37
dist: xenial
sudo: required
+ - python: 3.7
+ env: TOXENV=py37-nobrotli
+ dist: xenial
+ sudo: required
- python: 3.8-dev
env: TOXENV=py38
dist: xenial

MODIFY
CHANGES.rst
CHANGES.rst
@@ -18,6 +18,10 @@ dev (master)
* Require and validate certificates by default when using HTTPS (Pull #1507)
+* Added support for Brotli content encoding. It is enabled automatically if
+ ``brotlipy`` package is installed which can be requested with
+ ``urllib3[brotli]`` extra. (Pull #1532)
+
* ... [Short description of non-trivial change.] (Issue #)

MODIFY
appveyor.yml
appveyor.yml
@@ -13,6 +13,12 @@ environment:
TOXENV: "py27"
TOXPY27: "%PYTHON%\\python.exe"
+ - PYTHON: "C:\\Python27-x64"
+ PYTHON_VERSION: "2.7.x"
+ PYTHON_ARCH: "64"
+ TOXENV: "py27-nobrotli"
+ TOXPY27: "%PYTHON%\\python.exe"
+
- PYTHON: "C:\\Python34-x64"
PYTHON_VERSION: "3.4.x"
PYTHON_ARCH: "64"
@@ -37,6 +43,12 @@ environment:
TOXENV: "py37"
TOXPY37: "%PYTHON%\\python.exe"
+ - PYTHON: "C:\\Python37-x64"
+ PYTHON_VERSION: "3.7.x"
+ PYTHON_ARCH: "64"
+ TOXENV: "py37-nobrotli"
+ TOXPY37: "%PYTHON%\\python.exe"
+
cache:
- C:\Users\appveyor\AppData\Local\pip\Cache

MODIFY
setup.py
setup.py
@@ -64,6 +64,9 @@ setup(name='urllib3',
],
test_suite='test',
extras_require={
+ 'brotli': [
+ 'brotlipy>=0.6.0',
+ ],
'secure': [
'pyOpenSSL>=0.14',
'cryptography>=1.3.4',

MODIFY
src/urllib3/response.py
src/urllib3/response.py
@@ -6,6 +6,11 @@ import logging
from socket import timeout as SocketTimeout
from socket import error as SocketError
+try:
+ import brotli
+except ImportError:
+ brotli = None
+
from ._collections import HTTPHeaderDict
from .exceptions import (
BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError,
@@ -90,6 +95,18 @@ class GzipDecoder(object):
self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
+if brotli is not None:
+ class BrotliDecoder(object):
+ def __init__(self):
+ self._obj = brotli.Decompressor()
+
+ def __getattr__(self, name):
+ return getattr(self._obj, name)
+
+ def decompress(self, data):
+ return self._obj.decompress(data)
+
+
class MultiDecoder(object):
"""
From RFC7231:
@@ -118,6 +135,9 @@ def _get_decoder(mode):
if mode == 'gzip':
return GzipDecoder()
+ if brotli is not None and mode == 'br':
+ return BrotliDecoder()
+
return DeflateDecoder()
@@ -155,6 +175,8 @@ class HTTPResponse(io.IOBase):
"""
CONTENT_DECODERS = ['gzip', 'deflate']
+ if brotli is not None:
+ CONTENT_DECODERS += ['br']
REDIRECT_STATUSES = [301, 302, 303, 307, 308]
def __init__(self, body='', headers=None, status=0, version=0, reason=None,
@@ -317,20 +339,26 @@ class HTTPResponse(io.IOBase):
if len(encodings):
self._decoder = _get_decoder(content_encoding)
+ DECODER_ERROR_CLASSES = (IOError, zlib.error)
+ if brotli is not None:
+ DECODER_ERROR_CLASSES += (brotli.Error,)
+
def _decode(self, data, decode_content, flush_decoder):
"""
Decode the data passed in and potentially flush the decoder.
"""
+ if not decode_content:
+ return data
+
try:
- if decode_content and self._decoder:
+ if self._decoder:
data = self._decoder.decompress(data)
- except (IOError, zlib.error) as e:
+ except self.DECODER_ERROR_CLASSES as e:
content_encoding = self.headers.get('content-encoding', '').lower()
raise DecodeError(
"Received response with content-encoding: %s, but "
"failed to decode it." % content_encoding, e)
-
- if flush_decoder and decode_content:
+ if flush_decoder:
data += self._flush_decoder()
return data

MODIFY
src/urllib3/util/request.py
src/urllib3/util/request.py
@@ -5,6 +5,13 @@ from ..packages.six import b, integer_types
from ..exceptions import UnrewindableBodyError
ACCEPT_ENCODING = 'gzip,deflate'
+try:
+ import brotli as _unused_module_brotli # noqa: F401
+except ImportError:
+ pass
+else:
+ ACCEPT_ENCODING += ',br'
+
_FAILEDTELL = object()

MODIFY
test/__init__.py
test/__init__.py
@@ -8,6 +8,10 @@ import ssl
import os
import pytest
+try:
+ import brotli
+except ImportError:
+ brotli = None
from urllib3.exceptions import HTTPWarning
from urllib3.packages import six
@@ -74,6 +78,16 @@ def onlyPy3(test):
return wrapper
+def onlyBrotlipy():
+ return pytest.mark.skipif(
+ brotli is None, reason='only run if brotlipy is present')
+
+
+def notBrotlipy():
+ return pytest.mark.skipif(
+ brotli is not None, reason='only run if brotlipy is absent')
+
+
def notSecureTransport(test):
"""Skips this test when SecureTransport is in use."""

MODIFY
test/test_response.py
test/test_response.py
@@ -6,7 +6,7 @@ from io import BytesIO, BufferedReader
import pytest
import mock
-from urllib3.response import HTTPResponse
+from urllib3.response import HTTPResponse, brotli
from urllib3.exceptions import (
DecodeError, ResponseNotChunked, ProtocolError, InvalidHeader
)
@@ -14,6 +14,8 @@ from urllib3.packages.six.moves import http_client as httplib
from urllib3.util.retry import Retry, RequestHistory
from urllib3.util.response import is_fp_closed
+from test import onlyBrotlipy
+
from base64 import b64decode
# A known random (i.e, not-too-compressible) payload generated with:
@@ -208,6 +210,35 @@ class TestResponse(object):
assert r.data == b'foofoofoo'
+ @onlyBrotlipy()
+ def test_decode_brotli(self):
+ data = brotli.compress(b'foo')
+
+ fp = BytesIO(data)
+ r = HTTPResponse(fp, headers={'content-encoding': 'br'})
+ assert r.data == b'foo'
+
+ @onlyBrotlipy()
+ def test_chunked_decoding_brotli(self):
+ data = brotli.compress(b'foobarbaz')
+
+ fp = BytesIO(data)
+ r = HTTPResponse(
+ fp, headers={'content-encoding': 'br'}, preload_content=False)
+
+ ret = b''
+ for _ in range(100):
+ ret += r.read(1)
+ if r.closed:
+ break
+ assert ret == b'foobarbaz'
+
+ @onlyBrotlipy()
+ def test_decode_brotli_error(self):
+ fp = BytesIO(b'foo')
+ with pytest.raises(DecodeError):
+ HTTPResponse(fp, headers={'content-encoding': 'br'})
+
def test_multi_decoding_deflate_deflate(self):
data = zlib.compress(zlib.compress(b'foo'))

MODIFY
test/test_util.py
test/test_util.py
@@ -44,7 +44,7 @@ from urllib3.packages import six
from . import clear_warnings
-from test import onlyPy3, onlyPy2
+from test import onlyPy3, onlyPy2, onlyBrotlipy, notBrotlipy
# This number represents a time in seconds, it doesn't mean anything in
# isolation. Setting to a high-ish value to avoid conflicts with the smaller
@@ -300,14 +300,30 @@ class TestUtil(object):
parse_url(b"https://www.google.com/")
@pytest.mark.parametrize('kwargs, expected', [
- ({'accept_encoding': True},
- {'accept-encoding': 'gzip,deflate'}),
+ pytest.param(
+ {'accept_encoding': True},
+ {'accept-encoding': 'gzip,deflate,br'},
+ marks=onlyBrotlipy(),
+ ),
+ pytest.param(
+ {'accept_encoding': True},
+ {'accept-encoding': 'gzip,deflate'},
+ marks=notBrotlipy(),
+ ),
({'accept_encoding': 'foo,bar'},
{'accept-encoding': 'foo,bar'}),
({'accept_encoding': ['foo', 'bar']},
{'accept-encoding': 'foo,bar'}),
- ({'accept_encoding': True, 'user_agent': 'banana'},
- {'accept-encoding': 'gzip,deflate', 'user-agent': 'banana'}),
+ pytest.param(
+ {'accept_encoding': True, 'user_agent': 'banana'},
+ {'accept-encoding': 'gzip,deflate,br', 'user-agent': 'banana'},
+ marks=onlyBrotlipy(),
+ ),
+ pytest.param(
+ {'accept_encoding': True, 'user_agent': 'banana'},
+ {'accept-encoding': 'gzip,deflate', 'user-agent': 'banana'},
+ marks=notBrotlipy(),
+ ),
({'user_agent': 'banana'},
{'user-agent': 'banana'}),
({'keep_alive': True},

MODIFY
tox.ini
tox.ini
[tox]
-envlist = flake8-py3, py27, py34, py35, py36, py37, py38, pypy
+envlist = flake8-py3, py27, py34, py35, py36, py37, py38, pypy, py{27,37}-nobrotli
[testenv]
deps= -r{toxinidir}/dev-requirements.txt
-extras= socks,secure
+extras = socks,secure,brotli
commands =
# Print out the python version and bitness
pip --version
@@ -21,6 +21,12 @@ setenv =
PYTHONWARNINGS=always::DeprecationWarning
passenv = CFLAGS LDFLAGS TRAVIS APPVEYOR CRYPTOGRAPHY_OSX_NO_LINK_FLAGS TRAVIS_INFRA
+[testenv:py27-nobrotli]
+extras = socks,secure
+
+[testenv:py37-nobrotli]
+extras = socks,secure
+
[testenv:gae]
basepython = python2.7
deps=

Add support for brotli content encoding via brotlipy package (#1532)