Handle gzip responses with multiple members
MODIFY
test/test_response.py
test/test_response.py
import socket
+import zlib
from io import BytesIO, BufferedReader
@@ -96,7 +97,6 @@ class TestResponse(object):
assert r.read() == b''
def test_decode_deflate(self):
- import zlib
data = zlib.compress(b'foo')
fp = BytesIO(data)
@@ -105,7 +105,6 @@ class TestResponse(object):
assert r.data == b'foo'
def test_decode_deflate_case_insensitve(self):
- import zlib
data = zlib.compress(b'foo')
fp = BytesIO(data)
@@ -114,7 +113,6 @@ class TestResponse(object):
assert r.data == b'foo'
def test_chunked_decoding_deflate(self):
- import zlib
data = zlib.compress(b'foo')
fp = BytesIO(data)
@@ -132,7 +130,6 @@ class TestResponse(object):
assert r.read() == b''
def test_chunked_decoding_deflate2(self):
- import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS)
data = compress.compress(b'foo')
data += compress.flush()
@@ -150,7 +147,6 @@ class TestResponse(object):
assert r.read() == b''
def test_chunked_decoding_gzip(self):
- import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
data = compress.compress(b'foo')
data += compress.flush()
@@ -165,6 +161,53 @@ class TestResponse(object):
assert r.read() == b''
assert r.read() == b''
+ def test_decode_gzip_multi_member(self):
+ compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
+ data = compress.compress(b'foo')
+ data += compress.flush()
+ data = data * 3
+
+ fp = BytesIO(data)
+ r = HTTPResponse(fp, headers={'content-encoding': 'gzip'})
+
+ assert r.data == b'foofoofoo'
+
+ def test_decode_gzip_error(self):
+ fp = BytesIO(b'foo')
+ with pytest.raises(DecodeError):
+ HTTPResponse(fp, headers={'content-encoding': 'gzip'})
+
+ def test_decode_gzip_swallow_garbage(self):
+ # When data comes from multiple calls to read(), data after
+ # the first zlib error (here triggered by garbage) should be
+ # ignored.
+ compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
+ data = compress.compress(b'foo')
+ data += compress.flush()
+ data = data * 3 + b'foo'
+
+ fp = BytesIO(data)
+ r = HTTPResponse(
+ fp, headers={'content-encoding': 'gzip'}, preload_content=False)
+ ret = b''
+ for _ in range(100):
+ ret += r.read(1)
+ if r.closed:
+ break
+
+ assert ret == b'foofoofoo'
+
+ def test_chunked_decoding_gzip_swallow_garbage(self):
+ compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
+ data = compress.compress(b'foo')
+ data += compress.flush()
+ data = data * 3 + b'foo'
+
+ fp = BytesIO(data)
+ r = HTTPResponse(fp, headers={'content-encoding': 'gzip'})
+
+ assert r.data == b'foofoofoo'
+
def test_body_blob(self):
resp = HTTPResponse(b'foo')
assert resp.data == b'foo'
@@ -292,7 +335,6 @@ class TestResponse(object):
next(stream)
def test_gzipped_streaming(self):
- import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
data = compress.compress(b'foo')
data += compress.flush()
@@ -308,7 +350,6 @@ class TestResponse(object):
next(stream)
def test_gzipped_streaming_tell(self):
- import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
uncompressed_data = b'foo'
data = compress.compress(uncompressed_data)
@@ -331,8 +372,6 @@ class TestResponse(object):
def test_deflate_streaming_tell_intermediate_point(self):
# Ensure that ``tell()`` returns the correct number of bytes when
# part-way through streaming compressed content.
- import zlib
-
NUMBER_OF_READS = 10
class MockCompressedDataReading(BytesIO):
@@ -382,7 +421,6 @@ class TestResponse(object):
assert len(ZLIB_PAYLOAD) == end_of_stream
def test_deflate_streaming(self):
- import zlib
data = zlib.compress(b'foo')
fp = BytesIO(data)
@@ -396,7 +434,6 @@ class TestResponse(object):
next(stream)
def test_deflate2_streaming(self):
- import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, -zlib.MAX_WBITS)
data = compress.compress(b'foo')
data += compress.flush()
@@ -528,7 +565,6 @@ class TestResponse(object):
"""Show that we can decode the gizpped and chunked body."""
def stream():
# Set up a generator to chunk the gzipped body
- import zlib
compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
data = compress.compress(b'foobar')
data += compress.flush()

MODIFY
urllib3/response.py
urllib3/response.py
@@ -52,18 +52,42 @@ class DeflateDecoder(object):
self._data = None
+class GzipDecoderState(object):
+
+ FIRST_MEMBER = 0
+ OTHER_MEMBERS = 1
+ SWALLOW_DATA = 2
+
+
class GzipDecoder(object):
def __init__(self):
self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
+ self._state = GzipDecoderState.FIRST_MEMBER
def __getattr__(self, name):
return getattr(self._obj, name)
def decompress(self, data):
+ ret = binary_type()
+ if self._state == GzipDecoderState.SWALLOW_DATA or not data:
+ return ret
+ while True:
+ try:
+ ret += self._obj.decompress(data)
+ except zlib.error:
+ previous_state = self._state
+ # Ignore data after the first error
+ self._state = GzipDecoderState.SWALLOW_DATA
+ if previous_state == GzipDecoderState.OTHER_MEMBERS:
+ # Allow trailing garbage acceptable in other gzip clients
+ return ret
+ raise
+ data = self._obj.unused_data
if not data:
- return data
- return self._obj.decompress(data)
+ return ret
+ self._state = GzipDecoderState.OTHER_MEMBERS
+ self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS)
def _get_decoder(mode):

Handle gzip responses with multiple members

Signed-off-by: Isaac To <isaacto@gmail.com>