Create a more efficient HTTPResponse.__iter__() method
MODIFY
CHANGES.rst
CHANGES.rst
@@ -4,12 +4,18 @@ Changes
dev (master)
------------
+* Implemented a more efficient ``HTTPResponse.__iter__()`` method (Issue #1483)
+
+* ... [Short description of non-trivial change.] (Issue #)
+
+
+1.24.1 (2018-11-02)
+-------------------
+
* Remove quadratic behavior within ``GzipDecoder.decompress()`` (Issue #1467)
* Restored functionality of `ciphers` parameter for `create_urllib3_context()`. (Issue #1462)
-* ... [Short description of non-trivial change.] (Issue #)
-
1.24 (2018-10-16)
-----------------

MODIFY
src/urllib3/response.py
src/urllib3/response.py
@@ -705,3 +705,20 @@ class HTTPResponse(io.IOBase):
return self.retries.history[-1].redirect_location
else:
return self._request_url
+
+ def __iter__(self):
+ buffer = [b""]
+ for chunk in self.stream(decode_content=True):
+ if b"\n" in chunk:
+ chunk = chunk.split(b"\n")
+ yield b"".join(buffer) + chunk[0] + b"\n"
+ for x in chunk[1:-1]:
+ yield x + b"\n"
+ if chunk[-1]:
+ buffer = [chunk[-1]]
+ else:
+ buffer = []
+ else:
+ buffer.append(chunk)
+ if buffer:
+ yield b"".join(buffer)

MODIFY
test/test_response.py
test/test_response.py
@@ -708,6 +708,41 @@ class TestResponse(object):
resp = HTTPResponse(fp, retries=retry)
assert resp.geturl() == 'https://www.example.com'
+ @pytest.mark.parametrize(
+ ["payload", "expected_stream"],
+ [(b"", [b""]),
+ (b"\n", [b"\n"]),
+ (b"abc\ndef", [b"abc\n", b"def"]),
+ (b"Hello\nworld\n\n\n!", [b"Hello\n", b"world\n", b"\n", b"\n", b"!"])]
+ )
+ def test__iter__(self, payload, expected_stream):
+ actual_stream = []
+ for chunk in HTTPResponse(BytesIO(payload), preload_content=False):
+ actual_stream.append(chunk)
+
+ assert actual_stream == expected_stream
+
+ def test__iter__decode_content(self):
+ def stream():
+ # Set up a generator to chunk the gzipped body
+ compress = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
+ data = compress.compress(b'foo\nbar')
+ data += compress.flush()
+ for i in range(0, len(data), 2):
+ yield data[i:i + 2]
+
+ fp = MockChunkedEncodingResponse(list(stream()))
+ r = httplib.HTTPResponse(MockSock)
+ r.fp = fp
+ headers = {'transfer-encoding': 'chunked', 'content-encoding': 'gzip'}
+ resp = HTTPResponse(r, preload_content=False, headers=headers)
+
+ data = b''
+ for c in resp:
+ data += c
+
+ assert b'foo\nbar' == data
+
class MockChunkedEncodingResponse(object):

MODIFY
test/with_dummyserver/test_connectionpool.py
test/with_dummyserver/test_connectionpool.py
@@ -512,6 +512,23 @@ class TestConnectionPool(HTTPDummyServerTestCase):
self.assertEqual(body[i], expected_body[i])
+ def test_post_with_multipart__iter__(self):
+ data = {'hello': 'world'}
+ r = self.pool.request('POST', '/echo',
+ fields=data,
+ preload_content=False,
+ multipart_boundary="boundary",
+ encode_multipart=True)
+
+ chunks = [chunk for chunk in r]
+ assert chunks == [
+ b"--boundary\r\n",
+ b'Content-Disposition: form-data; name="hello"\r\n',
+ b'\r\n',
+ b'world\r\n',
+ b"--boundary--\r\n"
+ ]
+
def test_check_gzip(self):
r = self.pool.request('GET', '/encodingrequest',
headers={'accept-encoding': 'gzip'})

Create a more efficient HTTPResponse.__iter__() method (#1484)