Add optional auto_close parameter to HTTPResponse
MODIFY
CHANGES.rst
CHANGES.rst
@@ -14,6 +14,9 @@ dev (master)
* Fix issue where URLs containing invalid characters within ``Url.auth`` would
raise an exception instead of percent-encoding those characters.
+* Add support for ``HTTPResponse.auto_close = False`` which makes HTTP responses
+ work well with BufferedReaders and other ``io`` module features. (Pull #1652)
+
1.25.3 (2019-05-23)
-------------------

MODIFY
docs/user-guide.rst
docs/user-guide.rst
@@ -77,6 +77,20 @@ to a byte string representing the response content::
.. note:: For larger responses, it's sometimes better to :ref:`stream <stream>`
the response.
+Using io Wrappers with Response content
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Sometimes you want to use :class:`io.TextIOWrapper` or similar objects like a CSV reader
+directly with :class:`~response.HTTPResponse` data. Making these two interfaces play nice
+together requires using the :attr:`~response.HTTPResponse.auto_close` attribute by setting it
+to ``False``. By default HTTP responses are closed after reading all bytes, this disables that behavior::
+
+ >>> import io
+ >>> r = http.request('GET', 'https://example.com', preload_content=False)
+ >>> r.auto_close = False
+ >>> for line in io.TextIOWrapper(r):
+ >>> print(line)
+
.. _request_data:
Request data

MODIFY
src/urllib3/response.py
src/urllib3/response.py
@@ -208,6 +208,7 @@ class HTTPResponse(io.IOBase):
enforce_content_length=False,
request_method=None,
request_url=None,
+ auto_close=True,
):
if isinstance(headers, HTTPHeaderDict):
@@ -221,6 +222,7 @@ class HTTPResponse(io.IOBase):
self.decode_content = decode_content
self.retries = retries
self.enforce_content_length = enforce_content_length
+ self.auto_close = auto_close
self._decoder = None
self._body = None
@@ -493,16 +495,16 @@ class HTTPResponse(io.IOBase):
return
flush_decoder = False
- data = None
+ fp_closed = getattr(self._fp, "closed", False)
with self._error_catcher():
if amt is None:
# cStringIO doesn't like amt=None
- data = self._fp.read()
+ data = self._fp.read() if not fp_closed else b""
flush_decoder = True
else:
cache_content = False
- data = self._fp.read(amt)
+ data = self._fp.read(amt) if not fp_closed else b""
if (
amt != 0 and not data
): # Platform-specific: Buggy versions of Python.
@@ -615,9 +617,14 @@ class HTTPResponse(io.IOBase):
if self._connection:
self._connection.close()
+ if not self.auto_close:
+ io.IOBase.close(self)
+
@property
def closed(self):
- if self._fp is None:
+ if not self.auto_close:
+ return io.IOBase.closed.__get__(self)
+ elif self._fp is None:
return True
elif hasattr(self._fp, "isclosed"):
return self._fp.isclosed()
@@ -638,7 +645,11 @@ class HTTPResponse(io.IOBase):
)
def flush(self):
- if self._fp is not None and hasattr(self._fp, "flush"):
+ if (
+ self._fp is not None
+ and hasattr(self._fp, "flush")
+ and not getattr(self._fp, "closed", False)
+ ):
return self._fp.flush()
def readable(self):

Add optional auto_close parameter to HTTPResponse (#1652)