We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
response.json()?
python-requests have response.json() that decodes json body and returns appropriate Python objects. Does it make sense to have something like this in Scrapy? 
Add json response
My first PR for Scrapy that is supposed to fix GH https://github.com/scrapy/scrapy/issues/2444

Adds a `JsonResponse` class and a `json()` function similar to the one for the "requests" library [here](https://github.com/psf/requests/blob/master/requests/models.py#L874-L898)

Test manually by 
```
# from the Scrapy package itself install Scrapy first  
python setup.py install

# then run the Scrapy shell
scrapy shell https://api.github.com/events
>> response.json
```

Fixes #2444


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/http/response/text.py`
Content:
```
1 """
2 This module implements the TextResponse class which adds encoding handling and
3 discovering (through HTTP headers) to base Response class.
4 
5 See documentation in docs/topics/request-response.rst
6 """
7 
8 import warnings
9 from contextlib import suppress
10 from typing import Generator
11 from urllib.parse import urljoin
12 
13 import parsel
14 from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
15                             http_content_type_encoding, resolve_encoding)
16 from w3lib.html import strip_html5_whitespace
17 
18 from scrapy.exceptions import ScrapyDeprecationWarning
19 from scrapy.http import Request
20 from scrapy.http.response import Response
21 from scrapy.utils.python import memoizemethod_noargs, to_unicode
22 from scrapy.utils.response import get_base_url
23 
24 
25 class TextResponse(Response):
26 
27     _DEFAULT_ENCODING = 'ascii'
28 
29     def __init__(self, *args, **kwargs):
30         self._encoding = kwargs.pop('encoding', None)
31         self._cached_benc = None
32         self._cached_ubody = None
33         self._cached_selector = None
34         super(TextResponse, self).__init__(*args, **kwargs)
35 
36     def _set_url(self, url):
37         if isinstance(url, str):
38             self._url = to_unicode(url, self.encoding)
39         else:
40             super(TextResponse, self)._set_url(url)
41 
42     def _set_body(self, body):
43         self._body = b''  # used by encoding detection
44         if isinstance(body, str):
45             if self._encoding is None:
46                 raise TypeError('Cannot convert unicode body - %s has no encoding' %
47                                 type(self).__name__)
48             self._body = body.encode(self._encoding)
49         else:
50             super(TextResponse, self)._set_body(body)
51 
52     def replace(self, *args, **kwargs):
53         kwargs.setdefault('encoding', self.encoding)
54         return Response.replace(self, *args, **kwargs)
55 
56     @property
57     def encoding(self):
58         return self._declared_encoding() or self._body_inferred_encoding()
59 
60     def _declared_encoding(self):
61         return self._encoding or self._headers_encoding() \
62             or self._body_declared_encoding()
63 
64     def body_as_unicode(self):
65         """Return body as unicode"""
66         warnings.warn('Response.body_as_unicode() is deprecated, '
67                       'please use Response.text instead.',
68                       ScrapyDeprecationWarning)
69         return self.text
70 
71     @property
72     def text(self):
73         """ Body as unicode """
74         # access self.encoding before _cached_ubody to make sure
75         # _body_inferred_encoding is called
76         benc = self.encoding
77         if self._cached_ubody is None:
78             charset = 'charset=%s' % benc
79             self._cached_ubody = html_to_unicode(charset, self.body)[1]
80         return self._cached_ubody
81 
82     def urljoin(self, url):
83         """Join this Response's url with a possible relative url to form an
84         absolute interpretation of the latter."""
85         return urljoin(get_base_url(self), url)
86 
87     @memoizemethod_noargs
88     def _headers_encoding(self):
89         content_type = self.headers.get(b'Content-Type', b'')
90         return http_content_type_encoding(to_unicode(content_type))
91 
92     def _body_inferred_encoding(self):
93         if self._cached_benc is None:
94             content_type = to_unicode(self.headers.get(b'Content-Type', b''))
95             benc, ubody = html_to_unicode(content_type, self.body,
96                                           auto_detect_fun=self._auto_detect_fun,
97                                           default_encoding=self._DEFAULT_ENCODING)
98             self._cached_benc = benc
99             self._cached_ubody = ubody
100         return self._cached_benc
101 
102     def _auto_detect_fun(self, text):
103         for enc in (self._DEFAULT_ENCODING, 'utf-8', 'cp1252'):
104             try:
105                 text.decode(enc)
106             except UnicodeError:
107                 continue
108             return resolve_encoding(enc)
109 
110     @memoizemethod_noargs
111     def _body_declared_encoding(self):
112         return html_body_declared_encoding(self.body)
113 
114     @property
115     def selector(self):
116         from scrapy.selector import Selector
117         if self._cached_selector is None:
118             self._cached_selector = Selector(self)
119         return self._cached_selector
120 
121     def xpath(self, query, **kwargs):
122         return self.selector.xpath(query, **kwargs)
123 
124     def css(self, query):
125         return self.selector.css(query)
126 
127     def follow(self, url, callback=None, method='GET', headers=None, body=None,
128                cookies=None, meta=None, encoding=None, priority=0,
129                dont_filter=False, errback=None, cb_kwargs=None, flags=None):
130         # type: (...) -> Request
131         """
132         Return a :class:`~.Request` instance to follow a link ``url``.
133         It accepts the same arguments as ``Request.__init__`` method,
134         but ``url`` can be not only an absolute URL, but also
135 
136         * a relative URL
137         * a :class:`~scrapy.link.Link` object, e.g. the result of
138           :ref:`topics-link-extractors`
139         * a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g.
140           ``response.css('a.my_link')[0]``
141         * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.
142           ``response.css('a::attr(href)')[0]`` or
143           ``response.xpath('//img/@src')[0]``
144 
145         See :ref:`response-follow-example` for usage examples.
146         """
147         if isinstance(url, parsel.Selector):
148             url = _url_from_selector(url)
149         elif isinstance(url, parsel.SelectorList):
150             raise ValueError("SelectorList is not supported")
151         encoding = self.encoding if encoding is None else encoding
152         return super(TextResponse, self).follow(
153             url=url,
154             callback=callback,
155             method=method,
156             headers=headers,
157             body=body,
158             cookies=cookies,
159             meta=meta,
160             encoding=encoding,
161             priority=priority,
162             dont_filter=dont_filter,
163             errback=errback,
164             cb_kwargs=cb_kwargs,
165             flags=flags,
166         )
167 
168     def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,
169                    cookies=None, meta=None, encoding=None, priority=0,
170                    dont_filter=False, errback=None, cb_kwargs=None, flags=None,
171                    css=None, xpath=None):
172         # type: (...) -> Generator[Request, None, None]
173         """
174         A generator that produces :class:`~.Request` instances to follow all
175         links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s
176         ``__init__`` method, except that each ``urls`` element does not need to be
177         an absolute URL, it can be any of the following:
178 
179         * a relative URL
180         * a :class:`~scrapy.link.Link` object, e.g. the result of
181           :ref:`topics-link-extractors`
182         * a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g.
183           ``response.css('a.my_link')[0]``
184         * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.
185           ``response.css('a::attr(href)')[0]`` or
186           ``response.xpath('//img/@src')[0]``
187 
188         In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction
189         within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted).
190 
191         Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or
192         using the ``css`` or ``xpath`` parameters, this method will not produce requests for
193         selectors from which links cannot be obtained (for instance, anchor tags without an
194         ``href`` attribute)
195         """
196         arguments = [x for x in (urls, css, xpath) if x is not None]
197         if len(arguments) != 1:
198             raise ValueError(
199                 "Please supply exactly one of the following arguments: urls, css, xpath"
200             )
201         if not urls:
202             if css:
203                 urls = self.css(css)
204             if xpath:
205                 urls = self.xpath(xpath)
206         if isinstance(urls, parsel.SelectorList):
207             selectors = urls
208             urls = []
209             for sel in selectors:
210                 with suppress(_InvalidSelector):
211                     urls.append(_url_from_selector(sel))
212         return super(TextResponse, self).follow_all(
213             urls=urls,
214             callback=callback,
215             method=method,
216             headers=headers,
217             body=body,
218             cookies=cookies,
219             meta=meta,
220             encoding=encoding,
221             priority=priority,
222             dont_filter=dont_filter,
223             errback=errback,
224             cb_kwargs=cb_kwargs,
225             flags=flags,
226         )
227 
228 
229 class _InvalidSelector(ValueError):
230     """
231     Raised when a URL cannot be obtained from a Selector
232     """
233 
234 
235 def _url_from_selector(sel):
236     # type: (parsel.Selector) -> str
237     if isinstance(sel.root, str):
238         # e.g. ::attr(href) result
239         return strip_html5_whitespace(sel.root)
240     if not hasattr(sel.root, 'tag'):
241         raise _InvalidSelector("Unsupported selector: %s" % sel)
242     if sel.root.tag not in ('a', 'link'):
243         raise _InvalidSelector("Only <a> and <link> elements are supported; got <%s>" %
244                                sel.root.tag)
245     href = sel.root.get('href')
246     if href is None:
247         raise _InvalidSelector("<%s> element has no href attribute: %s" %
248                                (sel.root.tag, sel))
249     return strip_html5_whitespace(href)
250
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -5,6 +5,7 @@
 See documentation in docs/topics/request-response.rst
 """
 
+import json
 import warnings
 from contextlib import suppress
 from typing import Generator
@@ -21,10 +22,13 @@
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
 from scrapy.utils.response import get_base_url
 
+_NONE = object()
+
 
 class TextResponse(Response):
 
     _DEFAULT_ENCODING = 'ascii'
+    _cached_decoded_json = _NONE
 
     def __init__(self, *args, **kwargs):
         self._encoding = kwargs.pop('encoding', None)
@@ -68,6 +72,14 @@
                       ScrapyDeprecationWarning)
         return self.text
 
+    def json(self):
+        """
+        Deserialize a JSON document to a Python object.
+        """
+        if self._cached_decoded_json is _NONE:
+            self._cached_decoded_json = json.loads(self.text)
+        return self._cached_decoded_json
+
     @property
     def text(self):
         """ Body as unicode """

{"golden_diff": "diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py\n--- a/scrapy/http/response/text.py\n+++ b/scrapy/http/response/text.py\n@@ -5,6 +5,7 @@\n See documentation in docs/topics/request-response.rst\n \"\"\"\n \n+import json\n import warnings\n from contextlib import suppress\n from typing import Generator\n@@ -21,10 +22,13 @@\n from scrapy.utils.python import memoizemethod_noargs, to_unicode\n from scrapy.utils.response import get_base_url\n \n+_NONE = object()\n+\n \n class TextResponse(Response):\n \n     _DEFAULT_ENCODING = 'ascii'\n+    _cached_decoded_json = _NONE\n \n     def __init__(self, *args, **kwargs):\n         self._encoding = kwargs.pop('encoding', None)\n@@ -68,6 +72,14 @@\n                       ScrapyDeprecationWarning)\n         return self.text\n \n+    def json(self):\n+        \"\"\"\n+        Deserialize a JSON document to a Python object.\n+        \"\"\"\n+        if self._cached_decoded_json is _NONE:\n+            self._cached_decoded_json = json.loads(self.text)\n+        return self._cached_decoded_json\n+\n     @property\n     def text(self):\n         \"\"\" Body as unicode \"\"\"\n", "issue": "response.json()?\npython-requests have response.json() that decodes json body and returns appropriate Python objects. Does it make sense to have something like this in Scrapy? \nAdd json response\nMy first PR for Scrapy that is supposed to fix GH https://github.com/scrapy/scrapy/issues/2444\r\n\r\nAdds a `JsonResponse` class and a `json()` function similar to the one for the \"requests\" library [here](https://github.com/psf/requests/blob/master/requests/models.py#L874-L898)\r\n\r\nTest manually by \r\n```\r\n# from the Scrapy package itself install Scrapy first  \r\npython setup.py install\r\n\r\n# then run the Scrapy shell\r\nscrapy shell https://api.github.com/events\r\n>> response.json\r\n```\r\n\r\nFixes #2444\r\n\n", "before_files": [{"content": "\"\"\"\nThis module implements the TextResponse class which adds encoding handling and\ndiscovering (through HTTP headers) to base Response class.\n\nSee documentation in docs/topics/request-response.rst\n\"\"\"\n\nimport warnings\nfrom contextlib import suppress\nfrom typing import Generator\nfrom urllib.parse import urljoin\n\nimport parsel\nfrom w3lib.encoding import (html_body_declared_encoding, html_to_unicode,\n                            http_content_type_encoding, resolve_encoding)\nfrom w3lib.html import strip_html5_whitespace\n\nfrom scrapy.exceptions import ScrapyDeprecationWarning\nfrom scrapy.http import Request\nfrom scrapy.http.response import Response\nfrom scrapy.utils.python import memoizemethod_noargs, to_unicode\nfrom scrapy.utils.response import get_base_url\n\n\nclass TextResponse(Response):\n\n    _DEFAULT_ENCODING = 'ascii'\n\n    def __init__(self, *args, **kwargs):\n        self._encoding = kwargs.pop('encoding', None)\n        self._cached_benc = None\n        self._cached_ubody = None\n        self._cached_selector = None\n        super(TextResponse, self).__init__(*args, **kwargs)\n\n    def _set_url(self, url):\n        if isinstance(url, str):\n            self._url = to_unicode(url, self.encoding)\n        else:\n            super(TextResponse, self)._set_url(url)\n\n    def _set_body(self, body):\n        self._body = b''  # used by encoding detection\n        if isinstance(body, str):\n            if self._encoding is None:\n                raise TypeError('Cannot convert unicode body - %s has no encoding' %\n                                type(self).__name__)\n            self._body = body.encode(self._encoding)\n        else:\n            super(TextResponse, self)._set_body(body)\n\n    def replace(self, *args, **kwargs):\n        kwargs.setdefault('encoding', self.encoding)\n        return Response.replace(self, *args, **kwargs)\n\n    @property\n    def encoding(self):\n        return self._declared_encoding() or self._body_inferred_encoding()\n\n    def _declared_encoding(self):\n        return self._encoding or self._headers_encoding() \\\n            or self._body_declared_encoding()\n\n    def body_as_unicode(self):\n        \"\"\"Return body as unicode\"\"\"\n        warnings.warn('Response.body_as_unicode() is deprecated, '\n                      'please use Response.text instead.',\n                      ScrapyDeprecationWarning)\n        return self.text\n\n    @property\n    def text(self):\n        \"\"\" Body as unicode \"\"\"\n        # access self.encoding before _cached_ubody to make sure\n        # _body_inferred_encoding is called\n        benc = self.encoding\n        if self._cached_ubody is None:\n            charset = 'charset=%s' % benc\n            self._cached_ubody = html_to_unicode(charset, self.body)[1]\n        return self._cached_ubody\n\n    def urljoin(self, url):\n        \"\"\"Join this Response's url with a possible relative url to form an\n        absolute interpretation of the latter.\"\"\"\n        return urljoin(get_base_url(self), url)\n\n    @memoizemethod_noargs\n    def _headers_encoding(self):\n        content_type = self.headers.get(b'Content-Type', b'')\n        return http_content_type_encoding(to_unicode(content_type))\n\n    def _body_inferred_encoding(self):\n        if self._cached_benc is None:\n            content_type = to_unicode(self.headers.get(b'Content-Type', b''))\n            benc, ubody = html_to_unicode(content_type, self.body,\n                                          auto_detect_fun=self._auto_detect_fun,\n                                          default_encoding=self._DEFAULT_ENCODING)\n            self._cached_benc = benc\n            self._cached_ubody = ubody\n        return self._cached_benc\n\n    def _auto_detect_fun(self, text):\n        for enc in (self._DEFAULT_ENCODING, 'utf-8', 'cp1252'):\n            try:\n                text.decode(enc)\n            except UnicodeError:\n                continue\n            return resolve_encoding(enc)\n\n    @memoizemethod_noargs\n    def _body_declared_encoding(self):\n        return html_body_declared_encoding(self.body)\n\n    @property\n    def selector(self):\n        from scrapy.selector import Selector\n        if self._cached_selector is None:\n            self._cached_selector = Selector(self)\n        return self._cached_selector\n\n    def xpath(self, query, **kwargs):\n        return self.selector.xpath(query, **kwargs)\n\n    def css(self, query):\n        return self.selector.css(query)\n\n    def follow(self, url, callback=None, method='GET', headers=None, body=None,\n               cookies=None, meta=None, encoding=None, priority=0,\n               dont_filter=False, errback=None, cb_kwargs=None, flags=None):\n        # type: (...) -> Request\n        \"\"\"\n        Return a :class:`~.Request` instance to follow a link ``url``.\n        It accepts the same arguments as ``Request.__init__`` method,\n        but ``url`` can be not only an absolute URL, but also\n\n        * a relative URL\n        * a :class:`~scrapy.link.Link` object, e.g. the result of\n          :ref:`topics-link-extractors`\n        * a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g.\n          ``response.css('a.my_link')[0]``\n        * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.\n          ``response.css('a::attr(href)')[0]`` or\n          ``response.xpath('//img/@src')[0]``\n\n        See :ref:`response-follow-example` for usage examples.\n        \"\"\"\n        if isinstance(url, parsel.Selector):\n            url = _url_from_selector(url)\n        elif isinstance(url, parsel.SelectorList):\n            raise ValueError(\"SelectorList is not supported\")\n        encoding = self.encoding if encoding is None else encoding\n        return super(TextResponse, self).follow(\n            url=url,\n            callback=callback,\n            method=method,\n            headers=headers,\n            body=body,\n            cookies=cookies,\n            meta=meta,\n            encoding=encoding,\n            priority=priority,\n            dont_filter=dont_filter,\n            errback=errback,\n            cb_kwargs=cb_kwargs,\n            flags=flags,\n        )\n\n    def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,\n                   cookies=None, meta=None, encoding=None, priority=0,\n                   dont_filter=False, errback=None, cb_kwargs=None, flags=None,\n                   css=None, xpath=None):\n        # type: (...) -> Generator[Request, None, None]\n        \"\"\"\n        A generator that produces :class:`~.Request` instances to follow all\n        links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s\n        ``__init__`` method, except that each ``urls`` element does not need to be\n        an absolute URL, it can be any of the following:\n\n        * a relative URL\n        * a :class:`~scrapy.link.Link` object, e.g. the result of\n          :ref:`topics-link-extractors`\n        * a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g.\n          ``response.css('a.my_link')[0]``\n        * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.\n          ``response.css('a::attr(href)')[0]`` or\n          ``response.xpath('//img/@src')[0]``\n\n        In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction\n        within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted).\n\n        Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or\n        using the ``css`` or ``xpath`` parameters, this method will not produce requests for\n        selectors from which links cannot be obtained (for instance, anchor tags without an\n        ``href`` attribute)\n        \"\"\"\n        arguments = [x for x in (urls, css, xpath) if x is not None]\n        if len(arguments) != 1:\n            raise ValueError(\n                \"Please supply exactly one of the following arguments: urls, css, xpath\"\n            )\n        if not urls:\n            if css:\n                urls = self.css(css)\n            if xpath:\n                urls = self.xpath(xpath)\n        if isinstance(urls, parsel.SelectorList):\n            selectors = urls\n            urls = []\n            for sel in selectors:\n                with suppress(_InvalidSelector):\n                    urls.append(_url_from_selector(sel))\n        return super(TextResponse, self).follow_all(\n            urls=urls,\n            callback=callback,\n            method=method,\n            headers=headers,\n            body=body,\n            cookies=cookies,\n            meta=meta,\n            encoding=encoding,\n            priority=priority,\n            dont_filter=dont_filter,\n            errback=errback,\n            cb_kwargs=cb_kwargs,\n            flags=flags,\n        )\n\n\nclass _InvalidSelector(ValueError):\n    \"\"\"\n    Raised when a URL cannot be obtained from a Selector\n    \"\"\"\n\n\ndef _url_from_selector(sel):\n    # type: (parsel.Selector) -> str\n    if isinstance(sel.root, str):\n        # e.g. ::attr(href) result\n        return strip_html5_whitespace(sel.root)\n    if not hasattr(sel.root, 'tag'):\n        raise _InvalidSelector(\"Unsupported selector: %s\" % sel)\n    if sel.root.tag not in ('a', 'link'):\n        raise _InvalidSelector(\"Only <a> and <link> elements are supported; got <%s>\" %\n                               sel.root.tag)\n    href = sel.root.get('href')\n    if href is None:\n        raise _InvalidSelector(\"<%s> element has no href attribute: %s\" %\n                               (sel.root.tag, sel))\n    return strip_html5_whitespace(href)\n", "path": "scrapy/http/response/text.py"}], "after_files": [{"content": "\"\"\"\nThis module implements the TextResponse class which adds encoding handling and\ndiscovering (through HTTP headers) to base Response class.\n\nSee documentation in docs/topics/request-response.rst\n\"\"\"\n\nimport json\nimport warnings\nfrom contextlib import suppress\nfrom typing import Generator\nfrom urllib.parse import urljoin\n\nimport parsel\nfrom w3lib.encoding import (html_body_declared_encoding, html_to_unicode,\n                            http_content_type_encoding, resolve_encoding)\nfrom w3lib.html import strip_html5_whitespace\n\nfrom scrapy.exceptions import ScrapyDeprecationWarning\nfrom scrapy.http import Request\nfrom scrapy.http.response import Response\nfrom scrapy.utils.python import memoizemethod_noargs, to_unicode\nfrom scrapy.utils.response import get_base_url\n\n_NONE = object()\n\n\nclass TextResponse(Response):\n\n    _DEFAULT_ENCODING = 'ascii'\n    _cached_decoded_json = _NONE\n\n    def __init__(self, *args, **kwargs):\n        self._encoding = kwargs.pop('encoding', None)\n        self._cached_benc = None\n        self._cached_ubody = None\n        self._cached_selector = None\n        super(TextResponse, self).__init__(*args, **kwargs)\n\n    def _set_url(self, url):\n        if isinstance(url, str):\n            self._url = to_unicode(url, self.encoding)\n        else:\n            super(TextResponse, self)._set_url(url)\n\n    def _set_body(self, body):\n        self._body = b''  # used by encoding detection\n        if isinstance(body, str):\n            if self._encoding is None:\n                raise TypeError('Cannot convert unicode body - %s has no encoding' %\n                                type(self).__name__)\n            self._body = body.encode(self._encoding)\n        else:\n            super(TextResponse, self)._set_body(body)\n\n    def replace(self, *args, **kwargs):\n        kwargs.setdefault('encoding', self.encoding)\n        return Response.replace(self, *args, **kwargs)\n\n    @property\n    def encoding(self):\n        return self._declared_encoding() or self._body_inferred_encoding()\n\n    def _declared_encoding(self):\n        return self._encoding or self._headers_encoding() \\\n            or self._body_declared_encoding()\n\n    def body_as_unicode(self):\n        \"\"\"Return body as unicode\"\"\"\n        warnings.warn('Response.body_as_unicode() is deprecated, '\n                      'please use Response.text instead.',\n                      ScrapyDeprecationWarning)\n        return self.text\n\n    def json(self):\n        \"\"\"\n        Deserialize a JSON document to a Python object.\n        \"\"\"\n        if self._cached_decoded_json is _NONE:\n            self._cached_decoded_json = json.loads(self.text)\n        return self._cached_decoded_json\n\n    @property\n    def text(self):\n        \"\"\" Body as unicode \"\"\"\n        # access self.encoding before _cached_ubody to make sure\n        # _body_inferred_encoding is called\n        benc = self.encoding\n        if self._cached_ubody is None:\n            charset = 'charset=%s' % benc\n            self._cached_ubody = html_to_unicode(charset, self.body)[1]\n        return self._cached_ubody\n\n    def urljoin(self, url):\n        \"\"\"Join this Response's url with a possible relative url to form an\n        absolute interpretation of the latter.\"\"\"\n        return urljoin(get_base_url(self), url)\n\n    @memoizemethod_noargs\n    def _headers_encoding(self):\n        content_type = self.headers.get(b'Content-Type', b'')\n        return http_content_type_encoding(to_unicode(content_type))\n\n    def _body_inferred_encoding(self):\n        if self._cached_benc is None:\n            content_type = to_unicode(self.headers.get(b'Content-Type', b''))\n            benc, ubody = html_to_unicode(content_type, self.body,\n                                          auto_detect_fun=self._auto_detect_fun,\n                                          default_encoding=self._DEFAULT_ENCODING)\n            self._cached_benc = benc\n            self._cached_ubody = ubody\n        return self._cached_benc\n\n    def _auto_detect_fun(self, text):\n        for enc in (self._DEFAULT_ENCODING, 'utf-8', 'cp1252'):\n            try:\n                text.decode(enc)\n            except UnicodeError:\n                continue\n            return resolve_encoding(enc)\n\n    @memoizemethod_noargs\n    def _body_declared_encoding(self):\n        return html_body_declared_encoding(self.body)\n\n    @property\n    def selector(self):\n        from scrapy.selector import Selector\n        if self._cached_selector is None:\n            self._cached_selector = Selector(self)\n        return self._cached_selector\n\n    def xpath(self, query, **kwargs):\n        return self.selector.xpath(query, **kwargs)\n\n    def css(self, query):\n        return self.selector.css(query)\n\n    def follow(self, url, callback=None, method='GET', headers=None, body=None,\n               cookies=None, meta=None, encoding=None, priority=0,\n               dont_filter=False, errback=None, cb_kwargs=None, flags=None):\n        # type: (...) -> Request\n        \"\"\"\n        Return a :class:`~.Request` instance to follow a link ``url``.\n        It accepts the same arguments as ``Request.__init__`` method,\n        but ``url`` can be not only an absolute URL, but also\n\n        * a relative URL\n        * a :class:`~scrapy.link.Link` object, e.g. the result of\n          :ref:`topics-link-extractors`\n        * a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g.\n          ``response.css('a.my_link')[0]``\n        * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.\n          ``response.css('a::attr(href)')[0]`` or\n          ``response.xpath('//img/@src')[0]``\n\n        See :ref:`response-follow-example` for usage examples.\n        \"\"\"\n        if isinstance(url, parsel.Selector):\n            url = _url_from_selector(url)\n        elif isinstance(url, parsel.SelectorList):\n            raise ValueError(\"SelectorList is not supported\")\n        encoding = self.encoding if encoding is None else encoding\n        return super(TextResponse, self).follow(\n            url=url,\n            callback=callback,\n            method=method,\n            headers=headers,\n            body=body,\n            cookies=cookies,\n            meta=meta,\n            encoding=encoding,\n            priority=priority,\n            dont_filter=dont_filter,\n            errback=errback,\n            cb_kwargs=cb_kwargs,\n            flags=flags,\n        )\n\n    def follow_all(self, urls=None, callback=None, method='GET', headers=None, body=None,\n                   cookies=None, meta=None, encoding=None, priority=0,\n                   dont_filter=False, errback=None, cb_kwargs=None, flags=None,\n                   css=None, xpath=None):\n        # type: (...) -> Generator[Request, None, None]\n        \"\"\"\n        A generator that produces :class:`~.Request` instances to follow all\n        links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s\n        ``__init__`` method, except that each ``urls`` element does not need to be\n        an absolute URL, it can be any of the following:\n\n        * a relative URL\n        * a :class:`~scrapy.link.Link` object, e.g. the result of\n          :ref:`topics-link-extractors`\n        * a :class:`~scrapy.selector.Selector` object for a ``<link>`` or ``<a>`` element, e.g.\n          ``response.css('a.my_link')[0]``\n        * an attribute :class:`~scrapy.selector.Selector` (not SelectorList), e.g.\n          ``response.css('a::attr(href)')[0]`` or\n          ``response.xpath('//img/@src')[0]``\n\n        In addition, ``css`` and ``xpath`` arguments are accepted to perform the link extraction\n        within the ``follow_all`` method (only one of ``urls``, ``css`` and ``xpath`` is accepted).\n\n        Note that when passing a ``SelectorList`` as argument for the ``urls`` parameter or\n        using the ``css`` or ``xpath`` parameters, this method will not produce requests for\n        selectors from which links cannot be obtained (for instance, anchor tags without an\n        ``href`` attribute)\n        \"\"\"\n        arguments = [x for x in (urls, css, xpath) if x is not None]\n        if len(arguments) != 1:\n            raise ValueError(\n                \"Please supply exactly one of the following arguments: urls, css, xpath\"\n            )\n        if not urls:\n            if css:\n                urls = self.css(css)\n            if xpath:\n                urls = self.xpath(xpath)\n        if isinstance(urls, parsel.SelectorList):\n            selectors = urls\n            urls = []\n            for sel in selectors:\n                with suppress(_InvalidSelector):\n                    urls.append(_url_from_selector(sel))\n        return super(TextResponse, self).follow_all(\n            urls=urls,\n            callback=callback,\n            method=method,\n            headers=headers,\n            body=body,\n            cookies=cookies,\n            meta=meta,\n            encoding=encoding,\n            priority=priority,\n            dont_filter=dont_filter,\n            errback=errback,\n            cb_kwargs=cb_kwargs,\n            flags=flags,\n        )\n\n\nclass _InvalidSelector(ValueError):\n    \"\"\"\n    Raised when a URL cannot be obtained from a Selector\n    \"\"\"\n\n\ndef _url_from_selector(sel):\n    # type: (parsel.Selector) -> str\n    if isinstance(sel.root, str):\n        # e.g. ::attr(href) result\n        return strip_html5_whitespace(sel.root)\n    if not hasattr(sel.root, 'tag'):\n        raise _InvalidSelector(\"Unsupported selector: %s\" % sel)\n    if sel.root.tag not in ('a', 'link'):\n        raise _InvalidSelector(\"Only <a> and <link> elements are supported; got <%s>\" %\n                               sel.root.tag)\n    href = sel.root.get('href')\n    if href is None:\n        raise _InvalidSelector(\"<%s> element has no href attribute: %s\" %\n                               (sel.root.tag, sel))\n    return strip_html5_whitespace(href)\n", "path": "scrapy/http/response/text.py"}]}