We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
is_generator_with_return_value raises IndentationError with a flush left doc string
### Description

Code that is accepted by the python interpreter raises when fed through `textwrap.dedent`

### Steps to Reproduce

1. Create `is_generator_bug.py` with the content below (which I simplified from [the `is_generator_with_return_value` method body](https://github.com/scrapy/scrapy/blob/2.0.1/scrapy/utils/misc.py#L186-L187)
2. Run `python is_generator_bug.py`
3. Observe the kaboom

```python
import ast
import inspect
from textwrap import dedent
class Bob:
    def doit(self):
        """
this line is flush left
        """
        if True:
            yield 1234

if __name__ == '__main__':
    b = Bob()
    c = b.doit
    if inspect.isgeneratorfunction(c):
        tree = ast.parse(dedent(inspect.getsource(c)))
```

**Expected behavior:** [What you expect to happen]

No Error

**Actual behavior:** [What actually happens]

```console
$ python3.7 is_generator_bug.py
Traceback (most recent call last):
  File "is_generator_bug.py", line 16, in <module>
    tree = ast.parse(dedent(inspect.getsource(c)))
  File "/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ast.py", line 35, in parse
    return compile(source, filename, mode, PyCF_ONLY_AST)
  File "<unknown>", line 1
    def doit(self):
    ^
IndentationError: unexpected indent
```

**Reproduces how often:** [What percentage of the time does it reproduce?]

100%

### Versions

```
Scrapy       : 2.0.1
lxml         : 4.5.0.0
libxml2      : 2.9.10
cssselect    : 1.1.0
parsel       : 1.5.2
w3lib        : 1.21.0
Twisted      : 20.3.0
Python       : 3.7.7 (default, Mar 11 2020, 23:30:22) - [Clang 10.0.0 (clang-1000.11.45.5)]
pyOpenSSL    : 19.1.0 (OpenSSL 1.1.1d  10 Sep 2019)
cryptography : 2.8
Platform     : Darwin-17.7.0-x86_64-i386-64bit
```

### Additional context


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/utils/misc.py`
Content:
```
1 """Helper functions which don't fit anywhere else"""
2 import ast
3 import inspect
4 import os
5 import re
6 import hashlib
7 import warnings
8 from collections import deque
9 from contextlib import contextmanager
10 from importlib import import_module
11 from pkgutil import iter_modules
12 from textwrap import dedent
13 
14 from w3lib.html import replace_entities
15 
16 from scrapy.utils.datatypes import LocalWeakReferencedCache
17 from scrapy.utils.python import flatten, to_unicode
18 from scrapy.item import _BaseItem
19 from scrapy.utils.deprecate import ScrapyDeprecationWarning
20 
21 
22 _ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes
23 
24 
25 def arg_to_iter(arg):
26     """Convert an argument to an iterable. The argument can be a None, single
27     value, or an iterable.
28 
29     Exception: if arg is a dict, [arg] will be returned
30     """
31     if arg is None:
32         return []
33     elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):
34         return arg
35     else:
36         return [arg]
37 
38 
39 def load_object(path):
40     """Load an object given its absolute object path, and return it.
41 
42     The object can be the import path of a class, function, variable or an
43     instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'.
44 
45     If ``path`` is not a string, but is a callable object, such as a class or
46     a function, then return it as is.
47     """
48 
49     if not isinstance(path, str):
50         if callable(path):
51             return path
52         else:
53             raise TypeError("Unexpected argument type, expected string "
54                             "or object, got: %s" % type(path))
55 
56     try:
57         dot = path.rindex('.')
58     except ValueError:
59         raise ValueError(f"Error loading object '{path}': not a full path")
60 
61     module, name = path[:dot], path[dot + 1:]
62     mod = import_module(module)
63 
64     try:
65         obj = getattr(mod, name)
66     except AttributeError:
67         raise NameError(f"Module '{module}' doesn't define any object named '{name}'")
68 
69     return obj
70 
71 
72 def walk_modules(path):
73     """Loads a module and all its submodules from the given module path and
74     returns them. If *any* module throws an exception while importing, that
75     exception is thrown back.
76 
77     For example: walk_modules('scrapy.utils')
78     """
79 
80     mods = []
81     mod = import_module(path)
82     mods.append(mod)
83     if hasattr(mod, '__path__'):
84         for _, subpath, ispkg in iter_modules(mod.__path__):
85             fullpath = path + '.' + subpath
86             if ispkg:
87                 mods += walk_modules(fullpath)
88             else:
89                 submod = import_module(fullpath)
90                 mods.append(submod)
91     return mods
92 
93 
94 def extract_regex(regex, text, encoding='utf-8'):
95     """Extract a list of unicode strings from the given text/encoding using the following policies:
96 
97     * if the regex contains a named group called "extract" that will be returned
98     * if the regex contains multiple numbered groups, all those will be returned (flattened)
99     * if the regex doesn't contain any group the entire regex matching is returned
100     """
101     warnings.warn(
102         "scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.",
103         ScrapyDeprecationWarning,
104         stacklevel=2
105     )
106 
107     if isinstance(regex, str):
108         regex = re.compile(regex, re.UNICODE)
109 
110     try:
111         strings = [regex.search(text).group('extract')]   # named group
112     except Exception:
113         strings = regex.findall(text)    # full regex or numbered groups
114     strings = flatten(strings)
115 
116     if isinstance(text, str):
117         return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
118     else:
119         return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
120                 for s in strings]
121 
122 
123 def md5sum(file):
124     """Calculate the md5 checksum of a file-like object without reading its
125     whole content in memory.
126 
127     >>> from io import BytesIO
128     >>> md5sum(BytesIO(b'file content to hash'))
129     '784406af91dd5a54fbb9c84c2236595a'
130     """
131     m = hashlib.md5()
132     while True:
133         d = file.read(8096)
134         if not d:
135             break
136         m.update(d)
137     return m.hexdigest()
138 
139 
140 def rel_has_nofollow(rel):
141     """Return True if link rel attribute has nofollow type"""
142     return rel is not None and 'nofollow' in rel.split()
143 
144 
145 def create_instance(objcls, settings, crawler, *args, **kwargs):
146     """Construct a class instance using its ``from_crawler`` or
147     ``from_settings`` constructors, if available.
148 
149     At least one of ``settings`` and ``crawler`` needs to be different from
150     ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
151     If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
152     tried.
153 
154     ``*args`` and ``**kwargs`` are forwarded to the constructors.
155 
156     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
157 
158     .. versionchanged:: 2.2
159        Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
160        extension has not been implemented correctly).
161     """
162     if settings is None:
163         if crawler is None:
164             raise ValueError("Specify at least one of settings and crawler.")
165         settings = crawler.settings
166     if crawler and hasattr(objcls, 'from_crawler'):
167         instance = objcls.from_crawler(crawler, *args, **kwargs)
168         method_name = 'from_crawler'
169     elif hasattr(objcls, 'from_settings'):
170         instance = objcls.from_settings(settings, *args, **kwargs)
171         method_name = 'from_settings'
172     else:
173         instance = objcls(*args, **kwargs)
174         method_name = '__new__'
175     if instance is None:
176         raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
177     return instance
178 
179 
180 @contextmanager
181 def set_environ(**kwargs):
182     """Temporarily set environment variables inside the context manager and
183     fully restore previous environment afterwards
184     """
185 
186     original_env = {k: os.environ.get(k) for k in kwargs}
187     os.environ.update(kwargs)
188     try:
189         yield
190     finally:
191         for k, v in original_env.items():
192             if v is None:
193                 del os.environ[k]
194             else:
195                 os.environ[k] = v
196 
197 
198 def walk_callable(node):
199     """Similar to ``ast.walk``, but walks only function body and skips nested
200     functions defined within the node.
201     """
202     todo = deque([node])
203     walked_func_def = False
204     while todo:
205         node = todo.popleft()
206         if isinstance(node, ast.FunctionDef):
207             if walked_func_def:
208                 continue
209             walked_func_def = True
210         todo.extend(ast.iter_child_nodes(node))
211         yield node
212 
213 
214 _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
215 
216 
217 def is_generator_with_return_value(callable):
218     """
219     Returns True if a callable is a generator function which includes a
220     'return' statement with a value different than None, False otherwise
221     """
222     if callable in _generator_callbacks_cache:
223         return _generator_callbacks_cache[callable]
224 
225     def returns_none(return_node):
226         value = return_node.value
227         return value is None or isinstance(value, ast.NameConstant) and value.value is None
228 
229     if inspect.isgeneratorfunction(callable):
230         tree = ast.parse(dedent(inspect.getsource(callable)))
231         for node in walk_callable(tree):
232             if isinstance(node, ast.Return) and not returns_none(node):
233                 _generator_callbacks_cache[callable] = True
234                 return _generator_callbacks_cache[callable]
235 
236     _generator_callbacks_cache[callable] = False
237     return _generator_callbacks_cache[callable]
238 
239 
240 def warn_on_generator_with_return_value(spider, callable):
241     """
242     Logs a warning if a callable is a generator function and includes
243     a 'return' statement with a value different than None
244     """
245     if is_generator_with_return_value(callable):
246         warnings.warn(
247             f'The "{spider.__class__.__name__}.{callable.__name__}" method is '
248             'a generator and includes a "return" statement with a value '
249             'different than None. This could lead to unexpected behaviour. Please see '
250             'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '
251             'for details about the semantics of the "return" statement within generators',
252             stacklevel=2,
253         )
254
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -9,7 +9,6 @@
 from contextlib import contextmanager
 from importlib import import_module
 from pkgutil import iter_modules
-from textwrap import dedent
 
 from w3lib.html import replace_entities
 
@@ -227,7 +226,8 @@
         return value is None or isinstance(value, ast.NameConstant) and value.value is None
 
     if inspect.isgeneratorfunction(callable):
-        tree = ast.parse(dedent(inspect.getsource(callable)))
+        code = re.sub(r"^[\t ]+", "", inspect.getsource(callable))
+        tree = ast.parse(code)
         for node in walk_callable(tree):
             if isinstance(node, ast.Return) and not returns_none(node):
                 _generator_callbacks_cache[callable] = True
@@ -242,12 +242,23 @@
     Logs a warning if a callable is a generator function and includes
     a 'return' statement with a value different than None
     """
-    if is_generator_with_return_value(callable):
+    try:
+        if is_generator_with_return_value(callable):
+            warnings.warn(
+                f'The "{spider.__class__.__name__}.{callable.__name__}" method is '
+                'a generator and includes a "return" statement with a value '
+                'different than None. This could lead to unexpected behaviour. Please see '
+                'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '
+                'for details about the semantics of the "return" statement within generators',
+                stacklevel=2,
+            )
+    except IndentationError:
+        callable_name = spider.__class__.__name__ + "." + callable.__name__
         warnings.warn(
-            f'The "{spider.__class__.__name__}.{callable.__name__}" method is '
-            'a generator and includes a "return" statement with a value '
-            'different than None. This could lead to unexpected behaviour. Please see '
-            'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '
-            'for details about the semantics of the "return" statement within generators',
+            f'Unable to determine whether or not "{callable_name}" is a generator with a return value. '
+            'This will not prevent your code from working, but it prevents Scrapy from detecting '
+            f'potential issues in your implementation of "{callable_name}". Please, report this in the '
+            'Scrapy issue tracker (https://github.com/scrapy/scrapy/issues), '
+            f'including the code of "{callable_name}"',
             stacklevel=2,
         )

{"golden_diff": "diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py\n--- a/scrapy/utils/misc.py\n+++ b/scrapy/utils/misc.py\n@@ -9,7 +9,6 @@\n from contextlib import contextmanager\n from importlib import import_module\n from pkgutil import iter_modules\n-from textwrap import dedent\n \n from w3lib.html import replace_entities\n \n@@ -227,7 +226,8 @@\n         return value is None or isinstance(value, ast.NameConstant) and value.value is None\n \n     if inspect.isgeneratorfunction(callable):\n-        tree = ast.parse(dedent(inspect.getsource(callable)))\n+        code = re.sub(r\"^[\\t ]+\", \"\", inspect.getsource(callable))\n+        tree = ast.parse(code)\n         for node in walk_callable(tree):\n             if isinstance(node, ast.Return) and not returns_none(node):\n                 _generator_callbacks_cache[callable] = True\n@@ -242,12 +242,23 @@\n     Logs a warning if a callable is a generator function and includes\n     a 'return' statement with a value different than None\n     \"\"\"\n-    if is_generator_with_return_value(callable):\n+    try:\n+        if is_generator_with_return_value(callable):\n+            warnings.warn(\n+                f'The \"{spider.__class__.__name__}.{callable.__name__}\" method is '\n+                'a generator and includes a \"return\" statement with a value '\n+                'different than None. This could lead to unexpected behaviour. Please see '\n+                'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '\n+                'for details about the semantics of the \"return\" statement within generators',\n+                stacklevel=2,\n+            )\n+    except IndentationError:\n+        callable_name = spider.__class__.__name__ + \".\" + callable.__name__\n         warnings.warn(\n-            f'The \"{spider.__class__.__name__}.{callable.__name__}\" method is '\n-            'a generator and includes a \"return\" statement with a value '\n-            'different than None. This could lead to unexpected behaviour. Please see '\n-            'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '\n-            'for details about the semantics of the \"return\" statement within generators',\n+            f'Unable to determine whether or not \"{callable_name}\" is a generator with a return value. '\n+            'This will not prevent your code from working, but it prevents Scrapy from detecting '\n+            f'potential issues in your implementation of \"{callable_name}\". Please, report this in the '\n+            'Scrapy issue tracker (https://github.com/scrapy/scrapy/issues), '\n+            f'including the code of \"{callable_name}\"',\n             stacklevel=2,\n         )\n", "issue": "is_generator_with_return_value raises IndentationError with a flush left doc string\n### Description\r\n\r\nCode that is accepted by the python interpreter raises when fed through `textwrap.dedent`\r\n\r\n### Steps to Reproduce\r\n\r\n1. Create `is_generator_bug.py` with the content below (which I simplified from [the `is_generator_with_return_value` method body](https://github.com/scrapy/scrapy/blob/2.0.1/scrapy/utils/misc.py#L186-L187)\r\n2. Run `python is_generator_bug.py`\r\n3. Observe the kaboom\r\n\r\n```python\r\nimport ast\r\nimport inspect\r\nfrom textwrap import dedent\r\nclass Bob:\r\n    def doit(self):\r\n        \"\"\"\r\nthis line is flush left\r\n        \"\"\"\r\n        if True:\r\n            yield 1234\r\n\r\nif __name__ == '__main__':\r\n    b = Bob()\r\n    c = b.doit\r\n    if inspect.isgeneratorfunction(c):\r\n        tree = ast.parse(dedent(inspect.getsource(c)))\r\n```\r\n\r\n**Expected behavior:** [What you expect to happen]\r\n\r\nNo Error\r\n\r\n**Actual behavior:** [What actually happens]\r\n\r\n```console\r\n$ python3.7 is_generator_bug.py\r\nTraceback (most recent call last):\r\n  File \"is_generator_bug.py\", line 16, in <module>\r\n    tree = ast.parse(dedent(inspect.getsource(c)))\r\n  File \"/usr/local/Cellar/python/3.7.7/Frameworks/Python.framework/Versions/3.7/lib/python3.7/ast.py\", line 35, in parse\r\n    return compile(source, filename, mode, PyCF_ONLY_AST)\r\n  File \"<unknown>\", line 1\r\n    def doit(self):\r\n    ^\r\nIndentationError: unexpected indent\r\n```\r\n\r\n**Reproduces how often:** [What percentage of the time does it reproduce?]\r\n\r\n100%\r\n\r\n### Versions\r\n\r\n```\r\nScrapy       : 2.0.1\r\nlxml         : 4.5.0.0\r\nlibxml2      : 2.9.10\r\ncssselect    : 1.1.0\r\nparsel       : 1.5.2\r\nw3lib        : 1.21.0\r\nTwisted      : 20.3.0\r\nPython       : 3.7.7 (default, Mar 11 2020, 23:30:22) - [Clang 10.0.0 (clang-1000.11.45.5)]\r\npyOpenSSL    : 19.1.0 (OpenSSL 1.1.1d  10 Sep 2019)\r\ncryptography : 2.8\r\nPlatform     : Darwin-17.7.0-x86_64-i386-64bit\r\n```\r\n\r\n### Additional context\r\n\n", "before_files": [{"content": "\"\"\"Helper functions which don't fit anywhere else\"\"\"\nimport ast\nimport inspect\nimport os\nimport re\nimport hashlib\nimport warnings\nfrom collections import deque\nfrom contextlib import contextmanager\nfrom importlib import import_module\nfrom pkgutil import iter_modules\nfrom textwrap import dedent\n\nfrom w3lib.html import replace_entities\n\nfrom scrapy.utils.datatypes import LocalWeakReferencedCache\nfrom scrapy.utils.python import flatten, to_unicode\nfrom scrapy.item import _BaseItem\nfrom scrapy.utils.deprecate import ScrapyDeprecationWarning\n\n\n_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes\n\n\ndef arg_to_iter(arg):\n    \"\"\"Convert an argument to an iterable. The argument can be a None, single\n    value, or an iterable.\n\n    Exception: if arg is a dict, [arg] will be returned\n    \"\"\"\n    if arg is None:\n        return []\n    elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):\n        return arg\n    else:\n        return [arg]\n\n\ndef load_object(path):\n    \"\"\"Load an object given its absolute object path, and return it.\n\n    The object can be the import path of a class, function, variable or an\n    instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'.\n\n    If ``path`` is not a string, but is a callable object, such as a class or\n    a function, then return it as is.\n    \"\"\"\n\n    if not isinstance(path, str):\n        if callable(path):\n            return path\n        else:\n            raise TypeError(\"Unexpected argument type, expected string \"\n                            \"or object, got: %s\" % type(path))\n\n    try:\n        dot = path.rindex('.')\n    except ValueError:\n        raise ValueError(f\"Error loading object '{path}': not a full path\")\n\n    module, name = path[:dot], path[dot + 1:]\n    mod = import_module(module)\n\n    try:\n        obj = getattr(mod, name)\n    except AttributeError:\n        raise NameError(f\"Module '{module}' doesn't define any object named '{name}'\")\n\n    return obj\n\n\ndef walk_modules(path):\n    \"\"\"Loads a module and all its submodules from the given module path and\n    returns them. If *any* module throws an exception while importing, that\n    exception is thrown back.\n\n    For example: walk_modules('scrapy.utils')\n    \"\"\"\n\n    mods = []\n    mod = import_module(path)\n    mods.append(mod)\n    if hasattr(mod, '__path__'):\n        for _, subpath, ispkg in iter_modules(mod.__path__):\n            fullpath = path + '.' + subpath\n            if ispkg:\n                mods += walk_modules(fullpath)\n            else:\n                submod = import_module(fullpath)\n                mods.append(submod)\n    return mods\n\n\ndef extract_regex(regex, text, encoding='utf-8'):\n    \"\"\"Extract a list of unicode strings from the given text/encoding using the following policies:\n\n    * if the regex contains a named group called \"extract\" that will be returned\n    * if the regex contains multiple numbered groups, all those will be returned (flattened)\n    * if the regex doesn't contain any group the entire regex matching is returned\n    \"\"\"\n    warnings.warn(\n        \"scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.\",\n        ScrapyDeprecationWarning,\n        stacklevel=2\n    )\n\n    if isinstance(regex, str):\n        regex = re.compile(regex, re.UNICODE)\n\n    try:\n        strings = [regex.search(text).group('extract')]   # named group\n    except Exception:\n        strings = regex.findall(text)    # full regex or numbered groups\n    strings = flatten(strings)\n\n    if isinstance(text, str):\n        return [replace_entities(s, keep=['lt', 'amp']) for s in strings]\n    else:\n        return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])\n                for s in strings]\n\n\ndef md5sum(file):\n    \"\"\"Calculate the md5 checksum of a file-like object without reading its\n    whole content in memory.\n\n    >>> from io import BytesIO\n    >>> md5sum(BytesIO(b'file content to hash'))\n    '784406af91dd5a54fbb9c84c2236595a'\n    \"\"\"\n    m = hashlib.md5()\n    while True:\n        d = file.read(8096)\n        if not d:\n            break\n        m.update(d)\n    return m.hexdigest()\n\n\ndef rel_has_nofollow(rel):\n    \"\"\"Return True if link rel attribute has nofollow type\"\"\"\n    return rel is not None and 'nofollow' in rel.split()\n\n\ndef create_instance(objcls, settings, crawler, *args, **kwargs):\n    \"\"\"Construct a class instance using its ``from_crawler`` or\n    ``from_settings`` constructors, if available.\n\n    At least one of ``settings`` and ``crawler`` needs to be different from\n    ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.\n    If ``crawler`` is ``None``, only the ``from_settings`` constructor will be\n    tried.\n\n    ``*args`` and ``**kwargs`` are forwarded to the constructors.\n\n    Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.\n\n    .. versionchanged:: 2.2\n       Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an\n       extension has not been implemented correctly).\n    \"\"\"\n    if settings is None:\n        if crawler is None:\n            raise ValueError(\"Specify at least one of settings and crawler.\")\n        settings = crawler.settings\n    if crawler and hasattr(objcls, 'from_crawler'):\n        instance = objcls.from_crawler(crawler, *args, **kwargs)\n        method_name = 'from_crawler'\n    elif hasattr(objcls, 'from_settings'):\n        instance = objcls.from_settings(settings, *args, **kwargs)\n        method_name = 'from_settings'\n    else:\n        instance = objcls(*args, **kwargs)\n        method_name = '__new__'\n    if instance is None:\n        raise TypeError(f\"{objcls.__qualname__}.{method_name} returned None\")\n    return instance\n\n\n@contextmanager\ndef set_environ(**kwargs):\n    \"\"\"Temporarily set environment variables inside the context manager and\n    fully restore previous environment afterwards\n    \"\"\"\n\n    original_env = {k: os.environ.get(k) for k in kwargs}\n    os.environ.update(kwargs)\n    try:\n        yield\n    finally:\n        for k, v in original_env.items():\n            if v is None:\n                del os.environ[k]\n            else:\n                os.environ[k] = v\n\n\ndef walk_callable(node):\n    \"\"\"Similar to ``ast.walk``, but walks only function body and skips nested\n    functions defined within the node.\n    \"\"\"\n    todo = deque([node])\n    walked_func_def = False\n    while todo:\n        node = todo.popleft()\n        if isinstance(node, ast.FunctionDef):\n            if walked_func_def:\n                continue\n            walked_func_def = True\n        todo.extend(ast.iter_child_nodes(node))\n        yield node\n\n\n_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)\n\n\ndef is_generator_with_return_value(callable):\n    \"\"\"\n    Returns True if a callable is a generator function which includes a\n    'return' statement with a value different than None, False otherwise\n    \"\"\"\n    if callable in _generator_callbacks_cache:\n        return _generator_callbacks_cache[callable]\n\n    def returns_none(return_node):\n        value = return_node.value\n        return value is None or isinstance(value, ast.NameConstant) and value.value is None\n\n    if inspect.isgeneratorfunction(callable):\n        tree = ast.parse(dedent(inspect.getsource(callable)))\n        for node in walk_callable(tree):\n            if isinstance(node, ast.Return) and not returns_none(node):\n                _generator_callbacks_cache[callable] = True\n                return _generator_callbacks_cache[callable]\n\n    _generator_callbacks_cache[callable] = False\n    return _generator_callbacks_cache[callable]\n\n\ndef warn_on_generator_with_return_value(spider, callable):\n    \"\"\"\n    Logs a warning if a callable is a generator function and includes\n    a 'return' statement with a value different than None\n    \"\"\"\n    if is_generator_with_return_value(callable):\n        warnings.warn(\n            f'The \"{spider.__class__.__name__}.{callable.__name__}\" method is '\n            'a generator and includes a \"return\" statement with a value '\n            'different than None. This could lead to unexpected behaviour. Please see '\n            'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '\n            'for details about the semantics of the \"return\" statement within generators',\n            stacklevel=2,\n        )\n", "path": "scrapy/utils/misc.py"}], "after_files": [{"content": "\"\"\"Helper functions which don't fit anywhere else\"\"\"\nimport ast\nimport inspect\nimport os\nimport re\nimport hashlib\nimport warnings\nfrom collections import deque\nfrom contextlib import contextmanager\nfrom importlib import import_module\nfrom pkgutil import iter_modules\n\nfrom w3lib.html import replace_entities\n\nfrom scrapy.utils.datatypes import LocalWeakReferencedCache\nfrom scrapy.utils.python import flatten, to_unicode\nfrom scrapy.item import _BaseItem\nfrom scrapy.utils.deprecate import ScrapyDeprecationWarning\n\n\n_ITERABLE_SINGLE_VALUES = dict, _BaseItem, str, bytes\n\n\ndef arg_to_iter(arg):\n    \"\"\"Convert an argument to an iterable. The argument can be a None, single\n    value, or an iterable.\n\n    Exception: if arg is a dict, [arg] will be returned\n    \"\"\"\n    if arg is None:\n        return []\n    elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):\n        return arg\n    else:\n        return [arg]\n\n\ndef load_object(path):\n    \"\"\"Load an object given its absolute object path, and return it.\n\n    The object can be the import path of a class, function, variable or an\n    instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'.\n\n    If ``path`` is not a string, but is a callable object, such as a class or\n    a function, then return it as is.\n    \"\"\"\n\n    if not isinstance(path, str):\n        if callable(path):\n            return path\n        else:\n            raise TypeError(\"Unexpected argument type, expected string \"\n                            \"or object, got: %s\" % type(path))\n\n    try:\n        dot = path.rindex('.')\n    except ValueError:\n        raise ValueError(f\"Error loading object '{path}': not a full path\")\n\n    module, name = path[:dot], path[dot + 1:]\n    mod = import_module(module)\n\n    try:\n        obj = getattr(mod, name)\n    except AttributeError:\n        raise NameError(f\"Module '{module}' doesn't define any object named '{name}'\")\n\n    return obj\n\n\ndef walk_modules(path):\n    \"\"\"Loads a module and all its submodules from the given module path and\n    returns them. If *any* module throws an exception while importing, that\n    exception is thrown back.\n\n    For example: walk_modules('scrapy.utils')\n    \"\"\"\n\n    mods = []\n    mod = import_module(path)\n    mods.append(mod)\n    if hasattr(mod, '__path__'):\n        for _, subpath, ispkg in iter_modules(mod.__path__):\n            fullpath = path + '.' + subpath\n            if ispkg:\n                mods += walk_modules(fullpath)\n            else:\n                submod = import_module(fullpath)\n                mods.append(submod)\n    return mods\n\n\ndef extract_regex(regex, text, encoding='utf-8'):\n    \"\"\"Extract a list of unicode strings from the given text/encoding using the following policies:\n\n    * if the regex contains a named group called \"extract\" that will be returned\n    * if the regex contains multiple numbered groups, all those will be returned (flattened)\n    * if the regex doesn't contain any group the entire regex matching is returned\n    \"\"\"\n    warnings.warn(\n        \"scrapy.utils.misc.extract_regex has moved to parsel.utils.extract_regex.\",\n        ScrapyDeprecationWarning,\n        stacklevel=2\n    )\n\n    if isinstance(regex, str):\n        regex = re.compile(regex, re.UNICODE)\n\n    try:\n        strings = [regex.search(text).group('extract')]   # named group\n    except Exception:\n        strings = regex.findall(text)    # full regex or numbered groups\n    strings = flatten(strings)\n\n    if isinstance(text, str):\n        return [replace_entities(s, keep=['lt', 'amp']) for s in strings]\n    else:\n        return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])\n                for s in strings]\n\n\ndef md5sum(file):\n    \"\"\"Calculate the md5 checksum of a file-like object without reading its\n    whole content in memory.\n\n    >>> from io import BytesIO\n    >>> md5sum(BytesIO(b'file content to hash'))\n    '784406af91dd5a54fbb9c84c2236595a'\n    \"\"\"\n    m = hashlib.md5()\n    while True:\n        d = file.read(8096)\n        if not d:\n            break\n        m.update(d)\n    return m.hexdigest()\n\n\ndef rel_has_nofollow(rel):\n    \"\"\"Return True if link rel attribute has nofollow type\"\"\"\n    return rel is not None and 'nofollow' in rel.split()\n\n\ndef create_instance(objcls, settings, crawler, *args, **kwargs):\n    \"\"\"Construct a class instance using its ``from_crawler`` or\n    ``from_settings`` constructors, if available.\n\n    At least one of ``settings`` and ``crawler`` needs to be different from\n    ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.\n    If ``crawler`` is ``None``, only the ``from_settings`` constructor will be\n    tried.\n\n    ``*args`` and ``**kwargs`` are forwarded to the constructors.\n\n    Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.\n\n    .. versionchanged:: 2.2\n       Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an\n       extension has not been implemented correctly).\n    \"\"\"\n    if settings is None:\n        if crawler is None:\n            raise ValueError(\"Specify at least one of settings and crawler.\")\n        settings = crawler.settings\n    if crawler and hasattr(objcls, 'from_crawler'):\n        instance = objcls.from_crawler(crawler, *args, **kwargs)\n        method_name = 'from_crawler'\n    elif hasattr(objcls, 'from_settings'):\n        instance = objcls.from_settings(settings, *args, **kwargs)\n        method_name = 'from_settings'\n    else:\n        instance = objcls(*args, **kwargs)\n        method_name = '__new__'\n    if instance is None:\n        raise TypeError(f\"{objcls.__qualname__}.{method_name} returned None\")\n    return instance\n\n\n@contextmanager\ndef set_environ(**kwargs):\n    \"\"\"Temporarily set environment variables inside the context manager and\n    fully restore previous environment afterwards\n    \"\"\"\n\n    original_env = {k: os.environ.get(k) for k in kwargs}\n    os.environ.update(kwargs)\n    try:\n        yield\n    finally:\n        for k, v in original_env.items():\n            if v is None:\n                del os.environ[k]\n            else:\n                os.environ[k] = v\n\n\ndef walk_callable(node):\n    \"\"\"Similar to ``ast.walk``, but walks only function body and skips nested\n    functions defined within the node.\n    \"\"\"\n    todo = deque([node])\n    walked_func_def = False\n    while todo:\n        node = todo.popleft()\n        if isinstance(node, ast.FunctionDef):\n            if walked_func_def:\n                continue\n            walked_func_def = True\n        todo.extend(ast.iter_child_nodes(node))\n        yield node\n\n\n_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)\n\n\ndef is_generator_with_return_value(callable):\n    \"\"\"\n    Returns True if a callable is a generator function which includes a\n    'return' statement with a value different than None, False otherwise\n    \"\"\"\n    if callable in _generator_callbacks_cache:\n        return _generator_callbacks_cache[callable]\n\n    def returns_none(return_node):\n        value = return_node.value\n        return value is None or isinstance(value, ast.NameConstant) and value.value is None\n\n    if inspect.isgeneratorfunction(callable):\n        code = re.sub(r\"^[\\t ]+\", \"\", inspect.getsource(callable))\n        tree = ast.parse(code)\n        for node in walk_callable(tree):\n            if isinstance(node, ast.Return) and not returns_none(node):\n                _generator_callbacks_cache[callable] = True\n                return _generator_callbacks_cache[callable]\n\n    _generator_callbacks_cache[callable] = False\n    return _generator_callbacks_cache[callable]\n\n\ndef warn_on_generator_with_return_value(spider, callable):\n    \"\"\"\n    Logs a warning if a callable is a generator function and includes\n    a 'return' statement with a value different than None\n    \"\"\"\n    try:\n        if is_generator_with_return_value(callable):\n            warnings.warn(\n                f'The \"{spider.__class__.__name__}.{callable.__name__}\" method is '\n                'a generator and includes a \"return\" statement with a value '\n                'different than None. This could lead to unexpected behaviour. Please see '\n                'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '\n                'for details about the semantics of the \"return\" statement within generators',\n                stacklevel=2,\n            )\n    except IndentationError:\n        callable_name = spider.__class__.__name__ + \".\" + callable.__name__\n        warnings.warn(\n            f'Unable to determine whether or not \"{callable_name}\" is a generator with a return value. '\n            'This will not prevent your code from working, but it prevents Scrapy from detecting '\n            f'potential issues in your implementation of \"{callable_name}\". Please, report this in the '\n            'Scrapy issue tracker (https://github.com/scrapy/scrapy/issues), '\n            f'including the code of \"{callable_name}\"',\n            stacklevel=2,\n        )\n", "path": "scrapy/utils/misc.py"}]}