We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Fail or warn if from_crawler() returns None
## Summary

Generate a warning or error if from_crawler() for a middleware/extension/etc. returns None

## Motivation

I created a custom extension and connected signals in the from_crawler() classmethod, but neglected to return the new extension instance.  Scrapy still reported the extension under "Enabled extensions", but none of the signals worked, since the instance was immediately garbage collected and its signals were silently disconnected.

This was of course an error on my part, but it would have saved me a lot of debugging if I had gotten a warning that from_crawler() was returning None, or if the extension were removed from the "Enabled extensions" list.

Would it be appropriate for utils.misc.create_instance() to raise an error or generate a warning if it's about to return None?  Or should MiddlewareManager treat create_instance() returning None the same as create_instance() raising NotConfigured?

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/utils/misc.py`
Content:
```
1 """Helper functions which don't fit anywhere else"""
2 import ast
3 import inspect
4 import os
5 import re
6 import hashlib
7 import warnings
8 from contextlib import contextmanager
9 from importlib import import_module
10 from pkgutil import iter_modules
11 from textwrap import dedent
12 
13 from w3lib.html import replace_entities
14 
15 from scrapy.utils.datatypes import LocalWeakReferencedCache
16 from scrapy.utils.python import flatten, to_unicode
17 from scrapy.item import BaseItem
18 
19 
20 _ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes
21 
22 
23 def arg_to_iter(arg):
24     """Convert an argument to an iterable. The argument can be a None, single
25     value, or an iterable.
26 
27     Exception: if arg is a dict, [arg] will be returned
28     """
29     if arg is None:
30         return []
31     elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):
32         return arg
33     else:
34         return [arg]
35 
36 
37 def load_object(path):
38     """Load an object given its absolute object path, and return it.
39 
40     object can be the import path of a class, function, variable or an
41     instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'
42     """
43 
44     try:
45         dot = path.rindex('.')
46     except ValueError:
47         raise ValueError("Error loading object '%s': not a full path" % path)
48 
49     module, name = path[:dot], path[dot + 1:]
50     mod = import_module(module)
51 
52     try:
53         obj = getattr(mod, name)
54     except AttributeError:
55         raise NameError("Module '%s' doesn't define any object named '%s'" % (module, name))
56 
57     return obj
58 
59 
60 def walk_modules(path):
61     """Loads a module and all its submodules from the given module path and
62     returns them. If *any* module throws an exception while importing, that
63     exception is thrown back.
64 
65     For example: walk_modules('scrapy.utils')
66     """
67 
68     mods = []
69     mod = import_module(path)
70     mods.append(mod)
71     if hasattr(mod, '__path__'):
72         for _, subpath, ispkg in iter_modules(mod.__path__):
73             fullpath = path + '.' + subpath
74             if ispkg:
75                 mods += walk_modules(fullpath)
76             else:
77                 submod = import_module(fullpath)
78                 mods.append(submod)
79     return mods
80 
81 
82 def extract_regex(regex, text, encoding='utf-8'):
83     """Extract a list of unicode strings from the given text/encoding using the following policies:
84 
85     * if the regex contains a named group called "extract" that will be returned
86     * if the regex contains multiple numbered groups, all those will be returned (flattened)
87     * if the regex doesn't contain any group the entire regex matching is returned
88     """
89 
90     if isinstance(regex, str):
91         regex = re.compile(regex, re.UNICODE)
92 
93     try:
94         strings = [regex.search(text).group('extract')]   # named group
95     except Exception:
96         strings = regex.findall(text)    # full regex or numbered groups
97     strings = flatten(strings)
98 
99     if isinstance(text, str):
100         return [replace_entities(s, keep=['lt', 'amp']) for s in strings]
101     else:
102         return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])
103                 for s in strings]
104 
105 
106 def md5sum(file):
107     """Calculate the md5 checksum of a file-like object without reading its
108     whole content in memory.
109 
110     >>> from io import BytesIO
111     >>> md5sum(BytesIO(b'file content to hash'))
112     '784406af91dd5a54fbb9c84c2236595a'
113     """
114     m = hashlib.md5()
115     while True:
116         d = file.read(8096)
117         if not d:
118             break
119         m.update(d)
120     return m.hexdigest()
121 
122 
123 def rel_has_nofollow(rel):
124     """Return True if link rel attribute has nofollow type"""
125     return rel is not None and 'nofollow' in rel.split()
126 
127 
128 def create_instance(objcls, settings, crawler, *args, **kwargs):
129     """Construct a class instance using its ``from_crawler`` or
130     ``from_settings`` constructors, if available.
131 
132     At least one of ``settings`` and ``crawler`` needs to be different from
133     ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.
134     If ``crawler`` is ``None``, only the ``from_settings`` constructor will be
135     tried.
136 
137     ``*args`` and ``**kwargs`` are forwarded to the constructors.
138 
139     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
140     """
141     if settings is None:
142         if crawler is None:
143             raise ValueError("Specify at least one of settings and crawler.")
144         settings = crawler.settings
145     if crawler and hasattr(objcls, 'from_crawler'):
146         return objcls.from_crawler(crawler, *args, **kwargs)
147     elif hasattr(objcls, 'from_settings'):
148         return objcls.from_settings(settings, *args, **kwargs)
149     else:
150         return objcls(*args, **kwargs)
151 
152 
153 @contextmanager
154 def set_environ(**kwargs):
155     """Temporarily set environment variables inside the context manager and
156     fully restore previous environment afterwards
157     """
158 
159     original_env = {k: os.environ.get(k) for k in kwargs}
160     os.environ.update(kwargs)
161     try:
162         yield
163     finally:
164         for k, v in original_env.items():
165             if v is None:
166                 del os.environ[k]
167             else:
168                 os.environ[k] = v
169 
170 
171 _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
172 
173 
174 def is_generator_with_return_value(callable):
175     """
176     Returns True if a callable is a generator function which includes a
177     'return' statement with a value different than None, False otherwise
178     """
179     if callable in _generator_callbacks_cache:
180         return _generator_callbacks_cache[callable]
181 
182     def returns_none(return_node):
183         value = return_node.value
184         return value is None or isinstance(value, ast.NameConstant) and value.value is None
185 
186     if inspect.isgeneratorfunction(callable):
187         tree = ast.parse(dedent(inspect.getsource(callable)))
188         for node in ast.walk(tree):
189             if isinstance(node, ast.Return) and not returns_none(node):
190                 _generator_callbacks_cache[callable] = True
191                 return _generator_callbacks_cache[callable]
192 
193     _generator_callbacks_cache[callable] = False
194     return _generator_callbacks_cache[callable]
195 
196 
197 def warn_on_generator_with_return_value(spider, callable):
198     """
199     Logs a warning if a callable is a generator function and includes
200     a 'return' statement with a value different than None
201     """
202     if is_generator_with_return_value(callable):
203         warnings.warn(
204             'The "{}.{}" method is a generator and includes a "return" statement with a '
205             'value different than None. This could lead to unexpected behaviour. Please see '
206             'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '
207             'for details about the semantics of the "return" statement within generators'
208             .format(spider.__class__.__name__, callable.__name__), stacklevel=2,
209         )
210
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -137,17 +137,26 @@
     ``*args`` and ``**kwargs`` are forwarded to the constructors.
 
     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.
+
+    Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an
+    extension has not been implemented correctly).
     """
     if settings is None:
         if crawler is None:
             raise ValueError("Specify at least one of settings and crawler.")
         settings = crawler.settings
     if crawler and hasattr(objcls, 'from_crawler'):
-        return objcls.from_crawler(crawler, *args, **kwargs)
+        instance = objcls.from_crawler(crawler, *args, **kwargs)
+        method_name = 'from_crawler'
     elif hasattr(objcls, 'from_settings'):
-        return objcls.from_settings(settings, *args, **kwargs)
+        instance = objcls.from_settings(settings, *args, **kwargs)
+        method_name = 'from_settings'
     else:
-        return objcls(*args, **kwargs)
+        instance = objcls(*args, **kwargs)
+        method_name = '__new__'
+    if instance is None:
+        raise TypeError("%s.%s returned None" % (objcls.__qualname__, method_name))
+    return instance
 
 
 @contextmanager

{"golden_diff": "diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py\n--- a/scrapy/utils/misc.py\n+++ b/scrapy/utils/misc.py\n@@ -137,17 +137,26 @@\n     ``*args`` and ``**kwargs`` are forwarded to the constructors.\n \n     Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.\n+\n+    Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an\n+    extension has not been implemented correctly).\n     \"\"\"\n     if settings is None:\n         if crawler is None:\n             raise ValueError(\"Specify at least one of settings and crawler.\")\n         settings = crawler.settings\n     if crawler and hasattr(objcls, 'from_crawler'):\n-        return objcls.from_crawler(crawler, *args, **kwargs)\n+        instance = objcls.from_crawler(crawler, *args, **kwargs)\n+        method_name = 'from_crawler'\n     elif hasattr(objcls, 'from_settings'):\n-        return objcls.from_settings(settings, *args, **kwargs)\n+        instance = objcls.from_settings(settings, *args, **kwargs)\n+        method_name = 'from_settings'\n     else:\n-        return objcls(*args, **kwargs)\n+        instance = objcls(*args, **kwargs)\n+        method_name = '__new__'\n+    if instance is None:\n+        raise TypeError(\"%s.%s returned None\" % (objcls.__qualname__, method_name))\n+    return instance\n \n \n @contextmanager\n", "issue": "Fail or warn if from_crawler() returns None\n## Summary\r\n\r\nGenerate a warning or error if from_crawler() for a middleware/extension/etc. returns None\r\n\r\n## Motivation\r\n\r\nI created a custom extension and connected signals in the from_crawler() classmethod, but neglected to return the new extension instance.  Scrapy still reported the extension under \"Enabled extensions\", but none of the signals worked, since the instance was immediately garbage collected and its signals were silently disconnected.\r\n\r\nThis was of course an error on my part, but it would have saved me a lot of debugging if I had gotten a warning that from_crawler() was returning None, or if the extension were removed from the \"Enabled extensions\" list.\r\n\r\nWould it be appropriate for utils.misc.create_instance() to raise an error or generate a warning if it's about to return None?  Or should MiddlewareManager treat create_instance() returning None the same as create_instance() raising NotConfigured?\n", "before_files": [{"content": "\"\"\"Helper functions which don't fit anywhere else\"\"\"\nimport ast\nimport inspect\nimport os\nimport re\nimport hashlib\nimport warnings\nfrom contextlib import contextmanager\nfrom importlib import import_module\nfrom pkgutil import iter_modules\nfrom textwrap import dedent\n\nfrom w3lib.html import replace_entities\n\nfrom scrapy.utils.datatypes import LocalWeakReferencedCache\nfrom scrapy.utils.python import flatten, to_unicode\nfrom scrapy.item import BaseItem\n\n\n_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes\n\n\ndef arg_to_iter(arg):\n    \"\"\"Convert an argument to an iterable. The argument can be a None, single\n    value, or an iterable.\n\n    Exception: if arg is a dict, [arg] will be returned\n    \"\"\"\n    if arg is None:\n        return []\n    elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):\n        return arg\n    else:\n        return [arg]\n\n\ndef load_object(path):\n    \"\"\"Load an object given its absolute object path, and return it.\n\n    object can be the import path of a class, function, variable or an\n    instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'\n    \"\"\"\n\n    try:\n        dot = path.rindex('.')\n    except ValueError:\n        raise ValueError(\"Error loading object '%s': not a full path\" % path)\n\n    module, name = path[:dot], path[dot + 1:]\n    mod = import_module(module)\n\n    try:\n        obj = getattr(mod, name)\n    except AttributeError:\n        raise NameError(\"Module '%s' doesn't define any object named '%s'\" % (module, name))\n\n    return obj\n\n\ndef walk_modules(path):\n    \"\"\"Loads a module and all its submodules from the given module path and\n    returns them. If *any* module throws an exception while importing, that\n    exception is thrown back.\n\n    For example: walk_modules('scrapy.utils')\n    \"\"\"\n\n    mods = []\n    mod = import_module(path)\n    mods.append(mod)\n    if hasattr(mod, '__path__'):\n        for _, subpath, ispkg in iter_modules(mod.__path__):\n            fullpath = path + '.' + subpath\n            if ispkg:\n                mods += walk_modules(fullpath)\n            else:\n                submod = import_module(fullpath)\n                mods.append(submod)\n    return mods\n\n\ndef extract_regex(regex, text, encoding='utf-8'):\n    \"\"\"Extract a list of unicode strings from the given text/encoding using the following policies:\n\n    * if the regex contains a named group called \"extract\" that will be returned\n    * if the regex contains multiple numbered groups, all those will be returned (flattened)\n    * if the regex doesn't contain any group the entire regex matching is returned\n    \"\"\"\n\n    if isinstance(regex, str):\n        regex = re.compile(regex, re.UNICODE)\n\n    try:\n        strings = [regex.search(text).group('extract')]   # named group\n    except Exception:\n        strings = regex.findall(text)    # full regex or numbered groups\n    strings = flatten(strings)\n\n    if isinstance(text, str):\n        return [replace_entities(s, keep=['lt', 'amp']) for s in strings]\n    else:\n        return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])\n                for s in strings]\n\n\ndef md5sum(file):\n    \"\"\"Calculate the md5 checksum of a file-like object without reading its\n    whole content in memory.\n\n    >>> from io import BytesIO\n    >>> md5sum(BytesIO(b'file content to hash'))\n    '784406af91dd5a54fbb9c84c2236595a'\n    \"\"\"\n    m = hashlib.md5()\n    while True:\n        d = file.read(8096)\n        if not d:\n            break\n        m.update(d)\n    return m.hexdigest()\n\n\ndef rel_has_nofollow(rel):\n    \"\"\"Return True if link rel attribute has nofollow type\"\"\"\n    return rel is not None and 'nofollow' in rel.split()\n\n\ndef create_instance(objcls, settings, crawler, *args, **kwargs):\n    \"\"\"Construct a class instance using its ``from_crawler`` or\n    ``from_settings`` constructors, if available.\n\n    At least one of ``settings`` and ``crawler`` needs to be different from\n    ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.\n    If ``crawler`` is ``None``, only the ``from_settings`` constructor will be\n    tried.\n\n    ``*args`` and ``**kwargs`` are forwarded to the constructors.\n\n    Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.\n    \"\"\"\n    if settings is None:\n        if crawler is None:\n            raise ValueError(\"Specify at least one of settings and crawler.\")\n        settings = crawler.settings\n    if crawler and hasattr(objcls, 'from_crawler'):\n        return objcls.from_crawler(crawler, *args, **kwargs)\n    elif hasattr(objcls, 'from_settings'):\n        return objcls.from_settings(settings, *args, **kwargs)\n    else:\n        return objcls(*args, **kwargs)\n\n\n@contextmanager\ndef set_environ(**kwargs):\n    \"\"\"Temporarily set environment variables inside the context manager and\n    fully restore previous environment afterwards\n    \"\"\"\n\n    original_env = {k: os.environ.get(k) for k in kwargs}\n    os.environ.update(kwargs)\n    try:\n        yield\n    finally:\n        for k, v in original_env.items():\n            if v is None:\n                del os.environ[k]\n            else:\n                os.environ[k] = v\n\n\n_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)\n\n\ndef is_generator_with_return_value(callable):\n    \"\"\"\n    Returns True if a callable is a generator function which includes a\n    'return' statement with a value different than None, False otherwise\n    \"\"\"\n    if callable in _generator_callbacks_cache:\n        return _generator_callbacks_cache[callable]\n\n    def returns_none(return_node):\n        value = return_node.value\n        return value is None or isinstance(value, ast.NameConstant) and value.value is None\n\n    if inspect.isgeneratorfunction(callable):\n        tree = ast.parse(dedent(inspect.getsource(callable)))\n        for node in ast.walk(tree):\n            if isinstance(node, ast.Return) and not returns_none(node):\n                _generator_callbacks_cache[callable] = True\n                return _generator_callbacks_cache[callable]\n\n    _generator_callbacks_cache[callable] = False\n    return _generator_callbacks_cache[callable]\n\n\ndef warn_on_generator_with_return_value(spider, callable):\n    \"\"\"\n    Logs a warning if a callable is a generator function and includes\n    a 'return' statement with a value different than None\n    \"\"\"\n    if is_generator_with_return_value(callable):\n        warnings.warn(\n            'The \"{}.{}\" method is a generator and includes a \"return\" statement with a '\n            'value different than None. This could lead to unexpected behaviour. Please see '\n            'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '\n            'for details about the semantics of the \"return\" statement within generators'\n            .format(spider.__class__.__name__, callable.__name__), stacklevel=2,\n        )\n", "path": "scrapy/utils/misc.py"}], "after_files": [{"content": "\"\"\"Helper functions which don't fit anywhere else\"\"\"\nimport ast\nimport inspect\nimport os\nimport re\nimport hashlib\nimport warnings\nfrom contextlib import contextmanager\nfrom importlib import import_module\nfrom pkgutil import iter_modules\nfrom textwrap import dedent\n\nfrom w3lib.html import replace_entities\n\nfrom scrapy.utils.datatypes import LocalWeakReferencedCache\nfrom scrapy.utils.python import flatten, to_unicode\nfrom scrapy.item import BaseItem\n\n\n_ITERABLE_SINGLE_VALUES = dict, BaseItem, str, bytes\n\n\ndef arg_to_iter(arg):\n    \"\"\"Convert an argument to an iterable. The argument can be a None, single\n    value, or an iterable.\n\n    Exception: if arg is a dict, [arg] will be returned\n    \"\"\"\n    if arg is None:\n        return []\n    elif not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, '__iter__'):\n        return arg\n    else:\n        return [arg]\n\n\ndef load_object(path):\n    \"\"\"Load an object given its absolute object path, and return it.\n\n    object can be the import path of a class, function, variable or an\n    instance, e.g. 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware'\n    \"\"\"\n\n    try:\n        dot = path.rindex('.')\n    except ValueError:\n        raise ValueError(\"Error loading object '%s': not a full path\" % path)\n\n    module, name = path[:dot], path[dot + 1:]\n    mod = import_module(module)\n\n    try:\n        obj = getattr(mod, name)\n    except AttributeError:\n        raise NameError(\"Module '%s' doesn't define any object named '%s'\" % (module, name))\n\n    return obj\n\n\ndef walk_modules(path):\n    \"\"\"Loads a module and all its submodules from the given module path and\n    returns them. If *any* module throws an exception while importing, that\n    exception is thrown back.\n\n    For example: walk_modules('scrapy.utils')\n    \"\"\"\n\n    mods = []\n    mod = import_module(path)\n    mods.append(mod)\n    if hasattr(mod, '__path__'):\n        for _, subpath, ispkg in iter_modules(mod.__path__):\n            fullpath = path + '.' + subpath\n            if ispkg:\n                mods += walk_modules(fullpath)\n            else:\n                submod = import_module(fullpath)\n                mods.append(submod)\n    return mods\n\n\ndef extract_regex(regex, text, encoding='utf-8'):\n    \"\"\"Extract a list of unicode strings from the given text/encoding using the following policies:\n\n    * if the regex contains a named group called \"extract\" that will be returned\n    * if the regex contains multiple numbered groups, all those will be returned (flattened)\n    * if the regex doesn't contain any group the entire regex matching is returned\n    \"\"\"\n\n    if isinstance(regex, str):\n        regex = re.compile(regex, re.UNICODE)\n\n    try:\n        strings = [regex.search(text).group('extract')]   # named group\n    except Exception:\n        strings = regex.findall(text)    # full regex or numbered groups\n    strings = flatten(strings)\n\n    if isinstance(text, str):\n        return [replace_entities(s, keep=['lt', 'amp']) for s in strings]\n    else:\n        return [replace_entities(to_unicode(s, encoding), keep=['lt', 'amp'])\n                for s in strings]\n\n\ndef md5sum(file):\n    \"\"\"Calculate the md5 checksum of a file-like object without reading its\n    whole content in memory.\n\n    >>> from io import BytesIO\n    >>> md5sum(BytesIO(b'file content to hash'))\n    '784406af91dd5a54fbb9c84c2236595a'\n    \"\"\"\n    m = hashlib.md5()\n    while True:\n        d = file.read(8096)\n        if not d:\n            break\n        m.update(d)\n    return m.hexdigest()\n\n\ndef rel_has_nofollow(rel):\n    \"\"\"Return True if link rel attribute has nofollow type\"\"\"\n    return rel is not None and 'nofollow' in rel.split()\n\n\ndef create_instance(objcls, settings, crawler, *args, **kwargs):\n    \"\"\"Construct a class instance using its ``from_crawler`` or\n    ``from_settings`` constructors, if available.\n\n    At least one of ``settings`` and ``crawler`` needs to be different from\n    ``None``. If ``settings `` is ``None``, ``crawler.settings`` will be used.\n    If ``crawler`` is ``None``, only the ``from_settings`` constructor will be\n    tried.\n\n    ``*args`` and ``**kwargs`` are forwarded to the constructors.\n\n    Raises ``ValueError`` if both ``settings`` and ``crawler`` are ``None``.\n\n    Raises ``TypeError`` if the resulting instance is ``None`` (e.g. if an\n    extension has not been implemented correctly).\n    \"\"\"\n    if settings is None:\n        if crawler is None:\n            raise ValueError(\"Specify at least one of settings and crawler.\")\n        settings = crawler.settings\n    if crawler and hasattr(objcls, 'from_crawler'):\n        instance = objcls.from_crawler(crawler, *args, **kwargs)\n        method_name = 'from_crawler'\n    elif hasattr(objcls, 'from_settings'):\n        instance = objcls.from_settings(settings, *args, **kwargs)\n        method_name = 'from_settings'\n    else:\n        instance = objcls(*args, **kwargs)\n        method_name = '__new__'\n    if instance is None:\n        raise TypeError(\"%s.%s returned None\" % (objcls.__qualname__, method_name))\n    return instance\n\n\n@contextmanager\ndef set_environ(**kwargs):\n    \"\"\"Temporarily set environment variables inside the context manager and\n    fully restore previous environment afterwards\n    \"\"\"\n\n    original_env = {k: os.environ.get(k) for k in kwargs}\n    os.environ.update(kwargs)\n    try:\n        yield\n    finally:\n        for k, v in original_env.items():\n            if v is None:\n                del os.environ[k]\n            else:\n                os.environ[k] = v\n\n\n_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)\n\n\ndef is_generator_with_return_value(callable):\n    \"\"\"\n    Returns True if a callable is a generator function which includes a\n    'return' statement with a value different than None, False otherwise\n    \"\"\"\n    if callable in _generator_callbacks_cache:\n        return _generator_callbacks_cache[callable]\n\n    def returns_none(return_node):\n        value = return_node.value\n        return value is None or isinstance(value, ast.NameConstant) and value.value is None\n\n    if inspect.isgeneratorfunction(callable):\n        tree = ast.parse(dedent(inspect.getsource(callable)))\n        for node in ast.walk(tree):\n            if isinstance(node, ast.Return) and not returns_none(node):\n                _generator_callbacks_cache[callable] = True\n                return _generator_callbacks_cache[callable]\n\n    _generator_callbacks_cache[callable] = False\n    return _generator_callbacks_cache[callable]\n\n\ndef warn_on_generator_with_return_value(spider, callable):\n    \"\"\"\n    Logs a warning if a callable is a generator function and includes\n    a 'return' statement with a value different than None\n    \"\"\"\n    if is_generator_with_return_value(callable):\n        warnings.warn(\n            'The \"{}.{}\" method is a generator and includes a \"return\" statement with a '\n            'value different than None. This could lead to unexpected behaviour. Please see '\n            'https://docs.python.org/3/reference/simple_stmts.html#the-return-statement '\n            'for details about the semantics of the \"return\" statement within generators'\n            .format(spider.__class__.__name__, callable.__name__), stacklevel=2,\n        )\n", "path": "scrapy/utils/misc.py"}]}