We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Fixes issue with calling fetch in scrapy shell.
Reference Issue Fixes #5740 ,  #5742

You can recreate the issue with the following script.

```
import asyncio
import threading
from twisted.internet import asyncioreactor
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.reactor import get_asyncio_event_loop_policy

async def test_coro():
    pass

def test_deferred_from_coro():
    return deferred_from_coro(test_coro())

def trigger_warning_message():
    event_loop = get_asyncio_event_loop_policy().new_event_loop()
    asyncio.set_event_loop(event_loop)
    asyncioreactor.install()
    thread = threading.Thread(target=test_deferred_from_coro)
    thread.start()
    thread.join()

trigger_warning_message()
```

I was able to recreate this issue using `scrapy shell` and `fetch` on both windows and linux.  However it only occurs inside of a project with the `TWISTED_REACTOR` setting set to AsyncioSelectorReactor. 

What causes the issue is when `get_asyncio_event_loop_policy().get_event_loop()` is called, there is no event loop in the thread that the function is called from. Which raises an exception.  
asyncio exception in scrapy shell
We got a bug report that using `fetch(<url>)` in `scrapy shell` on Windows in a new project (so with `TWISTED_REACTOR` set to the asyncio one) raises "There is no current event loop" in `deferred_from_coro()`. The Python version is 3.9 and 3.10 and the Scrapy version is 2.7.1. It is possible that the recenty changed loop code indeed doesn't work correctly on Windows but I cannot test or fix it at this time.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/shell.py`
Content:
```
1 """Scrapy Shell
2 
3 See documentation in docs/topics/shell.rst
4 
5 """
6 import os
7 import signal
8 
9 from itemadapter import is_item
10 from twisted.internet import threads, defer
11 from twisted.python import threadable
12 from w3lib.url import any_to_uri
13 
14 from scrapy.crawler import Crawler
15 from scrapy.exceptions import IgnoreRequest
16 from scrapy.http import Request, Response
17 from scrapy.settings import Settings
18 from scrapy.spiders import Spider
19 from scrapy.utils.conf import get_config
20 from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
21 from scrapy.utils.datatypes import SequenceExclude
22 from scrapy.utils.misc import load_object
23 from scrapy.utils.response import open_in_browser
24 
25 
26 class Shell:
27 
28     relevant_classes = (Crawler, Spider, Request, Response, Settings)
29 
30     def __init__(self, crawler, update_vars=None, code=None):
31         self.crawler = crawler
32         self.update_vars = update_vars or (lambda x: None)
33         self.item_class = load_object(crawler.settings['DEFAULT_ITEM_CLASS'])
34         self.spider = None
35         self.inthread = not threadable.isInIOThread()
36         self.code = code
37         self.vars = {}
38 
39     def start(self, url=None, request=None, response=None, spider=None, redirect=True):
40         # disable accidental Ctrl-C key press from shutting down the engine
41         signal.signal(signal.SIGINT, signal.SIG_IGN)
42         if url:
43             self.fetch(url, spider, redirect=redirect)
44         elif request:
45             self.fetch(request, spider)
46         elif response:
47             request = response.request
48             self.populate_vars(response, request, spider)
49         else:
50             self.populate_vars()
51         if self.code:
52             print(eval(self.code, globals(), self.vars))
53         else:
54             """
55             Detect interactive shell setting in scrapy.cfg
56             e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg
57             [settings]
58             # shell can be one of ipython, bpython or python;
59             # to be used as the interactive python console, if available.
60             # (default is ipython, fallbacks in the order listed above)
61             shell = python
62             """
63             cfg = get_config()
64             section, option = 'settings', 'shell'
65             env = os.environ.get('SCRAPY_PYTHON_SHELL')
66             shells = []
67             if env:
68                 shells += env.strip().lower().split(',')
69             elif cfg.has_option(section, option):
70                 shells += [cfg.get(section, option).strip().lower()]
71             else:  # try all by default
72                 shells += DEFAULT_PYTHON_SHELLS.keys()
73             # always add standard shell as fallback
74             shells += ['python']
75             start_python_console(self.vars, shells=shells,
76                                  banner=self.vars.pop('banner', ''))
77 
78     def _schedule(self, request, spider):
79         spider = self._open_spider(request, spider)
80         d = _request_deferred(request)
81         d.addCallback(lambda x: (x, spider))
82         self.crawler.engine.crawl(request)
83         return d
84 
85     def _open_spider(self, request, spider):
86         if self.spider:
87             return self.spider
88 
89         if spider is None:
90             spider = self.crawler.spider or self.crawler._create_spider()
91 
92         self.crawler.spider = spider
93         self.crawler.engine.open_spider(spider, close_if_idle=False)
94         self.spider = spider
95         return spider
96 
97     def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):
98         from twisted.internet import reactor
99         if isinstance(request_or_url, Request):
100             request = request_or_url
101         else:
102             url = any_to_uri(request_or_url)
103             request = Request(url, dont_filter=True, **kwargs)
104             if redirect:
105                 request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))
106             else:
107                 request.meta['handle_httpstatus_all'] = True
108         response = None
109         try:
110             response, spider = threads.blockingCallFromThread(
111                 reactor, self._schedule, request, spider)
112         except IgnoreRequest:
113             pass
114         self.populate_vars(response, request, spider)
115 
116     def populate_vars(self, response=None, request=None, spider=None):
117         import scrapy
118 
119         self.vars['scrapy'] = scrapy
120         self.vars['crawler'] = self.crawler
121         self.vars['item'] = self.item_class()
122         self.vars['settings'] = self.crawler.settings
123         self.vars['spider'] = spider
124         self.vars['request'] = request
125         self.vars['response'] = response
126         if self.inthread:
127             self.vars['fetch'] = self.fetch
128         self.vars['view'] = open_in_browser
129         self.vars['shelp'] = self.print_help
130         self.update_vars(self.vars)
131         if not self.code:
132             self.vars['banner'] = self.get_help()
133 
134     def print_help(self):
135         print(self.get_help())
136 
137     def get_help(self):
138         b = []
139         b.append("Available Scrapy objects:")
140         b.append("  scrapy     scrapy module (contains scrapy.Request, scrapy.Selector, etc)")
141         for k, v in sorted(self.vars.items()):
142             if self._is_relevant(v):
143                 b.append(f"  {k:<10} {v}")
144         b.append("Useful shortcuts:")
145         if self.inthread:
146             b.append("  fetch(url[, redirect=True]) "
147                      "Fetch URL and update local objects (by default, redirects are followed)")
148             b.append("  fetch(req)                  "
149                      "Fetch a scrapy.Request and update local objects ")
150         b.append("  shelp()           Shell help (print this help)")
151         b.append("  view(response)    View response in a browser")
152 
153         return "\n".join(f"[s] {line}" for line in b)
154 
155     def _is_relevant(self, value):
156         return isinstance(value, self.relevant_classes) or is_item(value)
157 
158 
159 def inspect_response(response, spider):
160     """Open a shell to inspect the given response"""
161     Shell(spider.crawler).start(response=response, spider=spider)
162 
163 
164 def _request_deferred(request):
165     """Wrap a request inside a Deferred.
166 
167     This function is harmful, do not use it until you know what you are doing.
168 
169     This returns a Deferred whose first pair of callbacks are the request
170     callback and errback. The Deferred also triggers when the request
171     callback/errback is executed (i.e. when the request is downloaded)
172 
173     WARNING: Do not call request.replace() until after the deferred is called.
174     """
175     request_callback = request.callback
176     request_errback = request.errback
177 
178     def _restore_callbacks(result):
179         request.callback = request_callback
180         request.errback = request_errback
181         return result
182 
183     d = defer.Deferred()
184     d.addBoth(_restore_callbacks)
185     if request.callback:
186         d.addCallbacks(request.callback, request.errback)
187 
188     request.callback, request.errback = d.callback, d.errback
189     return d
190
```

Path: `scrapy/utils/reactor.py`
Content:
```
1 import asyncio
2 import sys
3 from contextlib import suppress
4 
5 from twisted.internet import asyncioreactor, error
6 
7 from scrapy.utils.misc import load_object
8 
9 
10 def listen_tcp(portrange, host, factory):
11     """Like reactor.listenTCP but tries different ports in a range."""
12     from twisted.internet import reactor
13     if len(portrange) > 2:
14         raise ValueError(f"invalid portrange: {portrange}")
15     if not portrange:
16         return reactor.listenTCP(0, factory, interface=host)
17     if not hasattr(portrange, '__iter__'):
18         return reactor.listenTCP(portrange, factory, interface=host)
19     if len(portrange) == 1:
20         return reactor.listenTCP(portrange[0], factory, interface=host)
21     for x in range(portrange[0], portrange[1] + 1):
22         try:
23             return reactor.listenTCP(x, factory, interface=host)
24         except error.CannotListenError:
25             if x == portrange[1]:
26                 raise
27 
28 
29 class CallLaterOnce:
30     """Schedule a function to be called in the next reactor loop, but only if
31     it hasn't been already scheduled since the last time it ran.
32     """
33 
34     def __init__(self, func, *a, **kw):
35         self._func = func
36         self._a = a
37         self._kw = kw
38         self._call = None
39 
40     def schedule(self, delay=0):
41         from twisted.internet import reactor
42         if self._call is None:
43             self._call = reactor.callLater(delay, self)
44 
45     def cancel(self):
46         if self._call:
47             self._call.cancel()
48 
49     def __call__(self):
50         self._call = None
51         return self._func(*self._a, **self._kw)
52 
53 
54 def get_asyncio_event_loop_policy():
55     policy = asyncio.get_event_loop_policy()
56     if (
57         sys.version_info >= (3, 8)
58         and sys.platform == "win32"
59         and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy)
60     ):
61         policy = asyncio.WindowsSelectorEventLoopPolicy()
62         asyncio.set_event_loop_policy(policy)
63 
64     return policy
65 
66 
67 def install_reactor(reactor_path, event_loop_path=None):
68     """Installs the :mod:`~twisted.internet.reactor` with the specified
69     import path. Also installs the asyncio event loop with the specified import
70     path if the asyncio reactor is enabled"""
71     reactor_class = load_object(reactor_path)
72     if reactor_class is asyncioreactor.AsyncioSelectorReactor:
73         with suppress(error.ReactorAlreadyInstalledError):
74             policy = get_asyncio_event_loop_policy()
75             if event_loop_path is not None:
76                 event_loop_class = load_object(event_loop_path)
77                 event_loop = event_loop_class()
78                 asyncio.set_event_loop(event_loop)
79             else:
80                 event_loop = policy.get_event_loop()
81 
82             asyncioreactor.install(eventloop=event_loop)
83     else:
84         *module, _ = reactor_path.split(".")
85         installer_path = module + ["install"]
86         installer = load_object(".".join(installer_path))
87         with suppress(error.ReactorAlreadyInstalledError):
88             installer()
89 
90 
91 def verify_installed_reactor(reactor_path):
92     """Raises :exc:`Exception` if the installed
93     :mod:`~twisted.internet.reactor` does not match the specified import
94     path."""
95     from twisted.internet import reactor
96     reactor_class = load_object(reactor_path)
97     if not reactor.__class__ == reactor_class:
98         msg = ("The installed reactor "
99                f"({reactor.__module__}.{reactor.__class__.__name__}) does not "
100                f"match the requested one ({reactor_path})")
101         raise Exception(msg)
102 
103 
104 def verify_installed_asyncio_event_loop(loop_path):
105     from twisted.internet import reactor
106     loop_class = load_object(loop_path)
107     if isinstance(reactor._asyncioEventloop, loop_class):
108         return
109     installed = (
110         f"{reactor._asyncioEventloop.__class__.__module__}"
111         f".{reactor._asyncioEventloop.__class__.__qualname__}"
112     )
113     specified = f"{loop_class.__module__}.{loop_class.__qualname__}"
114     raise Exception(
115         "Scrapy found an asyncio Twisted reactor already "
116         f"installed, and its event loop class ({installed}) does "
117         "not match the one specified in the ASYNCIO_EVENT_LOOP "
118         f"setting ({specified})"
119     )
120 
121 
122 def is_asyncio_reactor_installed():
123     from twisted.internet import reactor
124     return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)
125
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/shell.py b/scrapy/shell.py
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -21,6 +21,7 @@
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.misc import load_object
 from scrapy.utils.response import open_in_browser
+from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
 
 
 class Shell:
@@ -76,6 +77,10 @@
                                  banner=self.vars.pop('banner', ''))
 
     def _schedule(self, request, spider):
+        if is_asyncio_reactor_installed():
+            # set the asyncio event loop for the current thread
+            event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP']
+            set_asyncio_event_loop(event_loop_path)
         spider = self._open_spider(request, spider)
         d = _request_deferred(request)
         d.addCallback(lambda x: (x, spider))
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -71,14 +71,7 @@
     reactor_class = load_object(reactor_path)
     if reactor_class is asyncioreactor.AsyncioSelectorReactor:
         with suppress(error.ReactorAlreadyInstalledError):
-            policy = get_asyncio_event_loop_policy()
-            if event_loop_path is not None:
-                event_loop_class = load_object(event_loop_path)
-                event_loop = event_loop_class()
-                asyncio.set_event_loop(event_loop)
-            else:
-                event_loop = policy.get_event_loop()
-
+            event_loop = set_asyncio_event_loop(event_loop_path)
             asyncioreactor.install(eventloop=event_loop)
     else:
         *module, _ = reactor_path.split(".")
@@ -88,6 +81,25 @@
             installer()
 
 
+def set_asyncio_event_loop(event_loop_path):
+    """Sets and returns the event loop with specified import path."""
+    policy = get_asyncio_event_loop_policy()
+    if event_loop_path is not None:
+        event_loop_class = load_object(event_loop_path)
+        event_loop = event_loop_class()
+        asyncio.set_event_loop(event_loop)
+    else:
+        try:
+            event_loop = policy.get_event_loop()
+        except RuntimeError:
+            # `get_event_loop` is expected to fail when called from a new thread
+            # with no asyncio event loop yet installed. Such is the case when
+            # called from `scrapy shell`
+            event_loop = policy.new_event_loop()
+            asyncio.set_event_loop(event_loop)
+    return event_loop
+
+
 def verify_installed_reactor(reactor_path):
     """Raises :exc:`Exception` if the installed
     :mod:`~twisted.internet.reactor` does not match the specified import

{"golden_diff": "diff --git a/scrapy/shell.py b/scrapy/shell.py\n--- a/scrapy/shell.py\n+++ b/scrapy/shell.py\n@@ -21,6 +21,7 @@\n from scrapy.utils.datatypes import SequenceExclude\n from scrapy.utils.misc import load_object\n from scrapy.utils.response import open_in_browser\n+from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop\n \n \n class Shell:\n@@ -76,6 +77,10 @@\n                                  banner=self.vars.pop('banner', ''))\n \n     def _schedule(self, request, spider):\n+        if is_asyncio_reactor_installed():\n+            # set the asyncio event loop for the current thread\n+            event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP']\n+            set_asyncio_event_loop(event_loop_path)\n         spider = self._open_spider(request, spider)\n         d = _request_deferred(request)\n         d.addCallback(lambda x: (x, spider))\ndiff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py\n--- a/scrapy/utils/reactor.py\n+++ b/scrapy/utils/reactor.py\n@@ -71,14 +71,7 @@\n     reactor_class = load_object(reactor_path)\n     if reactor_class is asyncioreactor.AsyncioSelectorReactor:\n         with suppress(error.ReactorAlreadyInstalledError):\n-            policy = get_asyncio_event_loop_policy()\n-            if event_loop_path is not None:\n-                event_loop_class = load_object(event_loop_path)\n-                event_loop = event_loop_class()\n-                asyncio.set_event_loop(event_loop)\n-            else:\n-                event_loop = policy.get_event_loop()\n-\n+            event_loop = set_asyncio_event_loop(event_loop_path)\n             asyncioreactor.install(eventloop=event_loop)\n     else:\n         *module, _ = reactor_path.split(\".\")\n@@ -88,6 +81,25 @@\n             installer()\n \n \n+def set_asyncio_event_loop(event_loop_path):\n+    \"\"\"Sets and returns the event loop with specified import path.\"\"\"\n+    policy = get_asyncio_event_loop_policy()\n+    if event_loop_path is not None:\n+        event_loop_class = load_object(event_loop_path)\n+        event_loop = event_loop_class()\n+        asyncio.set_event_loop(event_loop)\n+    else:\n+        try:\n+            event_loop = policy.get_event_loop()\n+        except RuntimeError:\n+            # `get_event_loop` is expected to fail when called from a new thread\n+            # with no asyncio event loop yet installed. Such is the case when\n+            # called from `scrapy shell`\n+            event_loop = policy.new_event_loop()\n+            asyncio.set_event_loop(event_loop)\n+    return event_loop\n+\n+\n def verify_installed_reactor(reactor_path):\n     \"\"\"Raises :exc:`Exception` if the installed\n     :mod:`~twisted.internet.reactor` does not match the specified import\n", "issue": "Fixes issue with calling fetch in scrapy shell.\nReference Issue Fixes #5740 ,  #5742\r\n\r\nYou can recreate the issue with the following script.\r\n\r\n```\r\nimport asyncio\r\nimport threading\r\nfrom twisted.internet import asyncioreactor\r\nfrom scrapy.utils.defer import deferred_from_coro\r\nfrom scrapy.utils.reactor import get_asyncio_event_loop_policy\r\n\r\nasync def test_coro():\r\n    pass\r\n\r\ndef test_deferred_from_coro():\r\n    return deferred_from_coro(test_coro())\r\n\r\ndef trigger_warning_message():\r\n    event_loop = get_asyncio_event_loop_policy().new_event_loop()\r\n    asyncio.set_event_loop(event_loop)\r\n    asyncioreactor.install()\r\n    thread = threading.Thread(target=test_deferred_from_coro)\r\n    thread.start()\r\n    thread.join()\r\n\r\ntrigger_warning_message()\r\n```\r\n\r\nI was able to recreate this issue using `scrapy shell` and `fetch` on both windows and linux.  However it only occurs inside of a project with the `TWISTED_REACTOR` setting set to AsyncioSelectorReactor. \r\n\r\nWhat causes the issue is when `get_asyncio_event_loop_policy().get_event_loop()` is called, there is no event loop in the thread that the function is called from. Which raises an exception.  \nasyncio exception in scrapy shell\nWe got a bug report that using `fetch(<url>)` in `scrapy shell` on Windows in a new project (so with `TWISTED_REACTOR` set to the asyncio one) raises \"There is no current event loop\" in `deferred_from_coro()`. The Python version is 3.9 and 3.10 and the Scrapy version is 2.7.1. It is possible that the recenty changed loop code indeed doesn't work correctly on Windows but I cannot test or fix it at this time.\n", "before_files": [{"content": "\"\"\"Scrapy Shell\n\nSee documentation in docs/topics/shell.rst\n\n\"\"\"\nimport os\nimport signal\n\nfrom itemadapter import is_item\nfrom twisted.internet import threads, defer\nfrom twisted.python import threadable\nfrom w3lib.url import any_to_uri\n\nfrom scrapy.crawler import Crawler\nfrom scrapy.exceptions import IgnoreRequest\nfrom scrapy.http import Request, Response\nfrom scrapy.settings import Settings\nfrom scrapy.spiders import Spider\nfrom scrapy.utils.conf import get_config\nfrom scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console\nfrom scrapy.utils.datatypes import SequenceExclude\nfrom scrapy.utils.misc import load_object\nfrom scrapy.utils.response import open_in_browser\n\n\nclass Shell:\n\n    relevant_classes = (Crawler, Spider, Request, Response, Settings)\n\n    def __init__(self, crawler, update_vars=None, code=None):\n        self.crawler = crawler\n        self.update_vars = update_vars or (lambda x: None)\n        self.item_class = load_object(crawler.settings['DEFAULT_ITEM_CLASS'])\n        self.spider = None\n        self.inthread = not threadable.isInIOThread()\n        self.code = code\n        self.vars = {}\n\n    def start(self, url=None, request=None, response=None, spider=None, redirect=True):\n        # disable accidental Ctrl-C key press from shutting down the engine\n        signal.signal(signal.SIGINT, signal.SIG_IGN)\n        if url:\n            self.fetch(url, spider, redirect=redirect)\n        elif request:\n            self.fetch(request, spider)\n        elif response:\n            request = response.request\n            self.populate_vars(response, request, spider)\n        else:\n            self.populate_vars()\n        if self.code:\n            print(eval(self.code, globals(), self.vars))\n        else:\n            \"\"\"\n            Detect interactive shell setting in scrapy.cfg\n            e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg\n            [settings]\n            # shell can be one of ipython, bpython or python;\n            # to be used as the interactive python console, if available.\n            # (default is ipython, fallbacks in the order listed above)\n            shell = python\n            \"\"\"\n            cfg = get_config()\n            section, option = 'settings', 'shell'\n            env = os.environ.get('SCRAPY_PYTHON_SHELL')\n            shells = []\n            if env:\n                shells += env.strip().lower().split(',')\n            elif cfg.has_option(section, option):\n                shells += [cfg.get(section, option).strip().lower()]\n            else:  # try all by default\n                shells += DEFAULT_PYTHON_SHELLS.keys()\n            # always add standard shell as fallback\n            shells += ['python']\n            start_python_console(self.vars, shells=shells,\n                                 banner=self.vars.pop('banner', ''))\n\n    def _schedule(self, request, spider):\n        spider = self._open_spider(request, spider)\n        d = _request_deferred(request)\n        d.addCallback(lambda x: (x, spider))\n        self.crawler.engine.crawl(request)\n        return d\n\n    def _open_spider(self, request, spider):\n        if self.spider:\n            return self.spider\n\n        if spider is None:\n            spider = self.crawler.spider or self.crawler._create_spider()\n\n        self.crawler.spider = spider\n        self.crawler.engine.open_spider(spider, close_if_idle=False)\n        self.spider = spider\n        return spider\n\n    def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):\n        from twisted.internet import reactor\n        if isinstance(request_or_url, Request):\n            request = request_or_url\n        else:\n            url = any_to_uri(request_or_url)\n            request = Request(url, dont_filter=True, **kwargs)\n            if redirect:\n                request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))\n            else:\n                request.meta['handle_httpstatus_all'] = True\n        response = None\n        try:\n            response, spider = threads.blockingCallFromThread(\n                reactor, self._schedule, request, spider)\n        except IgnoreRequest:\n            pass\n        self.populate_vars(response, request, spider)\n\n    def populate_vars(self, response=None, request=None, spider=None):\n        import scrapy\n\n        self.vars['scrapy'] = scrapy\n        self.vars['crawler'] = self.crawler\n        self.vars['item'] = self.item_class()\n        self.vars['settings'] = self.crawler.settings\n        self.vars['spider'] = spider\n        self.vars['request'] = request\n        self.vars['response'] = response\n        if self.inthread:\n            self.vars['fetch'] = self.fetch\n        self.vars['view'] = open_in_browser\n        self.vars['shelp'] = self.print_help\n        self.update_vars(self.vars)\n        if not self.code:\n            self.vars['banner'] = self.get_help()\n\n    def print_help(self):\n        print(self.get_help())\n\n    def get_help(self):\n        b = []\n        b.append(\"Available Scrapy objects:\")\n        b.append(\"  scrapy     scrapy module (contains scrapy.Request, scrapy.Selector, etc)\")\n        for k, v in sorted(self.vars.items()):\n            if self._is_relevant(v):\n                b.append(f\"  {k:<10} {v}\")\n        b.append(\"Useful shortcuts:\")\n        if self.inthread:\n            b.append(\"  fetch(url[, redirect=True]) \"\n                     \"Fetch URL and update local objects (by default, redirects are followed)\")\n            b.append(\"  fetch(req)                  \"\n                     \"Fetch a scrapy.Request and update local objects \")\n        b.append(\"  shelp()           Shell help (print this help)\")\n        b.append(\"  view(response)    View response in a browser\")\n\n        return \"\\n\".join(f\"[s] {line}\" for line in b)\n\n    def _is_relevant(self, value):\n        return isinstance(value, self.relevant_classes) or is_item(value)\n\n\ndef inspect_response(response, spider):\n    \"\"\"Open a shell to inspect the given response\"\"\"\n    Shell(spider.crawler).start(response=response, spider=spider)\n\n\ndef _request_deferred(request):\n    \"\"\"Wrap a request inside a Deferred.\n\n    This function is harmful, do not use it until you know what you are doing.\n\n    This returns a Deferred whose first pair of callbacks are the request\n    callback and errback. The Deferred also triggers when the request\n    callback/errback is executed (i.e. when the request is downloaded)\n\n    WARNING: Do not call request.replace() until after the deferred is called.\n    \"\"\"\n    request_callback = request.callback\n    request_errback = request.errback\n\n    def _restore_callbacks(result):\n        request.callback = request_callback\n        request.errback = request_errback\n        return result\n\n    d = defer.Deferred()\n    d.addBoth(_restore_callbacks)\n    if request.callback:\n        d.addCallbacks(request.callback, request.errback)\n\n    request.callback, request.errback = d.callback, d.errback\n    return d\n", "path": "scrapy/shell.py"}, {"content": "import asyncio\nimport sys\nfrom contextlib import suppress\n\nfrom twisted.internet import asyncioreactor, error\n\nfrom scrapy.utils.misc import load_object\n\n\ndef listen_tcp(portrange, host, factory):\n    \"\"\"Like reactor.listenTCP but tries different ports in a range.\"\"\"\n    from twisted.internet import reactor\n    if len(portrange) > 2:\n        raise ValueError(f\"invalid portrange: {portrange}\")\n    if not portrange:\n        return reactor.listenTCP(0, factory, interface=host)\n    if not hasattr(portrange, '__iter__'):\n        return reactor.listenTCP(portrange, factory, interface=host)\n    if len(portrange) == 1:\n        return reactor.listenTCP(portrange[0], factory, interface=host)\n    for x in range(portrange[0], portrange[1] + 1):\n        try:\n            return reactor.listenTCP(x, factory, interface=host)\n        except error.CannotListenError:\n            if x == portrange[1]:\n                raise\n\n\nclass CallLaterOnce:\n    \"\"\"Schedule a function to be called in the next reactor loop, but only if\n    it hasn't been already scheduled since the last time it ran.\n    \"\"\"\n\n    def __init__(self, func, *a, **kw):\n        self._func = func\n        self._a = a\n        self._kw = kw\n        self._call = None\n\n    def schedule(self, delay=0):\n        from twisted.internet import reactor\n        if self._call is None:\n            self._call = reactor.callLater(delay, self)\n\n    def cancel(self):\n        if self._call:\n            self._call.cancel()\n\n    def __call__(self):\n        self._call = None\n        return self._func(*self._a, **self._kw)\n\n\ndef get_asyncio_event_loop_policy():\n    policy = asyncio.get_event_loop_policy()\n    if (\n        sys.version_info >= (3, 8)\n        and sys.platform == \"win32\"\n        and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy)\n    ):\n        policy = asyncio.WindowsSelectorEventLoopPolicy()\n        asyncio.set_event_loop_policy(policy)\n\n    return policy\n\n\ndef install_reactor(reactor_path, event_loop_path=None):\n    \"\"\"Installs the :mod:`~twisted.internet.reactor` with the specified\n    import path. Also installs the asyncio event loop with the specified import\n    path if the asyncio reactor is enabled\"\"\"\n    reactor_class = load_object(reactor_path)\n    if reactor_class is asyncioreactor.AsyncioSelectorReactor:\n        with suppress(error.ReactorAlreadyInstalledError):\n            policy = get_asyncio_event_loop_policy()\n            if event_loop_path is not None:\n                event_loop_class = load_object(event_loop_path)\n                event_loop = event_loop_class()\n                asyncio.set_event_loop(event_loop)\n            else:\n                event_loop = policy.get_event_loop()\n\n            asyncioreactor.install(eventloop=event_loop)\n    else:\n        *module, _ = reactor_path.split(\".\")\n        installer_path = module + [\"install\"]\n        installer = load_object(\".\".join(installer_path))\n        with suppress(error.ReactorAlreadyInstalledError):\n            installer()\n\n\ndef verify_installed_reactor(reactor_path):\n    \"\"\"Raises :exc:`Exception` if the installed\n    :mod:`~twisted.internet.reactor` does not match the specified import\n    path.\"\"\"\n    from twisted.internet import reactor\n    reactor_class = load_object(reactor_path)\n    if not reactor.__class__ == reactor_class:\n        msg = (\"The installed reactor \"\n               f\"({reactor.__module__}.{reactor.__class__.__name__}) does not \"\n               f\"match the requested one ({reactor_path})\")\n        raise Exception(msg)\n\n\ndef verify_installed_asyncio_event_loop(loop_path):\n    from twisted.internet import reactor\n    loop_class = load_object(loop_path)\n    if isinstance(reactor._asyncioEventloop, loop_class):\n        return\n    installed = (\n        f\"{reactor._asyncioEventloop.__class__.__module__}\"\n        f\".{reactor._asyncioEventloop.__class__.__qualname__}\"\n    )\n    specified = f\"{loop_class.__module__}.{loop_class.__qualname__}\"\n    raise Exception(\n        \"Scrapy found an asyncio Twisted reactor already \"\n        f\"installed, and its event loop class ({installed}) does \"\n        \"not match the one specified in the ASYNCIO_EVENT_LOOP \"\n        f\"setting ({specified})\"\n    )\n\n\ndef is_asyncio_reactor_installed():\n    from twisted.internet import reactor\n    return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)\n", "path": "scrapy/utils/reactor.py"}], "after_files": [{"content": "\"\"\"Scrapy Shell\n\nSee documentation in docs/topics/shell.rst\n\n\"\"\"\nimport os\nimport signal\n\nfrom itemadapter import is_item\nfrom twisted.internet import threads, defer\nfrom twisted.python import threadable\nfrom w3lib.url import any_to_uri\n\nfrom scrapy.crawler import Crawler\nfrom scrapy.exceptions import IgnoreRequest\nfrom scrapy.http import Request, Response\nfrom scrapy.settings import Settings\nfrom scrapy.spiders import Spider\nfrom scrapy.utils.conf import get_config\nfrom scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console\nfrom scrapy.utils.datatypes import SequenceExclude\nfrom scrapy.utils.misc import load_object\nfrom scrapy.utils.response import open_in_browser\nfrom scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop\n\n\nclass Shell:\n\n    relevant_classes = (Crawler, Spider, Request, Response, Settings)\n\n    def __init__(self, crawler, update_vars=None, code=None):\n        self.crawler = crawler\n        self.update_vars = update_vars or (lambda x: None)\n        self.item_class = load_object(crawler.settings['DEFAULT_ITEM_CLASS'])\n        self.spider = None\n        self.inthread = not threadable.isInIOThread()\n        self.code = code\n        self.vars = {}\n\n    def start(self, url=None, request=None, response=None, spider=None, redirect=True):\n        # disable accidental Ctrl-C key press from shutting down the engine\n        signal.signal(signal.SIGINT, signal.SIG_IGN)\n        if url:\n            self.fetch(url, spider, redirect=redirect)\n        elif request:\n            self.fetch(request, spider)\n        elif response:\n            request = response.request\n            self.populate_vars(response, request, spider)\n        else:\n            self.populate_vars()\n        if self.code:\n            print(eval(self.code, globals(), self.vars))\n        else:\n            \"\"\"\n            Detect interactive shell setting in scrapy.cfg\n            e.g.: ~/.config/scrapy.cfg or ~/.scrapy.cfg\n            [settings]\n            # shell can be one of ipython, bpython or python;\n            # to be used as the interactive python console, if available.\n            # (default is ipython, fallbacks in the order listed above)\n            shell = python\n            \"\"\"\n            cfg = get_config()\n            section, option = 'settings', 'shell'\n            env = os.environ.get('SCRAPY_PYTHON_SHELL')\n            shells = []\n            if env:\n                shells += env.strip().lower().split(',')\n            elif cfg.has_option(section, option):\n                shells += [cfg.get(section, option).strip().lower()]\n            else:  # try all by default\n                shells += DEFAULT_PYTHON_SHELLS.keys()\n            # always add standard shell as fallback\n            shells += ['python']\n            start_python_console(self.vars, shells=shells,\n                                 banner=self.vars.pop('banner', ''))\n\n    def _schedule(self, request, spider):\n        if is_asyncio_reactor_installed():\n            # set the asyncio event loop for the current thread\n            event_loop_path = self.crawler.settings['ASYNCIO_EVENT_LOOP']\n            set_asyncio_event_loop(event_loop_path)\n        spider = self._open_spider(request, spider)\n        d = _request_deferred(request)\n        d.addCallback(lambda x: (x, spider))\n        self.crawler.engine.crawl(request)\n        return d\n\n    def _open_spider(self, request, spider):\n        if self.spider:\n            return self.spider\n\n        if spider is None:\n            spider = self.crawler.spider or self.crawler._create_spider()\n\n        self.crawler.spider = spider\n        self.crawler.engine.open_spider(spider, close_if_idle=False)\n        self.spider = spider\n        return spider\n\n    def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):\n        from twisted.internet import reactor\n        if isinstance(request_or_url, Request):\n            request = request_or_url\n        else:\n            url = any_to_uri(request_or_url)\n            request = Request(url, dont_filter=True, **kwargs)\n            if redirect:\n                request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400))\n            else:\n                request.meta['handle_httpstatus_all'] = True\n        response = None\n        try:\n            response, spider = threads.blockingCallFromThread(\n                reactor, self._schedule, request, spider)\n        except IgnoreRequest:\n            pass\n        self.populate_vars(response, request, spider)\n\n    def populate_vars(self, response=None, request=None, spider=None):\n        import scrapy\n\n        self.vars['scrapy'] = scrapy\n        self.vars['crawler'] = self.crawler\n        self.vars['item'] = self.item_class()\n        self.vars['settings'] = self.crawler.settings\n        self.vars['spider'] = spider\n        self.vars['request'] = request\n        self.vars['response'] = response\n        if self.inthread:\n            self.vars['fetch'] = self.fetch\n        self.vars['view'] = open_in_browser\n        self.vars['shelp'] = self.print_help\n        self.update_vars(self.vars)\n        if not self.code:\n            self.vars['banner'] = self.get_help()\n\n    def print_help(self):\n        print(self.get_help())\n\n    def get_help(self):\n        b = []\n        b.append(\"Available Scrapy objects:\")\n        b.append(\"  scrapy     scrapy module (contains scrapy.Request, scrapy.Selector, etc)\")\n        for k, v in sorted(self.vars.items()):\n            if self._is_relevant(v):\n                b.append(f\"  {k:<10} {v}\")\n        b.append(\"Useful shortcuts:\")\n        if self.inthread:\n            b.append(\"  fetch(url[, redirect=True]) \"\n                     \"Fetch URL and update local objects (by default, redirects are followed)\")\n            b.append(\"  fetch(req)                  \"\n                     \"Fetch a scrapy.Request and update local objects \")\n        b.append(\"  shelp()           Shell help (print this help)\")\n        b.append(\"  view(response)    View response in a browser\")\n\n        return \"\\n\".join(f\"[s] {line}\" for line in b)\n\n    def _is_relevant(self, value):\n        return isinstance(value, self.relevant_classes) or is_item(value)\n\n\ndef inspect_response(response, spider):\n    \"\"\"Open a shell to inspect the given response\"\"\"\n    Shell(spider.crawler).start(response=response, spider=spider)\n\n\ndef _request_deferred(request):\n    \"\"\"Wrap a request inside a Deferred.\n\n    This function is harmful, do not use it until you know what you are doing.\n\n    This returns a Deferred whose first pair of callbacks are the request\n    callback and errback. The Deferred also triggers when the request\n    callback/errback is executed (i.e. when the request is downloaded)\n\n    WARNING: Do not call request.replace() until after the deferred is called.\n    \"\"\"\n    request_callback = request.callback\n    request_errback = request.errback\n\n    def _restore_callbacks(result):\n        request.callback = request_callback\n        request.errback = request_errback\n        return result\n\n    d = defer.Deferred()\n    d.addBoth(_restore_callbacks)\n    if request.callback:\n        d.addCallbacks(request.callback, request.errback)\n\n    request.callback, request.errback = d.callback, d.errback\n    return d\n", "path": "scrapy/shell.py"}, {"content": "import asyncio\nimport sys\nfrom contextlib import suppress\n\nfrom twisted.internet import asyncioreactor, error\n\nfrom scrapy.utils.misc import load_object\n\n\ndef listen_tcp(portrange, host, factory):\n    \"\"\"Like reactor.listenTCP but tries different ports in a range.\"\"\"\n    from twisted.internet import reactor\n    if len(portrange) > 2:\n        raise ValueError(f\"invalid portrange: {portrange}\")\n    if not portrange:\n        return reactor.listenTCP(0, factory, interface=host)\n    if not hasattr(portrange, '__iter__'):\n        return reactor.listenTCP(portrange, factory, interface=host)\n    if len(portrange) == 1:\n        return reactor.listenTCP(portrange[0], factory, interface=host)\n    for x in range(portrange[0], portrange[1] + 1):\n        try:\n            return reactor.listenTCP(x, factory, interface=host)\n        except error.CannotListenError:\n            if x == portrange[1]:\n                raise\n\n\nclass CallLaterOnce:\n    \"\"\"Schedule a function to be called in the next reactor loop, but only if\n    it hasn't been already scheduled since the last time it ran.\n    \"\"\"\n\n    def __init__(self, func, *a, **kw):\n        self._func = func\n        self._a = a\n        self._kw = kw\n        self._call = None\n\n    def schedule(self, delay=0):\n        from twisted.internet import reactor\n        if self._call is None:\n            self._call = reactor.callLater(delay, self)\n\n    def cancel(self):\n        if self._call:\n            self._call.cancel()\n\n    def __call__(self):\n        self._call = None\n        return self._func(*self._a, **self._kw)\n\n\ndef get_asyncio_event_loop_policy():\n    policy = asyncio.get_event_loop_policy()\n    if (\n        sys.version_info >= (3, 8)\n        and sys.platform == \"win32\"\n        and not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy)\n    ):\n        policy = asyncio.WindowsSelectorEventLoopPolicy()\n        asyncio.set_event_loop_policy(policy)\n\n    return policy\n\n\ndef install_reactor(reactor_path, event_loop_path=None):\n    \"\"\"Installs the :mod:`~twisted.internet.reactor` with the specified\n    import path. Also installs the asyncio event loop with the specified import\n    path if the asyncio reactor is enabled\"\"\"\n    reactor_class = load_object(reactor_path)\n    if reactor_class is asyncioreactor.AsyncioSelectorReactor:\n        with suppress(error.ReactorAlreadyInstalledError):\n            event_loop = set_asyncio_event_loop(event_loop_path)\n            asyncioreactor.install(eventloop=event_loop)\n    else:\n        *module, _ = reactor_path.split(\".\")\n        installer_path = module + [\"install\"]\n        installer = load_object(\".\".join(installer_path))\n        with suppress(error.ReactorAlreadyInstalledError):\n            installer()\n\n\ndef set_asyncio_event_loop(event_loop_path):\n    \"\"\"Sets and returns the event loop with specified import path.\"\"\"\n    policy = get_asyncio_event_loop_policy()\n    if event_loop_path is not None:\n        event_loop_class = load_object(event_loop_path)\n        event_loop = event_loop_class()\n        asyncio.set_event_loop(event_loop)\n    else:\n        try:\n            event_loop = policy.get_event_loop()\n        except RuntimeError:\n            # `get_event_loop` is expected to fail when called from a new thread\n            # with no asyncio event loop yet installed. Such is the case when\n            # called from `scrapy shell`\n            event_loop = policy.new_event_loop()\n            asyncio.set_event_loop(event_loop)\n    return event_loop\n\n\ndef verify_installed_reactor(reactor_path):\n    \"\"\"Raises :exc:`Exception` if the installed\n    :mod:`~twisted.internet.reactor` does not match the specified import\n    path.\"\"\"\n    from twisted.internet import reactor\n    reactor_class = load_object(reactor_path)\n    if not reactor.__class__ == reactor_class:\n        msg = (\"The installed reactor \"\n               f\"({reactor.__module__}.{reactor.__class__.__name__}) does not \"\n               f\"match the requested one ({reactor_path})\")\n        raise Exception(msg)\n\n\ndef verify_installed_asyncio_event_loop(loop_path):\n    from twisted.internet import reactor\n    loop_class = load_object(loop_path)\n    if isinstance(reactor._asyncioEventloop, loop_class):\n        return\n    installed = (\n        f\"{reactor._asyncioEventloop.__class__.__module__}\"\n        f\".{reactor._asyncioEventloop.__class__.__qualname__}\"\n    )\n    specified = f\"{loop_class.__module__}.{loop_class.__qualname__}\"\n    raise Exception(\n        \"Scrapy found an asyncio Twisted reactor already \"\n        f\"installed, and its event loop class ({installed}) does \"\n        \"not match the one specified in the ASYNCIO_EVENT_LOOP \"\n        f\"setting ({specified})\"\n    )\n\n\ndef is_asyncio_reactor_installed():\n    from twisted.internet import reactor\n    return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)\n", "path": "scrapy/utils/reactor.py"}]}