We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Enabled extensions, middlewares, pipelines
I found that this information isn't being printed anymore.

The responsible of this bug is [this line](https://github.com/scrapy/scrapy/pull/816/files#diff-fee03a44ad4de98d9361d89947c8aba3R83), seems that `spider` is `None` on `eventDict` at the moment the components are instantiated.

I'm not sure how to fix it because I 'm not quite sure what it is attempting to block.

/cc @curita


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/crawler.py`
Content:
```
1 import six
2 import signal
3 import warnings
4 
5 from twisted.internet import reactor, defer
6 
7 from scrapy.core.engine import ExecutionEngine
8 from scrapy.resolver import CachingThreadedResolver
9 from scrapy.extension import ExtensionManager
10 from scrapy.signalmanager import SignalManager
11 from scrapy.exceptions import ScrapyDeprecationWarning
12 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
13 from scrapy.utils.misc import load_object
14 from scrapy import log, signals
15 
16 
17 class Crawler(object):
18 
19     def __init__(self, spidercls, settings):
20         self.spidercls = spidercls
21         self.settings = settings
22         self.signals = SignalManager(self)
23         self.stats = load_object(self.settings['STATS_CLASS'])(self)
24         lf_cls = load_object(self.settings['LOG_FORMATTER'])
25         self.logformatter = lf_cls.from_crawler(self)
26         self.extensions = ExtensionManager.from_crawler(self)
27 
28         self.crawling = False
29         self.spider = None
30         self.engine = None
31 
32     @property
33     def spiders(self):
34         if not hasattr(self, '_spiders'):
35             warnings.warn("Crawler.spiders is deprecated, use "
36                           "CrawlerRunner.spiders or instantiate "
37                           "scrapy.spidermanager.SpiderManager with your "
38                           "settings.",
39                           category=ScrapyDeprecationWarning, stacklevel=2)
40             spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
41             self._spiders = spman_cls.from_settings(self.settings)
42         return self._spiders
43 
44     @defer.inlineCallbacks
45     def crawl(self, *args, **kwargs):
46         assert not self.crawling, "Crawling already taking place"
47         self.crawling = True
48 
49         try:
50             self.spider = self._create_spider(*args, **kwargs)
51             self.engine = self._create_engine()
52             start_requests = iter(self.spider.start_requests())
53             yield self.engine.open_spider(self.spider, start_requests)
54             yield defer.maybeDeferred(self.engine.start)
55         except Exception:
56             self.crawling = False
57             raise
58 
59     def _create_spider(self, *args, **kwargs):
60         return self.spidercls.from_crawler(self, *args, **kwargs)
61 
62     def _create_engine(self):
63         return ExecutionEngine(self, lambda _: self.stop())
64 
65     @defer.inlineCallbacks
66     def stop(self):
67         if self.crawling:
68             self.crawling = False
69             yield defer.maybeDeferred(self.engine.stop)
70 
71 
72 class CrawlerRunner(object):
73 
74     def __init__(self, settings):
75         self.settings = settings
76         smcls = load_object(settings['SPIDER_MANAGER_CLASS'])
77         self.spiders = smcls.from_settings(settings.frozencopy())
78         self.crawlers = set()
79         self.crawl_deferreds = set()
80 
81     def crawl(self, spidercls, *args, **kwargs):
82         crawler = self._create_logged_crawler(spidercls)
83         self.crawlers.add(crawler)
84 
85         d = crawler.crawl(*args, **kwargs)
86         self.crawl_deferreds.add(d)
87         return d
88 
89     def _create_logged_crawler(self, spidercls):
90         crawler = self._create_crawler(spidercls)
91         log_observer = log.start_from_crawler(crawler)
92         if log_observer:
93             crawler.signals.connect(log_observer.stop, signals.engine_stopped)
94         return crawler
95 
96     def _create_crawler(self, spidercls):
97         if isinstance(spidercls, six.string_types):
98             spidercls = self.spiders.load(spidercls)
99 
100         crawler_settings = self.settings.copy()
101         spidercls.update_settings(crawler_settings)
102         crawler_settings.freeze()
103 
104         crawler = Crawler(spidercls, crawler_settings)
105         return crawler
106 
107     def stop(self):
108         return defer.DeferredList(c.stop() for c in self.crawlers)
109 
110 
111 class CrawlerProcess(CrawlerRunner):
112     """A class to run multiple scrapy crawlers in a process simultaneously"""
113 
114     def __init__(self, settings):
115         super(CrawlerProcess, self).__init__(settings)
116         install_shutdown_handlers(self._signal_shutdown)
117         self.stopping = False
118 
119     def _signal_shutdown(self, signum, _):
120         install_shutdown_handlers(self._signal_kill)
121         signame = signal_names[signum]
122         log.msg(format="Received %(signame)s, shutting down gracefully. Send again to force ",
123                 level=log.INFO, signame=signame)
124         reactor.callFromThread(self.stop)
125 
126     def _signal_kill(self, signum, _):
127         install_shutdown_handlers(signal.SIG_IGN)
128         signame = signal_names[signum]
129         log.msg(format='Received %(signame)s twice, forcing unclean shutdown',
130                 level=log.INFO, signame=signame)
131         self._stop_logging()
132         reactor.callFromThread(self._stop_reactor)
133 
134     def start(self, stop_after_crawl=True, start_reactor=True):
135         self.log_observer = log.start_from_settings(self.settings)
136         log.scrapy_info(self.settings)
137         if start_reactor:
138             self._start_reactor(stop_after_crawl)
139 
140     def _start_reactor(self, stop_after_crawl=True):
141         if stop_after_crawl:
142             d = defer.DeferredList(self.crawl_deferreds)
143             if d.called:
144                 # Don't start the reactor if the deferreds are already fired
145                 return
146             d.addBoth(lambda _: self._stop_reactor())
147         if self.settings.getbool('DNSCACHE_ENABLED'):
148             reactor.installResolver(CachingThreadedResolver(reactor))
149         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
150         reactor.run(installSignalHandlers=False)  # blocking call
151 
152     def _stop_logging(self):
153         self.log_observer.stop()
154 
155     def _stop_reactor(self, _=None):
156         try:
157             reactor.stop()
158         except RuntimeError:  # raised if already stopped or in shutdown stage
159             pass
160
```

Path: `scrapy/commands/shell.py`
Content:
```
1 """
2 Scrapy Shell
3 
4 See documentation in docs/topics/shell.rst
5 """
6 
7 from threading import Thread
8 
9 from scrapy.command import ScrapyCommand
10 from scrapy.shell import Shell
11 from scrapy.http import Request
12 from scrapy import log
13 from scrapy.utils.spider import spidercls_for_request, DefaultSpider
14 
15 
16 class Command(ScrapyCommand):
17 
18     requires_project = False
19     default_settings = {'KEEP_ALIVE': True, 'LOGSTATS_INTERVAL': 0}
20 
21     def syntax(self):
22         return "[url|file]"
23 
24     def short_desc(self):
25         return "Interactive scraping console"
26 
27     def long_desc(self):
28         return "Interactive console for scraping the given url"
29 
30     def add_options(self, parser):
31         ScrapyCommand.add_options(self, parser)
32         parser.add_option("-c", dest="code",
33             help="evaluate the code in the shell, print the result and exit")
34         parser.add_option("--spider", dest="spider",
35             help="use this spider")
36 
37     def update_vars(self, vars):
38         """You can use this function to update the Scrapy objects that will be
39         available in the shell
40         """
41         pass
42 
43     def run(self, args, opts):
44         url = args[0] if args else None
45         spiders = self.crawler_process.spiders
46 
47         spidercls = DefaultSpider
48         if opts.spider:
49             spidercls = spiders.load(opts.spider)
50         elif url:
51             spidercls = spidercls_for_request(spiders, Request(url),
52                                               spidercls, log_multiple=True)
53 
54         # The crawler is created this way since the Shell manually handles the
55         # crawling engine, so the set up in the crawl method won't work
56         crawler = self.crawler_process._create_logged_crawler(spidercls)
57         # The Shell class needs a persistent engine in the crawler
58         crawler.engine = crawler._create_engine()
59         crawler.engine.start()
60 
61         self.crawler_process.start(start_reactor=False)
62         self._start_crawler_thread()
63 
64         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
65         shell.start(url=url)
66 
67     def _start_crawler_thread(self):
68         t = Thread(target=self.crawler_process._start_reactor,
69                    kwargs={'stop_after_crawl': False})
70         t.daemon = True
71         t.start()
72
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -58,14 +58,13 @@
         crawler.engine = crawler._create_engine()
         crawler.engine.start()
 
-        self.crawler_process.start(start_reactor=False)
         self._start_crawler_thread()
 
         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
         shell.start(url=url)
 
     def _start_crawler_thread(self):
-        t = Thread(target=self.crawler_process._start_reactor,
+        t = Thread(target=self.crawler_process.start,
                    kwargs={'stop_after_crawl': False})
         t.daemon = True
         t.start()
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -115,6 +115,8 @@
         super(CrawlerProcess, self).__init__(settings)
         install_shutdown_handlers(self._signal_shutdown)
         self.stopping = False
+        self.log_observer = log.start_from_settings(self.settings)
+        log.scrapy_info(settings)
 
     def _signal_shutdown(self, signum, _):
         install_shutdown_handlers(self._signal_kill)
@@ -131,13 +133,7 @@
         self._stop_logging()
         reactor.callFromThread(self._stop_reactor)
 
-    def start(self, stop_after_crawl=True, start_reactor=True):
-        self.log_observer = log.start_from_settings(self.settings)
-        log.scrapy_info(self.settings)
-        if start_reactor:
-            self._start_reactor(stop_after_crawl)
-
-    def _start_reactor(self, stop_after_crawl=True):
+    def start(self, stop_after_crawl=True):
         if stop_after_crawl:
             d = defer.DeferredList(self.crawl_deferreds)
             if d.called:

{"golden_diff": "diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py\n--- a/scrapy/commands/shell.py\n+++ b/scrapy/commands/shell.py\n@@ -58,14 +58,13 @@\n         crawler.engine = crawler._create_engine()\n         crawler.engine.start()\n \n-        self.crawler_process.start(start_reactor=False)\n         self._start_crawler_thread()\n \n         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)\n         shell.start(url=url)\n \n     def _start_crawler_thread(self):\n-        t = Thread(target=self.crawler_process._start_reactor,\n+        t = Thread(target=self.crawler_process.start,\n                    kwargs={'stop_after_crawl': False})\n         t.daemon = True\n         t.start()\ndiff --git a/scrapy/crawler.py b/scrapy/crawler.py\n--- a/scrapy/crawler.py\n+++ b/scrapy/crawler.py\n@@ -115,6 +115,8 @@\n         super(CrawlerProcess, self).__init__(settings)\n         install_shutdown_handlers(self._signal_shutdown)\n         self.stopping = False\n+        self.log_observer = log.start_from_settings(self.settings)\n+        log.scrapy_info(settings)\n \n     def _signal_shutdown(self, signum, _):\n         install_shutdown_handlers(self._signal_kill)\n@@ -131,13 +133,7 @@\n         self._stop_logging()\n         reactor.callFromThread(self._stop_reactor)\n \n-    def start(self, stop_after_crawl=True, start_reactor=True):\n-        self.log_observer = log.start_from_settings(self.settings)\n-        log.scrapy_info(self.settings)\n-        if start_reactor:\n-            self._start_reactor(stop_after_crawl)\n-\n-    def _start_reactor(self, stop_after_crawl=True):\n+    def start(self, stop_after_crawl=True):\n         if stop_after_crawl:\n             d = defer.DeferredList(self.crawl_deferreds)\n             if d.called:\n", "issue": "Enabled extensions, middlewares, pipelines\nI found that this information isn't being printed anymore.\n\nThe responsible of this bug is [this line](https://github.com/scrapy/scrapy/pull/816/files#diff-fee03a44ad4de98d9361d89947c8aba3R83), seems that `spider` is `None` on `eventDict` at the moment the components are instantiated.\n\nI'm not sure how to fix it because I 'm not quite sure what it is attempting to block.\n\n/cc @curita\n\n", "before_files": [{"content": "import six\nimport signal\nimport warnings\n\nfrom twisted.internet import reactor, defer\n\nfrom scrapy.core.engine import ExecutionEngine\nfrom scrapy.resolver import CachingThreadedResolver\nfrom scrapy.extension import ExtensionManager\nfrom scrapy.signalmanager import SignalManager\nfrom scrapy.exceptions import ScrapyDeprecationWarning\nfrom scrapy.utils.ossignal import install_shutdown_handlers, signal_names\nfrom scrapy.utils.misc import load_object\nfrom scrapy import log, signals\n\n\nclass Crawler(object):\n\n    def __init__(self, spidercls, settings):\n        self.spidercls = spidercls\n        self.settings = settings\n        self.signals = SignalManager(self)\n        self.stats = load_object(self.settings['STATS_CLASS'])(self)\n        lf_cls = load_object(self.settings['LOG_FORMATTER'])\n        self.logformatter = lf_cls.from_crawler(self)\n        self.extensions = ExtensionManager.from_crawler(self)\n\n        self.crawling = False\n        self.spider = None\n        self.engine = None\n\n    @property\n    def spiders(self):\n        if not hasattr(self, '_spiders'):\n            warnings.warn(\"Crawler.spiders is deprecated, use \"\n                          \"CrawlerRunner.spiders or instantiate \"\n                          \"scrapy.spidermanager.SpiderManager with your \"\n                          \"settings.\",\n                          category=ScrapyDeprecationWarning, stacklevel=2)\n            spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])\n            self._spiders = spman_cls.from_settings(self.settings)\n        return self._spiders\n\n    @defer.inlineCallbacks\n    def crawl(self, *args, **kwargs):\n        assert not self.crawling, \"Crawling already taking place\"\n        self.crawling = True\n\n        try:\n            self.spider = self._create_spider(*args, **kwargs)\n            self.engine = self._create_engine()\n            start_requests = iter(self.spider.start_requests())\n            yield self.engine.open_spider(self.spider, start_requests)\n            yield defer.maybeDeferred(self.engine.start)\n        except Exception:\n            self.crawling = False\n            raise\n\n    def _create_spider(self, *args, **kwargs):\n        return self.spidercls.from_crawler(self, *args, **kwargs)\n\n    def _create_engine(self):\n        return ExecutionEngine(self, lambda _: self.stop())\n\n    @defer.inlineCallbacks\n    def stop(self):\n        if self.crawling:\n            self.crawling = False\n            yield defer.maybeDeferred(self.engine.stop)\n\n\nclass CrawlerRunner(object):\n\n    def __init__(self, settings):\n        self.settings = settings\n        smcls = load_object(settings['SPIDER_MANAGER_CLASS'])\n        self.spiders = smcls.from_settings(settings.frozencopy())\n        self.crawlers = set()\n        self.crawl_deferreds = set()\n\n    def crawl(self, spidercls, *args, **kwargs):\n        crawler = self._create_logged_crawler(spidercls)\n        self.crawlers.add(crawler)\n\n        d = crawler.crawl(*args, **kwargs)\n        self.crawl_deferreds.add(d)\n        return d\n\n    def _create_logged_crawler(self, spidercls):\n        crawler = self._create_crawler(spidercls)\n        log_observer = log.start_from_crawler(crawler)\n        if log_observer:\n            crawler.signals.connect(log_observer.stop, signals.engine_stopped)\n        return crawler\n\n    def _create_crawler(self, spidercls):\n        if isinstance(spidercls, six.string_types):\n            spidercls = self.spiders.load(spidercls)\n\n        crawler_settings = self.settings.copy()\n        spidercls.update_settings(crawler_settings)\n        crawler_settings.freeze()\n\n        crawler = Crawler(spidercls, crawler_settings)\n        return crawler\n\n    def stop(self):\n        return defer.DeferredList(c.stop() for c in self.crawlers)\n\n\nclass CrawlerProcess(CrawlerRunner):\n    \"\"\"A class to run multiple scrapy crawlers in a process simultaneously\"\"\"\n\n    def __init__(self, settings):\n        super(CrawlerProcess, self).__init__(settings)\n        install_shutdown_handlers(self._signal_shutdown)\n        self.stopping = False\n\n    def _signal_shutdown(self, signum, _):\n        install_shutdown_handlers(self._signal_kill)\n        signame = signal_names[signum]\n        log.msg(format=\"Received %(signame)s, shutting down gracefully. Send again to force \",\n                level=log.INFO, signame=signame)\n        reactor.callFromThread(self.stop)\n\n    def _signal_kill(self, signum, _):\n        install_shutdown_handlers(signal.SIG_IGN)\n        signame = signal_names[signum]\n        log.msg(format='Received %(signame)s twice, forcing unclean shutdown',\n                level=log.INFO, signame=signame)\n        self._stop_logging()\n        reactor.callFromThread(self._stop_reactor)\n\n    def start(self, stop_after_crawl=True, start_reactor=True):\n        self.log_observer = log.start_from_settings(self.settings)\n        log.scrapy_info(self.settings)\n        if start_reactor:\n            self._start_reactor(stop_after_crawl)\n\n    def _start_reactor(self, stop_after_crawl=True):\n        if stop_after_crawl:\n            d = defer.DeferredList(self.crawl_deferreds)\n            if d.called:\n                # Don't start the reactor if the deferreds are already fired\n                return\n            d.addBoth(lambda _: self._stop_reactor())\n        if self.settings.getbool('DNSCACHE_ENABLED'):\n            reactor.installResolver(CachingThreadedResolver(reactor))\n        reactor.addSystemEventTrigger('before', 'shutdown', self.stop)\n        reactor.run(installSignalHandlers=False)  # blocking call\n\n    def _stop_logging(self):\n        self.log_observer.stop()\n\n    def _stop_reactor(self, _=None):\n        try:\n            reactor.stop()\n        except RuntimeError:  # raised if already stopped or in shutdown stage\n            pass\n", "path": "scrapy/crawler.py"}, {"content": "\"\"\"\nScrapy Shell\n\nSee documentation in docs/topics/shell.rst\n\"\"\"\n\nfrom threading import Thread\n\nfrom scrapy.command import ScrapyCommand\nfrom scrapy.shell import Shell\nfrom scrapy.http import Request\nfrom scrapy import log\nfrom scrapy.utils.spider import spidercls_for_request, DefaultSpider\n\n\nclass Command(ScrapyCommand):\n\n    requires_project = False\n    default_settings = {'KEEP_ALIVE': True, 'LOGSTATS_INTERVAL': 0}\n\n    def syntax(self):\n        return \"[url|file]\"\n\n    def short_desc(self):\n        return \"Interactive scraping console\"\n\n    def long_desc(self):\n        return \"Interactive console for scraping the given url\"\n\n    def add_options(self, parser):\n        ScrapyCommand.add_options(self, parser)\n        parser.add_option(\"-c\", dest=\"code\",\n            help=\"evaluate the code in the shell, print the result and exit\")\n        parser.add_option(\"--spider\", dest=\"spider\",\n            help=\"use this spider\")\n\n    def update_vars(self, vars):\n        \"\"\"You can use this function to update the Scrapy objects that will be\n        available in the shell\n        \"\"\"\n        pass\n\n    def run(self, args, opts):\n        url = args[0] if args else None\n        spiders = self.crawler_process.spiders\n\n        spidercls = DefaultSpider\n        if opts.spider:\n            spidercls = spiders.load(opts.spider)\n        elif url:\n            spidercls = spidercls_for_request(spiders, Request(url),\n                                              spidercls, log_multiple=True)\n\n        # The crawler is created this way since the Shell manually handles the\n        # crawling engine, so the set up in the crawl method won't work\n        crawler = self.crawler_process._create_logged_crawler(spidercls)\n        # The Shell class needs a persistent engine in the crawler\n        crawler.engine = crawler._create_engine()\n        crawler.engine.start()\n\n        self.crawler_process.start(start_reactor=False)\n        self._start_crawler_thread()\n\n        shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)\n        shell.start(url=url)\n\n    def _start_crawler_thread(self):\n        t = Thread(target=self.crawler_process._start_reactor,\n                   kwargs={'stop_after_crawl': False})\n        t.daemon = True\n        t.start()\n", "path": "scrapy/commands/shell.py"}], "after_files": [{"content": "import six\nimport signal\nimport warnings\n\nfrom twisted.internet import reactor, defer\n\nfrom scrapy.core.engine import ExecutionEngine\nfrom scrapy.resolver import CachingThreadedResolver\nfrom scrapy.extension import ExtensionManager\nfrom scrapy.signalmanager import SignalManager\nfrom scrapy.exceptions import ScrapyDeprecationWarning\nfrom scrapy.utils.ossignal import install_shutdown_handlers, signal_names\nfrom scrapy.utils.misc import load_object\nfrom scrapy import log, signals\n\n\nclass Crawler(object):\n\n    def __init__(self, spidercls, settings):\n        self.spidercls = spidercls\n        self.settings = settings\n        self.signals = SignalManager(self)\n        self.stats = load_object(self.settings['STATS_CLASS'])(self)\n        lf_cls = load_object(self.settings['LOG_FORMATTER'])\n        self.logformatter = lf_cls.from_crawler(self)\n        self.extensions = ExtensionManager.from_crawler(self)\n\n        self.crawling = False\n        self.spider = None\n        self.engine = None\n\n    @property\n    def spiders(self):\n        if not hasattr(self, '_spiders'):\n            warnings.warn(\"Crawler.spiders is deprecated, use \"\n                          \"CrawlerRunner.spiders or instantiate \"\n                          \"scrapy.spidermanager.SpiderManager with your \"\n                          \"settings.\",\n                          category=ScrapyDeprecationWarning, stacklevel=2)\n            spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])\n            self._spiders = spman_cls.from_settings(self.settings)\n        return self._spiders\n\n    @defer.inlineCallbacks\n    def crawl(self, *args, **kwargs):\n        assert not self.crawling, \"Crawling already taking place\"\n        self.crawling = True\n\n        try:\n            self.spider = self._create_spider(*args, **kwargs)\n            self.engine = self._create_engine()\n            start_requests = iter(self.spider.start_requests())\n            yield self.engine.open_spider(self.spider, start_requests)\n            yield defer.maybeDeferred(self.engine.start)\n        except Exception:\n            self.crawling = False\n            raise\n\n    def _create_spider(self, *args, **kwargs):\n        return self.spidercls.from_crawler(self, *args, **kwargs)\n\n    def _create_engine(self):\n        return ExecutionEngine(self, lambda _: self.stop())\n\n    @defer.inlineCallbacks\n    def stop(self):\n        if self.crawling:\n            self.crawling = False\n            yield defer.maybeDeferred(self.engine.stop)\n\n\nclass CrawlerRunner(object):\n\n    def __init__(self, settings):\n        self.settings = settings\n        smcls = load_object(settings['SPIDER_MANAGER_CLASS'])\n        self.spiders = smcls.from_settings(settings.frozencopy())\n        self.crawlers = set()\n        self.crawl_deferreds = set()\n\n    def crawl(self, spidercls, *args, **kwargs):\n        crawler = self._create_logged_crawler(spidercls)\n        self.crawlers.add(crawler)\n\n        d = crawler.crawl(*args, **kwargs)\n        self.crawl_deferreds.add(d)\n        return d\n\n    def _create_logged_crawler(self, spidercls):\n        crawler = self._create_crawler(spidercls)\n        log_observer = log.start_from_crawler(crawler)\n        if log_observer:\n            crawler.signals.connect(log_observer.stop, signals.engine_stopped)\n        return crawler\n\n    def _create_crawler(self, spidercls):\n        if isinstance(spidercls, six.string_types):\n            spidercls = self.spiders.load(spidercls)\n\n        crawler_settings = self.settings.copy()\n        spidercls.update_settings(crawler_settings)\n        crawler_settings.freeze()\n\n        crawler = Crawler(spidercls, crawler_settings)\n        return crawler\n\n    def stop(self):\n        return defer.DeferredList(c.stop() for c in self.crawlers)\n\n\nclass CrawlerProcess(CrawlerRunner):\n    \"\"\"A class to run multiple scrapy crawlers in a process simultaneously\"\"\"\n\n    def __init__(self, settings):\n        super(CrawlerProcess, self).__init__(settings)\n        install_shutdown_handlers(self._signal_shutdown)\n        self.stopping = False\n        self.log_observer = log.start_from_settings(self.settings)\n        log.scrapy_info(settings)\n\n    def _signal_shutdown(self, signum, _):\n        install_shutdown_handlers(self._signal_kill)\n        signame = signal_names[signum]\n        log.msg(format=\"Received %(signame)s, shutting down gracefully. Send again to force \",\n                level=log.INFO, signame=signame)\n        reactor.callFromThread(self.stop)\n\n    def _signal_kill(self, signum, _):\n        install_shutdown_handlers(signal.SIG_IGN)\n        signame = signal_names[signum]\n        log.msg(format='Received %(signame)s twice, forcing unclean shutdown',\n                level=log.INFO, signame=signame)\n        self._stop_logging()\n        reactor.callFromThread(self._stop_reactor)\n\n    def start(self, stop_after_crawl=True):\n        if stop_after_crawl:\n            d = defer.DeferredList(self.crawl_deferreds)\n            if d.called:\n                # Don't start the reactor if the deferreds are already fired\n                return\n            d.addBoth(lambda _: self._stop_reactor())\n        if self.settings.getbool('DNSCACHE_ENABLED'):\n            reactor.installResolver(CachingThreadedResolver(reactor))\n        reactor.addSystemEventTrigger('before', 'shutdown', self.stop)\n        reactor.run(installSignalHandlers=False)  # blocking call\n\n    def _stop_logging(self):\n        self.log_observer.stop()\n\n    def _stop_reactor(self, _=None):\n        try:\n            reactor.stop()\n        except RuntimeError:  # raised if already stopped or in shutdown stage\n            pass\n", "path": "scrapy/crawler.py"}, {"content": "\"\"\"\nScrapy Shell\n\nSee documentation in docs/topics/shell.rst\n\"\"\"\n\nfrom threading import Thread\n\nfrom scrapy.command import ScrapyCommand\nfrom scrapy.shell import Shell\nfrom scrapy.http import Request\nfrom scrapy import log\nfrom scrapy.utils.spider import spidercls_for_request, DefaultSpider\n\n\nclass Command(ScrapyCommand):\n\n    requires_project = False\n    default_settings = {'KEEP_ALIVE': True, 'LOGSTATS_INTERVAL': 0}\n\n    def syntax(self):\n        return \"[url|file]\"\n\n    def short_desc(self):\n        return \"Interactive scraping console\"\n\n    def long_desc(self):\n        return \"Interactive console for scraping the given url\"\n\n    def add_options(self, parser):\n        ScrapyCommand.add_options(self, parser)\n        parser.add_option(\"-c\", dest=\"code\",\n            help=\"evaluate the code in the shell, print the result and exit\")\n        parser.add_option(\"--spider\", dest=\"spider\",\n            help=\"use this spider\")\n\n    def update_vars(self, vars):\n        \"\"\"You can use this function to update the Scrapy objects that will be\n        available in the shell\n        \"\"\"\n        pass\n\n    def run(self, args, opts):\n        url = args[0] if args else None\n        spiders = self.crawler_process.spiders\n\n        spidercls = DefaultSpider\n        if opts.spider:\n            spidercls = spiders.load(opts.spider)\n        elif url:\n            spidercls = spidercls_for_request(spiders, Request(url),\n                                              spidercls, log_multiple=True)\n\n        # The crawler is created this way since the Shell manually handles the\n        # crawling engine, so the set up in the crawl method won't work\n        crawler = self.crawler_process._create_logged_crawler(spidercls)\n        # The Shell class needs a persistent engine in the crawler\n        crawler.engine = crawler._create_engine()\n        crawler.engine.start()\n\n        self._start_crawler_thread()\n\n        shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)\n        shell.start(url=url)\n\n    def _start_crawler_thread(self):\n        t = Thread(target=self.crawler_process.start,\n                   kwargs={'stop_after_crawl': False})\n        t.daemon = True\n        t.start()\n", "path": "scrapy/commands/shell.py"}]}