We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Scrapy parse doesn't support async callbacks with yield
When running `scrapy parse http://localhost:8080 -c parse --spider test` is rising error: `TypeError: 'async_generator' object is not iterable`. 

Simple Code of Spider:
```
import scrapy

class TestSpider(scrapy.Spider):
    name = 'test'
    allowed_domains = ['localhost']
    start_urls = ['http://localhost:8080']

    async def parse(self, response):
        for x in range(1, 5):
            yield ({"test": x})
```

It is happened when there is `yield` inside `async function`.

`scrapy crawl test` - works fine
Scrapy parse doesn't support async callbacks with yield
When running `scrapy parse http://localhost:8080 -c parse --spider test` is rising error: `TypeError: 'async_generator' object is not iterable`. 

Simple Code of Spider:
```
import scrapy

class TestSpider(scrapy.Spider):
    name = 'test'
    allowed_domains = ['localhost']
    start_urls = ['http://localhost:8080']

    async def parse(self, response):
        for x in range(1, 5):
            yield ({"test": x})
```

It is happened when there is `yield` inside `async function`.

`scrapy crawl test` - works fine
Scrapy parse doesn't support async callbacks with yield
When running `scrapy parse http://localhost:8080 -c parse --spider test` is rising error: `TypeError: 'async_generator' object is not iterable`. 

Simple Code of Spider:
```
import scrapy

class TestSpider(scrapy.Spider):
    name = 'test'
    allowed_domains = ['localhost']
    start_urls = ['http://localhost:8080']

    async def parse(self, response):
        for x in range(1, 5):
            yield ({"test": x})
```

It is happened when there is `yield` inside `async function`.

`scrapy crawl test` - works fine

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/commands/parse.py`
Content:
```
1 import json
2 import logging
3 from typing import Dict
4 
5 from itemadapter import ItemAdapter, is_item
6 from twisted.internet.defer import maybeDeferred
7 from w3lib.url import is_url
8 
9 from scrapy.commands import BaseRunSpiderCommand
10 from scrapy.exceptions import UsageError
11 from scrapy.http import Request
12 from scrapy.utils import display
13 from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
14 
15 logger = logging.getLogger(__name__)
16 
17 
18 class Command(BaseRunSpiderCommand):
19     requires_project = True
20 
21     spider = None
22     items: Dict[int, list] = {}
23     requests: Dict[int, list] = {}
24 
25     first_response = None
26 
27     def syntax(self):
28         return "[options] <url>"
29 
30     def short_desc(self):
31         return "Parse URL (using its spider) and print the results"
32 
33     def add_options(self, parser):
34         BaseRunSpiderCommand.add_options(self, parser)
35         parser.add_argument(
36             "--spider",
37             dest="spider",
38             default=None,
39             help="use this spider without looking for one",
40         )
41         parser.add_argument(
42             "--pipelines", action="store_true", help="process items through pipelines"
43         )
44         parser.add_argument(
45             "--nolinks",
46             dest="nolinks",
47             action="store_true",
48             help="don't show links to follow (extracted requests)",
49         )
50         parser.add_argument(
51             "--noitems",
52             dest="noitems",
53             action="store_true",
54             help="don't show scraped items",
55         )
56         parser.add_argument(
57             "--nocolour",
58             dest="nocolour",
59             action="store_true",
60             help="avoid using pygments to colorize the output",
61         )
62         parser.add_argument(
63             "-r",
64             "--rules",
65             dest="rules",
66             action="store_true",
67             help="use CrawlSpider rules to discover the callback",
68         )
69         parser.add_argument(
70             "-c",
71             "--callback",
72             dest="callback",
73             help="use this callback for parsing, instead looking for a callback",
74         )
75         parser.add_argument(
76             "-m",
77             "--meta",
78             dest="meta",
79             help="inject extra meta into the Request, it must be a valid raw json string",
80         )
81         parser.add_argument(
82             "--cbkwargs",
83             dest="cbkwargs",
84             help="inject extra callback kwargs into the Request, it must be a valid raw json string",
85         )
86         parser.add_argument(
87             "-d",
88             "--depth",
89             dest="depth",
90             type=int,
91             default=1,
92             help="maximum depth for parsing requests [default: %(default)s]",
93         )
94         parser.add_argument(
95             "-v",
96             "--verbose",
97             dest="verbose",
98             action="store_true",
99             help="print each depth level one by one",
100         )
101 
102     @property
103     def max_level(self):
104         max_items, max_requests = 0, 0
105         if self.items:
106             max_items = max(self.items)
107         if self.requests:
108             max_requests = max(self.requests)
109         return max(max_items, max_requests)
110 
111     def add_items(self, lvl, new_items):
112         old_items = self.items.get(lvl, [])
113         self.items[lvl] = old_items + new_items
114 
115     def add_requests(self, lvl, new_reqs):
116         old_reqs = self.requests.get(lvl, [])
117         self.requests[lvl] = old_reqs + new_reqs
118 
119     def print_items(self, lvl=None, colour=True):
120         if lvl is None:
121             items = [item for lst in self.items.values() for item in lst]
122         else:
123             items = self.items.get(lvl, [])
124 
125         print("# Scraped Items ", "-" * 60)
126         display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
127 
128     def print_requests(self, lvl=None, colour=True):
129         if lvl is None:
130             if self.requests:
131                 requests = self.requests[max(self.requests)]
132             else:
133                 requests = []
134         else:
135             requests = self.requests.get(lvl, [])
136 
137         print("# Requests ", "-" * 65)
138         display.pprint(requests, colorize=colour)
139 
140     def print_results(self, opts):
141         colour = not opts.nocolour
142 
143         if opts.verbose:
144             for level in range(1, self.max_level + 1):
145                 print(f"\n>>> DEPTH LEVEL: {level} <<<")
146                 if not opts.noitems:
147                     self.print_items(level, colour)
148                 if not opts.nolinks:
149                     self.print_requests(level, colour)
150         else:
151             print(f"\n>>> STATUS DEPTH LEVEL {self.max_level} <<<")
152             if not opts.noitems:
153                 self.print_items(colour=colour)
154             if not opts.nolinks:
155                 self.print_requests(colour=colour)
156 
157     def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):
158         items, requests = [], []
159         for x in spider_output:
160             if is_item(x):
161                 items.append(x)
162             elif isinstance(x, Request):
163                 requests.append(x)
164         return items, requests, opts, depth, spider, callback
165 
166     def run_callback(self, response, callback, cb_kwargs=None):
167         cb_kwargs = cb_kwargs or {}
168         d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))
169         return d
170 
171     def get_callback_from_rules(self, spider, response):
172         if getattr(spider, "rules", None):
173             for rule in spider.rules:
174                 if rule.link_extractor.matches(response.url):
175                     return rule.callback or "parse"
176         else:
177             logger.error(
178                 "No CrawlSpider rules found in spider %(spider)r, "
179                 "please specify a callback to use for parsing",
180                 {"spider": spider.name},
181             )
182 
183     def set_spidercls(self, url, opts):
184         spider_loader = self.crawler_process.spider_loader
185         if opts.spider:
186             try:
187                 self.spidercls = spider_loader.load(opts.spider)
188             except KeyError:
189                 logger.error(
190                     "Unable to find spider: %(spider)s", {"spider": opts.spider}
191                 )
192         else:
193             self.spidercls = spidercls_for_request(spider_loader, Request(url))
194             if not self.spidercls:
195                 logger.error("Unable to find spider for: %(url)s", {"url": url})
196 
197         def _start_requests(spider):
198             yield self.prepare_request(spider, Request(url), opts)
199 
200         if self.spidercls:
201             self.spidercls.start_requests = _start_requests
202 
203     def start_parsing(self, url, opts):
204         self.crawler_process.crawl(self.spidercls, **opts.spargs)
205         self.pcrawler = list(self.crawler_process.crawlers)[0]
206         self.crawler_process.start()
207 
208         if not self.first_response:
209             logger.error("No response downloaded for: %(url)s", {"url": url})
210 
211     def scraped_data(self, args):
212         items, requests, opts, depth, spider, callback = args
213         if opts.pipelines:
214             itemproc = self.pcrawler.engine.scraper.itemproc
215             for item in items:
216                 itemproc.process_item(item, spider)
217         self.add_items(depth, items)
218         self.add_requests(depth, requests)
219 
220         scraped_data = items if opts.output else []
221         if depth < opts.depth:
222             for req in requests:
223                 req.meta["_depth"] = depth + 1
224                 req.meta["_callback"] = req.callback
225                 req.callback = callback
226             scraped_data += requests
227 
228         return scraped_data
229 
230     def prepare_request(self, spider, request, opts):
231         def callback(response, **cb_kwargs):
232             # memorize first request
233             if not self.first_response:
234                 self.first_response = response
235 
236             # determine real callback
237             cb = response.meta["_callback"]
238             if not cb:
239                 if opts.callback:
240                     cb = opts.callback
241                 elif opts.rules and self.first_response == response:
242                     cb = self.get_callback_from_rules(spider, response)
243 
244                     if not cb:
245                         logger.error(
246                             "Cannot find a rule that matches %(url)r in spider: %(spider)s",
247                             {"url": response.url, "spider": spider.name},
248                         )
249                         return
250                 else:
251                     cb = "parse"
252 
253             if not callable(cb):
254                 cb_method = getattr(spider, cb, None)
255                 if callable(cb_method):
256                     cb = cb_method
257                 else:
258                     logger.error(
259                         "Cannot find callback %(callback)r in spider: %(spider)s",
260                         {"callback": cb, "spider": spider.name},
261                     )
262                     return
263 
264             # parse items and requests
265             depth = response.meta["_depth"]
266 
267             d = self.run_callback(response, cb, cb_kwargs)
268             d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
269             d.addCallback(self.scraped_data)
270             return d
271 
272         # update request meta if any extra meta was passed through the --meta/-m opts.
273         if opts.meta:
274             request.meta.update(opts.meta)
275 
276         # update cb_kwargs if any extra values were was passed through the --cbkwargs option.
277         if opts.cbkwargs:
278             request.cb_kwargs.update(opts.cbkwargs)
279 
280         request.meta["_depth"] = 1
281         request.meta["_callback"] = request.callback
282         request.callback = callback
283         return request
284 
285     def process_options(self, args, opts):
286         BaseRunSpiderCommand.process_options(self, args, opts)
287 
288         self.process_request_meta(opts)
289         self.process_request_cb_kwargs(opts)
290 
291     def process_request_meta(self, opts):
292         if opts.meta:
293             try:
294                 opts.meta = json.loads(opts.meta)
295             except ValueError:
296                 raise UsageError(
297                     "Invalid -m/--meta value, pass a valid json string to -m or --meta. "
298                     'Example: --meta=\'{"foo" : "bar"}\'',
299                     print_help=False,
300                 )
301 
302     def process_request_cb_kwargs(self, opts):
303         if opts.cbkwargs:
304             try:
305                 opts.cbkwargs = json.loads(opts.cbkwargs)
306             except ValueError:
307                 raise UsageError(
308                     "Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
309                     'Example: --cbkwargs=\'{"foo" : "bar"}\'',
310                     print_help=False,
311                 )
312 
313     def run(self, args, opts):
314         # parse arguments
315         if not len(args) == 1 or not is_url(args[0]):
316             raise UsageError()
317         else:
318             url = args[0]
319 
320         # prepare spidercls
321         self.set_spidercls(url, opts)
322 
323         if self.spidercls and opts.depth > 0:
324             self.start_parsing(url, opts)
325             self.print_results(opts)
326
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,3 +1,4 @@
+import inspect
 import json
 import logging
 from typing import Dict
@@ -10,7 +11,11 @@
 from scrapy.exceptions import UsageError
 from scrapy.http import Request
 from scrapy.utils import display
-from scrapy.utils.spider import iterate_spider_output, spidercls_for_request
+from scrapy.utils.asyncgen import collect_asyncgen
+from scrapy.utils.defer import aiter_errback, deferred_from_coro
+from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.spider import spidercls_for_request
 
 logger = logging.getLogger(__name__)
 
@@ -108,6 +113,25 @@
             max_requests = max(self.requests)
         return max(max_items, max_requests)
 
+    def handle_exception(self, _failure):
+        logger.error(
+            "An error is caught while iterating the async iterable",
+            exc_info=failure_to_exc_info(_failure),
+        )
+
+    def iterate_spider_output(self, result):
+        if inspect.isasyncgen(result):
+            d = deferred_from_coro(
+                collect_asyncgen(aiter_errback(result, self.handle_exception))
+            )
+            d.addCallback(self.iterate_spider_output)
+            return d
+        if inspect.iscoroutine(result):
+            d = deferred_from_coro(result)
+            d.addCallback(self.iterate_spider_output)
+            return d
+        return arg_to_iter(deferred_from_coro(result))
+
     def add_items(self, lvl, new_items):
         old_items = self.items.get(lvl, [])
         self.items[lvl] = old_items + new_items
@@ -165,7 +189,7 @@
 
     def run_callback(self, response, callback, cb_kwargs=None):
         cb_kwargs = cb_kwargs or {}
-        d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))
+        d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))
         return d
 
     def get_callback_from_rules(self, spider, response):

{"golden_diff": "diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py\n--- a/scrapy/commands/parse.py\n+++ b/scrapy/commands/parse.py\n@@ -1,3 +1,4 @@\n+import inspect\n import json\n import logging\n from typing import Dict\n@@ -10,7 +11,11 @@\n from scrapy.exceptions import UsageError\n from scrapy.http import Request\n from scrapy.utils import display\n-from scrapy.utils.spider import iterate_spider_output, spidercls_for_request\n+from scrapy.utils.asyncgen import collect_asyncgen\n+from scrapy.utils.defer import aiter_errback, deferred_from_coro\n+from scrapy.utils.log import failure_to_exc_info\n+from scrapy.utils.misc import arg_to_iter\n+from scrapy.utils.spider import spidercls_for_request\n \n logger = logging.getLogger(__name__)\n \n@@ -108,6 +113,25 @@\n             max_requests = max(self.requests)\n         return max(max_items, max_requests)\n \n+    def handle_exception(self, _failure):\n+        logger.error(\n+            \"An error is caught while iterating the async iterable\",\n+            exc_info=failure_to_exc_info(_failure),\n+        )\n+\n+    def iterate_spider_output(self, result):\n+        if inspect.isasyncgen(result):\n+            d = deferred_from_coro(\n+                collect_asyncgen(aiter_errback(result, self.handle_exception))\n+            )\n+            d.addCallback(self.iterate_spider_output)\n+            return d\n+        if inspect.iscoroutine(result):\n+            d = deferred_from_coro(result)\n+            d.addCallback(self.iterate_spider_output)\n+            return d\n+        return arg_to_iter(deferred_from_coro(result))\n+\n     def add_items(self, lvl, new_items):\n         old_items = self.items.get(lvl, [])\n         self.items[lvl] = old_items + new_items\n@@ -165,7 +189,7 @@\n \n     def run_callback(self, response, callback, cb_kwargs=None):\n         cb_kwargs = cb_kwargs or {}\n-        d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))\n+        d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))\n         return d\n \n     def get_callback_from_rules(self, spider, response):\n", "issue": "Scrapy parse doesn't support async callbacks with yield\nWhen running `scrapy parse http://localhost:8080 -c parse --spider test` is rising error: `TypeError: 'async_generator' object is not iterable`. \r\n\r\nSimple Code of Spider:\r\n```\r\nimport scrapy\r\n\r\nclass TestSpider(scrapy.Spider):\r\n    name = 'test'\r\n    allowed_domains = ['localhost']\r\n    start_urls = ['http://localhost:8080']\r\n\r\n    async def parse(self, response):\r\n        for x in range(1, 5):\r\n            yield ({\"test\": x})\r\n```\r\n\r\nIt is happened when there is `yield` inside `async function`.\r\n\r\n`scrapy crawl test` - works fine\nScrapy parse doesn't support async callbacks with yield\nWhen running `scrapy parse http://localhost:8080 -c parse --spider test` is rising error: `TypeError: 'async_generator' object is not iterable`. \r\n\r\nSimple Code of Spider:\r\n```\r\nimport scrapy\r\n\r\nclass TestSpider(scrapy.Spider):\r\n    name = 'test'\r\n    allowed_domains = ['localhost']\r\n    start_urls = ['http://localhost:8080']\r\n\r\n    async def parse(self, response):\r\n        for x in range(1, 5):\r\n            yield ({\"test\": x})\r\n```\r\n\r\nIt is happened when there is `yield` inside `async function`.\r\n\r\n`scrapy crawl test` - works fine\nScrapy parse doesn't support async callbacks with yield\nWhen running `scrapy parse http://localhost:8080 -c parse --spider test` is rising error: `TypeError: 'async_generator' object is not iterable`. \r\n\r\nSimple Code of Spider:\r\n```\r\nimport scrapy\r\n\r\nclass TestSpider(scrapy.Spider):\r\n    name = 'test'\r\n    allowed_domains = ['localhost']\r\n    start_urls = ['http://localhost:8080']\r\n\r\n    async def parse(self, response):\r\n        for x in range(1, 5):\r\n            yield ({\"test\": x})\r\n```\r\n\r\nIt is happened when there is `yield` inside `async function`.\r\n\r\n`scrapy crawl test` - works fine\n", "before_files": [{"content": "import json\nimport logging\nfrom typing import Dict\n\nfrom itemadapter import ItemAdapter, is_item\nfrom twisted.internet.defer import maybeDeferred\nfrom w3lib.url import is_url\n\nfrom scrapy.commands import BaseRunSpiderCommand\nfrom scrapy.exceptions import UsageError\nfrom scrapy.http import Request\nfrom scrapy.utils import display\nfrom scrapy.utils.spider import iterate_spider_output, spidercls_for_request\n\nlogger = logging.getLogger(__name__)\n\n\nclass Command(BaseRunSpiderCommand):\n    requires_project = True\n\n    spider = None\n    items: Dict[int, list] = {}\n    requests: Dict[int, list] = {}\n\n    first_response = None\n\n    def syntax(self):\n        return \"[options] <url>\"\n\n    def short_desc(self):\n        return \"Parse URL (using its spider) and print the results\"\n\n    def add_options(self, parser):\n        BaseRunSpiderCommand.add_options(self, parser)\n        parser.add_argument(\n            \"--spider\",\n            dest=\"spider\",\n            default=None,\n            help=\"use this spider without looking for one\",\n        )\n        parser.add_argument(\n            \"--pipelines\", action=\"store_true\", help=\"process items through pipelines\"\n        )\n        parser.add_argument(\n            \"--nolinks\",\n            dest=\"nolinks\",\n            action=\"store_true\",\n            help=\"don't show links to follow (extracted requests)\",\n        )\n        parser.add_argument(\n            \"--noitems\",\n            dest=\"noitems\",\n            action=\"store_true\",\n            help=\"don't show scraped items\",\n        )\n        parser.add_argument(\n            \"--nocolour\",\n            dest=\"nocolour\",\n            action=\"store_true\",\n            help=\"avoid using pygments to colorize the output\",\n        )\n        parser.add_argument(\n            \"-r\",\n            \"--rules\",\n            dest=\"rules\",\n            action=\"store_true\",\n            help=\"use CrawlSpider rules to discover the callback\",\n        )\n        parser.add_argument(\n            \"-c\",\n            \"--callback\",\n            dest=\"callback\",\n            help=\"use this callback for parsing, instead looking for a callback\",\n        )\n        parser.add_argument(\n            \"-m\",\n            \"--meta\",\n            dest=\"meta\",\n            help=\"inject extra meta into the Request, it must be a valid raw json string\",\n        )\n        parser.add_argument(\n            \"--cbkwargs\",\n            dest=\"cbkwargs\",\n            help=\"inject extra callback kwargs into the Request, it must be a valid raw json string\",\n        )\n        parser.add_argument(\n            \"-d\",\n            \"--depth\",\n            dest=\"depth\",\n            type=int,\n            default=1,\n            help=\"maximum depth for parsing requests [default: %(default)s]\",\n        )\n        parser.add_argument(\n            \"-v\",\n            \"--verbose\",\n            dest=\"verbose\",\n            action=\"store_true\",\n            help=\"print each depth level one by one\",\n        )\n\n    @property\n    def max_level(self):\n        max_items, max_requests = 0, 0\n        if self.items:\n            max_items = max(self.items)\n        if self.requests:\n            max_requests = max(self.requests)\n        return max(max_items, max_requests)\n\n    def add_items(self, lvl, new_items):\n        old_items = self.items.get(lvl, [])\n        self.items[lvl] = old_items + new_items\n\n    def add_requests(self, lvl, new_reqs):\n        old_reqs = self.requests.get(lvl, [])\n        self.requests[lvl] = old_reqs + new_reqs\n\n    def print_items(self, lvl=None, colour=True):\n        if lvl is None:\n            items = [item for lst in self.items.values() for item in lst]\n        else:\n            items = self.items.get(lvl, [])\n\n        print(\"# Scraped Items \", \"-\" * 60)\n        display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)\n\n    def print_requests(self, lvl=None, colour=True):\n        if lvl is None:\n            if self.requests:\n                requests = self.requests[max(self.requests)]\n            else:\n                requests = []\n        else:\n            requests = self.requests.get(lvl, [])\n\n        print(\"# Requests \", \"-\" * 65)\n        display.pprint(requests, colorize=colour)\n\n    def print_results(self, opts):\n        colour = not opts.nocolour\n\n        if opts.verbose:\n            for level in range(1, self.max_level + 1):\n                print(f\"\\n>>> DEPTH LEVEL: {level} <<<\")\n                if not opts.noitems:\n                    self.print_items(level, colour)\n                if not opts.nolinks:\n                    self.print_requests(level, colour)\n        else:\n            print(f\"\\n>>> STATUS DEPTH LEVEL {self.max_level} <<<\")\n            if not opts.noitems:\n                self.print_items(colour=colour)\n            if not opts.nolinks:\n                self.print_requests(colour=colour)\n\n    def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):\n        items, requests = [], []\n        for x in spider_output:\n            if is_item(x):\n                items.append(x)\n            elif isinstance(x, Request):\n                requests.append(x)\n        return items, requests, opts, depth, spider, callback\n\n    def run_callback(self, response, callback, cb_kwargs=None):\n        cb_kwargs = cb_kwargs or {}\n        d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))\n        return d\n\n    def get_callback_from_rules(self, spider, response):\n        if getattr(spider, \"rules\", None):\n            for rule in spider.rules:\n                if rule.link_extractor.matches(response.url):\n                    return rule.callback or \"parse\"\n        else:\n            logger.error(\n                \"No CrawlSpider rules found in spider %(spider)r, \"\n                \"please specify a callback to use for parsing\",\n                {\"spider\": spider.name},\n            )\n\n    def set_spidercls(self, url, opts):\n        spider_loader = self.crawler_process.spider_loader\n        if opts.spider:\n            try:\n                self.spidercls = spider_loader.load(opts.spider)\n            except KeyError:\n                logger.error(\n                    \"Unable to find spider: %(spider)s\", {\"spider\": opts.spider}\n                )\n        else:\n            self.spidercls = spidercls_for_request(spider_loader, Request(url))\n            if not self.spidercls:\n                logger.error(\"Unable to find spider for: %(url)s\", {\"url\": url})\n\n        def _start_requests(spider):\n            yield self.prepare_request(spider, Request(url), opts)\n\n        if self.spidercls:\n            self.spidercls.start_requests = _start_requests\n\n    def start_parsing(self, url, opts):\n        self.crawler_process.crawl(self.spidercls, **opts.spargs)\n        self.pcrawler = list(self.crawler_process.crawlers)[0]\n        self.crawler_process.start()\n\n        if not self.first_response:\n            logger.error(\"No response downloaded for: %(url)s\", {\"url\": url})\n\n    def scraped_data(self, args):\n        items, requests, opts, depth, spider, callback = args\n        if opts.pipelines:\n            itemproc = self.pcrawler.engine.scraper.itemproc\n            for item in items:\n                itemproc.process_item(item, spider)\n        self.add_items(depth, items)\n        self.add_requests(depth, requests)\n\n        scraped_data = items if opts.output else []\n        if depth < opts.depth:\n            for req in requests:\n                req.meta[\"_depth\"] = depth + 1\n                req.meta[\"_callback\"] = req.callback\n                req.callback = callback\n            scraped_data += requests\n\n        return scraped_data\n\n    def prepare_request(self, spider, request, opts):\n        def callback(response, **cb_kwargs):\n            # memorize first request\n            if not self.first_response:\n                self.first_response = response\n\n            # determine real callback\n            cb = response.meta[\"_callback\"]\n            if not cb:\n                if opts.callback:\n                    cb = opts.callback\n                elif opts.rules and self.first_response == response:\n                    cb = self.get_callback_from_rules(spider, response)\n\n                    if not cb:\n                        logger.error(\n                            \"Cannot find a rule that matches %(url)r in spider: %(spider)s\",\n                            {\"url\": response.url, \"spider\": spider.name},\n                        )\n                        return\n                else:\n                    cb = \"parse\"\n\n            if not callable(cb):\n                cb_method = getattr(spider, cb, None)\n                if callable(cb_method):\n                    cb = cb_method\n                else:\n                    logger.error(\n                        \"Cannot find callback %(callback)r in spider: %(spider)s\",\n                        {\"callback\": cb, \"spider\": spider.name},\n                    )\n                    return\n\n            # parse items and requests\n            depth = response.meta[\"_depth\"]\n\n            d = self.run_callback(response, cb, cb_kwargs)\n            d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)\n            d.addCallback(self.scraped_data)\n            return d\n\n        # update request meta if any extra meta was passed through the --meta/-m opts.\n        if opts.meta:\n            request.meta.update(opts.meta)\n\n        # update cb_kwargs if any extra values were was passed through the --cbkwargs option.\n        if opts.cbkwargs:\n            request.cb_kwargs.update(opts.cbkwargs)\n\n        request.meta[\"_depth\"] = 1\n        request.meta[\"_callback\"] = request.callback\n        request.callback = callback\n        return request\n\n    def process_options(self, args, opts):\n        BaseRunSpiderCommand.process_options(self, args, opts)\n\n        self.process_request_meta(opts)\n        self.process_request_cb_kwargs(opts)\n\n    def process_request_meta(self, opts):\n        if opts.meta:\n            try:\n                opts.meta = json.loads(opts.meta)\n            except ValueError:\n                raise UsageError(\n                    \"Invalid -m/--meta value, pass a valid json string to -m or --meta. \"\n                    'Example: --meta=\\'{\"foo\" : \"bar\"}\\'',\n                    print_help=False,\n                )\n\n    def process_request_cb_kwargs(self, opts):\n        if opts.cbkwargs:\n            try:\n                opts.cbkwargs = json.loads(opts.cbkwargs)\n            except ValueError:\n                raise UsageError(\n                    \"Invalid --cbkwargs value, pass a valid json string to --cbkwargs. \"\n                    'Example: --cbkwargs=\\'{\"foo\" : \"bar\"}\\'',\n                    print_help=False,\n                )\n\n    def run(self, args, opts):\n        # parse arguments\n        if not len(args) == 1 or not is_url(args[0]):\n            raise UsageError()\n        else:\n            url = args[0]\n\n        # prepare spidercls\n        self.set_spidercls(url, opts)\n\n        if self.spidercls and opts.depth > 0:\n            self.start_parsing(url, opts)\n            self.print_results(opts)\n", "path": "scrapy/commands/parse.py"}], "after_files": [{"content": "import inspect\nimport json\nimport logging\nfrom typing import Dict\n\nfrom itemadapter import ItemAdapter, is_item\nfrom twisted.internet.defer import maybeDeferred\nfrom w3lib.url import is_url\n\nfrom scrapy.commands import BaseRunSpiderCommand\nfrom scrapy.exceptions import UsageError\nfrom scrapy.http import Request\nfrom scrapy.utils import display\nfrom scrapy.utils.asyncgen import collect_asyncgen\nfrom scrapy.utils.defer import aiter_errback, deferred_from_coro\nfrom scrapy.utils.log import failure_to_exc_info\nfrom scrapy.utils.misc import arg_to_iter\nfrom scrapy.utils.spider import spidercls_for_request\n\nlogger = logging.getLogger(__name__)\n\n\nclass Command(BaseRunSpiderCommand):\n    requires_project = True\n\n    spider = None\n    items: Dict[int, list] = {}\n    requests: Dict[int, list] = {}\n\n    first_response = None\n\n    def syntax(self):\n        return \"[options] <url>\"\n\n    def short_desc(self):\n        return \"Parse URL (using its spider) and print the results\"\n\n    def add_options(self, parser):\n        BaseRunSpiderCommand.add_options(self, parser)\n        parser.add_argument(\n            \"--spider\",\n            dest=\"spider\",\n            default=None,\n            help=\"use this spider without looking for one\",\n        )\n        parser.add_argument(\n            \"--pipelines\", action=\"store_true\", help=\"process items through pipelines\"\n        )\n        parser.add_argument(\n            \"--nolinks\",\n            dest=\"nolinks\",\n            action=\"store_true\",\n            help=\"don't show links to follow (extracted requests)\",\n        )\n        parser.add_argument(\n            \"--noitems\",\n            dest=\"noitems\",\n            action=\"store_true\",\n            help=\"don't show scraped items\",\n        )\n        parser.add_argument(\n            \"--nocolour\",\n            dest=\"nocolour\",\n            action=\"store_true\",\n            help=\"avoid using pygments to colorize the output\",\n        )\n        parser.add_argument(\n            \"-r\",\n            \"--rules\",\n            dest=\"rules\",\n            action=\"store_true\",\n            help=\"use CrawlSpider rules to discover the callback\",\n        )\n        parser.add_argument(\n            \"-c\",\n            \"--callback\",\n            dest=\"callback\",\n            help=\"use this callback for parsing, instead looking for a callback\",\n        )\n        parser.add_argument(\n            \"-m\",\n            \"--meta\",\n            dest=\"meta\",\n            help=\"inject extra meta into the Request, it must be a valid raw json string\",\n        )\n        parser.add_argument(\n            \"--cbkwargs\",\n            dest=\"cbkwargs\",\n            help=\"inject extra callback kwargs into the Request, it must be a valid raw json string\",\n        )\n        parser.add_argument(\n            \"-d\",\n            \"--depth\",\n            dest=\"depth\",\n            type=int,\n            default=1,\n            help=\"maximum depth for parsing requests [default: %(default)s]\",\n        )\n        parser.add_argument(\n            \"-v\",\n            \"--verbose\",\n            dest=\"verbose\",\n            action=\"store_true\",\n            help=\"print each depth level one by one\",\n        )\n\n    @property\n    def max_level(self):\n        max_items, max_requests = 0, 0\n        if self.items:\n            max_items = max(self.items)\n        if self.requests:\n            max_requests = max(self.requests)\n        return max(max_items, max_requests)\n\n    def handle_exception(self, _failure):\n        logger.error(\n            \"An error is caught while iterating the async iterable\",\n            exc_info=failure_to_exc_info(_failure),\n        )\n\n    def iterate_spider_output(self, result):\n        if inspect.isasyncgen(result):\n            d = deferred_from_coro(\n                collect_asyncgen(aiter_errback(result, self.handle_exception))\n            )\n            d.addCallback(self.iterate_spider_output)\n            return d\n        if inspect.iscoroutine(result):\n            d = deferred_from_coro(result)\n            d.addCallback(self.iterate_spider_output)\n            return d\n        return arg_to_iter(deferred_from_coro(result))\n\n    def add_items(self, lvl, new_items):\n        old_items = self.items.get(lvl, [])\n        self.items[lvl] = old_items + new_items\n\n    def add_requests(self, lvl, new_reqs):\n        old_reqs = self.requests.get(lvl, [])\n        self.requests[lvl] = old_reqs + new_reqs\n\n    def print_items(self, lvl=None, colour=True):\n        if lvl is None:\n            items = [item for lst in self.items.values() for item in lst]\n        else:\n            items = self.items.get(lvl, [])\n\n        print(\"# Scraped Items \", \"-\" * 60)\n        display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)\n\n    def print_requests(self, lvl=None, colour=True):\n        if lvl is None:\n            if self.requests:\n                requests = self.requests[max(self.requests)]\n            else:\n                requests = []\n        else:\n            requests = self.requests.get(lvl, [])\n\n        print(\"# Requests \", \"-\" * 65)\n        display.pprint(requests, colorize=colour)\n\n    def print_results(self, opts):\n        colour = not opts.nocolour\n\n        if opts.verbose:\n            for level in range(1, self.max_level + 1):\n                print(f\"\\n>>> DEPTH LEVEL: {level} <<<\")\n                if not opts.noitems:\n                    self.print_items(level, colour)\n                if not opts.nolinks:\n                    self.print_requests(level, colour)\n        else:\n            print(f\"\\n>>> STATUS DEPTH LEVEL {self.max_level} <<<\")\n            if not opts.noitems:\n                self.print_items(colour=colour)\n            if not opts.nolinks:\n                self.print_requests(colour=colour)\n\n    def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):\n        items, requests = [], []\n        for x in spider_output:\n            if is_item(x):\n                items.append(x)\n            elif isinstance(x, Request):\n                requests.append(x)\n        return items, requests, opts, depth, spider, callback\n\n    def run_callback(self, response, callback, cb_kwargs=None):\n        cb_kwargs = cb_kwargs or {}\n        d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))\n        return d\n\n    def get_callback_from_rules(self, spider, response):\n        if getattr(spider, \"rules\", None):\n            for rule in spider.rules:\n                if rule.link_extractor.matches(response.url):\n                    return rule.callback or \"parse\"\n        else:\n            logger.error(\n                \"No CrawlSpider rules found in spider %(spider)r, \"\n                \"please specify a callback to use for parsing\",\n                {\"spider\": spider.name},\n            )\n\n    def set_spidercls(self, url, opts):\n        spider_loader = self.crawler_process.spider_loader\n        if opts.spider:\n            try:\n                self.spidercls = spider_loader.load(opts.spider)\n            except KeyError:\n                logger.error(\n                    \"Unable to find spider: %(spider)s\", {\"spider\": opts.spider}\n                )\n        else:\n            self.spidercls = spidercls_for_request(spider_loader, Request(url))\n            if not self.spidercls:\n                logger.error(\"Unable to find spider for: %(url)s\", {\"url\": url})\n\n        def _start_requests(spider):\n            yield self.prepare_request(spider, Request(url), opts)\n\n        if self.spidercls:\n            self.spidercls.start_requests = _start_requests\n\n    def start_parsing(self, url, opts):\n        self.crawler_process.crawl(self.spidercls, **opts.spargs)\n        self.pcrawler = list(self.crawler_process.crawlers)[0]\n        self.crawler_process.start()\n\n        if not self.first_response:\n            logger.error(\"No response downloaded for: %(url)s\", {\"url\": url})\n\n    def scraped_data(self, args):\n        items, requests, opts, depth, spider, callback = args\n        if opts.pipelines:\n            itemproc = self.pcrawler.engine.scraper.itemproc\n            for item in items:\n                itemproc.process_item(item, spider)\n        self.add_items(depth, items)\n        self.add_requests(depth, requests)\n\n        scraped_data = items if opts.output else []\n        if depth < opts.depth:\n            for req in requests:\n                req.meta[\"_depth\"] = depth + 1\n                req.meta[\"_callback\"] = req.callback\n                req.callback = callback\n            scraped_data += requests\n\n        return scraped_data\n\n    def prepare_request(self, spider, request, opts):\n        def callback(response, **cb_kwargs):\n            # memorize first request\n            if not self.first_response:\n                self.first_response = response\n\n            # determine real callback\n            cb = response.meta[\"_callback\"]\n            if not cb:\n                if opts.callback:\n                    cb = opts.callback\n                elif opts.rules and self.first_response == response:\n                    cb = self.get_callback_from_rules(spider, response)\n\n                    if not cb:\n                        logger.error(\n                            \"Cannot find a rule that matches %(url)r in spider: %(spider)s\",\n                            {\"url\": response.url, \"spider\": spider.name},\n                        )\n                        return\n                else:\n                    cb = \"parse\"\n\n            if not callable(cb):\n                cb_method = getattr(spider, cb, None)\n                if callable(cb_method):\n                    cb = cb_method\n                else:\n                    logger.error(\n                        \"Cannot find callback %(callback)r in spider: %(spider)s\",\n                        {\"callback\": cb, \"spider\": spider.name},\n                    )\n                    return\n\n            # parse items and requests\n            depth = response.meta[\"_depth\"]\n\n            d = self.run_callback(response, cb, cb_kwargs)\n            d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)\n            d.addCallback(self.scraped_data)\n            return d\n\n        # update request meta if any extra meta was passed through the --meta/-m opts.\n        if opts.meta:\n            request.meta.update(opts.meta)\n\n        # update cb_kwargs if any extra values were was passed through the --cbkwargs option.\n        if opts.cbkwargs:\n            request.cb_kwargs.update(opts.cbkwargs)\n\n        request.meta[\"_depth\"] = 1\n        request.meta[\"_callback\"] = request.callback\n        request.callback = callback\n        return request\n\n    def process_options(self, args, opts):\n        BaseRunSpiderCommand.process_options(self, args, opts)\n\n        self.process_request_meta(opts)\n        self.process_request_cb_kwargs(opts)\n\n    def process_request_meta(self, opts):\n        if opts.meta:\n            try:\n                opts.meta = json.loads(opts.meta)\n            except ValueError:\n                raise UsageError(\n                    \"Invalid -m/--meta value, pass a valid json string to -m or --meta. \"\n                    'Example: --meta=\\'{\"foo\" : \"bar\"}\\'',\n                    print_help=False,\n                )\n\n    def process_request_cb_kwargs(self, opts):\n        if opts.cbkwargs:\n            try:\n                opts.cbkwargs = json.loads(opts.cbkwargs)\n            except ValueError:\n                raise UsageError(\n                    \"Invalid --cbkwargs value, pass a valid json string to --cbkwargs. \"\n                    'Example: --cbkwargs=\\'{\"foo\" : \"bar\"}\\'',\n                    print_help=False,\n                )\n\n    def run(self, args, opts):\n        # parse arguments\n        if not len(args) == 1 or not is_url(args[0]):\n            raise UsageError()\n        else:\n            url = args[0]\n\n        # prepare spidercls\n        self.set_spidercls(url, opts)\n\n        if self.spidercls and opts.depth > 0:\n            self.start_parsing(url, opts)\n            self.print_results(opts)\n", "path": "scrapy/commands/parse.py"}]}