acrawler.py
async def crawl(self, root_urls):
        """Create and run worker tasks to process the `frontier` concurrently"""
        await self.scheduler.setup()
        for url in root_urls:
            parsed_url = urllib.parse.urlsplit(url)
            self.sites.add(parsed_url.netloc)
            await self.scheduler.add_to_frontier(url)

        # This method's implementation is modestly modified from the boilerplate
        # in https://docs.python.org/3/library/asyncio-queue.html#examples
        tasks = []
        for i in range(self.num_workers):
            task = asyncio.create_task(self.worker(f'worker-{i}'))
            tasks.append(task)

        # Wait until the frontier queue is fully processed.
        await self.scheduler.join()
        
        # Cancel our worker tasks.
        for task in tasks:
            task.cancel()
        
        # Wait until all worker tasks are cancelled.
        await asyncio.gather(*tasks, return_exceptions=True)
Create and run worker tasks to process the `frontier` concurrently
Create and run worker tasks to process the `frontier` concurrently
Create and run worker tasks to process the ` frontier ` concurrently
async def crawl(self, root_urls):
        await self.scheduler.setup()
        for url in root_urls:
            parsed_url = urllib.parse.urlsplit(url)
            self.sites.add(parsed_url.netloc)
            await self.scheduler.add_to_frontier(url)
        tasks = []
        for i in range(self.num_workers):
            task = asyncio.create_task(self.worker(f'worker-{i}'))
            tasks.append(task)
        await self.scheduler.join()
        for task in tasks:
            task.cancel()
        await asyncio.gather(*tasks, return_exceptions=True)
async def crawl ( self , root_urls ) : await self . scheduler . setup ( ) for url in root_urls : parsed_url = urllib . parse . urlsplit ( url ) self . sites . add ( parsed_url . netloc ) await self . scheduler . add_to_frontier ( url ) tasks = [ ] for i in range ( self . num_workers ) : task = asyncio . create_task ( self . worker ( f'worker-{i}' ) ) tasks . append ( task ) await self . scheduler . join ( ) for task in tasks : task . cancel ( ) await asyncio . gather ( * tasks , return_exceptions = True )
Create and run worker tasks to process the `frontier` concurrently
Create and run worker tasks to process the ` frontier ` concurrently
"""Create and run worker tasks to process the `frontier` concurrently""" # This method's implementation is modestly modified from the boilerplate # in https://docs.python.org/3/library/asyncio-queue.html#examples # Wait until the frontier queue is fully processed. # Cancel our worker tasks. # Wait until all worker tasks are cancelled.
self
root_urls