acrawler.py
async def crawl_next(self, session, url):
        """Crawls the next url from the `frontier`, processing tags for the sitemap"""
        tag_parser = TagParser({"a", "img"})

        # TODO: support 301, error handling in general here
        async for chunk in fetch(session, url):
            for tag in self.process_sitemap_tags(url, tag_parser, chunk):
                yield tag
                # Filter entries placed on the exploration frontier such
                # that they are all prefixed by one of the root sites
                if tag.name == "a":
                    parsed_tag_url = urllib.parse.urlsplit(tag.url)
                    if parsed_tag_url.netloc in self.sites:
                        await self.scheduler.add_to_frontier(tag.url)
Crawls the next url from the `frontier`, processing tags for the sitemap
Crawls the next url from the `frontier`, processing tags for the sitemap
Crawls the next url from the ` frontier ` processing tags for the sitemap
async def crawl_next(self, session, url):
        tag_parser = TagParser({"a", "img"})
        async for chunk in fetch(session, url):
            for tag in self.process_sitemap_tags(url, tag_parser, chunk):
                yield tag
                if tag.name == "a":
                    parsed_tag_url = urllib.parse.urlsplit(tag.url)
                    if parsed_tag_url.netloc in self.sites:
                        await self.scheduler.add_to_frontier(tag.url)
async def crawl_next ( self , session , url ) : tag_parser = TagParser ( { "a" , "img" } ) async for chunk in fetch ( session , url ) : for tag in self . process_sitemap_tags ( url , tag_parser , chunk ) : yield tag if tag . name == "a" : parsed_tag_url = urllib . parse . urlsplit ( tag . url ) if parsed_tag_url . netloc in self . sites : await self . scheduler . add_to_frontier ( tag . url )
Crawls the next url from the `frontier`, processing tags for the sitemap
Crawls the next url from the ` frontier ` processing tags for the sitemap
"""Crawls the next url from the `frontier`, processing tags for the sitemap""" # TODO: support 301, error handling in general here # Filter entries placed on the exploration frontier such # that they are all prefixed by one of the root sites
self
session
url