acrawler.py
def process_sitemap_tags(self, url, tag_parser, chunk):
        """Yields sitemap tags and added to `frontier` if under `roots`"""
        # TODO: This method should be refactored so it is a separate pluggable
        # factory, much like session_maker and serializer. This work will
        # require revisiting the tag_parser/chunk calling convention from
        # crawl_next.
        for tag in tag_parser.consume(chunk):
            if tag.name == "a" and "href" in tag.attrs:
                tag.url = resolve_url(url, tag.attrs["href"])

            elif tag.name == "img" and "src" in tag.attrs:
                tag.url = resolve_url(url, tag.attrs["src"])

            if tag.url is not None:
                # If there's now a url defined, then it is part of the sitemap
                yield tag
Yields sitemap tags and added to `frontier` if under `roots`
Yields sitemap tags and added to `frontier` if under `roots`
Yields sitemap tags and added to ` frontier ` if under ` roots `
def process_sitemap_tags(self, url, tag_parser, chunk):
        for tag in tag_parser.consume(chunk):
            if tag.name == "a" and "href" in tag.attrs:
                tag.url = resolve_url(url, tag.attrs["href"])
            elif tag.name == "img" and "src" in tag.attrs:
                tag.url = resolve_url(url, tag.attrs["src"])
            if tag.url is not None:
                yield tag
def process_sitemap_tags ( self , url , tag_parser , chunk ) : for tag in tag_parser . consume ( chunk ) : if tag . name == "a" and "href" in tag . attrs : tag . url = resolve_url ( url , tag . attrs [ "href" ] ) elif tag . name == "img" and "src" in tag . attrs : tag . url = resolve_url ( url , tag . attrs [ "src" ] ) if tag . url is not None : yield tag
Yields sitemap tags and added to `frontier` if under `roots`
Yields sitemap tags and added to ` frontier ` if under ` roots `
"""Yields sitemap tags and added to `frontier` if under `roots`""" # TODO: This method should be refactored so it is a separate pluggable # factory, much like session_maker and serializer. This work will # require revisiting the tag_parser/chunk calling convention from # crawl_next. # If there's now a url defined, then it is part of the sitemap
self
url
tag_parser
chunk