acrawler.py
async def main(argv):
    """Runs a crawler under an event loop"""
    args = parse_args(argv)
    yaml = YAML()
    yaml.register_class(Tag)

    def serializer(objs):
        # NOTE: outputing a list takes advantage of YAML's
        # serialization for lists, which is both concatable and
        # tailable
        yaml.dump(objs, sys.stdout)

    if args.redis:
        scheduler = RedisScheduler(args.redis)
    else:
        scheduler = SimpleScheduler()
    async with scheduler as open_scheduler:
        crawler = Crawler(
            open_scheduler, aiohttp.ClientSession, serializer,
            args.max_pages, args.num_workers)
        await crawler.crawl(args.roots)
Runs a crawler under an event loop
Runs a crawler under an event loop
Runs a crawler under an event loop
async def main(argv):
    args = parse_args(argv)
    yaml = YAML()
    yaml.register_class(Tag)
    def serializer(objs):
        yaml.dump(objs, sys.stdout)
    if args.redis:
        scheduler = RedisScheduler(args.redis)
    else:
        scheduler = SimpleScheduler()
    async with scheduler as open_scheduler:
        crawler = Crawler(
            open_scheduler, aiohttp.ClientSession, serializer,
            args.max_pages, args.num_workers)
        await crawler.crawl(args.roots)
async def main ( argv ) : args = parse_args ( argv ) yaml = YAML ( ) yaml . register_class ( Tag ) def serializer ( objs ) : yaml . dump ( objs , sys . stdout ) if args . redis : scheduler = RedisScheduler ( args . redis ) else : scheduler = SimpleScheduler ( ) async with scheduler as open_scheduler : crawler = Crawler ( open_scheduler , aiohttp . ClientSession , serializer , args . max_pages , args . num_workers ) await crawler . crawl ( args . roots )
Runs a crawler under an event loop
Runs a crawler under an event loop
"""Runs a crawler under an event loop""" # NOTE: outputing a list takes advantage of YAML's # serialization for lists, which is both concatable and # tailable
argv