def crawl():
    """
    Initialize all crawlers (and indexers).

    Start the:
    1. GitHub crawler, :class:`crawler.GitHubCrawler`.
    2. Bitbucket crawler, :class:`crawler.BitbucketCrawler`.
    3. Git indexer, :class:`bitshift.crawler.indexer.GitIndexer`.
    """

    _configure_logging()
    time.sleep(5)

    repo_clone_queue = Queue.Queue(maxsize=MAX_URL_QUEUE_SIZE)
    run_event = Event()
    run_event.set()
    threads = [GitIndexer(repo_clone_queue, run_event)]

    if sys.argv[1:]:
        names = sys.argv[1:]
        ranks = GitHubCrawler.get_ranks(names)
        for name in names:
            repo = GitRepository("https://github.com/" + name, name, "GitHub",
                                 ranks[name])
            repo_clone_queue.put(repo)
    else:
        threads += [GitHubCrawler(repo_clone_queue, run_event),
                    BitbucketCrawler(repo_clone_queue, run_event)]

    for thread in threads:
        thread.start()

    try:
        while 1:
            time.sleep(0.1)
    except __HOLE__:
        run_event.clear()
        with repo_clone_queue.mutex:
            repo_clone_queue.queue.clear()
        for thread in threads:
            thread.join()
dataset/ETHPy150Open earwig/bitshift/bitshift/crawler/crawl.py/crawl