def main():
    url_help = """
    url to query in the index:
    For prefix, use:
    http://example.com/*

    For domain query, use:
    *.example.com
    """

    field_list_help = """
    select fields to include: eg, --fl url,timestamp
    """

    parser = ArgumentParser('CDX Index API Client')

    parser.add_argument('url',
                        help=url_help)

    parser.add_argument('-n', '--show-num-pages', action='store_true',
                        help='Show Number of Pages only and exit')

    parser.add_argument('-p', '--processes', type=int,
                        help='Number of worker processes to use')

    parser.add_argument('--fl',
                        help=field_list_help)

    parser.add_argument('-j', '--json', action='store_true',
                        help='Use json output instead of cdx(j)')

    parser.add_argument('-z', '--gzipped', action='store_true',
                        help='Storge gzipped results, with .gz extensions')

    parser.add_argument('-o', '--output-prefix',
                        help='Custom output prefix, append with -NN for each page')

    parser.add_argument('-d', '--directory',
                        help='Specify custom output directory')

    parser.add_argument('--page-size', type=int,
                        help='size of each page in blocks, >=1')

    group = parser.add_mutually_exclusive_group()
    group.add_argument('-c', '--coll', default='CC-MAIN-2015-06',
                       help='The index collection to use')

    group.add_argument('--cdx-server-url',
                       help='Set endpoint for CDX Server API')

    parser.add_argument('--timeout', default=30, type=int,
                        help='HTTP read timeout before retry')

    parser.add_argument('--max-retries', default=5, type=int,
                        help='Number of retry attempts')

    parser.add_argument('-v', '--verbose', action='store_true',
                        help='Verbose logging of debug msgs')

    parser.add_argument('--pages', type=int, nargs='*',
                        help=('Get only the specified result page(s) instead ' +
                              'of all results'))

    parser.add_argument('--header', nargs='*',
                        help='Add custom header to request')

    parser.add_argument('--in-order', action='store_true',
                        help='Fetch pages in order (default is to shuffle page list)')

    # Logging
    r = parser.parse_args()

    if r.verbose:
        level = logging.DEBUG
    else:
        level = logging.INFO


    logging.basicConfig(format='%(asctime)s: [%(levelname)s]: %(message)s',
                        level=level)

    logging.getLogger("requests").setLevel(logging.WARNING)

    if r.cdx_server_url:
        api_url = r.cdx_server_url
    else:
        api_url = DEF_API_BASE + r.coll + '-index'

    logging.debug('Getting Num Pages...')
    num_pages = get_num_pages(api_url, r.url, r.page_size)

    # Num Pages Only Query
    if r.show_num_pages:
        print(num_pages)
        return

    if num_pages == 0:
        print('No results found for: ' + r.url)

    # set output
    if not r.output_prefix:
        if r.url.startswith('*'):
            output_prefix = 'domain-' + r.url.strip('*.')
        elif r.url.endswith('*'):
            output_prefix = 'prefix-' + r.url.strip('*')
        elif r.url.startswith(('http://', 'https://', '//')):
            output_prefix = r.url.split('//', 1)[-1]
        else:
            output_prefix = r.url

        output_prefix = output_prefix.strip('/')
        output_prefix = output_prefix.replace('/', '-')
        output_prefix = urllib.quote(output_prefix) + '-'
    else:
        output_prefix = r.output_prefix

    def get_page_job(page):
        job = {}
        job['api_url'] = api_url
        job['url'] = r.url
        job['page'] = page
        job['num_pages'] = num_pages
        job['output_prefix'] = output_prefix
        job['fl'] = r.fl
        job['json'] = r.json
        job['page_size'] = r.page_size
        job['timeout'] = r.timeout
        job['max_retries'] = r.max_retries
        job['gzipped'] = r.gzipped
        job['headers'] = r.header
        job['dir'] = r.directory
        return job

    if r.pages:
        page_list = r.pages
        logging.info('Fetching pages {0} of {1}'.format(r.pages, r.url))
        num_pages = len(page_list)
    else:
        page_list = range(0, num_pages)
        logging.info('Fetching {0} pages of {1}'.format(num_pages, r.url))

    if num_pages == 1:
        fetch_result_page(get_page_job(page_list[0]))
        return

    # set num workers based on proesses
    if not r.processes:
        try:
            num_workers = cpu_count() * 2
        except __HOLE__:
            num_workers = 4
    else:
        num_workers = r.processes

    num_workers = min(num_workers, num_pages)

    # generate page jobs
    job_list = map(get_page_job, page_list)

    run_workers(num_workers, job_list, not r.in_order)
dataset/ETHPy150Open ikreymer/cdx-index-client/cdx-index-client.py/main