def download_file(url, dest=None, chunk_size=1024, replace="ask",
                  label="Downloading {dest_basename} ({size:.2f}MB)",
                  expected_extension=None):
    """Download a file from a given URL and display progress.

    :param dest: If the destination exists and is a directory, the filename
        will be guessed from the Content-Disposition header. If the destination
        is an existing file, the user will either be prompted to overwrite, or
        the file will be replaced (depending on the value of **replace**). If
        the destination does not exist, it will be used as the filename.
    :param int chunk_size: bytes read in at a time.
    :param replace: If `False`, an existing destination file will not be
        overwritten.
    :param label: a string which is formatted and displayed as the progress bar
        label. Variables provided include *dest_basename*, *dest*, and *size*.
    :param expected_extension: if set, the filename will be sanitized to ensure
        it has the given extension. The extension should not start with a dot
        (`.`).
    """
    dest = Path(dest or url.split("/")[-1])
    response = get(url, stream=True)
    if (dest.exists()
            and dest.is_dir()
            and "Content-Disposition" in response.headers):
        content_disposition = rfc6266.parse_requests_response(response)
        if expected_extension is not None:
            filename = content_disposition.filename_sanitized(
                expected_extension)
        filename = secure_filename(filename)
        dest = dest / filename
    if dest.exists() and not dest.is_dir():
        if (replace is False
                or replace == "ask"
                and not click.confirm("Replace {}?".format(dest))):
            return str(dest)
    size = int(response.headers.get("content-length", 0))
    label = label.format(dest=dest, dest_basename=dest.name,
                         size=size/1024.0/1024)
    with click.open_file(str(dest), "wb") as f:
        content_iter = response.iter_content(chunk_size=chunk_size)
        with click.progressbar(content_iter, length=size/1024,
                               label=label) as bar:
            for chunk in bar:
                if chunk:
                    f.write(chunk)
                    f.flush()
    return str(dest)
def find_many_example_files(
        top_directory_name, shuffled=True, first_spc_date_string=None,
        last_spc_date_string=None, first_batch_number=None,
        last_batch_number=None, raise_error_if_any_missing=True):
    """Looks for many files with input examples.

    :param top_directory_name: See doc for `find_example_file`.
    :param shuffled: Same.
    :param first_spc_date_string: [used only if `shuffled = False`]
        First SPC date (format "yyyymmdd").  This method will look for all SPC
        dates from `first_spc_date_string`...`last_spc_date_string`.
    :param last_spc_date_string: See above.
    :param first_batch_number: [used only if `shuffled = True`]
        First batch number (integer).  This method will look for all batches
        from `first_batch_number`...`last_batch_number`.
    :param last_batch_number: See above.
    :param raise_error_if_any_missing: Boolean flag.  If *any* desired file is
        not found and `raise_error_if_any_missing = True`, this method will
        error out.
    :return: example_file_names: 1-D list of paths to example files.
    :raises: ValueError: if no files are found.
    """

    error_checking.assert_is_boolean(shuffled)

    if shuffled:
        error_checking.assert_is_integer(first_batch_number)
        error_checking.assert_is_integer(last_batch_number)
        error_checking.assert_is_geq(first_batch_number, 0)
        error_checking.assert_is_geq(last_batch_number, first_batch_number)

        example_file_pattern = (
            '{0:s}/batches{1:s}-{1:s}/input_examples_batch{1:s}.nc'
        ).format(top_directory_name, BATCH_NUMBER_REGEX)

        example_file_names = glob.glob(example_file_pattern)

        if len(example_file_names) > 0:
            batch_numbers = numpy.array(
                [_file_name_to_batch_number(f) for f in example_file_names],
                dtype=int)
            good_indices = numpy.where(numpy.logical_and(
                batch_numbers >= first_batch_number,
                batch_numbers <= last_batch_number
            ))[0]

            example_file_names = [example_file_names[k] for k in good_indices]

        if len(example_file_names) == 0:
            error_string = (
                'Cannot find any files with batch number from {0:d}...{1:d}.'
            ).format(first_batch_number, last_batch_number)
            raise ValueError(error_string)

        return example_file_names

    spc_date_strings = time_conversion.get_spc_dates_in_range(
        first_spc_date_string=first_spc_date_string,
        last_spc_date_string=last_spc_date_string)

    example_file_names = []
    for this_spc_date_string in spc_date_strings:
        this_file_name = find_example_file(
            top_directory_name=top_directory_name, shuffled=False,
            spc_date_string=this_spc_date_string,
            raise_error_if_missing=raise_error_if_any_missing)

        if not os.path.isfile(this_file_name):
            continue
        example_file_names.append(this_file_name)

    if len(example_file_names) == 0:
        error_string = (
            'Cannot find any file with SPC date from {0:s} to {1:s}.'
        ).format(first_spc_date_string, last_spc_date_string)

        raise ValueError(error_string)

    return example_file_names