def download_file(url, dest=None, chunk_size=1024, replace="ask",
                  label="Downloading {dest_basename} ({size:.2f}MB)",
                  expected_extension=None):
    """Download a file from a given URL and display progress.

    :param dest: If the destination exists and is a directory, the filename
        will be guessed from the Content-Disposition header. If the destination
        is an existing file, the user will either be prompted to overwrite, or
        the file will be replaced (depending on the value of **replace**). If
        the destination does not exist, it will be used as the filename.
    :param int chunk_size: bytes read in at a time.
    :param replace: If `False`, an existing destination file will not be
        overwritten.
    :param label: a string which is formatted and displayed as the progress bar
        label. Variables provided include *dest_basename*, *dest*, and *size*.
    :param expected_extension: if set, the filename will be sanitized to ensure
        it has the given extension. The extension should not start with a dot
        (`.`).
    """
    dest = Path(dest or url.split("/")[-1])
    response = get(url, stream=True)
    if (dest.exists()
            and dest.is_dir()
            and "Content-Disposition" in response.headers):
        content_disposition = rfc6266.parse_requests_response(response)
        if expected_extension is not None:
            filename = content_disposition.filename_sanitized(
                expected_extension)
        filename = secure_filename(filename)
        dest = dest / filename
    if dest.exists() and not dest.is_dir():
        if (replace is False
                or replace == "ask"
                and not click.confirm("Replace {}?".format(dest))):
            return str(dest)
    size = int(response.headers.get("content-length", 0))
    label = label.format(dest=dest, dest_basename=dest.name,
                         size=size/1024.0/1024)
    with click.open_file(str(dest), "wb") as f:
        content_iter = response.iter_content(chunk_size=chunk_size)
        with click.progressbar(content_iter, length=size/1024,
                               label=label) as bar:
            for chunk in bar:
                if chunk:
                    f.write(chunk)
                    f.flush()
    return str(dest)
def subset_examples(example_dict, indices_to_keep, create_new_dict=False):
    """Subsets examples in dictionary.

    :param example_dict: See doc for `write_example_file`.
    :param indices_to_keep: 1-D numpy array with indices of examples to keep.
    :param create_new_dict: Boolean flag.  If True, this method will create a
        new dictionary, leaving the input dictionary untouched.
    :return: example_dict: Same as input, but possibly with fewer examples.
    """

    error_checking.assert_is_integer_numpy_array(indices_to_keep)
    error_checking.assert_is_numpy_array(indices_to_keep, num_dimensions=1)
    error_checking.assert_is_boolean(create_new_dict)

    if not create_new_dict:
        for this_key in MAIN_KEYS:
            optional_key_missing = (
                this_key not in REQUIRED_MAIN_KEYS
                and this_key not in example_dict
            )

            if optional_key_missing:
                continue

            if this_key == TARGET_MATRIX_KEY:
                if this_key in example_dict:
                    example_dict[this_key] = (
                        example_dict[this_key][indices_to_keep, ...]
                    )
                else:
                    example_dict[TARGET_VALUES_KEY] = (
                        example_dict[TARGET_VALUES_KEY][indices_to_keep]
                    )

                continue

            if this_key == FULL_IDS_KEY:
                example_dict[this_key] = [
                    example_dict[this_key][k] for k in indices_to_keep
                ]
            else:
                example_dict[this_key] = example_dict[this_key][
                    indices_to_keep, ...]

        return example_dict

    new_example_dict = {}

    for this_key in METADATA_KEYS:
        sounding_key_missing = (
            this_key in [SOUNDING_FIELDS_KEY, SOUNDING_HEIGHTS_KEY]
            and this_key not in example_dict
        )

        if sounding_key_missing:
            continue

        if this_key == TARGET_NAMES_KEY:
            if this_key in example_dict:
                new_example_dict[this_key] = example_dict[this_key]
            else:
                new_example_dict[TARGET_NAME_KEY] = example_dict[
                    TARGET_NAME_KEY]

            continue

        new_example_dict[this_key] = example_dict[this_key]

    for this_key in MAIN_KEYS:
        optional_key_missing = (
            this_key not in REQUIRED_MAIN_KEYS
            and this_key not in example_dict
        )

        if optional_key_missing:
            continue

        if this_key == TARGET_MATRIX_KEY:
            if this_key in example_dict:
                new_example_dict[this_key] = (
                    example_dict[this_key][indices_to_keep, ...]
                )
            else:
                new_example_dict[TARGET_VALUES_KEY] = (
                    example_dict[TARGET_VALUES_KEY][indices_to_keep]
                )

            continue

        if this_key == FULL_IDS_KEY:
            new_example_dict[this_key] = [
                example_dict[this_key][k] for k in indices_to_keep
            ]
        else:
            new_example_dict[this_key] = example_dict[this_key][
                indices_to_keep, ...]

    return new_example_dict