def download_file(url, dest=None, chunk_size=1024, replace="ask",
                  label="Downloading {dest_basename} ({size:.2f}MB)",
                  expected_extension=None):
    """Download a file from a given URL and display progress.

    :param dest: If the destination exists and is a directory, the filename
        will be guessed from the Content-Disposition header. If the destination
        is an existing file, the user will either be prompted to overwrite, or
        the file will be replaced (depending on the value of **replace**). If
        the destination does not exist, it will be used as the filename.
    :param int chunk_size: bytes read in at a time.
    :param replace: If `False`, an existing destination file will not be
        overwritten.
    :param label: a string which is formatted and displayed as the progress bar
        label. Variables provided include *dest_basename*, *dest*, and *size*.
    :param expected_extension: if set, the filename will be sanitized to ensure
        it has the given extension. The extension should not start with a dot
        (`.`).
    """
    dest = Path(dest or url.split("/")[-1])
    response = get(url, stream=True)
    if (dest.exists()
            and dest.is_dir()
            and "Content-Disposition" in response.headers):
        content_disposition = rfc6266.parse_requests_response(response)
        if expected_extension is not None:
            filename = content_disposition.filename_sanitized(
                expected_extension)
        filename = secure_filename(filename)
        dest = dest / filename
    if dest.exists() and not dest.is_dir():
        if (replace is False
                or replace == "ask"
                and not click.confirm("Replace {}?".format(dest))):
            return str(dest)
    size = int(response.headers.get("content-length", 0))
    label = label.format(dest=dest, dest_basename=dest.name,
                         size=size/1024.0/1024)
    with click.open_file(str(dest), "wb") as f:
        content_iter = response.iter_content(chunk_size=chunk_size)
        with click.progressbar(content_iter, length=size/1024,
                               label=label) as bar:
            for chunk in bar:
                if chunk:
                    f.write(chunk)
                    f.flush()
    return str(dest)
def verify(opt):
    if opt['datatype'] == 'train':
        logging.warning("changing datatype from train to train:ordered")
        opt['datatype'] = 'train:ordered'
    opt.log()
    # create repeat label agent and assign it to the specified task
    agent = RepeatLabelAgent(opt)
    world = create_task(opt, agent)

    log_every_n_secs = opt.get('log_every_n_secs', -1)
    if log_every_n_secs <= 0:
        log_every_n_secs = float('inf')
    log_time = TimeLogger()

    counts = {}
    counts['missing_text'] = 0
    counts['missing_labels'] = 0
    counts['missing_label_candidates'] = 0
    counts['empty_string_label_candidates'] = 0
    counts['label_candidates_with_missing_label'] = 0
    counts['did_not_return_message'] = 0

    # Show some example dialogs.
    while not world.epoch_done():
        world.parley()

        act = world.acts[0]

        if not isinstance(act, Message):
            counts['did_not_return_message'] += 1

        if 'text' not in act and 'image' not in act:
            warn("warning: missing text field:\n", act, opt)
            counts['missing_text'] += 1

        if 'labels' not in act and 'eval_labels' not in act:
            warn("warning: missing labels/eval_labels field:\n", act, opt)
            counts['missing_labels'] += 1
        else:
            if 'label_candidates' not in act:
                counts['missing_label_candidates'] += 1
            else:
                labels = act.get('labels', act.get('eval_labels'))
                is_label_cand = {}
                for l in labels:
                    is_label_cand[l] = False
                for c in act['label_candidates']:
                    if c == '':
                        warn("warning: empty string label_candidate:\n", act, opt)
                        counts['empty_string_label_candidates'] += 1
                    if c in is_label_cand:
                        if is_label_cand[c] is True:
                            warn(
                                "warning: label mentioned twice in candidate_labels:\n",
                                act,
                                opt,
                            )
                        is_label_cand[c] = True
                for _, has in is_label_cand.items():
                    if has is False:
                        warn("warning: label missing in candidate_labels:\n", act, opt)
                        counts['label_candidates_with_missing_label'] += 1

        if log_time.time() > log_every_n_secs:
            text, log = report(world, counts, log_time)
            print(text)

    try:
        # print dataset size if available
        logging.info(
            f'Loaded {world.num_episodes()} episodes with a '
            f'total of {world.num_examples()} examples'
        )
    except AttributeError:
        pass

    counts['exs'] = int(world.report()['exs'])
    return counts