code: def verify(opt):
      if opt["datatype"] == "train":
          logging.warning(
              "changing datatype from train to train:ordered"
          )
          opt["datatype"] = "train:ordered"
  
      # create repeat label agent and assign it to the specified task
      opt["fixed_response"] = None
      agent = FixedResponseAgent(opt)
      world = create_task(opt, agent)
      opt.log()
  
      log_every_n_secs = opt.get("log_every_n_secs", -1)
      if log_every_n_secs <= 0:
          log_every_n_secs = float("inf")
      log_time = TimeLogger()
  
      dictionary = DictionaryAgent(opt)
      ignore_tokens = opt.get("ignore_tokens").split(",")
  
      counts = {}
      for t in {"input", "labels", "both"}:
          counts[f"{t}/tokens"] = 0
          counts[f"{t}/utterances"] = 0
          counts[f"{t}/avg_utterance_length"] = None
          counts[f"{t}/unique_tokens"] = 0
          counts[f"{t}/unique_utterances"] = 0
          # for counting the stats..
          counts[f"{t}/token_dict"] = {}
          counts[f"{t}/utterance_dict"] = {}
  
      def tokenize(txt):
          return dictionary.tokenize(txt)
  
      def keep_token(t):
          for s in ignore_tokens:
              if s != "" and s in t:
                  return False
          return True
  
      # max number of examples to evaluate
      max_cnt = (
          opt["num_examples"]
          if opt["num_examples"] > 0
          else float("inf")
      )
      cnt = 0
  
      # Show some example dialogs.
      while not world.epoch_done() and world.total_exs < max_cnt:
          world.parley()
          act = world.get_acts()[opt.get("agent")]
          if act.is_padding():
              continue
          for itype in {"input", "labels"}:
              if itype == "input":
                  if opt.get("new_line_new_utt"):
                      txts = act.get("text").split("\n")
                  else:
                      txts = [act.get("text", "")]
              else:
                  txts = act.get("labels", act.get("eval_labels", [""]))
  
              for txt in txts:
                  tokens = tokenize(txt)
                  retxt = [t for t in tokens if keep_token(t)]
                  counts[f"{itype}/tokens"] += len(retxt)
                  counts["both/tokens"] += len(retxt)
                  counts[f"{itype}/utterances"] += 1
                  counts["both/utterances"] += 1
                  counts[
                      f"{itype}/avg_utterance_length"
                  ] += AverageMetric(len(retxt), 1)
                  counts[f"both/avg_utterance_length"] += AverageMetric(
                      len(retxt), 1
                  )
                  for t in retxt:
                      if t not in counts[f"{itype}/token_dict"]:
                          counts[f"{itype}/unique_tokens"] += 1
                          counts[f"{itype}/token_dict"][t] = True
                      if t not in counts["both/token_dict"]:
                          counts["both/unique_tokens"] += 1
                          counts["both/token_dict"][t] = True
                  retxt = " ".join(retxt)
                  if retxt not in counts[f"{itype}/utterance_dict"]:
                      counts[f"{itype}/unique_utterances"] += 1
                      counts[f"{itype}/utterance_dict"][retxt] = True
                  if retxt not in counts["both/utterance_dict"]:
                      counts["both/unique_utterances"] += 1
                      counts["both/utterance_dict"][retxt] = True
  
          if log_time.time() > log_every_n_secs:
              report = _report(world, counts)
              cnt = report.pop("exs")
              text, log = log_time.log(
                  cnt, world.num_examples(), report
              )
              logging.info(text)
  
      try:
          # print dataset size if available
          logging.info(
              f"loaded {world.num_episodes()} episodes with a total "
              f"of {world.num_examples()} examples"
          )
      except AttributeError:
          pass
  
      retval = _report(world, counts)
      retval.pop("exs")
      return retval
  
file_path: ParlAI/parlai/scripts/data_stats.py
imports: from parlai.core.params import ParlaiParser
  from parlai.agents.fixed_response.fixed_response import FixedResponseAgent
  from parlai.core.worlds import create_task
  from parlai.utils.misc import TimeLogger, nice_report
  from parlai.core.metrics import AverageMetric
  from parlai.core.dict import DictionaryAgent
  from parlai.core.script import ParlaiScript, register_script
  import parlai.utils.logging
label_desc: Use image node_id 3 for calling a global function with example usage: verify(opt) and returns: retval, dictionary, True, False