docstr: Returns a sequence of tokens from the iterable.
docstr_tok: ["Returns","a","sequence","of","tokens","from","the","iterable","."]
code: def tokenize(self, text, building=False):
      """
      Returns a sequence of tokens from the iterable.
      """
      if self.lower:
          text = text.lower()
  
      if self.tokenizer == "re":
          return self.re_tokenize(text)
  
      elif self.tokenizer == "whitespace":
          return text.split(" ")
  
      word_tokens = (
          text.replace(".", " . ")
          .replace(". . .", "...")
          .replace(",", " , ")
          .replace(";", " ; ")
          .replace(":", " : ")
          .replace("!", " ! ")
          .replace("?", " ? ")
          .replace("  ", " ")
          .replace("  ", " ")
          .strip()
          .split(" ")
      )
  
      return word_tokens
  
file_path: ParlAI/projects/wizard_of_wikipedia/wizard_transformer_ranker/wizard_dict.py
imports: from parlai.core.build_data import modelzoo_path
  from parlai.core.dict import DictionaryAgent
  from collections import defaultdict
  import copy
  import os
  import re
label_desc: Use image node_id 2 for calling the WizardDictAgent obj's underlying member method code with example usage: obj.tokenize(text, building) and returns: word_tokens, self, text