class_docstr: Base class inherited by all tokenizer classes. This class
  handles downloading and loading pre-trained tokenizer and adding tokens to
  the vocabulary.
  
  Derived class can set up a few special tokens to be used in common scripts
  and internals: :attr:`bos_token`, :attr:`eos_token`, :attr:`unk_token`,
  :attr:`sep_token`, :attr:`pad_token`, :attr:`cls_token`,
  :attr:`mask_token`, and :attr:`additional_special_tokens`.
  
  We defined an :attr:`added_tokens_encoder` to add new tokens to the
  vocabulary without having to handle the specific vocabulary augmentation
  methods of the various underlying dictionary structures (`BPE`,
  `sentencepiece` ...).
class_docstr_tok: ["Base","class","inherited","by","all","tokenizer","classes",".","This","class","handles","downloading","and","loading","pre-trained","tokenizer","and","adding","tokens","to","the","vocabulary",".","Derived","class","can","set","up","a","few","special","tokens","to","be","used","in","common","scripts","and","internals",":",":","attr",":","`","bos_token","`",",",":","attr",":","`","eos_token","`",",",":","attr",":","`","unk_token","`",",",":","attr",":","`","sep_token","`",",",":","attr",":","`","pad_token","`",",",":","attr",":","`","cls_token","`",",",":","attr",":","`","mask_token","`",",","and",":","attr",":","`","additional_special_tokens","`",".","We","defined","an",":","attr",":","`","added_tokens_encoder","`","to","add","new","tokens","to","the","vocabulary","without","having","to","handle","the","specific","vocabulary","augmentation","methods","of","the","various","underlying","dictionary","structures","(","`","BPE","`",",","`","sentencepiece","`","...",")","."]
docstr: Instantiate a tokenizer from the vocabulary files or the saved
  tokenizer files.
  
  Args:
      pretrained_model_path: The path to a vocabulary file or a folder
          that contains the saved pre-trained tokenizer files.
      configs: Tokenizer configurations. You can overwrite the original
          tokenizer configurations saved in the configuration file
          by this dictionary.
  
  Returns:
      A tokenizer instance.
docstr_tok: ["Instantiate","a","tokenizer","from","the","vocabulary","files","or","the","saved","tokenizer","files",".","Args",":","pretrained_model_path",":","The","path","to","a","vocabulary","file","or","a","folder","that","contains","the","saved","pre-trained","tokenizer","files",".","configs",":","Tokenizer","configurations",".","You","can","overwrite","the","original","tokenizer","configurations","saved","in","the","configuration","file","by","this","dictionary",".","Returns",":","A","tokenizer","instance","."]
code: def load(
      cls, pretrained_model_path: str, configs: Optional[Dict] = None
  ):
      r"""Instantiate a tokenizer from the vocabulary files or the saved
      tokenizer files.
  
      Args:
          pretrained_model_path: The path to a vocabulary file or a folder
              that contains the saved pre-trained tokenizer files.
          configs: Tokenizer configurations. You can overwrite the original
              tokenizer configurations saved in the configuration file
              by this dictionary.
  
      Returns:
          A tokenizer instance.
      """
      vocab_files = {}
      # Look for the tokenizer main vocabulary files
      for file_id, file_name in cls._VOCAB_FILE_NAMES.items():
          full_file_name: Optional[str]
          if os.path.isdir(pretrained_model_path):
              # If a directory is provided we look for the standard file name
              full_file_name = os.path.join(
                  pretrained_model_path, file_name
              )
          else:
              # If a path to a file is provided we use it (will only work
              # for non-BPE tokenizer using a single vocabulary file)
              full_file_name = pretrained_model_path
          if not os.path.exists(full_file_name):
              print(
                  "Can't find file {}. We won't load it.".format(
                      full_file_name
                  )
              )
              full_file_name = None
          vocab_files[file_id] = full_file_name
  
      # Look for the additional tokens files
      all_vocab_files_names = {
          "added_tokens_file": ADDED_TOKENS_FILE,
          "special_tokens_map_file": SPECIAL_TOKENS_MAP_FILE,
          "config_file": CONFIG_FILE,
      }
  
      # If a path to a file was provided, get the parent directory
      saved_directory = pretrained_model_path
      if os.path.exists(saved_directory) and not os.path.isdir(
          saved_directory
      ):
          saved_directory = os.path.dirname(saved_directory)
  
      for file_id, file_name in all_vocab_files_names.items():
          full_file_name = os.path.join(saved_directory, file_name)
          if not os.path.exists(full_file_name):
              print(
                  "Can't find file {}. We won't load it.".format(
                      full_file_name
                  )
              )
              full_file_name = None
          vocab_files[file_id] = full_file_name
  
      if all(
          full_file_name is None
          for full_file_name in vocab_files.values()
      ):
          raise ValueError(
              "Can't find tokenizer files in {}.".format(
                  saved_directory
              )
          )
  
      kwargs: Dict[str, Any]
      if cls._IS_PRETRAINED:
          kwargs = {"pretrained_model_name": None}
      else:
          kwargs = {}
  
      added_tokens_file = vocab_files.pop("added_tokens_file", None)
      special_tokens_map_file = vocab_files.pop(
          "special_tokens_map_file", None
      )
      tokenizer_config_file = vocab_files.pop("config_file", None)
  
      for args_name, file_path in vocab_files.items():
          if args_name not in kwargs:
              kwargs[args_name] = file_path
  
      if special_tokens_map_file is not None:
          with open(special_tokens_map_file, encoding="utf-8") as f:
              special_tokens_map = json.load(f)
          for key, value in special_tokens_map.items():
              if key not in kwargs:
                  kwargs[key] = value
  
      if tokenizer_config_file is not None:
          with open(tokenizer_config_file, encoding="utf-8") as f:
              tokenizer_config = json.load(f)
          for key, value in tokenizer_config.items():
              if key not in kwargs:
                  kwargs[key] = value
  
      if configs is not None:
          for key, value in configs.items():
              kwargs[key] = value
  
      tokenizer = cls(hparams=kwargs)
  
      # Add supplementary tokens.
      if added_tokens_file is not None:
          with open(added_tokens_file, encoding="utf-8") as f:
              added_tok_encoder = json.load(f)
          added_tok_decoder = {
              v: k for k, v in added_tok_encoder.items()
          }
          tokenizer.added_tokens_encoder.update(added_tok_encoder)
          tokenizer.added_tokens_decoder.update(added_tok_decoder)
  
      return tokenizer
  
imports: from typing import Any, Dict, List, Optional, Tuple, overload
  import json
  import os
  import warnings
  from texar.tf.module_base import ModuleBase
label_desc: Use image node_id 2 for calling the TokenizerBase obj's underlying member method code with example usage: obj.load(cls, pretrained_model_path, configs) and returns: tokenizer