class_docstr: Base class inherited by all tokenizer classes. This class
  handles downloading and loading pre-trained tokenizer and adding tokens to
  the vocabulary.
  
  Derived class can set up a few special tokens to be used in common scripts
  and internals: :attr:`bos_token`, :attr:`eos_token`, :attr:`unk_token`,
  :attr:`sep_token`, :attr:`pad_token`, :attr:`cls_token`,
  :attr:`mask_token`, and :attr:`additional_special_tokens`.
  
  We defined an :attr:`added_tokens_encoder` to add new tokens to the
  vocabulary without having to handle the specific vocabulary augmentation
  methods of the various underlying dictionary structures (`BPE`,
  `sentencepiece` ...).
class_docstr_tok: ["Base","class","inherited","by","all","tokenizer","classes",".","This","class","handles","downloading","and","loading","pre-trained","tokenizer","and","adding","tokens","to","the","vocabulary",".","Derived","class","can","set","up","a","few","special","tokens","to","be","used","in","common","scripts","and","internals",":",":","attr",":","`","bos_token","`",",",":","attr",":","`","eos_token","`",",",":","attr",":","`","unk_token","`",",",":","attr",":","`","sep_token","`",",",":","attr",":","`","pad_token","`",",",":","attr",":","`","cls_token","`",",",":","attr",":","`","mask_token","`",",","and",":","attr",":","`","additional_special_tokens","`",".","We","defined","an",":","attr",":","`","added_tokens_encoder","`","to","add","new","tokens","to","the","vocabulary","without","having","to","handle","the","specific","vocabulary","augmentation","methods","of","the","various","underlying","dictionary","structures","(","`","BPE","`",",","`","sentencepiece","`","...",")","."]
docstr: Save the tokenizer vocabulary files (with added tokens), tokenizer
  configuration file and a dictionary mapping special token class
  attributes (:attr:`cls_token`, :attr:`unk_token`, ...) to their values
  (`<unk>`, `<cls>`, ...) to a directory, so that it can be re-loaded
  using the :meth:`~load`.
  
  Args:
      save_dir: The path to a folder in which the tokenizer files
          will be saved.
  
  Return:
      The paths to the vocabulary file, added token file, special token
      mapping file, and the configuration file.
docstr_tok: ["Save","the","tokenizer","vocabulary","files","(","with","added","tokens",")",",","tokenizer","configuration","file","and","a","dictionary","mapping","special","token","class","attributes","(",":","attr",":","`","cls_token","`",",",":","attr",":","`","unk_token","`",",","...",")","to","their","values","(","`","<","unk",">","`",",","`","<","cls",">","`",",","...",")","to","a","directory",",","so","that","it","can","be","re-loaded","using","the",":","meth",":","`","~load","`",".","Args",":","save_dir",":","The","path","to","a","folder","in","which","the","tokenizer","files","will","be","saved",".","Return",":","The","paths","to","the","vocabulary","file",",","added","token","file",",","special","token","mapping","file",",","and","the","configuration","file","."]
code: def save(self, save_dir: str) -> Tuple[str]:
      r"""Save the tokenizer vocabulary files (with added tokens), tokenizer
      configuration file and a dictionary mapping special token class
      attributes (:attr:`cls_token`, :attr:`unk_token`, ...) to their values
      (`<unk>`, `<cls>`, ...) to a directory, so that it can be re-loaded
      using the :meth:`~load`.
  
      Args:
          save_dir: The path to a folder in which the tokenizer files
              will be saved.
  
      Return:
          The paths to the vocabulary file, added token file, special token
          mapping file, and the configuration file.
      """
      if not os.path.isdir(save_dir):
          raise ValueError(
              "Saving directory ({}) should be a "
              "directory".format(save_dir)
          )
  
      special_tokens_map_file = os.path.join(
          save_dir, SPECIAL_TOKENS_MAP_FILE
      )
      added_tokens_file = os.path.join(save_dir, ADDED_TOKENS_FILE)
      config_file = os.path.join(save_dir, CONFIG_FILE)
  
      with open(special_tokens_map_file, "w", encoding="utf-8") as f:
          f.write(
              json.dumps(self.special_tokens_map, ensure_ascii=False)
          )
  
      with open(added_tokens_file, "w", encoding="utf-8") as f:
          if self.added_tokens_encoder:
              out_str = json.dumps(
                  self.added_tokens_encoder, ensure_ascii=False
              )
          else:
              out_str = "{}"
          f.write(out_str)
  
      with open(config_file, "w", encoding="utf-8") as f:
          if self.config:
              out_str = json.dumps(self.config, ensure_ascii=False)
          else:
              out_str = "{}"
          f.write(out_str)
  
      vocab_files = self.save_vocab(save_dir)
      return vocab_files + (
          special_tokens_map_file,
          added_tokens_file,
          config_file,
      )
  
imports: from typing import Any, Dict, List, Optional, Tuple, overload
  import json
  import os
  import warnings
  from texar.tf.module_base import ModuleBase
label_desc: Use image node_id 3 for calling the TokenizerBase obj's underlying member method code with example usage: obj.save(save_dir) and returns: unknown