class_docstr: Base class inherited by all tokenizer classes. This class
  handles downloading and loading pre-trained tokenizer and adding tokens to
  the vocabulary.
  
  Derived class can set up a few special tokens to be used in common scripts
  and internals: :attr:`bos_token`, :attr:`eos_token`, :attr:`unk_token`,
  :attr:`sep_token`, :attr:`pad_token`, :attr:`cls_token`,
  :attr:`mask_token`, and :attr:`additional_special_tokens`.
  
  We defined an :attr:`added_tokens_encoder` to add new tokens to the
  vocabulary without having to handle the specific vocabulary augmentation
  methods of the various underlying dictionary structures (`BPE`,
  `sentencepiece` ...).
class_docstr_tok: ["Base","class","inherited","by","all","tokenizer","classes",".","This","class","handles","downloading","and","loading","pre-trained","tokenizer","and","adding","tokens","to","the","vocabulary",".","Derived","class","can","set","up","a","few","special","tokens","to","be","used","in","common","scripts","and","internals",":",":","attr",":","`","bos_token","`",",",":","attr",":","`","eos_token","`",",",":","attr",":","`","unk_token","`",",",":","attr",":","`","sep_token","`",",",":","attr",":","`","pad_token","`",",",":","attr",":","`","cls_token","`",",",":","attr",":","`","mask_token","`",",","and",":","attr",":","`","additional_special_tokens","`",".","We","defined","an",":","attr",":","`","added_tokens_encoder","`","to","add","new","tokens","to","the","vocabulary","without","having","to","handle","the","specific","vocabulary","augmentation","methods","of","the","various","underlying","dictionary","structures","(","`","BPE","`",",","`","sentencepiece","`","...",")","."]
docstr: Add a dictionary of special tokens to the encoder and link them to
  class attributes. If the special tokens are not in the vocabulary, they
  are added to it and indexed starting from the last index of the
  current vocabulary.
  
  Args:
      special_tokens_dict: A dictionary mapping special token class
          attributes (:attr:`cls_token`, :attr:`unk_token`, ...) to their
          values (`<unk>`, `<cls>`, ...).
  
  Returns:
      Number of tokens added to the vocabulary which can be used to
      correspondingly increase the size of the associated model embedding
      matrices.
docstr_tok: ["Add","a","dictionary","of","special","tokens","to","the","encoder","and","link","them","to","class","attributes",".","If","the","special","tokens","are","not","in","the","vocabulary",",","they","are","added","to","it","and","indexed","starting","from","the","last","index","of","the","current","vocabulary",".","Args",":","special_tokens_dict",":","A","dictionary","mapping","special","token","class","attributes","(",":","attr",":","`","cls_token","`",",",":","attr",":","`","unk_token","`",",","...",")","to","their","values","(","`","<","unk",">","`",",","`","<","cls",">","`",",","...",")",".","Returns",":","Number","of","tokens","added","to","the","vocabulary","which","can","be","used","to","correspondingly","increase","the","size","of","the","associated","model","embedding","matrices","."]
code: def add_special_tokens(
      self, special_tokens_dict: Dict[str, str]
  ) -> int:
      r"""Add a dictionary of special tokens to the encoder and link them to
      class attributes. If the special tokens are not in the vocabulary, they
      are added to it and indexed starting from the last index of the
      current vocabulary.
  
      Args:
          special_tokens_dict: A dictionary mapping special token class
              attributes (:attr:`cls_token`, :attr:`unk_token`, ...) to their
              values (`<unk>`, `<cls>`, ...).
  
      Returns:
          Number of tokens added to the vocabulary which can be used to
          correspondingly increase the size of the associated model embedding
          matrices.
      """
      if not special_tokens_dict:
          return 0
  
      added_tokens = 0
      for key, value in special_tokens_dict.items():
          assert key in self._SPECIAL_TOKENS_ATTRIBUTES
          if key == "additional_special_tokens":
              assert isinstance(value, (list, tuple)) and all(
                  isinstance(t, str) for t in value
              )
              added_tokens += self.add_tokens(value)
          else:
              assert isinstance(value, str)
              added_tokens += self.add_tokens([value])
          setattr(self, key, value)
  
      return added_tokens
  
imports: from typing import Any, Dict, List, Optional, Tuple, overload
  import json
  import os
  import warnings
  from texar.tf.module_base import ModuleBase
label_desc: Use image node_id 8 for calling the TokenizerBase obj's underlying member method code with example usage: obj.add_special_tokens(special_tokens_dict) and returns: added_tokens, int