class_docstr: Base class inherited by all tokenizer classes. This class
  handles downloading and loading pre-trained tokenizer and adding tokens to
  the vocabulary.
  
  Derived class can set up a few special tokens to be used in common scripts
  and internals: :attr:`bos_token`, :attr:`eos_token`, :attr:`unk_token`,
  :attr:`sep_token`, :attr:`pad_token`, :attr:`cls_token`,
  :attr:`mask_token`, and :attr:`additional_special_tokens`.
  
  We defined an :attr:`added_tokens_encoder` to add new tokens to the
  vocabulary without having to handle the specific vocabulary augmentation
  methods of the various underlying dictionary structures (`BPE`,
  `sentencepiece` ...).
class_docstr_tok: ["Base","class","inherited","by","all","tokenizer","classes",".","This","class","handles","downloading","and","loading","pre-trained","tokenizer","and","adding","tokens","to","the","vocabulary",".","Derived","class","can","set","up","a","few","special","tokens","to","be","used","in","common","scripts","and","internals",":",":","attr",":","`","bos_token","`",",",":","attr",":","`","eos_token","`",",",":","attr",":","`","unk_token","`",",",":","attr",":","`","sep_token","`",",",":","attr",":","`","pad_token","`",",",":","attr",":","`","cls_token","`",",",":","attr",":","`","mask_token","`",",","and",":","attr",":","`","additional_special_tokens","`",".","We","defined","an",":","attr",":","`","added_tokens_encoder","`","to","add","new","tokens","to","the","vocabulary","without","having","to","handle","the","specific","vocabulary","augmentation","methods","of","the","various","underlying","dictionary","structures","(","`","BPE","`",",","`","sentencepiece","`","...",")","."]
docstr: Add a list of new tokens to the tokenizer class. If the new tokens
  are not in the vocabulary, they are added to the
  :attr:`added_tokens_encoder` with indices starting from the last index
  of the current vocabulary.
  
  Args:
      new_tokens: A list of new tokens.
  
  Returns:
      Number of tokens added to the vocabulary which can be used to
      correspondingly increase the size of the associated model embedding
      matrices.
docstr_tok: ["Add","a","list","of","new","tokens","to","the","tokenizer","class",".","If","the","new","tokens","are","not","in","the","vocabulary",",","they","are","added","to","the",":","attr",":","`","added_tokens_encoder","`","with","indices","starting","from","the","last","index","of","the","current","vocabulary",".","Args",":","new_tokens",":","A","list","of","new","tokens",".","Returns",":","Number","of","tokens","added","to","the","vocabulary","which","can","be","used","to","correspondingly","increase","the","size","of","the","associated","model","embedding","matrices","."]
code: def add_tokens(self, new_tokens: List[Optional[str]]) -> int:
      r"""Add a list of new tokens to the tokenizer class. If the new tokens
      are not in the vocabulary, they are added to the
      :attr:`added_tokens_encoder` with indices starting from the last index
      of the current vocabulary.
  
      Args:
          new_tokens: A list of new tokens.
  
      Returns:
          Number of tokens added to the vocabulary which can be used to
          correspondingly increase the size of the associated model embedding
          matrices.
      """
      if not new_tokens:
          return 0
  
      to_add_tokens = []
      for token in new_tokens:
          assert isinstance(token, str)
          if token != self.unk_token and (
              self.map_token_to_id(token)
              == self.map_token_to_id(self.unk_token)
          ):
              to_add_tokens.append(token)
  
      added_tok_encoder = dict(
          (tok, len(self) + i) for i, tok in enumerate(to_add_tokens)
      )
      added_tok_decoder = {v: k for k, v in added_tok_encoder.items()}
      self.added_tokens_encoder.update(added_tok_encoder)
      self.added_tokens_decoder.update(added_tok_decoder)
  
      return len(to_add_tokens)
  
imports: from typing import Any, Dict, List, Optional, Tuple, overload
  import json
  import os
  import warnings
  from texar.tf.module_base import ModuleBase
label_desc: Use image node_id 7 for calling the TokenizerBase obj's underlying member method code with example usage: obj.add_tokens(new_tokens) and returns: len, int