class_docstr: Base class for all slow tokenizers.
  
  Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`].
  
  Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading
  pretrained tokenizers as well as adding tokens to the vocabulary.
  
  This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the
  specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).
class_docstr_tok: ["Base","class","for","all","slow","tokenizers",".","Inherits","from","[","`","~tokenization_utils_base.PreTrainedTokenizerBase","`","]",".","Handle","all","the","shared","methods","for","tokenization","and","special","tokens","as","well","as","methods","downloading\/caching\/loading","pretrained","tokenizers","as","well","as","adding","tokens","to","the","vocabulary",".","This","class","also","contain","the","added","tokens","in","a","unified","way","on","top","of","all","tokenizers","so","we","do","n't","have","to","handle","the","specific","vocabulary","augmentation","methods","of","the","various","underlying","dictionary","structures","(","BPE",",","sentencepiece","...",")","."]
code: def __init__(self, **kwargs):
      # 1. Init the parent class
  
      self.tokens_trie = Trie()
  
      # 2. init `_added_tokens_decoder` if child class did not
      if not hasattr(self, "_added_tokens_decoder"):
          self._added_tokens_decoder: Dict[int, AddedToken] = {}
  
      # 3. if a `added_tokens_decoder` is passed, we are loading from a saved tokenizer, we overwrite
      self._added_tokens_decoder.update(
          kwargs.pop("added_tokens_decoder", {})
      )
      self._added_tokens_encoder: Dict[str, int] = {
          k.content: v for v, k in self._added_tokens_decoder.items()
      }
  
      # 4 init the parent class
      super().__init__(**kwargs)
  
      # 4. If some of the special tokens are not part of the vocab, we add them, at the end.
      # the order of addition is the same as self.SPECIAL_TOKENS_ATTRIBUTES following `tokenizers`
      self._add_tokens(
          [
              token
              for token in self.all_special_tokens_extended
              if token not in self._added_tokens_encoder
          ],
          special_tokens=True,
      )
  
      self._decode_use_source_tokenizer = False
  
imports: import bisect
  import itertools
  import re
  import unicodedata
  from collections import OrderedDict
  from typing import Any, Dict, List, Optional, Tuple, Union, overload
  from .tokenization_utils_base import ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING, INIT_TOKENIZER_DOCSTRING, AddedToken, BatchEncoding, EncodedInput, EncodedInputPair, PreTokenizedInput, PreTokenizedInputPair, PreTrainedTokenizerBase, TextInput, TextInputPair, TruncationStrategy
  from .utils import PaddingStrategy, TensorType, add_end_docstrings, logging
label_desc: Use image node_id 1 to create a new PreTrainedTokenizer object from inherited base classes: PreTrainedTokenizerBase with example: obj = PreTrainedTokenizer()