class_docstr: Base class for all fast tokenizers (wrapping HuggingFace tokenizers library).
  
  Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`].
  
  Handles all the shared methods for tokenization and special tokens, as well as methods for
  downloading/caching/loading pretrained tokenizers, as well as adding tokens to the vocabulary.
  
  This class also contains the added tokens in a unified way on top of all tokenizers so we don't have to handle the
  specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).
code: def __init__(self, *args, **kwargs):
      tokenizer_object = kwargs.pop("tokenizer_object", None)
      slow_tokenizer = kwargs.pop("__slow_tokenizer", None)
      fast_tokenizer_file = kwargs.pop("tokenizer_file", None)
      from_slow = kwargs.pop("from_slow", False)
      added_tokens_decoder = kwargs.pop("added_tokens_decoder", {})
  
      if (
          from_slow
          and slow_tokenizer is None
          and self.slow_tokenizer_class is None
      ):
          raise ValueError(
              "Cannot instantiate this tokenizer from a slow version. If it's based on sentencepiece, make sure you "
              "have sentencepiece installed."
          )
  
      if tokenizer_object is not None:
          fast_tokenizer = copy.deepcopy(tokenizer_object)
      elif fast_tokenizer_file is not None and not from_slow:
          # We have a serialization from tokenizers which let us directly build the backend
          fast_tokenizer = TokenizerFast.from_file(fast_tokenizer_file)
      elif slow_tokenizer is not None:
          # We need to convert a slow tokenizer to build the backend
          fast_tokenizer = convert_slow_tokenizer(slow_tokenizer)
      elif self.slow_tokenizer_class is not None:
          # We need to create and convert a slow tokenizer to build the backend
          slow_tokenizer = self.slow_tokenizer_class(*args, **kwargs)
          fast_tokenizer = convert_slow_tokenizer(slow_tokenizer)
      else:
          raise ValueError(
              "Couldn't instantiate the backend tokenizer from one of: \n"
              "(1) a `tokenizers` library serialization file, \n"
              "(2) a slow tokenizer instance to convert or \n"
              "(3) an equivalent slow tokenizer class to instantiate and convert. \n"
              "You need to have sentencepiece installed to convert a slow tokenizer to a fast one."
          )
  
      self._tokenizer = fast_tokenizer
  
      if slow_tokenizer is not None:
          kwargs.update(slow_tokenizer.init_kwargs)
  
      self._decode_use_source_tokenizer = False
  
      _truncation = self._tokenizer.truncation
  
      if _truncation is not None:
          self._tokenizer.enable_truncation(**_truncation)
          kwargs.setdefault("max_length", _truncation["max_length"])
          kwargs.setdefault("truncation_side", _truncation["direction"])
          kwargs.setdefault("stride", _truncation["stride"])
          kwargs.setdefault(
              "truncation_strategy", _truncation["strategy"]
          )
      else:
          self._tokenizer.no_truncation()
  
      _padding = self._tokenizer.padding
      if _padding is not None:
          self._tokenizer.enable_padding(**_padding)
          kwargs.setdefault("pad_token", _padding["pad_token"])
          kwargs.setdefault(
              "pad_token_type_id", _padding["pad_type_id"]
          )
          kwargs.setdefault("padding_side", _padding["direction"])
          kwargs.setdefault("max_length", _padding["length"])
          kwargs.setdefault(
              "pad_to_multiple_of", _padding["pad_to_multiple_of"]
          )
  
      # We call this after having initialized the backend tokenizer because we update it.
      super().__init__(**kwargs)
  
      # The following logic will be replace with a single add_tokens once a fix is pushed to tokenizers
      # allows converting a slow -> fast, non-legacy: if the `tokenizer.json` does not have all the added tokens
      # uses the information stored in `added_tokens_decoder`.
      # this is costly for fast tokenizers as we re-compute the regex again. But not all tokens are added tokens
      tokens_to_add = [
          token
          for index, token in sorted(
              added_tokens_decoder.items(), key=lambda x: x[0]
          )
          if token not in self.added_tokens_decoder
      ]
      encoder = list(self.added_tokens_encoder.keys()) + [
          str(token) for token in tokens_to_add
      ]
      # if some of the special tokens are strings, we check if we don't already have a token
      tokens_to_add += [
          token
          for token in self.all_special_tokens_extended
          if token not in encoder and token not in tokens_to_add
      ]
      if len(tokens_to_add) > 0:
          # super hack: if a token.special is set, tokenizer ignores it for now so FIXME @ArthurZ
          # Accumulate added tokens into batches of special/non-special tokens, because calling add_tokens() for
          # individual tokens would repeatedly rebuild a trie, which can be slow.
          is_last_special = None
          tokens = []
          special_tokens = self.all_special_tokens
          for token in tokens_to_add:
              is_special = (
                  (token.special or str(token) in special_tokens)
                  if isinstance(token, AddedToken)
                  else str(token) in special_tokens
              )
              if (
                  is_last_special is None
                  or is_last_special == is_special
              ):
                  tokens.append(token)
              else:
                  self._add_tokens(
                      tokens, special_tokens=is_last_special
                  )
                  tokens = [token]
              is_last_special = is_special
          if tokens:
              self._add_tokens(tokens, special_tokens=is_last_special)
  
imports: import copy
  import json
  import os
  from collections import defaultdict
  from typing import Any, Dict, List, Optional, Tuple, Union
  import tokenizers.pre_tokenizers
  from tokenizers import Encoding
  from tokenizers import Tokenizer
  from tokenizers.decoders import Decoder
  from tokenizers.trainers import BpeTrainer, UnigramTrainer, WordLevelTrainer, WordPieceTrainer
  from .convert_slow_tokenizer import convert_slow_tokenizer
  from .tokenization_utils import PreTrainedTokenizer
  from .tokenization_utils_base import INIT_TOKENIZER_DOCSTRING, AddedToken, BatchEncoding, PreTokenizedInput, PreTokenizedInputPair, PreTrainedTokenizerBase, SpecialTokensMixin, TextInput, TextInputPair, TruncationStrategy
  from .utils import PaddingStrategy, add_end_docstrings, logging
label_desc: Use image node_id 1 to create a new PreTrainedTokenizerFast object from inherited base classes: PreTrainedTokenizerBase with example: obj = PreTrainedTokenizerFast()