class_docstr: Base class for all fast tokenizers (wrapping HuggingFace tokenizers library).
  
  Inherits from [`~tokenization_utils_base.PreTrainedTokenizerBase`].
  
  Handles all the shared methods for tokenization and special tokens, as well as methods for
  downloading/caching/loading pretrained tokenizers, as well as adding tokens to the vocabulary.
  
  This class also contains the added tokens in a unified way on top of all tokenizers so we don't have to handle the
  specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).
class_docstr_tok: ["Base","class","for","all","fast","tokenizers","(","wrapping","HuggingFace","tokenizers","library",")",".","Inherits","from","[","`","~tokenization_utils_base.PreTrainedTokenizerBase","`","]",".","Handles","all","the","shared","methods","for","tokenization","and","special","tokens",",","as","well","as","methods","for","downloading\/caching\/loading","pretrained","tokenizers",",","as","well","as","adding","tokens","to","the","vocabulary",".","This","class","also","contains","the","added","tokens","in","a","unified","way","on","top","of","all","tokenizers","so","we","do","n't","have","to","handle","the","specific","vocabulary","augmentation","methods","of","the","various","underlying","dictionary","structures","(","BPE",",","sentencepiece","...",")","."]
code: def _batch_encode_plus(
      self,
      batch_text_or_text_pairs: Union[
          List[TextInput],
          List[TextInputPair],
          List[PreTokenizedInput],
          List[PreTokenizedInputPair],
      ],
      add_special_tokens: bool = True,
      padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
      truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
      max_length: Optional[int] = None,
      stride: int = 0,
      is_split_into_words: bool = False,
      pad_to_multiple_of: Optional[int] = None,
      return_tensors: Optional[str] = None,
      return_token_type_ids: Optional[bool] = None,
      return_attention_mask: Optional[bool] = None,
      return_overflowing_tokens: bool = False,
      return_special_tokens_mask: bool = False,
      return_offsets_mapping: bool = False,
      return_length: bool = False,
      verbose: bool = True,
  ) -> BatchEncoding:
      if not isinstance(batch_text_or_text_pairs, (tuple, list)):
          raise TypeError(
              f"batch_text_or_text_pairs has to be a list or a tuple (got {type(batch_text_or_text_pairs)})"
          )
  
      # Set the truncation and padding strategy and restore the initial configuration
      self.set_truncation_and_padding(
          padding_strategy=padding_strategy,
          truncation_strategy=truncation_strategy,
          max_length=max_length,
          stride=stride,
          pad_to_multiple_of=pad_to_multiple_of,
      )
  
      encodings = self._tokenizer.encode_batch(
          batch_text_or_text_pairs,
          add_special_tokens=add_special_tokens,
          is_pretokenized=is_split_into_words,
      )
  
      # Convert encoding to dict
      # `Tokens` has type: Tuple[
      #                       List[Dict[str, List[List[int]]]] or List[Dict[str, 2D-Tensor]],
      #                       List[EncodingFast]
      #                    ]
      # with nested dimensions corresponding to batch, overflows, sequence length
      tokens_and_encodings = [
          self._convert_encoding(
              encoding=encoding,
              return_token_type_ids=return_token_type_ids,
              return_attention_mask=return_attention_mask,
              return_overflowing_tokens=return_overflowing_tokens,
              return_special_tokens_mask=return_special_tokens_mask,
              return_offsets_mapping=return_offsets_mapping,
              return_length=return_length,
              verbose=verbose,
          )
          for encoding in encodings
      ]
  
      # Convert the output to have dict[list] from list[dict] and remove the additional overflows dimension
      # From (variable) shape (batch, overflows, sequence length) to ~ (batch * overflows, sequence length)
      # (we say ~ because the number of overflow varies with the example in the batch)
      #
      # To match each overflowing sample with the original sample in the batch
      # we add an overflow_to_sample_mapping array (see below)
      sanitized_tokens = {}
      for key in tokens_and_encodings[0][0].keys():
          stack = [
              e for item, _ in tokens_and_encodings for e in item[key]
          ]
          sanitized_tokens[key] = stack
      sanitized_encodings = [
          e for _, item in tokens_and_encodings for e in item
      ]
  
      # If returning overflowing tokens, we need to return a mapping
      # from the batch idx to the original sample
      if return_overflowing_tokens:
          overflow_to_sample_mapping = []
          for i, (toks, _) in enumerate(tokens_and_encodings):
              overflow_to_sample_mapping += [i] * len(toks["input_ids"])
          sanitized_tokens[
              "overflow_to_sample_mapping"
          ] = overflow_to_sample_mapping
  
      for input_ids in sanitized_tokens["input_ids"]:
          self._eventual_warn_about_too_long_sequence(
              input_ids, max_length, verbose
          )
      return BatchEncoding(
          sanitized_tokens,
          sanitized_encodings,
          tensor_type=return_tensors,
      )
  
imports: import copy
  import json
  import os
  from collections import defaultdict
  from typing import Any, Dict, List, Optional, Tuple, Union
  import tokenizers.pre_tokenizers
  from tokenizers import Encoding
  from tokenizers import Tokenizer
  from tokenizers.decoders import Decoder
  from tokenizers.trainers import BpeTrainer, UnigramTrainer, WordLevelTrainer, WordPieceTrainer
  from .convert_slow_tokenizer import convert_slow_tokenizer
  from .tokenization_utils import PreTrainedTokenizer
  from .tokenization_utils_base import INIT_TOKENIZER_DOCSTRING, AddedToken, BatchEncoding, PreTokenizedInput, PreTokenizedInputPair, PreTrainedTokenizerBase, SpecialTokensMixin, TextInput, TextInputPair, TruncationStrategy
  from .utils import PaddingStrategy, add_end_docstrings, logging
label_desc: Use image node_id 19 for calling the PreTrainedTokenizerFast obj's underlying member method code with example usage: obj._batch_encode_plus(batch_text_or_text_pairs, add_special_tokens, padding_strategy, truncation_strategy, max_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose) and returns: BatchEncoding