class_docstr: Base class for all slow tokenizers.
  
  Inherits from :class:`~transformers.tokenization_utils_base.PreTrainedTokenizerBase`.
  
  Handle all the shared methods for tokenization and special tokens as well as methods downloading/caching/loading
  pretrained tokenizers as well as adding tokens to the vocabulary.
  
  This class also contain the added tokens in a unified way on top of all tokenizers so we don't have to handle the
  specific vocabulary augmentation methods of the various underlying dictionary structures (BPE, sentencepiece...).
class_docstr_tok: ["Base","class","for","all","slow","tokenizers",".","Inherits","from",":","class",":","`","~transformers.tokenization_utils_base.PreTrainedTokenizerBase","`",".","Handle","all","the","shared","methods","for","tokenization","and","special","tokens","as","well","as","methods","downloading\/caching\/loading","pretrained","tokenizers","as","well","as","adding","tokens","to","the","vocabulary",".","This","class","also","contain","the","added","tokens","in","a","unified","way","on","top","of","all","tokenizers","so","we","do","n't","have","to","handle","the","specific","vocabulary","augmentation","methods","of","the","various","underlying","dictionary","structures","(","BPE",",","sentencepiece","...",")","."]
docstr: Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It
  adds special tokens, truncates sequences if overflowing while taking into account the special tokens and
  manages a moving window (with user defined stride) for overflowing tokens
  
  Args:
      batch_ids_pairs: list of tokenized input ids or input ids pairs
docstr_tok: ["Prepares","a","sequence","of","input","id",",","or","a","pair","of","sequences","of","inputs","ids","so","that","it","can","be","used","by","the","model",".","It","adds","special","tokens",",","truncates","sequences","if","overflowing","while","taking","into","account","the","special","tokens","and","manages","a","moving","window","(","with","user","defined","stride",")","for","overflowing","tokens","Args",":","batch_ids_pairs",":","list","of","tokenized","input","ids","or","input","ids","pairs"]
code: def _batch_prepare_for_model(
      self,
      batch_ids_pairs: List[
          Union[PreTokenizedInputPair, Tuple[List[int], None]]
      ],
      add_special_tokens: bool = True,
      padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
      truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
      max_length: Optional[int] = None,
      stride: int = 0,
      pad_to_multiple_of: Optional[int] = None,
      return_tensors: Optional[str] = None,
      return_token_type_ids: Optional[bool] = None,
      return_attention_mask: Optional[bool] = None,
      return_overflowing_tokens: bool = False,
      return_special_tokens_mask: bool = False,
      return_length: bool = False,
      verbose: bool = True,
  ) -> BatchEncoding:
      """
      Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It
      adds special tokens, truncates sequences if overflowing while taking into account the special tokens and
      manages a moving window (with user defined stride) for overflowing tokens
  
      Args:
          batch_ids_pairs: list of tokenized input ids or input ids pairs
      """
  
      batch_outputs = {}
      for first_ids, second_ids in batch_ids_pairs:
          outputs = self.prepare_for_model(
              first_ids,
              second_ids,
              add_special_tokens=add_special_tokens,
              padding=PaddingStrategy.DO_NOT_PAD.value,  # we pad in batch afterward
              truncation=truncation_strategy.value,
              max_length=max_length,
              stride=stride,
              pad_to_multiple_of=None,  # we pad in batch afterward
              return_attention_mask=False,  # we pad in batch afterward
              return_token_type_ids=return_token_type_ids,
              return_overflowing_tokens=return_overflowing_tokens,
              return_special_tokens_mask=return_special_tokens_mask,
              return_length=return_length,
              return_tensors=None,  # We convert the whole batch to tensors at the end
              prepend_batch_axis=False,
              verbose=verbose,
          )
  
          for key, value in outputs.items():
              if key not in batch_outputs:
                  batch_outputs[key] = []
              batch_outputs[key].append(value)
  
      batch_outputs = self.pad(
          batch_outputs,
          padding=padding_strategy.value,
          max_length=max_length,
          pad_to_multiple_of=pad_to_multiple_of,
          return_attention_mask=return_attention_mask,
      )
  
      batch_outputs = BatchEncoding(
          batch_outputs, tensor_type=return_tensors
      )
  
      return batch_outputs
  
imports: import bisect
  import itertools
  import re
  import unicodedata
  from collections import OrderedDict
  from typing import Any, Dict, List, Optional, Tuple, Union, overload
  from .file_utils import PaddingStrategy, TensorType, add_end_docstrings
  from .tokenization_utils_base import ENCODE_KWARGS_DOCSTRING, ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRING, INIT_TOKENIZER_DOCSTRING, AddedToken, BatchEncoding, EncodedInput, EncodedInputPair, PreTokenizedInput, PreTokenizedInputPair, PreTrainedTokenizerBase, TextInput, TextInputPair, TruncationStrategy
  from fastNLP.core.log import logger
label_desc: Use image node_id 16 for calling the PreTrainedTokenizer obj's underlying member method code with example usage: obj._batch_prepare_for_model(batch_ids_pairs, add_special_tokens, padding_strategy, truncation_strategy, max_length, stride, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_length, verbose) and returns: batch_outputs