class_docstr: Construct a LUKE tokenizer.
  
  This tokenizer inherits from [`RobertaTokenizer`] which contains most of the main methods. Users should refer to
  this superclass for more information regarding those methods. Compared to [`RobertaTokenizer`], [`LukeTokenizer`]
  also creates entity sequences, namely `entity_ids`, `entity_attention_mask`, `entity_token_type_ids`, and
  `entity_position_ids` to be used by the LUKE model.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
      merges_file (`str`):
          Path to the merges file.
      entity_vocab_file (`str`):
          Path to the entity vocabulary file.
      task (`str`, *optional*):
          Task for which you want to prepare sequences. One of `"entity_classification"`,
          `"entity_pair_classification"`, or `"entity_span_classification"`. If you specify this argument, the entity
          sequence is automatically created based on the given entity span(s).
      max_entity_length (`int`, *optional*, defaults to 32):
          The maximum length of `entity_ids`.
      max_mention_length (`int`, *optional*, defaults to 30):
          The maximum number of tokens inside an entity span.
      entity_token_1 (`str`, *optional*, defaults to `<ent>`):
          The special token used to represent an entity span in a word token sequence. This token is only used when
          `task` is set to `"entity_classification"` or `"entity_pair_classification"`.
      entity_token_2 (`str`, *optional*, defaults to `<ent2>`):
          The special token used to represent an entity span in a word token sequence. This token is only used when
          `task` is set to `"entity_pair_classification"`.
class_docstr_tok: ["Construct","a","LUKE","tokenizer",".","This","tokenizer","inherits","from","[","`","RobertaTokenizer","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Compared","to","[","`","RobertaTokenizer","`","]",",","[","`","LukeTokenizer","`","]","also","creates","entity","sequences",",","namely","`","entity_ids","`",",","`","entity_attention_mask","`",",","`","entity_token_type_ids","`",",","and","`","entity_position_ids","`","to","be","used","by","the","LUKE","model",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file",".","merges_file","(","`","str","`",")",":","Path","to","the","merges","file",".","entity_vocab_file","(","`","str","`",")",":","Path","to","the","entity","vocabulary","file",".","task","(","`","str","`",",","*","optional","*",")",":","Task","for","which","you","want","to","prepare","sequences",".","One","of","`","``","entity_classification","''","`",",","`","``","entity_pair_classification","''","`",",","or","`","``","entity_span_classification","''","`",".","If","you","specify","this","argument",",","the","entity","sequence","is","automatically","created","based","on","the","given","entity","span","(","s",")",".","max_entity_length","(","`","int","`",",","*","optional","*",",","defaults","to","32",")",":","The","maximum","length","of","`","entity_ids","`",".","max_mention_length","(","`","int","`",",","*","optional","*",",","defaults","to","30",")",":","The","maximum","number","of","tokens","inside","an","entity","span",".","entity_token_1","(","`","str","`",",","*","optional","*",",","defaults","to","`","<","ent",">","`",")",":","The","special","token","used","to","represent","an","entity","span","in","a","word","token","sequence",".","This","token","is","only","used","when","`","task","`","is","set","to","`","``","entity_classification","''","`","or","`","``","entity_pair_classification","''","`",".","entity_token_2","(","`","str","`",",","*","optional","*",",","defaults","to","`","<","ent2",">","`",")",":","The","special","token","used","to","represent","an","entity","span","in","a","word","token","sequence",".","This","token","is","only","used","when","`","task","`","is","set","to","`","``","entity_pair_classification","''","`","."]
docstr: Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It
  adds special tokens, truncates sequences if overflowing while taking into account the special tokens and
  manages a moving window (with user defined stride) for overflowing tokens
  
  
  Args:
      batch_ids_pairs: list of tokenized input ids or input ids pairs
      batch_entity_ids_pairs: list of entity ids or entity ids pairs
      batch_entity_token_spans_pairs: list of entity spans or entity spans pairs
      max_entity_length: The maximum length of the entity sequence.
docstr_tok: ["Prepares","a","sequence","of","input","id",",","or","a","pair","of","sequences","of","inputs","ids","so","that","it","can","be","used","by","the","model",".","It","adds","special","tokens",",","truncates","sequences","if","overflowing","while","taking","into","account","the","special","tokens","and","manages","a","moving","window","(","with","user","defined","stride",")","for","overflowing","tokens","Args",":","batch_ids_pairs",":","list","of","tokenized","input","ids","or","input","ids","pairs","batch_entity_ids_pairs",":","list","of","entity","ids","or","entity","ids","pairs","batch_entity_token_spans_pairs",":","list","of","entity","spans","or","entity","spans","pairs","max_entity_length",":","The","maximum","length","of","the","entity","sequence","."]
code: def _batch_prepare_for_model(
      self,
      batch_ids_pairs: List[Tuple[List[int], None]],
      batch_entity_ids_pairs: List[
          Tuple[Optional[List[int]], Optional[List[int]]]
      ],
      batch_entity_token_spans_pairs: List[
          Tuple[
              Optional[List[Tuple[int, int]]],
              Optional[List[Tuple[int, int]]],
          ]
      ],
      add_special_tokens: bool = True,
      padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
      truncation_strategy: TruncationStrategy = TruncationStrategy.DO_NOT_TRUNCATE,
      max_length: Optional[int] = None,
      max_entity_length: Optional[int] = None,
      stride: int = 0,
      pad_to_multiple_of: Optional[int] = None,
      return_tensors: Optional[str] = None,
      return_token_type_ids: Optional[bool] = None,
      return_attention_mask: Optional[bool] = None,
      return_overflowing_tokens: bool = False,
      return_special_tokens_mask: bool = False,
      return_length: bool = False,
      verbose: bool = True,
  ) -> BatchEncoding:
      """
      Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It
      adds special tokens, truncates sequences if overflowing while taking into account the special tokens and
      manages a moving window (with user defined stride) for overflowing tokens
  
  
      Args:
          batch_ids_pairs: list of tokenized input ids or input ids pairs
          batch_entity_ids_pairs: list of entity ids or entity ids pairs
          batch_entity_token_spans_pairs: list of entity spans or entity spans pairs
          max_entity_length: The maximum length of the entity sequence.
      """
  
      batch_outputs = {}
      for input_ids, entity_ids, entity_token_span_pairs in zip(
          batch_ids_pairs,
          batch_entity_ids_pairs,
          batch_entity_token_spans_pairs,
      ):
          first_ids, second_ids = input_ids
          first_entity_ids, second_entity_ids = entity_ids
          (
              first_entity_token_spans,
              second_entity_token_spans,
          ) = entity_token_span_pairs
          outputs = self.prepare_for_model(
              first_ids,
              second_ids,
              entity_ids=first_entity_ids,
              pair_entity_ids=second_entity_ids,
              entity_token_spans=first_entity_token_spans,
              pair_entity_token_spans=second_entity_token_spans,
              add_special_tokens=add_special_tokens,
              padding=PaddingStrategy.DO_NOT_PAD.value,  # we pad in batch afterward
              truncation=truncation_strategy.value,
              max_length=max_length,
              max_entity_length=max_entity_length,
              stride=stride,
              pad_to_multiple_of=None,  # we pad in batch afterward
              return_attention_mask=False,  # we pad in batch afterward
              return_token_type_ids=return_token_type_ids,
              return_overflowing_tokens=return_overflowing_tokens,
              return_special_tokens_mask=return_special_tokens_mask,
              return_length=return_length,
              return_tensors=None,  # We convert the whole batch to tensors at the end
              prepend_batch_axis=False,
              verbose=verbose,
          )
  
          for key, value in outputs.items():
              if key not in batch_outputs:
                  batch_outputs[key] = []
              batch_outputs[key].append(value)
  
      batch_outputs = self.pad(
          batch_outputs,
          padding=padding_strategy.value,
          max_length=max_length,
          pad_to_multiple_of=pad_to_multiple_of,
          return_attention_mask=return_attention_mask,
      )
  
      batch_outputs = BatchEncoding(
          batch_outputs, tensor_type=return_tensors
      )
  
      return batch_outputs
  
imports: import itertools
  import json
  import os
  from collections.abc import Mapping
  from typing import Dict, List, Optional, Tuple, Union
  import numpy
  from ...None import RobertaTokenizer
  from ...tokenization_utils_base import ENCODE_KWARGS_DOCSTRING, AddedToken, BatchEncoding, EncodedInput, PaddingStrategy, TensorType, TextInput, TextInputPair, TruncationStrategy, to_py_obj
  from ...utils import add_end_docstrings, is_tf_tensor, is_torch_tensor, logging
label_desc: Use image node_id 7 for calling the LukeTokenizer obj's underlying member method code with example usage: obj._batch_prepare_for_model(batch_ids_pairs, batch_entity_ids_pairs, batch_entity_token_spans_pairs, add_special_tokens, padding_strategy, truncation_strategy, max_length, max_entity_length, stride, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_length, verbose) and returns: batch_outputs