class_docstr: Construct a LUKE tokenizer.
  
  This tokenizer inherits from [`RobertaTokenizer`] which contains most of the main methods. Users should refer to
  this superclass for more information regarding those methods. Compared to [`RobertaTokenizer`], [`LukeTokenizer`]
  also creates entity sequences, namely `entity_ids`, `entity_attention_mask`, `entity_token_type_ids`, and
  `entity_position_ids` to be used by the LUKE model.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
      merges_file (`str`):
          Path to the merges file.
      entity_vocab_file (`str`):
          Path to the entity vocabulary file.
      task (`str`, *optional*):
          Task for which you want to prepare sequences. One of `"entity_classification"`,
          `"entity_pair_classification"`, or `"entity_span_classification"`. If you specify this argument, the entity
          sequence is automatically created based on the given entity span(s).
      max_entity_length (`int`, *optional*, defaults to 32):
          The maximum length of `entity_ids`.
      max_mention_length (`int`, *optional*, defaults to 30):
          The maximum number of tokens inside an entity span.
      entity_token_1 (`str`, *optional*, defaults to `<ent>`):
          The special token used to represent an entity span in a word token sequence. This token is only used when
          `task` is set to `"entity_classification"` or `"entity_pair_classification"`.
      entity_token_2 (`str`, *optional*, defaults to `<ent2>`):
          The special token used to represent an entity span in a word token sequence. This token is only used when
          `task` is set to `"entity_pair_classification"`.
class_docstr_tok: ["Construct","a","LUKE","tokenizer",".","This","tokenizer","inherits","from","[","`","RobertaTokenizer","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Compared","to","[","`","RobertaTokenizer","`","]",",","[","`","LukeTokenizer","`","]","also","creates","entity","sequences",",","namely","`","entity_ids","`",",","`","entity_attention_mask","`",",","`","entity_token_type_ids","`",",","and","`","entity_position_ids","`","to","be","used","by","the","LUKE","model",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file",".","merges_file","(","`","str","`",")",":","Path","to","the","merges","file",".","entity_vocab_file","(","`","str","`",")",":","Path","to","the","entity","vocabulary","file",".","task","(","`","str","`",",","*","optional","*",")",":","Task","for","which","you","want","to","prepare","sequences",".","One","of","`","``","entity_classification","''","`",",","`","``","entity_pair_classification","''","`",",","or","`","``","entity_span_classification","''","`",".","If","you","specify","this","argument",",","the","entity","sequence","is","automatically","created","based","on","the","given","entity","span","(","s",")",".","max_entity_length","(","`","int","`",",","*","optional","*",",","defaults","to","32",")",":","The","maximum","length","of","`","entity_ids","`",".","max_mention_length","(","`","int","`",",","*","optional","*",",","defaults","to","30",")",":","The","maximum","number","of","tokens","inside","an","entity","span",".","entity_token_1","(","`","str","`",",","*","optional","*",",","defaults","to","`","<","ent",">","`",")",":","The","special","token","used","to","represent","an","entity","span","in","a","word","token","sequence",".","This","token","is","only","used","when","`","task","`","is","set","to","`","``","entity_classification","''","`","or","`","``","entity_pair_classification","''","`",".","entity_token_2","(","`","str","`",",","*","optional","*",",","defaults","to","`","<","ent2",">","`",")",":","The","special","token","used","to","represent","an","entity","span","in","a","word","token","sequence",".","This","token","is","only","used","when","`","task","`","is","set","to","`","``","entity_pair_classification","''","`","."]
docstr: Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
  
  
  Args:
      encoded_inputs:
          Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
      max_length: maximum length of the returned list and optionally padding length (see below).
          Will truncate by taking into account the special tokens.
      max_entity_length: The maximum length of the entity sequence.
      padding_strategy: PaddingStrategy to use for padding.
  
  
          - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
          - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
          - PaddingStrategy.DO_NOT_PAD: Do not pad
          The tokenizer padding sides are defined in self.padding_side:
  
  
              - 'left': pads on the left of the sequences
              - 'right': pads on the right of the sequences
      pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
          This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
          >= 7.5 (Volta).
      return_attention_mask:
          (optional) Set to False to avoid returning attention mask (default: set to model specifics)
docstr_tok: ["Pad","encoded","inputs","(","on","left\/right","and","up","to","predefined","length","or","max","length","in","the","batch",")","Args",":","encoded_inputs",":","Dictionary","of","tokenized","inputs","(","`","List","[","int","]","`",")","or","batch","of","tokenized","inputs","(","`","List","[","List","[","int","]","]","`",")",".","max_length",":","maximum","length","of","the","returned","list","and","optionally","padding","length","(","see","below",")",".","Will","truncate","by","taking","into","account","the","special","tokens",".","max_entity_length",":","The","maximum","length","of","the","entity","sequence",".","padding_strategy",":","PaddingStrategy","to","use","for","padding",".","-","PaddingStrategy.LONGEST","Pad","to","the","longest","sequence","in","the","batch","-","PaddingStrategy.MAX_LENGTH",":","Pad","to","the","max","length","(","default",")","-","PaddingStrategy.DO_NOT_PAD",":","Do","not","pad","The","tokenizer","padding","sides","are","defined","in","self.padding_side",":","-","'left","'",":","pads","on","the","left","of","the","sequences","-","'right","'",":","pads","on","the","right","of","the","sequences","pad_to_multiple_of",":","(","optional",")","Integer","if","set","will","pad","the","sequence","to","a","multiple","of","the","provided","value",".","This","is","especially","useful","to","enable","the","use","of","Tensor","Core","on","NVIDIA","hardware","with","compute","capability",">","=","7.5","(","Volta",")",".","return_attention_mask",":","(","optional",")","Set","to","False","to","avoid","returning","attention","mask","(","default",":","set","to","model","specifics",")"]
code: def _pad(
      self,
      encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
      max_length: Optional[int] = None,
      max_entity_length: Optional[int] = None,
      padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
      pad_to_multiple_of: Optional[int] = None,
      return_attention_mask: Optional[bool] = None,
  ) -> dict:
      """
      Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
  
  
      Args:
          encoded_inputs:
              Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
          max_length: maximum length of the returned list and optionally padding length (see below).
              Will truncate by taking into account the special tokens.
          max_entity_length: The maximum length of the entity sequence.
          padding_strategy: PaddingStrategy to use for padding.
  
  
              - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
              - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
              - PaddingStrategy.DO_NOT_PAD: Do not pad
              The tokenizer padding sides are defined in self.padding_side:
  
  
                  - 'left': pads on the left of the sequences
                  - 'right': pads on the right of the sequences
          pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
              This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
              >= 7.5 (Volta).
          return_attention_mask:
              (optional) Set to False to avoid returning attention mask (default: set to model specifics)
      """
      entities_provided = bool("entity_ids" in encoded_inputs)
  
      # Load from model defaults
      if return_attention_mask is None:
          return_attention_mask = (
              "attention_mask" in self.model_input_names
          )
  
      if padding_strategy == PaddingStrategy.LONGEST:
          max_length = len(encoded_inputs["input_ids"])
          if entities_provided:
              max_entity_length = len(encoded_inputs["entity_ids"])
  
      if (
          max_length is not None
          and pad_to_multiple_of is not None
          and (max_length % pad_to_multiple_of != 0)
      ):
          max_length = (
              (max_length // pad_to_multiple_of) + 1
          ) * pad_to_multiple_of
  
      if (
          entities_provided
          and max_entity_length is not None
          and pad_to_multiple_of is not None
          and (max_entity_length % pad_to_multiple_of != 0)
      ):
          max_entity_length = (
              (max_entity_length // pad_to_multiple_of) + 1
          ) * pad_to_multiple_of
  
      needs_to_be_padded = (
          padding_strategy != PaddingStrategy.DO_NOT_PAD
          and (
              len(encoded_inputs["input_ids"]) != max_length
              or (
                  entities_provided
                  and len(encoded_inputs["entity_ids"])
                  != max_entity_length
              )
          )
      )
  
      # Initialize attention mask if not present.
      if (
          return_attention_mask
          and "attention_mask" not in encoded_inputs
      ):
          encoded_inputs["attention_mask"] = [1] * len(
              encoded_inputs["input_ids"]
          )
      if (
          entities_provided
          and return_attention_mask
          and "entity_attention_mask" not in encoded_inputs
      ):
          encoded_inputs["entity_attention_mask"] = [1] * len(
              encoded_inputs["entity_ids"]
          )
  
      if needs_to_be_padded:
          difference = max_length - len(encoded_inputs["input_ids"])
          if entities_provided:
              entity_difference = max_entity_length - len(
                  encoded_inputs["entity_ids"]
              )
          if self.padding_side == "right":
              if return_attention_mask:
                  encoded_inputs["attention_mask"] = (
                      encoded_inputs["attention_mask"]
                      + [0] * difference
                  )
                  if entities_provided:
                      encoded_inputs["entity_attention_mask"] = (
                          encoded_inputs["entity_attention_mask"]
                          + [0] * entity_difference
                      )
              if "token_type_ids" in encoded_inputs:
                  encoded_inputs["token_type_ids"] = (
                      encoded_inputs["token_type_ids"]
                      + [0] * difference
                  )
                  if entities_provided:
                      encoded_inputs["entity_token_type_ids"] = (
                          encoded_inputs["entity_token_type_ids"]
                          + [0] * entity_difference
                      )
              if "special_tokens_mask" in encoded_inputs:
                  encoded_inputs["special_tokens_mask"] = (
                      encoded_inputs["special_tokens_mask"]
                      + [1] * difference
                  )
              encoded_inputs["input_ids"] = (
                  encoded_inputs["input_ids"]
                  + [self.pad_token_id] * difference
              )
              if entities_provided:
                  encoded_inputs["entity_ids"] = (
                      encoded_inputs["entity_ids"]
                      + [self.entity_pad_token_id] * entity_difference
                  )
                  encoded_inputs["entity_position_ids"] = (
                      encoded_inputs["entity_position_ids"]
                      + [[-1] * self.max_mention_length]
                      * entity_difference
                  )
                  if self.task == "entity_span_classification":
                      encoded_inputs["entity_start_positions"] = (
                          encoded_inputs["entity_start_positions"]
                          + [0] * entity_difference
                      )
                      encoded_inputs["entity_end_positions"] = (
                          encoded_inputs["entity_end_positions"]
                          + [0] * entity_difference
                      )
  
          elif self.padding_side == "left":
              if return_attention_mask:
                  encoded_inputs["attention_mask"] = [
                      0
                  ] * difference + encoded_inputs["attention_mask"]
                  if entities_provided:
                      encoded_inputs["entity_attention_mask"] = [
                          0
                      ] * entity_difference + encoded_inputs[
                          "entity_attention_mask"
                      ]
              if "token_type_ids" in encoded_inputs:
                  encoded_inputs["token_type_ids"] = [
                      0
                  ] * difference + encoded_inputs["token_type_ids"]
                  if entities_provided:
                      encoded_inputs["entity_token_type_ids"] = [
                          0
                      ] * entity_difference + encoded_inputs[
                          "entity_token_type_ids"
                      ]
              if "special_tokens_mask" in encoded_inputs:
                  encoded_inputs["special_tokens_mask"] = [
                      1
                  ] * difference + encoded_inputs["special_tokens_mask"]
              encoded_inputs["input_ids"] = [
                  self.pad_token_id
              ] * difference + encoded_inputs["input_ids"]
              if entities_provided:
                  encoded_inputs["entity_ids"] = [
                      self.entity_pad_token_id
                  ] * entity_difference + encoded_inputs["entity_ids"]
                  encoded_inputs["entity_position_ids"] = [
                      [-1] * self.max_mention_length
                  ] * entity_difference + encoded_inputs[
                      "entity_position_ids"
                  ]
                  if self.task == "entity_span_classification":
                      encoded_inputs["entity_start_positions"] = [
                          0
                      ] * entity_difference + encoded_inputs[
                          "entity_start_positions"
                      ]
                      encoded_inputs["entity_end_positions"] = [
                          0
                      ] * entity_difference + encoded_inputs[
                          "entity_end_positions"
                      ]
          else:
              raise ValueError(
                  "Invalid padding strategy:" + str(self.padding_side)
              )
  
      return encoded_inputs
  
imports: import itertools
  import json
  import os
  from collections.abc import Mapping
  from typing import Dict, List, Optional, Tuple, Union
  import numpy
  from ...None import RobertaTokenizer
  from ...tokenization_utils_base import ENCODE_KWARGS_DOCSTRING, AddedToken, BatchEncoding, EncodedInput, PaddingStrategy, TensorType, TextInput, TextInputPair, TruncationStrategy, to_py_obj
  from ...utils import add_end_docstrings, is_tf_tensor, is_torch_tensor, logging
label_desc: Use image node_id 10 for calling the LukeTokenizer obj's underlying member method code with example usage: obj._pad(encoded_inputs, max_length, max_entity_length, padding_strategy, pad_to_multiple_of, return_attention_mask) and returns: encoded_inputs