class_docstr: Construct a LUKE tokenizer.
  
  This tokenizer inherits from [`RobertaTokenizer`] which contains most of the main methods. Users should refer to
  this superclass for more information regarding those methods. Compared to [`RobertaTokenizer`], [`LukeTokenizer`]
  also creates entity sequences, namely `entity_ids`, `entity_attention_mask`, `entity_token_type_ids`, and
  `entity_position_ids` to be used by the LUKE model.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
      merges_file (`str`):
          Path to the merges file.
      entity_vocab_file (`str`):
          Path to the entity vocabulary file.
      task (`str`, *optional*):
          Task for which you want to prepare sequences. One of `"entity_classification"`,
          `"entity_pair_classification"`, or `"entity_span_classification"`. If you specify this argument, the entity
          sequence is automatically created based on the given entity span(s).
      max_entity_length (`int`, *optional*, defaults to 32):
          The maximum length of `entity_ids`.
      max_mention_length (`int`, *optional*, defaults to 30):
          The maximum number of tokens inside an entity span.
      entity_token_1 (`str`, *optional*, defaults to `<ent>`):
          The special token used to represent an entity span in a word token sequence. This token is only used when
          `task` is set to `"entity_classification"` or `"entity_pair_classification"`.
      entity_token_2 (`str`, *optional*, defaults to `<ent2>`):
          The special token used to represent an entity span in a word token sequence. This token is only used when
          `task` is set to `"entity_pair_classification"`.
class_docstr_tok: ["Construct","a","LUKE","tokenizer",".","This","tokenizer","inherits","from","[","`","RobertaTokenizer","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Compared","to","[","`","RobertaTokenizer","`","]",",","[","`","LukeTokenizer","`","]","also","creates","entity","sequences",",","namely","`","entity_ids","`",",","`","entity_attention_mask","`",",","`","entity_token_type_ids","`",",","and","`","entity_position_ids","`","to","be","used","by","the","LUKE","model",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file",".","merges_file","(","`","str","`",")",":","Path","to","the","merges","file",".","entity_vocab_file","(","`","str","`",")",":","Path","to","the","entity","vocabulary","file",".","task","(","`","str","`",",","*","optional","*",")",":","Task","for","which","you","want","to","prepare","sequences",".","One","of","`","``","entity_classification","''","`",",","`","``","entity_pair_classification","''","`",",","or","`","``","entity_span_classification","''","`",".","If","you","specify","this","argument",",","the","entity","sequence","is","automatically","created","based","on","the","given","entity","span","(","s",")",".","max_entity_length","(","`","int","`",",","*","optional","*",",","defaults","to","32",")",":","The","maximum","length","of","`","entity_ids","`",".","max_mention_length","(","`","int","`",",","*","optional","*",",","defaults","to","30",")",":","The","maximum","number","of","tokens","inside","an","entity","span",".","entity_token_1","(","`","str","`",",","*","optional","*",",","defaults","to","`","<","ent",">","`",")",":","The","special","token","used","to","represent","an","entity","span","in","a","word","token","sequence",".","This","token","is","only","used","when","`","task","`","is","set","to","`","``","entity_classification","''","`","or","`","``","entity_pair_classification","''","`",".","entity_token_2","(","`","str","`",",","*","optional","*",",","defaults","to","`","<","ent2",">","`",")",":","The","special","token","used","to","represent","an","entity","span","in","a","word","token","sequence",".","This","token","is","only","used","when","`","task","`","is","set","to","`","``","entity_pair_classification","''","`","."]
docstr: Main method to tokenize and prepare for the model one or several sequence(s) or one or several pair(s) of
  sequences, depending on the task you want to prepare them for.
  
  Args:
      text (`str`, `List[str]`, `List[List[str]]`):
          The sequence or batch of sequences to be encoded. Each sequence must be a string. Note that this
          tokenizer does not support tokenization based on pretokenized strings.
      text_pair (`str`, `List[str]`, `List[List[str]]`):
          The sequence or batch of sequences to be encoded. Each sequence must be a string. Note that this
          tokenizer does not support tokenization based on pretokenized strings.
      entity_spans (`List[Tuple[int, int]]`, `List[List[Tuple[int, int]]]`, *optional*):
          The sequence or batch of sequences of entity spans to be encoded. Each sequence consists of tuples each
          with two integers denoting character-based start and end positions of entities. If you specify
          `"entity_classification"` or `"entity_pair_classification"` as the `task` argument in the constructor,
          the length of each sequence must be 1 or 2, respectively. If you specify `entities`, the length of each
          sequence must be equal to the length of each sequence of `entities`.
      entity_spans_pair (`List[Tuple[int, int]]`, `List[List[Tuple[int, int]]]`, *optional*):
          The sequence or batch of sequences of entity spans to be encoded. Each sequence consists of tuples each
          with two integers denoting character-based start and end positions of entities. If you specify the
          `task` argument in the constructor, this argument is ignored. If you specify `entities_pair`, the
          length of each sequence must be equal to the length of each sequence of `entities_pair`.
      entities (`List[str]`, `List[List[str]]`, *optional*):
          The sequence or batch of sequences of entities to be encoded. Each sequence consists of strings
          representing entities, i.e., special entities (e.g., [MASK]) or entity titles of Wikipedia (e.g., Los
          Angeles). This argument is ignored if you specify the `task` argument in the constructor. The length of
          each sequence must be equal to the length of each sequence of `entity_spans`. If you specify
          `entity_spans` without specifying this argument, the entity sequence or the batch of entity sequences
          is automatically constructed by filling it with the [MASK] entity.
      entities_pair (`List[str]`, `List[List[str]]`, *optional*):
          The sequence or batch of sequences of entities to be encoded. Each sequence consists of strings
          representing entities, i.e., special entities (e.g., [MASK]) or entity titles of Wikipedia (e.g., Los
          Angeles). This argument is ignored if you specify the `task` argument in the constructor. The length of
          each sequence must be equal to the length of each sequence of `entity_spans_pair`. If you specify
          `entity_spans_pair` without specifying this argument, the entity sequence or the batch of entity
          sequences is automatically constructed by filling it with the [MASK] entity.
      max_entity_length (`int`, *optional*):
          The maximum length of `entity_ids`.
docstr_tok: ["Main","method","to","tokenize","and","prepare","for","the","model","one","or","several","sequence","(","s",")","or","one","or","several","pair","(","s",")","of","sequences",",","depending","on","the","task","you","want","to","prepare","them","for",".","Args",":","text","(","`","str","`",",","`","List","[","str","]","`",",","`","List","[","List","[","str","]","]","`",")",":","The","sequence","or","batch","of","sequences","to","be","encoded",".","Each","sequence","must","be","a","string",".","Note","that","this","tokenizer","does","not","support","tokenization","based","on","pretokenized","strings",".","text_pair","(","`","str","`",",","`","List","[","str","]","`",",","`","List","[","List","[","str","]","]","`",")",":","The","sequence","or","batch","of","sequences","to","be","encoded",".","Each","sequence","must","be","a","string",".","Note","that","this","tokenizer","does","not","support","tokenization","based","on","pretokenized","strings",".","entity_spans","(","`","List","[","Tuple","[","int",",","int","]","]","`",",","`","List","[","List","[","Tuple","[","int",",","int","]","]","]","`",",","*","optional","*",")",":","The","sequence","or","batch","of","sequences","of","entity","spans","to","be","encoded",".","Each","sequence","consists","of","tuples","each","with","two","integers","denoting","character-based","start","and","end","positions","of","entities",".","If","you","specify","`","``","entity_classification","''","`","or","`","``","entity_pair_classification","''","`","as","the","`","task","`","argument","in","the","constructor",",","the","length","of","each","sequence","must","be","1","or","2",",","respectively",".","If","you","specify","`","entities","`",",","the","length","of","each","sequence","must","be","equal","to","the","length","of","each","sequence","of","`","entities","`",".","entity_spans_pair","(","`","List","[","Tuple","[","int",",","int","]","]","`",",","`","List","[","List","[","Tuple","[","int",",","int","]","]","]","`",",","*","optional","*",")",":","The","sequence","or","batch","of","sequences","of","entity","spans","to","be","encoded",".","Each","sequence","consists","of","tuples","each","with","two","integers","denoting","character-based","start","and","end","positions","of","entities",".","If","you","specify","the","`","task","`","argument","in","the","constructor",",","this","argument","is","ignored",".","If","you","specify","`","entities_pair","`",",","the","length","of","each","sequence","must","be","equal","to","the","length","of","each","sequence","of","`","entities_pair","`",".","entities","(","`","List","[","str","]","`",",","`","List","[","List","[","str","]","]","`",",","*","optional","*",")",":","The","sequence","or","batch","of","sequences","of","entities","to","be","encoded",".","Each","sequence","consists","of","strings","representing","entities",",","i.e.",",","special","entities","(","e.g.",",","[","MASK","]",")","or","entity","titles","of","Wikipedia","(","e.g.",",","Los","Angeles",")",".","This","argument","is","ignored","if","you","specify","the","`","task","`","argument","in","the","constructor",".","The","length","of","each","sequence","must","be","equal","to","the","length","of","each","sequence","of","`","entity_spans","`",".","If","you","specify","`","entity_spans","`","without","specifying","this","argument",",","the","entity","sequence","or","the","batch","of","entity","sequences","is","automatically","constructed","by","filling","it","with","the","[","MASK","]","entity",".","entities_pair","(","`","List","[","str","]","`",",","`","List","[","List","[","str","]","]","`",",","*","optional","*",")",":","The","sequence","or","batch","of","sequences","of","entities","to","be","encoded",".","Each","sequence","consists","of","strings","representing","entities",",","i.e.",",","special","entities","(","e.g.",",","[","MASK","]",")","or","entity","titles","of","Wikipedia","(","e.g.",",","Los","Angeles",")",".","This","argument","is","ignored","if","you","specify","the","`","task","`","argument","in","the","constructor",".","The","length","of","each","sequence","must","be","equal","to","the","length","of","each","sequence","of","`","entity_spans_pair","`",".","If","you","specify","`","entity_spans_pair","`","without","specifying","this","argument",",","the","entity","sequence","or","the","batch","of","entity","sequences","is","automatically","constructed","by","filling","it","with","the","[","MASK","]","entity",".","max_entity_length","(","`","int","`",",","*","optional","*",")",":","The","maximum","length","of","`","entity_ids","`","."]
code: def __call__(
      self,
      text: Union[TextInput, List[TextInput]],
      text_pair: Optional[Union[TextInput, List[TextInput]]] = None,
      entity_spans: Optional[
          Union[EntitySpanInput, List[EntitySpanInput]]
      ] = None,
      entity_spans_pair: Optional[
          Union[EntitySpanInput, List[EntitySpanInput]]
      ] = None,
      entities: Optional[Union[EntityInput, List[EntityInput]]] = None,
      entities_pair: Optional[
          Union[EntityInput, List[EntityInput]]
      ] = None,
      add_special_tokens: bool = True,
      padding: Union[bool, str, PaddingStrategy] = False,
      truncation: Union[bool, str, TruncationStrategy] = None,
      max_length: Optional[int] = None,
      max_entity_length: Optional[int] = None,
      stride: int = 0,
      is_split_into_words: Optional[bool] = False,
      pad_to_multiple_of: Optional[int] = None,
      return_tensors: Optional[Union[str, TensorType]] = None,
      return_token_type_ids: Optional[bool] = None,
      return_attention_mask: Optional[bool] = None,
      return_overflowing_tokens: bool = False,
      return_special_tokens_mask: bool = False,
      return_offsets_mapping: bool = False,
      return_length: bool = False,
      verbose: bool = True,
      **kwargs,
  ) -> BatchEncoding:
      """
      Main method to tokenize and prepare for the model one or several sequence(s) or one or several pair(s) of
      sequences, depending on the task you want to prepare them for.
  
      Args:
          text (`str`, `List[str]`, `List[List[str]]`):
              The sequence or batch of sequences to be encoded. Each sequence must be a string. Note that this
              tokenizer does not support tokenization based on pretokenized strings.
          text_pair (`str`, `List[str]`, `List[List[str]]`):
              The sequence or batch of sequences to be encoded. Each sequence must be a string. Note that this
              tokenizer does not support tokenization based on pretokenized strings.
          entity_spans (`List[Tuple[int, int]]`, `List[List[Tuple[int, int]]]`, *optional*):
              The sequence or batch of sequences of entity spans to be encoded. Each sequence consists of tuples each
              with two integers denoting character-based start and end positions of entities. If you specify
              `"entity_classification"` or `"entity_pair_classification"` as the `task` argument in the constructor,
              the length of each sequence must be 1 or 2, respectively. If you specify `entities`, the length of each
              sequence must be equal to the length of each sequence of `entities`.
          entity_spans_pair (`List[Tuple[int, int]]`, `List[List[Tuple[int, int]]]`, *optional*):
              The sequence or batch of sequences of entity spans to be encoded. Each sequence consists of tuples each
              with two integers denoting character-based start and end positions of entities. If you specify the
              `task` argument in the constructor, this argument is ignored. If you specify `entities_pair`, the
              length of each sequence must be equal to the length of each sequence of `entities_pair`.
          entities (`List[str]`, `List[List[str]]`, *optional*):
              The sequence or batch of sequences of entities to be encoded. Each sequence consists of strings
              representing entities, i.e., special entities (e.g., [MASK]) or entity titles of Wikipedia (e.g., Los
              Angeles). This argument is ignored if you specify the `task` argument in the constructor. The length of
              each sequence must be equal to the length of each sequence of `entity_spans`. If you specify
              `entity_spans` without specifying this argument, the entity sequence or the batch of entity sequences
              is automatically constructed by filling it with the [MASK] entity.
          entities_pair (`List[str]`, `List[List[str]]`, *optional*):
              The sequence or batch of sequences of entities to be encoded. Each sequence consists of strings
              representing entities, i.e., special entities (e.g., [MASK]) or entity titles of Wikipedia (e.g., Los
              Angeles). This argument is ignored if you specify the `task` argument in the constructor. The length of
              each sequence must be equal to the length of each sequence of `entity_spans_pair`. If you specify
              `entity_spans_pair` without specifying this argument, the entity sequence or the batch of entity
              sequences is automatically constructed by filling it with the [MASK] entity.
          max_entity_length (`int`, *optional*):
              The maximum length of `entity_ids`.
      """
      # Input type checking for clearer error
      is_valid_single_text = isinstance(text, str)
      is_valid_batch_text = isinstance(text, (list, tuple)) and (
          len(text) == 0 or (isinstance(text[0], str))
      )
      if not (is_valid_single_text or is_valid_batch_text):
          raise ValueError(
              "text input must be of type `str` (single example) or `List[str]` (batch)."
          )
  
      is_valid_single_text_pair = isinstance(text_pair, str)
      is_valid_batch_text_pair = isinstance(
          text_pair, (list, tuple)
      ) and (len(text_pair) == 0 or isinstance(text_pair[0], str))
      if not (
          text_pair is None
          or is_valid_single_text_pair
          or is_valid_batch_text_pair
      ):
          raise ValueError(
              "text_pair input must be of type `str` (single example) or `List[str]` (batch)."
          )
  
      is_batched = bool(isinstance(text, (list, tuple)))
  
      if is_batched:
          batch_text_or_text_pairs = (
              list(zip(text, text_pair))
              if text_pair is not None
              else text
          )
          if entities is None:
              batch_entities_or_entities_pairs = None
          else:
              batch_entities_or_entities_pairs = (
                  list(zip(entities, entities_pair))
                  if entities_pair is not None
                  else entities
              )
  
          if entity_spans is None:
              batch_entity_spans_or_entity_spans_pairs = None
          else:
              batch_entity_spans_or_entity_spans_pairs = (
                  list(zip(entity_spans, entity_spans_pair))
                  if entity_spans_pair is not None
                  else entity_spans
              )
  
          return self.batch_encode_plus(
              batch_text_or_text_pairs=batch_text_or_text_pairs,
              batch_entity_spans_or_entity_spans_pairs=batch_entity_spans_or_entity_spans_pairs,
              batch_entities_or_entities_pairs=batch_entities_or_entities_pairs,
              add_special_tokens=add_special_tokens,
              padding=padding,
              truncation=truncation,
              max_length=max_length,
              max_entity_length=max_entity_length,
              stride=stride,
              is_split_into_words=is_split_into_words,
              pad_to_multiple_of=pad_to_multiple_of,
              return_tensors=return_tensors,
              return_token_type_ids=return_token_type_ids,
              return_attention_mask=return_attention_mask,
              return_overflowing_tokens=return_overflowing_tokens,
              return_special_tokens_mask=return_special_tokens_mask,
              return_offsets_mapping=return_offsets_mapping,
              return_length=return_length,
              verbose=verbose,
              **kwargs,
          )
      else:
          return self.encode_plus(
              text=text,
              text_pair=text_pair,
              entity_spans=entity_spans,
              entity_spans_pair=entity_spans_pair,
              entities=entities,
              entities_pair=entities_pair,
              add_special_tokens=add_special_tokens,
              padding=padding,
              truncation=truncation,
              max_length=max_length,
              max_entity_length=max_entity_length,
              stride=stride,
              is_split_into_words=is_split_into_words,
              pad_to_multiple_of=pad_to_multiple_of,
              return_tensors=return_tensors,
              return_token_type_ids=return_token_type_ids,
              return_attention_mask=return_attention_mask,
              return_overflowing_tokens=return_overflowing_tokens,
              return_special_tokens_mask=return_special_tokens_mask,
              return_offsets_mapping=return_offsets_mapping,
              return_length=return_length,
              verbose=verbose,
              **kwargs,
          )
  
imports: import itertools
  import json
  import os
  from collections.abc import Mapping
  from typing import Dict, List, Optional, Tuple, Union
  import numpy
  from ...None import RobertaTokenizer
  from ...tokenization_utils_base import ENCODE_KWARGS_DOCSTRING, AddedToken, BatchEncoding, EncodedInput, PaddingStrategy, TensorType, TextInput, TextInputPair, TruncationStrategy, to_py_obj
  from ...utils import add_end_docstrings, is_tf_tensor, is_torch_tensor, logging
label_desc: Use image node_id 2 for calling the LukeTokenizer obj's underlying member method code with example usage: obj.__call__(text, text_pair, entity_spans, entity_spans_pair, entities, entities_pair, add_special_tokens, padding, truncation, max_length, max_entity_length, stride, is_split_into_words, pad_to_multiple_of, return_tensors, return_token_type_ids, return_attention_mask, return_overflowing_tokens, return_special_tokens_mask, return_offsets_mapping, return_length, verbose) and returns: self, self