class_docstr: Construct a LUKE tokenizer.
  
  This tokenizer inherits from [`RobertaTokenizer`] which contains most of the main methods. Users should refer to
  this superclass for more information regarding those methods. Compared to [`RobertaTokenizer`], [`LukeTokenizer`]
  also creates entity sequences, namely `entity_ids`, `entity_attention_mask`, `entity_token_type_ids`, and
  `entity_position_ids` to be used by the LUKE model.
  
  Args:
      vocab_file (`str`):
          Path to the vocabulary file.
      merges_file (`str`):
          Path to the merges file.
      entity_vocab_file (`str`):
          Path to the entity vocabulary file.
      task (`str`, *optional*):
          Task for which you want to prepare sequences. One of `"entity_classification"`,
          `"entity_pair_classification"`, or `"entity_span_classification"`. If you specify this argument, the entity
          sequence is automatically created based on the given entity span(s).
      max_entity_length (`int`, *optional*, defaults to 32):
          The maximum length of `entity_ids`.
      max_mention_length (`int`, *optional*, defaults to 30):
          The maximum number of tokens inside an entity span.
      entity_token_1 (`str`, *optional*, defaults to `<ent>`):
          The special token used to represent an entity span in a word token sequence. This token is only used when
          `task` is set to `"entity_classification"` or `"entity_pair_classification"`.
      entity_token_2 (`str`, *optional*, defaults to `<ent2>`):
          The special token used to represent an entity span in a word token sequence. This token is only used when
          `task` is set to `"entity_pair_classification"`.
class_docstr_tok: ["Construct","a","LUKE","tokenizer",".","This","tokenizer","inherits","from","[","`","RobertaTokenizer","`","]","which","contains","most","of","the","main","methods",".","Users","should","refer","to","this","superclass","for","more","information","regarding","those","methods",".","Compared","to","[","`","RobertaTokenizer","`","]",",","[","`","LukeTokenizer","`","]","also","creates","entity","sequences",",","namely","`","entity_ids","`",",","`","entity_attention_mask","`",",","`","entity_token_type_ids","`",",","and","`","entity_position_ids","`","to","be","used","by","the","LUKE","model",".","Args",":","vocab_file","(","`","str","`",")",":","Path","to","the","vocabulary","file",".","merges_file","(","`","str","`",")",":","Path","to","the","merges","file",".","entity_vocab_file","(","`","str","`",")",":","Path","to","the","entity","vocabulary","file",".","task","(","`","str","`",",","*","optional","*",")",":","Task","for","which","you","want","to","prepare","sequences",".","One","of","`","``","entity_classification","''","`",",","`","``","entity_pair_classification","''","`",",","or","`","``","entity_span_classification","''","`",".","If","you","specify","this","argument",",","the","entity","sequence","is","automatically","created","based","on","the","given","entity","span","(","s",")",".","max_entity_length","(","`","int","`",",","*","optional","*",",","defaults","to","32",")",":","The","maximum","length","of","`","entity_ids","`",".","max_mention_length","(","`","int","`",",","*","optional","*",",","defaults","to","30",")",":","The","maximum","number","of","tokens","inside","an","entity","span",".","entity_token_1","(","`","str","`",",","*","optional","*",",","defaults","to","`","<","ent",">","`",")",":","The","special","token","used","to","represent","an","entity","span","in","a","word","token","sequence",".","This","token","is","only","used","when","`","task","`","is","set","to","`","``","entity_classification","''","`","or","`","``","entity_pair_classification","''","`",".","entity_token_2","(","`","str","`",",","*","optional","*",",","defaults","to","`","<","ent2",">","`",")",":","The","special","token","used","to","represent","an","entity","span","in","a","word","token","sequence",".","This","token","is","only","used","when","`","task","`","is","set","to","`","``","entity_pair_classification","''","`","."]
code: def __init__(
      self,
      vocab_file,
      merges_file,
      entity_vocab_file,
      task=None,
      max_entity_length=32,
      max_mention_length=30,
      entity_token_1="<ent>",
      entity_token_2="<ent2>",
      entity_unk_token="[UNK]",
      entity_pad_token="[PAD]",
      entity_mask_token="[MASK]",
      entity_mask2_token="[MASK2]",
      **kwargs,
  ):
      # we add 2 special tokens for downstream tasks
      # for more information about lstrip and rstrip, see https://github.com/huggingface/transformers/pull/2778
      entity_token_1 = (
          AddedToken(entity_token_1, lstrip=False, rstrip=False)
          if isinstance(entity_token_1, str)
          else entity_token_1
      )
      entity_token_2 = (
          AddedToken(entity_token_2, lstrip=False, rstrip=False)
          if isinstance(entity_token_2, str)
          else entity_token_2
      )
      kwargs["additional_special_tokens"] = kwargs.get(
          "additional_special_tokens", []
      )
      kwargs["additional_special_tokens"] += [
          entity_token_1,
          entity_token_2,
      ]
  
      super().__init__(
          vocab_file=vocab_file,
          merges_file=merges_file,
          task=task,
          max_entity_length=32,
          max_mention_length=30,
          entity_token_1="<ent>",
          entity_token_2="<ent2>",
          entity_unk_token=entity_unk_token,
          entity_pad_token=entity_pad_token,
          entity_mask_token=entity_mask_token,
          entity_mask2_token=entity_mask2_token,
          **kwargs,
      )
  
      with open(
          entity_vocab_file, encoding="utf-8"
      ) as entity_vocab_handle:
          self.entity_vocab = json.load(entity_vocab_handle)
      for entity_special_token in [
          entity_unk_token,
          entity_pad_token,
          entity_mask_token,
          entity_mask2_token,
      ]:
          if entity_special_token not in self.entity_vocab:
              raise ValueError(
                  f"Specified entity special token ``{entity_special_token}`` is not found in entity_vocab. "
                  f"Probably an incorrect entity vocab file is loaded: {entity_vocab_file}."
              )
      self.entity_unk_token_id = self.entity_vocab[entity_unk_token]
      self.entity_pad_token_id = self.entity_vocab[entity_pad_token]
      self.entity_mask_token_id = self.entity_vocab[entity_mask_token]
      self.entity_mask2_token_id = self.entity_vocab[entity_mask2_token]
  
      self.task = task
      if task is None or task == "entity_span_classification":
          self.max_entity_length = max_entity_length
      elif task == "entity_classification":
          self.max_entity_length = 1
      elif task == "entity_pair_classification":
          self.max_entity_length = 2
      else:
          raise ValueError(
              f"Task {task} not supported. Select task from ['entity_classification', 'entity_pair_classification',"
              " 'entity_span_classification'] only."
          )
  
      self.max_mention_length = max_mention_length
  
imports: import itertools
  import json
  import os
  from collections.abc import Mapping
  from typing import Dict, List, Optional, Tuple, Union
  import numpy
  from ...None import RobertaTokenizer
  from ...tokenization_utils_base import ENCODE_KWARGS_DOCSTRING, AddedToken, BatchEncoding, EncodedInput, PaddingStrategy, TensorType, TextInput, TextInputPair, TruncationStrategy, to_py_obj
  from ...utils import add_end_docstrings, is_tf_tensor, is_torch_tensor, logging
label_desc: Use image node_id 1 to create a new LukeTokenizer object from inherited base classes: RobertaTokenizer with example: obj = LukeTokenizer(vocab_file, merges_file, entity_vocab_file, task, max_entity_length, max_mention_length, entity_token_1, entity_token_2, entity_unk_token, entity_pad_token, entity_mask_token, entity_mask2_token)