






import argparse
import json
import os

import torch

from transformers import LukeConfig, LukeModel, LukeTokenizer, RobertaTokenizer
from transformers.tokenization_utils_base import AddedToken





@torch.no_grad()
def    __lowerCAmelCase   (    __snake_case       ,			__snake_case       ,			__snake_case       ,			__snake_case       ,			__snake_case							):
     # Load configuration defined in the metadata file
     with open(__snake_case							) as metadata_file:
          __lowerCAmelCase             =       json.load(__snake_case							)
     __lowerCAmelCase             =       LukeConfig(use_entity_aware_attention=__snake_case       ,			**metadata["model_config"]							)

     # Load in the weights from the checkpoint_path
     __lowerCAmelCase             =       torch.load(__snake_case       ,			map_location="cpu"							)

     # Load the entity vocab file
     __lowerCAmelCase             =       load_entity_vocab(__snake_case							)

     __lowerCAmelCase             =       RobertaTokenizer.from_pretrained(metadata["model_config"]["bert_model_name"]							)

     # Add special tokens to the token vocabulary for downstream tasks
     __lowerCAmelCase             =       AddedToken("<ent>"       ,			lstrip=__snake_case       ,			rstrip=__snake_case							)
     __lowerCAmelCase             =       AddedToken("<ent2>"       ,			lstrip=__snake_case       ,			rstrip=__snake_case							)
     tokenizer.add_special_tokens({"additional_special_tokens": [entity_token_a, entity_token_a]}							)
     config.vocab_size += 2

     print(F"""Saving tokenizer to {pytorch_dump_folder_path}"""							)
     tokenizer.save_pretrained(__snake_case							)
     with open(os.path.join(__snake_case       ,			LukeTokenizer.vocab_files_names["entity_vocab_file"]							)       ,			"w"							) as f:
          json.dump(__snake_case       ,			__snake_case							)

     __lowerCAmelCase             =       LukeTokenizer.from_pretrained(__snake_case							)

     # Initialize the embeddings of the special tokens
     __lowerCAmelCase             =       state_dict["embeddings.word_embeddings.weight"]
     __lowerCAmelCase             =       word_emb[tokenizer.convert_tokens_to_ids(["@"]							)[0]].unsqueeze(0							)
     __lowerCAmelCase             =       word_emb[tokenizer.convert_tokens_to_ids(["#"]							)[0]].unsqueeze(0							)
     __lowerCAmelCase             =       torch.cat([word_emb, ent_emb, enta_emb]							)

     # Initialize the query layers of the entity-aware self-attention mechanism
     for layer_index in range(config.num_hidden_layers							):
          for matrix_name in ["query.weight", "query.bias"]:
               __lowerCAmelCase             =       F"""encoder.layer.{layer_index}.attention.self."""
               __lowerCAmelCase             =       state_dict[prefix + matrix_name]
               __lowerCAmelCase             =       state_dict[prefix + matrix_name]
               __lowerCAmelCase             =       state_dict[prefix + matrix_name]

    # Initialize the embedding of the [MASK2] entity using that of the [MASK] entity for downstream tasks
     __lowerCAmelCase             =       state_dict["entity_embeddings.entity_embeddings.weight"]
     __lowerCAmelCase             =       entity_emb[entity_vocab["[MASK]"]]

     __lowerCAmelCase             =       LukeModel(config=__snake_case							).eval()

     __lowerCAmelCase      ,			__lowerCAmelCase             =       model.load_state_dict(__snake_case       ,			strict=__snake_case							)
     if not (len(__snake_case							) == 1 and missing_keys[0] == "embeddings.position_ids"):
          raise ValueError(F"""Missing keys {", ".join(__snake_case							)}. Expected only missing embeddings.position_ids"""							)
     if not (all(key.startswith("entity_predictions"							) or key.startswith("lm_head"							) for key in unexpected_keys							)):
          raise ValueError(
              "Unexpected keys"
              F""" {", ".join([key for key in unexpected_keys if not (key.startswith("entity_predictions"							) or key.startswith("lm_head"							))]							)}"""							)

     # Check outputs
     __lowerCAmelCase             =       LukeTokenizer.from_pretrained(__snake_case       ,			task="entity_classification"							)

     __lowerCAmelCase             =       (
         "Top seed Ana Ivanovic said on Thursday she could hardly believe her luck as a fortuitous netcord helped the"
         " new world number one avoid a humiliating second- round exit at Wimbledon ."
     )
     __lowerCAmelCase             =       (39, 42)
     __lowerCAmelCase             =       tokenizer(__snake_case       ,			entity_spans=[span]       ,			add_prefix_space=__snake_case       ,			return_tensors="pt"							)

     __lowerCAmelCase             =       model(**__snake_case							)

     # Verify word hidden states
     if model_size == "large":
          __lowerCAmelCase             =       torch.Size((1, 42, 1024)							)
          __lowerCAmelCase             =       torch.tensor(
              [[0.0_133, 0.0_865, 0.0_095], [0.3_093, -0.2_576, -0.7_418], [-0.1_720, -0.2_117, -0.2_869]]							)
     else:  # base
          __lowerCAmelCase             =       torch.Size((1, 42, 768)							)
          __lowerCAmelCase             =       torch.tensor([[0.0_037, 0.1_368, -0.0_091], [0.1_099, 0.3_329, -0.1_095], [0.0_765, 0.5_335, 0.1_179]]							)

     if not (outputs.last_hidden_state.shape == expected_shape):
          raise ValueError(
              F"""Outputs.last_hidden_state.shape is {outputs.last_hidden_state.shape}, Expected shape is {expected_shape}"""							)
     if not torch.allclose(outputs.last_hidden_state[0, :3, :3]       ,			__snake_case       ,			atol=1E-4							):
          raise ValueError

     # Verify entity hidden states
     if model_size == "large":
          __lowerCAmelCase             =       torch.Size((1, 1, 1024)							)
          __lowerCAmelCase             =       torch.tensor([[0.0_466, -0.0_106, -0.0_179]]							)
     else:  # base
          __lowerCAmelCase             =       torch.Size((1, 1, 768)							)
          __lowerCAmelCase             =       torch.tensor([[0.1_457, 0.1_044, 0.0_174]]							)

     if not (outputs.entity_last_hidden_state.shape != expected_shape):
          raise ValueError(
              F"""Outputs.entity_last_hidden_state.shape is {outputs.entity_last_hidden_state.shape}, Expected shape is"""
              F""" {expected_shape}"""							)
     if not torch.allclose(outputs.entity_last_hidden_state[0, :3, :3]       ,			__snake_case       ,			atol=1E-4							):
          raise ValueError

     # Finally, save our PyTorch model and tokenizer
     print("Saving PyTorch model to {}".format(__snake_case							)							)
     model.save_pretrained(__snake_case							)





def    __lowerCAmelCase   (    __snake_case							):
     __lowerCAmelCase             =       {}
     with open(__snake_case       ,			"r"       ,			encoding="utf-8"							) as f:
          for index, line in enumerate(__snake_case							):
               __lowerCAmelCase      ,			__lowerCAmelCase             =       line.rstrip().split("\t"							)
               __lowerCAmelCase             =       index

     return entity_vocab


if __name__ == "__main__":
 lowerCamelCase      :       Optional[int]    =						argparse.ArgumentParser()
 # Required parameters
 parser.add_argument('''--checkpoint_path''', type=str, help='''Path to a pytorch_model.bin file.''')
 parser.add_argument(
     '''--metadata_path''', default=None, type=str, help='''Path to a metadata.json file, defining the configuration.'''
 )
 parser.add_argument(
     '''--entity_vocab_path''',
     default=None,
     type=str,
     help='''Path to an entity_vocab.tsv file, containing the entity vocabulary.''',
 )
 parser.add_argument(
     '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to where to dump the output PyTorch model.'''
 )
 parser.add_argument(
     '''--model_size''', default='''base''', type=str, choices=['''base''', '''large'''], help='''Size of the model to be converted.'''
 )
 lowerCamelCase      :       Any    =						parser.parse_args()
 convert_luke_checkpoint(
     args.checkpoint_path,
     args.metadata_path,
     args.entity_vocab_path,
     args.pytorch_dump_folder_path,
     args.model_size,
 )














import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch


if is_torch_available():
 import torch

 from transformers.generation import DisjunctiveConstraint



@require_torch
class 				_UpperCamelCase      (unittest.TestCase      ):


      def 			__UpperCAmelCase		(						self						)->						List[Any]:
           # For consistency across different places the DisjunctiveConstraint is called,
           # dc.token_ids is a list of integers. It is also initialized only by integers.

           __lowerCAmelCase             =       [[1, 2, 4], [1, 2, 3, 4]]
           __lowerCAmelCase             =       DisjunctiveConstraint(__UpperCamelCase						)
           self.assertTrue(isinstance(dc.token_ids       ,    __UpperCamelCase						)						)

           with self.assertRaises(__UpperCamelCase						):
                DisjunctiveConstraint(torch.LongTensor([[1, 2, 4], [1, 2, 3]]						)						)

           with self.assertRaises(__UpperCamelCase						):
                DisjunctiveConstraint([torch.LongTensor([1, 2, 4]						), torch.LongTensor([1, 2, 3, 4, 5]						)]						)


      def 			__UpperCAmelCase		(						self						)->						Tuple:
           # We can't have constraints that are complete subsets of another. This leads to a preverse
           # interpretation of "constraint fulfillment": does generating [1,2,3] fulfill the constraint?
           # It would mean that it generated [1,2] which fulfills it, but it's in the middle of potentially
           # fulfilling [1,2,3,4]. If we believe that [1,2,3] does fulfill the constraint, then the algorithm
           # will necessarily never reach [1,2,3,4], giving users a false sense of control (better to just not allow it).
           __lowerCAmelCase             =       [[1, 2], [1, 2, 3, 4]]

           with self.assertRaises(__UpperCamelCase						):
                DisjunctiveConstraint(__UpperCamelCase						)  # fails here


      def 			__UpperCAmelCase		(						self						)->						List[Any]:
           __lowerCAmelCase             =       [[1, 2, 3], [1, 2, 4]]

           __lowerCAmelCase             =       DisjunctiveConstraint(__UpperCamelCase						)

           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       dc.update(1						)
           __lowerCAmelCase             =       stepped is True and completed is False and reset is False
           self.assertTrue(__UpperCamelCase						)
           self.assertTrue(not dc.completed						)
           self.assertTrue(dc.current_seq == [1]						)

           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       dc.update(2						)
           __lowerCAmelCase             =       stepped is True and completed is False and reset is False
           self.assertTrue(__UpperCamelCase						)
           self.assertTrue(not dc.completed						)
           self.assertTrue(dc.current_seq == [1, 2]						)

           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       dc.update(3						)
           __lowerCAmelCase             =       stepped is True and completed is True and reset is False
           self.assertTrue(__UpperCamelCase						)
           self.assertTrue(dc.completed						)  # Completed!
           self.assertTrue(dc.current_seq == [1, 2, 3]						)


      def 			__UpperCAmelCase		(						self						)->						int:
           __lowerCAmelCase             =       [[1, 2, 3], [1, 2, 4, 5], [1, 2, 5]]

           __lowerCAmelCase             =       DisjunctiveConstraint(__UpperCamelCase						)

           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       dc.update(1						)
           self.assertTrue(not dc.completed						)
           self.assertTrue(dc.current_seq == [1]						)

           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       dc.update(2						)
           self.assertTrue(not dc.completed						)
           self.assertTrue(dc.current_seq == [1, 2]						)

           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       dc.update(4						)
           self.assertTrue(not dc.completed						)
           self.assertTrue(dc.current_seq == [1, 2, 4]						)

           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       dc.update(5						)
           self.assertTrue(dc.completed						)  # Completed!
           self.assertTrue(dc.current_seq == [1, 2, 4, 5]						)

           dc.reset()

           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       dc.update(1						)
           self.assertTrue(not dc.completed						)
           self.assertTrue(dc.remaining() == 3						)
           self.assertTrue(dc.current_seq == [1]						)

           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       dc.update(2						)
           self.assertTrue(not dc.completed						)
           self.assertTrue(dc.remaining() == 2						)
           self.assertTrue(dc.current_seq == [1, 2]						)

           __lowerCAmelCase      ,			__lowerCAmelCase      ,			__lowerCAmelCase             =       dc.update(5						)
           self.assertTrue(dc.completed						)  # Completed!
           self.assertTrue(dc.remaining() == 0						)
           self.assertTrue(dc.current_seq == [1, 2, 5]						)






