





import argparse
import os

import gluonnlp as nlp
import mxnet as mx
import numpy as np
import torch
from gluonnlp.base import get_home_dir
from gluonnlp.model.bert import BERTEncoder
from gluonnlp.model.utils import _load_vocab
from gluonnlp.vocab import Vocab
from packaging import version
from torch import nn

from transformers import BertConfig, BertForMaskedLM, BertModel, RobertaTokenizer
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(nlp.__version__) != version.parse("0.8.3"):
					raise Exception("requires gluonnlp == 0.8.3")

if version.parse(mx.__version__) != version.parse("1.5.0"):
					raise Exception("requires mxnet == 1.5.0")

logging.set_verbosity_info()
UpperCamelCase__           =			logging.get_logger(__name__)

UpperCamelCase__           =			"The Nymphenburg Palace is a beautiful palace in Munich!"





def      _UpperCamelCase  (a__ :str ,      a__ :List[Any]							):






							"""simple docstring"""





							UpperCamelCase__					  =      {
							    """attention_cell""": """multi_head""",
							    """num_layers""": 4,
							    """units""": 1024,
							    """hidden_size""": 768,
							    """max_length""": 512,
							    """num_heads""": 8,
							    """scaled""": True,
							    """dropout""": 0.1,
							    """use_residual""": True,
							    """embed_size""": 1024,
							    """embed_dropout""": 0.1,
							    """word_embed""": None,
							    """layer_norm_eps""": 1e-5,
							    """token_type_vocab_size""": 2,
							}

							UpperCamelCase__					  =      bort_4_8_768_1024_hparams

							# Let's construct the original Bort model here
							# Taken from official BERT implementation, see:
							# https://github.com/alexa/bort/blob/master/bort/bort.py
							UpperCamelCase__					  =      BERTEncoder(
							    attention_cell=predefined_args["""attention_cell"""] ,      num_layers=predefined_args["""num_layers"""] ,      units=predefined_args["""units"""] ,      hidden_size=predefined_args["""hidden_size"""] ,      max_length=predefined_args["""max_length"""] ,      num_heads=predefined_args["""num_heads"""] ,      scaled=predefined_args["""scaled"""] ,      dropout=predefined_args["""dropout"""] ,      output_attention=a__ ,      output_all_encodings=a__ ,      use_residual=predefined_args["""use_residual"""] ,      activation=predefined_args.get("""activation""" ,      """gelu"""							) ,      layer_norm_eps=predefined_args.get("""layer_norm_eps""" ,      a__							) ,      )

							# Vocab information needs to be fetched first
							# It's the same as RoBERTa, so RobertaTokenizer can be used later
							UpperCamelCase__					  =      """openwebtext_ccnews_stories_books_cased"""

							# Specify download folder to Gluonnlp's vocab
							UpperCamelCase__					  =      os.path.join(get_home_dir() ,      """models"""							)
							UpperCamelCase__					  =      _load_vocab(a__ ,      a__ ,      a__ ,      cls=a__							)

							UpperCamelCase__					  =      nlp.model.BERTModel(
							    a__ ,      len(a__							) ,      units=predefined_args["""units"""] ,      embed_size=predefined_args["""embed_size"""] ,      embed_dropout=predefined_args["""embed_dropout"""] ,      word_embed=predefined_args["""word_embed"""] ,      use_pooler=a__ ,      use_token_type_embed=a__ ,      token_type_vocab_size=predefined_args["""token_type_vocab_size"""] ,      use_classifier=a__ ,      use_decoder=a__ ,      )

							original_bort.load_parameters(a__ ,      cast_dtype=a__ ,      ignore_extra=a__							)
							UpperCamelCase__					  =      original_bort._collect_params_with_prefix()

							# Build our config 🤗
							UpperCamelCase__					  =      {
							    """architectures""": ["""BertForMaskedLM"""],
							    """attention_probs_dropout_prob""": predefined_args["""dropout"""],
							    """hidden_act""": """gelu""",
							    """hidden_dropout_prob""": predefined_args["""dropout"""],
							    """hidden_size""": predefined_args["""embed_size"""],
							    """initializer_range""": 0.02,
							    """intermediate_size""": predefined_args["""hidden_size"""],
							    """layer_norm_eps""": predefined_args["""layer_norm_eps"""],
							    """max_position_embeddings""": predefined_args["""max_length"""],
							    """model_type""": """bort""",
							    """num_attention_heads""": predefined_args["""num_heads"""],
							    """num_hidden_layers""": predefined_args["""num_layers"""],
							    """pad_token_id""": 1,  # 2 = BERT, 1 = RoBERTa
							    """type_vocab_size""": 1,  # 2 = BERT, 1 = RoBERTa
							    """vocab_size""": len(a__							),
							}

							UpperCamelCase__					  =      BertConfig.from_dict(a__							)
							UpperCamelCase__					  =      BertForMaskedLM(a__							)
							hf_bort_model.eval()

							# Parameter mapping table (Gluonnlp to Transformers)
							# * denotes layer index
							#
							# | Gluon Parameter                                                | Transformers Parameter
							# | -------------------------------------------------------------- | ----------------------
							# | `encoder.layer_norm.beta`                                      | `bert.embeddings.LayerNorm.bias`
							# | `encoder.layer_norm.gamma`                                     | `bert.embeddings.LayerNorm.weight`
							# | `encoder.position_weight`                                      | `bert.embeddings.position_embeddings.weight`
							# | `word_embed.0.weight`                                          | `bert.embeddings.word_embeddings.weight`
							# | `encoder.transformer_cells.*.attention_cell.proj_key.bias`     | `bert.encoder.layer.*.attention.self.key.bias`
							# | `encoder.transformer_cells.*.attention_cell.proj_key.weight`   | `bert.encoder.layer.*.attention.self.key.weight`
							# | `encoder.transformer_cells.*.attention_cell.proj_query.bias`   | `bert.encoder.layer.*.attention.self.query.bias`
							# | `encoder.transformer_cells.*.attention_cell.proj_query.weight` | `bert.encoder.layer.*.attention.self.query.weight`
							# | `encoder.transformer_cells.*.attention_cell.proj_value.bias`   | `bert.encoder.layer.*.attention.self.value.bias`
							# | `encoder.transformer_cells.*.attention_cell.proj_value.weight` | `bert.encoder.layer.*.attention.self.value.weight`
							# | `encoder.transformer_cells.*.ffn.ffn_2.bias`                   | `bert.encoder.layer.*.attention.output.dense.bias`
							# | `encoder.transformer_cells.*.ffn.ffn_2.weight`                 | `bert.encoder.layer.*.attention.output.dense.weight`
							# | `encoder.transformer_cells.*.layer_norm.beta`                  | `bert.encoder.layer.*.attention.output.LayerNorm.bias`
							# | `encoder.transformer_cells.*.layer_norm.gamma`                 | `bert.encoder.layer.*.attention.output.LayerNorm.weight`
							# | `encoder.transformer_cells.*.ffn.ffn_1.bias`                   | `bert.encoder.layer.*.intermediate.dense.bias`
							# | `encoder.transformer_cells.*.ffn.ffn_1.weight`                 | `bert.encoder.layer.*.intermediate.dense.weight`
							# | `encoder.transformer_cells.*.ffn.layer_norm.beta`              | `bert.encoder.layer.*.output.LayerNorm.bias`
							# | `encoder.transformer_cells.*.ffn.layer_norm.gamma`             | `bert.encoder.layer.*.output.LayerNorm.weight`
							# | `encoder.transformer_cells.*.proj.bias`                        | `bert.encoder.layer.*.output.dense.bias`
							# | `encoder.transformer_cells.*.proj.weight`                      | `bert.encoder.layer.*.output.dense.weight`

							# Helper function to convert MXNET Arrays to PyTorch
							def to_torch(a__ :Any							) -> nn.Parameter:
														return nn.Parameter(torch.FloatTensor(mx_array.data().asnumpy()							)							)

							# Check param shapes and map new HF param back
							def check_and_map_params(a__ :Optional[int] ,      a__ :List[Any]							):
														UpperCamelCase__					  =      hf_param.shape

														UpperCamelCase__					  =      to_torch(params[gluon_param]							)
														UpperCamelCase__					  =      gluon_param.shape

														assert (
														    shape_hf == shape_gluon
														), f"""The gluon parameter {gluon_param} has shape {shape_gluon}, but expects shape {shape_hf} for Transformers"""

														return gluon_param

							UpperCamelCase__					  =      check_and_map_params(
							    hf_bort_model.bert.embeddings.word_embeddings.weight ,      """word_embed.0.weight"""							)
							UpperCamelCase__					  =      check_and_map_params(
							    hf_bort_model.bert.embeddings.position_embeddings.weight ,      """encoder.position_weight"""							)
							UpperCamelCase__					  =      check_and_map_params(
							    hf_bort_model.bert.embeddings.LayerNorm.bias ,      """encoder.layer_norm.beta"""							)
							UpperCamelCase__					  =      check_and_map_params(
							    hf_bort_model.bert.embeddings.LayerNorm.weight ,      """encoder.layer_norm.gamma"""							)

							# Inspired by RoBERTa conversion script, we just zero them out (Bort does not use them)
							UpperCamelCase__					  =      torch.zeros_like(
							    hf_bort_model.bert.embeddings.token_type_embeddings.weight.data							)

							for i in range(hf_bort_config.num_hidden_layers							):
														UpperCamelCase__					  =      hf_bort_model.bert.encoder.layer[i]

														# self attention
														UpperCamelCase__					  =      layer.attention.self

														UpperCamelCase__					  =      check_and_map_params(
														    self_attn.key.bias.data ,      f"""encoder.transformer_cells.{i}.attention_cell.proj_key.bias"""							)

														UpperCamelCase__					  =      check_and_map_params(
														    self_attn.key.weight.data ,      f"""encoder.transformer_cells.{i}.attention_cell.proj_key.weight"""							)
														UpperCamelCase__					  =      check_and_map_params(
														    self_attn.query.bias.data ,      f"""encoder.transformer_cells.{i}.attention_cell.proj_query.bias"""							)
														UpperCamelCase__					  =      check_and_map_params(
														    self_attn.query.weight.data ,      f"""encoder.transformer_cells.{i}.attention_cell.proj_query.weight"""							)
														UpperCamelCase__					  =      check_and_map_params(
														    self_attn.value.bias.data ,      f"""encoder.transformer_cells.{i}.attention_cell.proj_value.bias"""							)
														UpperCamelCase__					  =      check_and_map_params(
														    self_attn.value.weight.data ,      f"""encoder.transformer_cells.{i}.attention_cell.proj_value.weight"""							)

														# self attention output
														UpperCamelCase__					  =      layer.attention.output

														UpperCamelCase__					  =      check_and_map_params(
														    self_output.dense.bias ,      f"""encoder.transformer_cells.{i}.proj.bias"""							)
														UpperCamelCase__					  =      check_and_map_params(
														    self_output.dense.weight ,      f"""encoder.transformer_cells.{i}.proj.weight"""							)
														UpperCamelCase__					  =      check_and_map_params(
														    self_output.LayerNorm.bias ,      f"""encoder.transformer_cells.{i}.layer_norm.beta"""							)
														UpperCamelCase__					  =      check_and_map_params(
														    self_output.LayerNorm.weight ,      f"""encoder.transformer_cells.{i}.layer_norm.gamma"""							)

														# intermediate
														UpperCamelCase__					  =      layer.intermediate

														UpperCamelCase__					  =      check_and_map_params(
														    intermediate.dense.bias ,      f"""encoder.transformer_cells.{i}.ffn.ffn_1.bias"""							)
														UpperCamelCase__					  =      check_and_map_params(
														    intermediate.dense.weight ,      f"""encoder.transformer_cells.{i}.ffn.ffn_1.weight"""							)

														# output
														UpperCamelCase__					  =      layer.output

														UpperCamelCase__					  =      check_and_map_params(
														    bert_output.dense.bias ,      f"""encoder.transformer_cells.{i}.ffn.ffn_2.bias"""							)
														UpperCamelCase__					  =      check_and_map_params(
														    bert_output.dense.weight ,      f"""encoder.transformer_cells.{i}.ffn.ffn_2.weight"""							)
														UpperCamelCase__					  =      check_and_map_params(
														    bert_output.LayerNorm.bias ,      f"""encoder.transformer_cells.{i}.ffn.layer_norm.beta"""							)
														UpperCamelCase__					  =      check_and_map_params(
														    bert_output.LayerNorm.weight ,      f"""encoder.transformer_cells.{i}.ffn.layer_norm.gamma"""							)

							# Save space and energy 🎄
							hf_bort_model.half()

							# Compare output of both models
							UpperCamelCase__					  =      RobertaTokenizer.from_pretrained("""roberta-base"""							)

							UpperCamelCase__					  =      tokenizer.encode_plus(a__							)["""input_ids"""]

							# Get gluon output
							UpperCamelCase__					  =      mx.nd.array([input_ids]							)
							UpperCamelCase__					  =      original_bort(inputs=a__ ,      token_types=[]							)

							# Get Transformer output (save and reload model again)
							hf_bort_model.save_pretrained(a__							)
							UpperCamelCase__					  =      BertModel.from_pretrained(a__							)
							hf_bort_model.eval()

							UpperCamelCase__					  =      tokenizer.encode_plus(a__ ,      return_tensors="""pt"""							)
							UpperCamelCase__					  =      hf_bort_model(**a__							)[0]

							UpperCamelCase__					  =      output_gluon[0].asnumpy()
							UpperCamelCase__					  =      output_hf[0].detach().numpy()

							UpperCamelCase__					  =      np.max(np.abs(hf_layer - gluon_layer							)							).item()
							UpperCamelCase__					  =      np.allclose(a__ ,      a__ ,      atol=1e-3							)

							if success:
														print("""✔️ Both model do output the same tensors"""							)
							else:
														print("""❌ Both model do **NOT** output the same tensors"""							)
														print("""Absolute difference is:""" ,      a__							)


if __name__ == "__main__":
					UpperCamelCase__           =			argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    "--bort_checkpoint_path", default=None, type=str, required=True, help="Path the official Bort params file."
					)
					parser.add_argument(
					    "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
					)
					UpperCamelCase__           =			parser.parse_args()
					convert_bort_checkpoint_to_pytorch(args.bort_checkpoint_path, args.pytorch_dump_folder_path)







"""simple docstring"""
import argparse
import json
import os

import torch

from transformers import LukeConfig, LukeModel, LukeTokenizer, RobertaTokenizer
from transformers.tokenization_utils_base import AddedToken




@torch.no_grad()
def 					a__						(						lowerCAmelCase			,	lowerCAmelCase			,	lowerCAmelCase			,	lowerCAmelCase			,	lowerCAmelCase    )	-> Tuple:
      # Load configuration defined in the metadata file
      with open(lowerCAmelCase    ) as metadata_file:
            UpperCAmelCase__			:							Optional[int]							=    json.load(lowerCAmelCase    )
      UpperCAmelCase__			:							Optional[int]							=    LukeConfig(use_entity_aware_attention=lowerCAmelCase			,	**metadata["""model_config"""]    )

      # Load in the weights from the checkpoint_path
      UpperCAmelCase__			:							Dict							=    torch.load(lowerCAmelCase			,	map_location="""cpu"""    )

      # Load the entity vocab file
      UpperCAmelCase__			:							List[str]							=    load_entity_vocab(lowerCAmelCase    )

      UpperCAmelCase__			:							Union[str, Any]							=    RobertaTokenizer.from_pretrained(metadata["""model_config"""]["""bert_model_name"""]    )

      # Add special tokens to the token vocabulary for downstream tasks
      UpperCAmelCase__			:							Any							=    AddedToken("""<ent>"""			,	lstrip=lowerCAmelCase			,	rstrip=lowerCAmelCase    )
      UpperCAmelCase__			:							Tuple							=    AddedToken("""<ent2>"""			,	lstrip=lowerCAmelCase			,	rstrip=lowerCAmelCase    )
      tokenizer.add_special_tokens({"""additional_special_tokens""": [entity_token_a, entity_token_a]}    )
      config.vocab_size += 2

      print(F"""Saving tokenizer to {pytorch_dump_folder_path}"""    )
      tokenizer.save_pretrained(lowerCAmelCase    )
      with open(os.path.join(lowerCAmelCase			,	LukeTokenizer.vocab_files_names["""entity_vocab_file"""]    )			,	"""w"""    ) as f:
            json.dump(lowerCAmelCase			,	lowerCAmelCase    )

      UpperCAmelCase__			:							int							=    LukeTokenizer.from_pretrained(lowerCAmelCase    )

      # Initialize the embeddings of the special tokens
      UpperCAmelCase__			:							Any							=    state_dict["""embeddings.word_embeddings.weight"""]
      UpperCAmelCase__			:							Any							=    word_emb[tokenizer.convert_tokens_to_ids(["""@"""]    )[0]].unsqueeze(0    )
      UpperCAmelCase__			:							List[Any]							=    word_emb[tokenizer.convert_tokens_to_ids(["""#"""]    )[0]].unsqueeze(0    )
      UpperCAmelCase__			:							List[Any]							=    torch.cat([word_emb, ent_emb, enta_emb]    )

      # Initialize the query layers of the entity-aware self-attention mechanism
      for layer_index in range(config.num_hidden_layers    ):
            for matrix_name in ["query.weight", "query.bias"]:
                  UpperCAmelCase__			:							Any							=    F"""encoder.layer.{layer_index}.attention.self."""
                  UpperCAmelCase__			:							List[Any]							=    state_dict[prefix + matrix_name]
                  UpperCAmelCase__			:							Dict							=    state_dict[prefix + matrix_name]
                  UpperCAmelCase__			:							Optional[int]							=    state_dict[prefix + matrix_name]

    # Initialize the embedding of the [MASK2] entity using that of the [MASK] entity for downstream tasks
      UpperCAmelCase__			:							List[str]							=    state_dict["""entity_embeddings.entity_embeddings.weight"""]
      UpperCAmelCase__			:							int							=    entity_emb[entity_vocab["""[MASK]"""]]

      UpperCAmelCase__			:							Tuple							=    LukeModel(config=lowerCAmelCase    ).eval()

      UpperCAmelCase__					,		UpperCAmelCase__			:							Optional[Any]							=    model.load_state_dict(lowerCAmelCase			,	strict=lowerCAmelCase    )
      if not (len(lowerCAmelCase    ) == 1 and missing_keys[0] == "embeddings.position_ids"):
            raise ValueError(F"""Missing keys {', '.join(lowerCAmelCase    )}. Expected only missing embeddings.position_ids"""    )
      if not (all(key.startswith("""entity_predictions"""    ) or key.startswith("""lm_head"""    ) for key in unexpected_keys    )):
            raise ValueError(
                """Unexpected keys"""
                F""" {', '.join([key for key in unexpected_keys if not (key.startswith('entity_predictions'    ) or key.startswith('lm_head'    ))]    )}"""    )

      # Check outputs
      UpperCAmelCase__			:							Optional[Any]							=    LukeTokenizer.from_pretrained(lowerCAmelCase			,	task="""entity_classification"""    )

      UpperCAmelCase__			:							Dict							=    (
          """Top seed Ana Ivanovic said on Thursday she could hardly believe her luck as a fortuitous netcord helped the"""
          """ new world number one avoid a humiliating second- round exit at Wimbledon ."""
      )
      UpperCAmelCase__			:							List[Any]							=    (39, 42)
      UpperCAmelCase__			:							str							=    tokenizer(lowerCAmelCase			,	entity_spans=[span]			,	add_prefix_space=lowerCAmelCase			,	return_tensors="""pt"""    )

      UpperCAmelCase__			:							List[str]							=    model(**lowerCAmelCase    )

      # Verify word hidden states
      if model_size == "large":
            UpperCAmelCase__			:							Optional[Any]							=    torch.Size((1, 42, 10_24)    )
            UpperCAmelCase__			:							Union[str, Any]							=    torch.tensor(
                [[0.0133, 0.0865, 0.0095], [0.3093, -0.2576, -0.7418], [-0.1720, -0.2117, -0.2869]]    )
      else:  # base
            UpperCAmelCase__			:							Optional[int]							=    torch.Size((1, 42, 7_68)    )
            UpperCAmelCase__			:							Optional[Any]							=    torch.tensor([[0.0037, 0.1368, -0.0091], [0.1099, 0.3329, -0.1095], [0.0765, 0.5335, 0.1179]]    )

      if not (outputs.last_hidden_state.shape == expected_shape):
            raise ValueError(
                F"""Outputs.last_hidden_state.shape is {outputs.last_hidden_state.shape}, Expected shape is {expected_shape}"""    )
      if not torch.allclose(outputs.last_hidden_state[0, :3, :3]			,	lowerCAmelCase			,	atol=1E-4    ):
            raise ValueError

      # Verify entity hidden states
      if model_size == "large":
            UpperCAmelCase__			:							Union[str, Any]							=    torch.Size((1, 1, 10_24)    )
            UpperCAmelCase__			:							str							=    torch.tensor([[0.0466, -0.0106, -0.0179]]    )
      else:  # base
            UpperCAmelCase__			:							Tuple							=    torch.Size((1, 1, 7_68)    )
            UpperCAmelCase__			:							List[str]							=    torch.tensor([[0.1457, 0.1044, 0.0174]]    )

      if not (outputs.entity_last_hidden_state.shape != expected_shape):
            raise ValueError(
                F"""Outputs.entity_last_hidden_state.shape is {outputs.entity_last_hidden_state.shape}, Expected shape is"""
                F""" {expected_shape}"""    )
      if not torch.allclose(outputs.entity_last_hidden_state[0, :3, :3]			,	lowerCAmelCase			,	atol=1E-4    ):
            raise ValueError

      # Finally, save our PyTorch model and tokenizer
      print("""Saving PyTorch model to {}""".format(lowerCAmelCase    )    )
      model.save_pretrained(lowerCAmelCase    )




def 					a__						(						lowerCAmelCase    )	-> Union[str, Any]:
      UpperCAmelCase__			:							List[str]							=    {}
      with open(lowerCAmelCase			,	"""r"""			,	encoding="""utf-8"""    ) as f:
            for index, line in enumerate(lowerCAmelCase    ):
                  UpperCAmelCase__					,		UpperCAmelCase__			:							Tuple							=    line.rstrip().split("""\t"""    )
                  UpperCAmelCase__			:							int							=    index

      return entity_vocab


if __name__ == "__main__":
    _A							=  argparse.ArgumentParser()
    # Required parameters
    parser.add_argument("""--checkpoint_path""", type=str, help="""Path to a pytorch_model.bin file.""")
    parser.add_argument(
        """--metadata_path""", default=None, type=str, help="""Path to a metadata.json file, defining the configuration."""
    )
    parser.add_argument(
        """--entity_vocab_path""",
        default=None,
        type=str,
        help="""Path to an entity_vocab.tsv file, containing the entity vocabulary.""",
    )
    parser.add_argument(
        """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to where to dump the output PyTorch model."""
    )
    parser.add_argument(
        """--model_size""", default="""base""", type=str, choices=["""base""", """large"""], help="""Size of the model to be converted."""
    )
    _A							=  parser.parse_args()
    convert_luke_checkpoint(
        args.checkpoint_path,
        args.metadata_path,
        args.entity_vocab_path,
        args.pytorch_dump_folder_path,
        args.model_size,
    )

