


"""simple docstring"""



import argparse
import os

import gluonnlp as nlp
import mxnet as mx
import numpy as np
import torch
from gluonnlp.base import get_home_dir
from gluonnlp.model.bert import BERTEncoder
from gluonnlp.model.utils import _load_vocab
from gluonnlp.vocab import Vocab
from packaging import version
from torch import nn

from transformers import BertConfig, BertForMaskedLM, BertModel, RobertaTokenizer
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(nlp.__version__) != version.parse("""0.8.3"""):
  raise Exception("""requires gluonnlp == 0.8.3""")

if version.parse(mx.__version__) != version.parse("""1.5.0"""):
  raise Exception("""requires mxnet == 1.5.0""")

logging.set_verbosity_info()
__snake_case       =       logging.get_logger(__name__)

__snake_case       =       """The Nymphenburg Palace is a beautiful palace in Munich!"""







def     __lowerCAmelCase		(		lowercase    :    Any		,				lowercase    :    Dict					)      ->		str:


    """simple docstring"""





    snake_case       : List[Any]    =       {
        "attention_cell": "multi_head",
        "num_layers": 4,
        "units": 1024,
        "hidden_size": 768,
        "max_length": 512,
        "num_heads": 8,
        "scaled": True,
        "dropout": 0.1,
        "use_residual": True,
        "embed_size": 1024,
        "embed_dropout": 0.1,
        "word_embed": None,
        "layer_norm_eps": 1e-5,
        "token_type_vocab_size": 2,
    }

    snake_case       : List[Any]    =       bort_4_8_768_1024_hparams

    # Let's construct the original Bort model here
    # Taken from official BERT implementation, see:
    # https://github.com/alexa/bort/blob/master/bort/bort.py
    snake_case       : Optional[int]    =       BERTEncoder(
        attention_cell=predefined_args["attention_cell"]		,				num_layers=predefined_args["num_layers"]		,				units=predefined_args["units"]		,				hidden_size=predefined_args["hidden_size"]		,				max_length=predefined_args["max_length"]		,				num_heads=predefined_args["num_heads"]		,				scaled=predefined_args["scaled"]		,				dropout=predefined_args["dropout"]		,				output_attention=lowercase__		,				output_all_encodings=lowercase__		,				use_residual=predefined_args["use_residual"]		,				activation=predefined_args.get("activation"		,				"gelu"					)		,				layer_norm_eps=predefined_args.get("layer_norm_eps"		,				lowercase__					)		,				)

    # Vocab information needs to be fetched first
    # It's the same as RoBERTa, so RobertaTokenizer can be used later
    snake_case       : int    =       "openwebtext_ccnews_stories_books_cased"

    # Specify download folder to Gluonnlp's vocab
    snake_case       : Union[str, Any]    =       os.path.join(get_home_dir()		,				"models"					)
    snake_case       : Dict    =       _load_vocab(lowercase__		,				lowercase__		,				lowercase__		,				cls=lowercase__					)

    snake_case       : List[Any]    =       nlp.model.BERTModel(
        lowercase__		,				len(lowercase__					)		,				units=predefined_args["units"]		,				embed_size=predefined_args["embed_size"]		,				embed_dropout=predefined_args["embed_dropout"]		,				word_embed=predefined_args["word_embed"]		,				use_pooler=lowercase__		,				use_token_type_embed=lowercase__		,				token_type_vocab_size=predefined_args["token_type_vocab_size"]		,				use_classifier=lowercase__		,				use_decoder=lowercase__		,				)

    original_bort.load_parameters(lowercase__		,				cast_dtype=lowercase__		,				ignore_extra=lowercase__					)
    snake_case       : List[Any]    =       original_bort._collect_params_with_prefix()

    # Build our config 🤗
    snake_case       : int    =       {
        "architectures": ["BertForMaskedLM"],
        "attention_probs_dropout_prob": predefined_args["dropout"],
        "hidden_act": "gelu",
        "hidden_dropout_prob": predefined_args["dropout"],
        "hidden_size": predefined_args["embed_size"],
        "initializer_range": 0.02,
        "intermediate_size": predefined_args["hidden_size"],
        "layer_norm_eps": predefined_args["layer_norm_eps"],
        "max_position_embeddings": predefined_args["max_length"],
        "model_type": "bort",
        "num_attention_heads": predefined_args["num_heads"],
        "num_hidden_layers": predefined_args["num_layers"],
        "pad_token_id": 1,  # 2 = BERT, 1 = RoBERTa
        "type_vocab_size": 1,  # 2 = BERT, 1 = RoBERTa
        "vocab_size": len(lowercase__					),
    }

    snake_case       : str    =       BertConfig.from_dict(lowercase__					)
    snake_case       : str    =       BertForMaskedLM(lowercase__					)
    hf_bort_model.eval()

    # Parameter mapping table (Gluonnlp to Transformers)
    # * denotes layer index
    #
    # | Gluon Parameter                                                | Transformers Parameter
    # | -------------------------------------------------------------- | ----------------------
    # | `encoder.layer_norm.beta`                                      | `bert.embeddings.LayerNorm.bias`
    # | `encoder.layer_norm.gamma`                                     | `bert.embeddings.LayerNorm.weight`
    # | `encoder.position_weight`                                      | `bert.embeddings.position_embeddings.weight`
    # | `word_embed.0.weight`                                          | `bert.embeddings.word_embeddings.weight`
    # | `encoder.transformer_cells.*.attention_cell.proj_key.bias`     | `bert.encoder.layer.*.attention.self.key.bias`
    # | `encoder.transformer_cells.*.attention_cell.proj_key.weight`   | `bert.encoder.layer.*.attention.self.key.weight`
    # | `encoder.transformer_cells.*.attention_cell.proj_query.bias`   | `bert.encoder.layer.*.attention.self.query.bias`
    # | `encoder.transformer_cells.*.attention_cell.proj_query.weight` | `bert.encoder.layer.*.attention.self.query.weight`
    # | `encoder.transformer_cells.*.attention_cell.proj_value.bias`   | `bert.encoder.layer.*.attention.self.value.bias`
    # | `encoder.transformer_cells.*.attention_cell.proj_value.weight` | `bert.encoder.layer.*.attention.self.value.weight`
    # | `encoder.transformer_cells.*.ffn.ffn_2.bias`                   | `bert.encoder.layer.*.attention.output.dense.bias`
    # | `encoder.transformer_cells.*.ffn.ffn_2.weight`                 | `bert.encoder.layer.*.attention.output.dense.weight`
    # | `encoder.transformer_cells.*.layer_norm.beta`                  | `bert.encoder.layer.*.attention.output.LayerNorm.bias`
    # | `encoder.transformer_cells.*.layer_norm.gamma`                 | `bert.encoder.layer.*.attention.output.LayerNorm.weight`
    # | `encoder.transformer_cells.*.ffn.ffn_1.bias`                   | `bert.encoder.layer.*.intermediate.dense.bias`
    # | `encoder.transformer_cells.*.ffn.ffn_1.weight`                 | `bert.encoder.layer.*.intermediate.dense.weight`
    # | `encoder.transformer_cells.*.ffn.layer_norm.beta`              | `bert.encoder.layer.*.output.LayerNorm.bias`
    # | `encoder.transformer_cells.*.ffn.layer_norm.gamma`             | `bert.encoder.layer.*.output.LayerNorm.weight`
    # | `encoder.transformer_cells.*.proj.bias`                        | `bert.encoder.layer.*.output.dense.bias`
    # | `encoder.transformer_cells.*.proj.weight`                      | `bert.encoder.layer.*.output.dense.weight`

    # Helper function to convert MXNET Arrays to PyTorch
    def to_torch(lowercase    :    List[str]					) -> nn.Parameter:
        return nn.Parameter(torch.FloatTensor(mx_array.data().asnumpy()					)					)

    # Check param shapes and map new HF param back
    def check_and_map_params(lowercase    :    Dict		,				lowercase    :    List[str]					):
        snake_case       : List[str]    =       hf_param.shape

        snake_case       : Union[str, Any]    =       to_torch(params[gluon_param]					)
        snake_case       : Optional[Any]    =       gluon_param.shape

        assert (
            shape_hf == shape_gluon
        ), F'The gluon parameter {gluon_param} has shape {shape_gluon}, but expects shape {shape_hf} for Transformers'

        return gluon_param

    snake_case       : List[Any]    =       check_and_map_params(
        hf_bort_model.bert.embeddings.word_embeddings.weight		,				"word_embed.0.weight"					)
    snake_case       : int    =       check_and_map_params(
        hf_bort_model.bert.embeddings.position_embeddings.weight		,				"encoder.position_weight"					)
    snake_case       : Union[str, Any]    =       check_and_map_params(
        hf_bort_model.bert.embeddings.LayerNorm.bias		,				"encoder.layer_norm.beta"					)
    snake_case       : List[str]    =       check_and_map_params(
        hf_bort_model.bert.embeddings.LayerNorm.weight		,				"encoder.layer_norm.gamma"					)

    # Inspired by RoBERTa conversion script, we just zero them out (Bort does not use them)
    snake_case       : Any    =       torch.zeros_like(
        hf_bort_model.bert.embeddings.token_type_embeddings.weight.data					)

    for i in range(hf_bort_config.num_hidden_layers					):
        snake_case       : BertLayer    =       hf_bort_model.bert.encoder.layer[i]

        # self attention
        snake_case       : BertSelfAttention    =       layer.attention.self

        snake_case       : Dict    =       check_and_map_params(
            self_attn.key.bias.data		,				F'encoder.transformer_cells.{i}.attention_cell.proj_key.bias'					)

        snake_case       : Dict    =       check_and_map_params(
            self_attn.key.weight.data		,				F'encoder.transformer_cells.{i}.attention_cell.proj_key.weight'					)
        snake_case       : str    =       check_and_map_params(
            self_attn.query.bias.data		,				F'encoder.transformer_cells.{i}.attention_cell.proj_query.bias'					)
        snake_case       : Dict    =       check_and_map_params(
            self_attn.query.weight.data		,				F'encoder.transformer_cells.{i}.attention_cell.proj_query.weight'					)
        snake_case       : Dict    =       check_and_map_params(
            self_attn.value.bias.data		,				F'encoder.transformer_cells.{i}.attention_cell.proj_value.bias'					)
        snake_case       : int    =       check_and_map_params(
            self_attn.value.weight.data		,				F'encoder.transformer_cells.{i}.attention_cell.proj_value.weight'					)

        # self attention output
        snake_case       : BertSelfOutput    =       layer.attention.output

        snake_case       : str    =       check_and_map_params(
            self_output.dense.bias		,				F'encoder.transformer_cells.{i}.proj.bias'					)
        snake_case       : List[str]    =       check_and_map_params(
            self_output.dense.weight		,				F'encoder.transformer_cells.{i}.proj.weight'					)
        snake_case       : Any    =       check_and_map_params(
            self_output.LayerNorm.bias		,				F'encoder.transformer_cells.{i}.layer_norm.beta'					)
        snake_case       : Optional[int]    =       check_and_map_params(
            self_output.LayerNorm.weight		,				F'encoder.transformer_cells.{i}.layer_norm.gamma'					)

        # intermediate
        snake_case       : BertIntermediate    =       layer.intermediate

        snake_case       : Dict    =       check_and_map_params(
            intermediate.dense.bias		,				F'encoder.transformer_cells.{i}.ffn.ffn_1.bias'					)
        snake_case       : Union[str, Any]    =       check_and_map_params(
            intermediate.dense.weight		,				F'encoder.transformer_cells.{i}.ffn.ffn_1.weight'					)

        # output
        snake_case       : BertOutput    =       layer.output

        snake_case       : Dict    =       check_and_map_params(
            bert_output.dense.bias		,				F'encoder.transformer_cells.{i}.ffn.ffn_2.bias'					)
        snake_case       : Optional[int]    =       check_and_map_params(
            bert_output.dense.weight		,				F'encoder.transformer_cells.{i}.ffn.ffn_2.weight'					)
        snake_case       : Any    =       check_and_map_params(
            bert_output.LayerNorm.bias		,				F'encoder.transformer_cells.{i}.ffn.layer_norm.beta'					)
        snake_case       : Optional[Any]    =       check_and_map_params(
            bert_output.LayerNorm.weight		,				F'encoder.transformer_cells.{i}.ffn.layer_norm.gamma'					)

    # Save space and energy 🎄
    hf_bort_model.half()

    # Compare output of both models
    snake_case       : Dict    =       RobertaTokenizer.from_pretrained("roberta-base"					)

    snake_case       : int    =       tokenizer.encode_plus(lowercase__					)["input_ids"]

    # Get gluon output
    snake_case       : List[Any]    =       mx.nd.array([input_ids]					)
    snake_case       : Tuple    =       original_bort(inputs=lowercase__		,				token_types=[]					)

    # Get Transformer output (save and reload model again)
    hf_bort_model.save_pretrained(lowercase__					)
    snake_case       : Tuple    =       BertModel.from_pretrained(lowercase__					)
    hf_bort_model.eval()

    snake_case       : List[str]    =       tokenizer.encode_plus(lowercase__		,				return_tensors="pt"					)
    snake_case       : Any    =       hf_bort_model(**lowercase__					)[0]

    snake_case       : Tuple    =       output_gluon[0].asnumpy()
    snake_case       : List[Any]    =       output_hf[0].detach().numpy()

    snake_case       : Any    =       np.max(np.abs(hf_layer - gluon_layer					)					).item()
    snake_case       : str    =       np.allclose(lowercase__		,				lowercase__		,				atol=1e-3					)

    if success:
        print("✔️ Both model do output the same tensors"					)
    else:
        print("❌ Both model do **NOT** output the same tensors"					)
        print("Absolute difference is:"		,				lowercase__					)


if __name__ == "__main__":
  __snake_case       =       argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      """--bort_checkpoint_path""", default=None, type=str, required=True, help="""Path the official Bort params file."""
  )
  parser.add_argument(
      """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
  )
  __snake_case       =       parser.parse_args()
  convert_bort_checkpoint_to_pytorch(args.bort_checkpoint_path, args.pytorch_dump_folder_path)








"""simple docstring"""

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


_UpperCamelCase		     =    logging.get_logger(__name__)

_UpperCamelCase		     =    {
    """microsoft/focalnet-tiny""": """https://huggingface.co/microsoft/focalnet-tiny/resolve/main/config.json""",
}






class 			__a					(						__magic_name__       ,	__magic_name__       ):





       """simple docstring"""


       __UpperCamelCase					:      Optional[Any]    =						'focalnet'






       def __init__(   self			,					snake_case=224			,					snake_case=4			,					snake_case=3			,					snake_case=96			,					snake_case=False			,					snake_case=[192, 384, 768, 768]			,					snake_case=[2, 2, 6, 2]			,					snake_case=[2, 2, 2, 2]			,					snake_case=[3, 3, 3, 3]			,					snake_case="gelu"			,					snake_case=4.0			,					snake_case=0.0			,					snake_case=0.1			,					snake_case=False			,					snake_case=1e-4			,					snake_case=False			,					snake_case=False			,					snake_case=False			,					snake_case=0.02			,					snake_case=1e-5			,					snake_case=32			,					snake_case=None			,					snake_case=None			,					**snake_case			,					):





        """simple docstring"""

        super().__init__(**snake_case   )

        lowerCAmelCase__   :				Optional[Any] 			=				image_size
        lowerCAmelCase__   :				Union[str, Any] 			=				patch_size
        lowerCAmelCase__   :				List[str] 			=				num_channels
        lowerCAmelCase__   :				List[str] 			=				embed_dim
        lowerCAmelCase__   :				List[Any] 			=				use_conv_embed
        lowerCAmelCase__   :				List[str] 			=				hidden_sizes
        lowerCAmelCase__   :				List[Any] 			=				depths
        lowerCAmelCase__   :				Union[str, Any] 			=				focal_levels
        lowerCAmelCase__   :				Union[str, Any] 			=				focal_windows
        lowerCAmelCase__   :				int 			=				hidden_act
        lowerCAmelCase__   :				List[Any] 			=				mlp_ratio
        lowerCAmelCase__   :				str 			=				hidden_dropout_prob
        lowerCAmelCase__   :				Union[str, Any] 			=				drop_path_rate
        lowerCAmelCase__   :				Tuple 			=				use_layerscale
        lowerCAmelCase__   :				Tuple 			=				layerscale_value
        lowerCAmelCase__   :				str 			=				use_post_layernorm
        lowerCAmelCase__   :				str 			=				use_post_layernorm_in_modulation
        lowerCAmelCase__   :				Union[str, Any] 			=				normalize_modulator
        lowerCAmelCase__   :				Optional[int] 			=				initializer_range
        lowerCAmelCase__   :				List[Any] 			=				layer_norm_eps
        lowerCAmelCase__   :				str 			=				encoder_stride
        lowerCAmelCase__   :				Union[str, Any] 			=				["stem"] + [F"""stage{idx}""" for idx in range(1			,					len(self.depths   ) + 1   )]
        lowerCAmelCase__			,						lowerCAmelCase__   :				Union[str, Any] 			=				get_aligned_output_features_output_indices(
            out_features=snake_case			,					out_indices=snake_case			,					stage_names=self.stage_names   )






