
'''simple docstring'''





import unittest

import numpy as np

from transformers.testing_utils import require_pytesseract, require_torch
from transformers.utils import is_pytesseract_available, is_torch_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
						import torch

if is_pytesseract_available():
						from PIL import Image

						from transformers import LayoutLMvaImageProcessor

class 						a   (				unittest.TestCase    ):




				def __init__(							self	:				Any,    SCREAMING_SNAKE_CASE_	:				Any,    SCREAMING_SNAKE_CASE_	:				str=7,    SCREAMING_SNAKE_CASE_	:				List[str]=3,    SCREAMING_SNAKE_CASE_	:				Dict=18,    SCREAMING_SNAKE_CASE_	:				Optional[int]=30,    SCREAMING_SNAKE_CASE_	:				Union[str, Any]=4_00,    SCREAMING_SNAKE_CASE_	:				List[str]=True,    SCREAMING_SNAKE_CASE_	:				Tuple=None,    SCREAMING_SNAKE_CASE_	:				Tuple=True,    ):
										snake_case       :     Optional[int] =    size if size is not None else {"""height""": 18, """width""": 18}
										snake_case       :     str =    parent
										snake_case       :     List[str] =    batch_size
										snake_case       :     str =    num_channels
										snake_case       :     Dict =    image_size
										snake_case       :     Dict =    min_resolution
										snake_case       :     str =    max_resolution
										snake_case       :     Dict =    do_resize
										snake_case       :     List[str] =    size
										snake_case       :     Union[str, Any] =    apply_ocr




				def 	__snake_case							(							self	:				List[str]      ):
										return {"do_resize": self.do_resize, "size": self.size, "apply_ocr": self.apply_ocr}




@require_torch
@require_pytesseract
class 						a   (				SCREAMING_SNAKE_CASE__				,unittest.TestCase    ):
				_snake_case		    = LayoutLMvaImageProcessor if is_pytesseract_available() else None




				def 	__snake_case							(							self	:				List[Any]      ):
										snake_case       :     Any =    LayoutLMvaImageProcessingTester(self      )




				@property
				def 	__snake_case							(							self	:				Tuple      ):
										return self.image_processor_tester.prepare_image_processor_dict()




				def 	__snake_case							(							self	:				List[str]      ):
										snake_case       :     List[Any] =    self.image_processing_class(**self.image_processor_dict      )
										self.assertTrue(hasattr(_lowercase,    '''do_resize'''      )      )
										self.assertTrue(hasattr(_lowercase,    '''size'''      )      )
										self.assertTrue(hasattr(_lowercase,    '''apply_ocr'''      )      )




				def 	__snake_case							(							self	:				Tuple      ):
										snake_case       :     Optional[int] =    self.image_processing_class.from_dict(self.image_processor_dict      )
										self.assertEqual(image_processor.size,    {'''height''': 18, '''width''': 18}      )

										snake_case       :     Tuple =    self.image_processing_class.from_dict(self.image_processor_dict,    size=42      )
										self.assertEqual(image_processor.size,    {'''height''': 42, '''width''': 42}      )




				def 	__snake_case							(							self	:				str      ):
										pass




				def 	__snake_case							(							self	:				str      ):
										snake_case       :     Union[str, Any] =    self.image_processing_class(**self.image_processor_dict      )
										# create random PIL images
										snake_case       :     Optional[int] =    prepare_image_inputs(self.image_processor_tester,    equal_resolution=_lowercase      )
										for image in image_inputs:
																self.assertIsInstance(_lowercase,    Image.Image      )

										# Test not batched input
										snake_case       :     Optional[Any] =    image_processing(image_inputs[0],    return_tensors='''pt'''      )
										self.assertEqual(
										    encoding.pixel_values.shape,    (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.size['''height'''],
										        self.image_processor_tester.size['''width'''],
										    ),    )

										self.assertIsInstance(encoding.words,    _lowercase      )
										self.assertIsInstance(encoding.boxes,    _lowercase      )

										# Test batched
										snake_case       :     Union[str, Any] =    image_processing(_lowercase,    return_tensors='''pt'''      ).pixel_values
										self.assertEqual(
										    encoded_images.shape,    (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.size['''height'''],
										        self.image_processor_tester.size['''width'''],
										    ),    )




				def 	__snake_case							(							self	:				Any      ):
										snake_case       :     int =    self.image_processing_class(**self.image_processor_dict      )
										# create random numpy tensors
										snake_case       :     Dict =    prepare_image_inputs(self.image_processor_tester,    equal_resolution=_lowercase,    numpify=_lowercase      )
										for image in image_inputs:
																self.assertIsInstance(_lowercase,    np.ndarray      )

										# Test not batched input
										snake_case       :     Tuple =    image_processing(image_inputs[0],    return_tensors='''pt'''      ).pixel_values
										self.assertEqual(
										    encoded_images.shape,    (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.size['''height'''],
										        self.image_processor_tester.size['''width'''],
										    ),    )

										# Test batched
										snake_case       :     Union[str, Any] =    image_processing(_lowercase,    return_tensors='''pt'''      ).pixel_values
										self.assertEqual(
										    encoded_images.shape,    (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.size['''height'''],
										        self.image_processor_tester.size['''width'''],
										    ),    )




				def 	__snake_case							(							self	:				Union[str, Any]      ):
										snake_case       :     Optional[Any] =    self.image_processing_class(**self.image_processor_dict      )
										# create random PyTorch tensors
										snake_case       :     List[Any] =    prepare_image_inputs(self.image_processor_tester,    equal_resolution=_lowercase,    torchify=_lowercase      )
										for image in image_inputs:
																self.assertIsInstance(_lowercase,    torch.Tensor      )

										# Test not batched input
										snake_case       :     int =    image_processing(image_inputs[0],    return_tensors='''pt'''      ).pixel_values
										self.assertEqual(
										    encoded_images.shape,    (
										        1,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.size['''height'''],
										        self.image_processor_tester.size['''width'''],
										    ),    )

										# Test batched
										snake_case       :     Dict =    image_processing(_lowercase,    return_tensors='''pt'''      ).pixel_values
										self.assertEqual(
										    encoded_images.shape,    (
										        self.image_processor_tester.batch_size,
										        self.image_processor_tester.num_channels,
										        self.image_processor_tester.size['''height'''],
										        self.image_processor_tester.size['''width'''],
										    ),    )




				def 	__snake_case							(							self	:				Dict      ):
										snake_case       :     Optional[Any] =    LayoutLMvaImageProcessor()

										from datasets import load_dataset

										snake_case       :     Dict =    load_dataset('''hf-internal-testing/fixtures_docvqa''',    split='''test'''      )

										snake_case       :     Any =    Image.open(ds[0]['''file''']      ).convert('''RGB'''      )

										snake_case       :     List[str] =    image_processing(_lowercase,    return_tensors='''pt'''      )

										self.assertEqual(encoding.pixel_values.shape,    (1, 3, 2_24, 2_24)      )
										self.assertEqual(len(encoding.words      ),    len(encoding.boxes      )      )

										# fmt: off
										# the words and boxes were obtained with Tesseract 4.1.1
										snake_case       :     Optional[int] =    [["""11:14""", """to""", """11:39""", """a.m""", """11:39""", """to""", """11:44""", """a.m.""", """11:44""", """a.m.""", """to""", """12:25""", """p.m.""", """12:25""", """to""", """12:58""", """p.m.""", """12:58""", """to""", """4:00""", """p.m.""", """2:00""", """to""", """5:00""", """p.m.""", """Coffee""", """Break""", """Coffee""", """will""", """be""", """served""", """for""", """men""", """and""", """women""", """in""", """the""", """lobby""", """adjacent""", """to""", """exhibit""", """area.""", """Please""", """move""", """into""", """exhibit""", """area.""", """(Exhibits""", """Open)""", """TRRF""", """GENERAL""", """SESSION""", """(PART""", """|)""", """Presiding:""", """Lee""", """A.""", """Waller""", """TRRF""", """Vice""", """President""", """“Introductory""", """Remarks”""", """Lee""", """A.""", """Waller,""", """TRRF""", """Vice""", """Presi-""", """dent""", """Individual""", """Interviews""", """with""", """TRRF""", """Public""", """Board""", """Members""", """and""", """Sci-""", """entific""", """Advisory""", """Council""", """Mem-""", """bers""", """Conducted""", """by""", """TRRF""", """Treasurer""", """Philip""", """G.""", """Kuehn""", """to""", """get""", """answers""", """which""", """the""", """public""", """refrigerated""", """warehousing""", """industry""", """is""", """looking""", """for.""", """Plus""", """questions""", """from""", """the""", """floor.""", """Dr.""", """Emil""", """M.""", """Mrak,""", """University""", """of""", """Cal-""", """ifornia,""", """Chairman,""", """TRRF""", """Board;""", """Sam""", """R.""", """Cecil,""", """University""", """of""", """Georgia""", """College""", """of""", """Agriculture;""", """Dr.""", """Stanley""", """Charm,""", """Tufts""", """University""", """School""", """of""", """Medicine;""", """Dr.""", """Robert""", """H.""", """Cotton,""", """ITT""", """Continental""", """Baking""", """Company;""", """Dr.""", """Owen""", """Fennema,""", """University""", """of""", """Wis-""", """consin;""", """Dr.""", """Robert""", """E.""", """Hardenburg,""", """USDA.""", """Questions""", """and""", """Answers""", """Exhibits""", """Open""", """Capt.""", """Jack""", """Stoney""", """Room""", """TRRF""", """Scientific""", """Advisory""", """Council""", """Meeting""", """Ballroom""", """Foyer"""]]  # noqa: E231
										snake_case       :     int =    [[[1_41, 57, 2_14, 69], [2_28, 58, 2_52, 69], [1_41, 75, 2_16, 88], [2_30, 79, 2_80, 88], [1_42, 2_60, 2_18, 2_73], [2_30, 2_61, 2_55, 2_73], [1_43, 2_79, 2_18, 2_90], [2_31, 2_82, 2_90, 2_91], [1_43, 3_42, 2_18, 3_54], [2_31, 3_45, 2_89, 3_55], [2_02, 3_62, 2_27, 3_73], [1_43, 3_79, 2_20, 3_92], [2_31, 3_82, 2_91, 3_94], [1_44, 7_14, 2_20, 7_26], [2_31, 7_15, 2_56, 7_26], [1_44, 7_32, 2_20, 7_45], [2_32, 7_36, 2_91, 7_47], [1_44, 7_69, 2_18, 7_82], [2_31, 7_70, 2_56, 7_82], [1_41, 7_88, 2_02, 8_01], [2_15, 7_91, 2_74, 8_04], [1_43, 8_26, 2_04, 8_38], [2_15, 8_26, 2_40, 8_38], [1_42, 8_44, 2_02, 8_57], [2_15, 8_47, 2_74, 8_59], [3_34, 57, 4_27, 69], [4_40, 57, 5_22, 69], [3_69, 75, 4_61, 88], [4_69, 75, 5_16, 88], [5_28, 76, 5_62, 88], [5_70, 76, 6_67, 88], [6_75, 75, 7_11, 87], [7_21, 79, 7_78, 88], [7_89, 75, 8_40, 88], [3_69, 97, 4_70, 1_07], [4_84, 94, 5_07, 1_06], [5_18, 94, 5_62, 1_07], [5_76, 94, 6_55, 1_10], [6_68, 94, 7_92, 1_09], [8_04, 95, 8_29, 1_07], [3_69, 1_13, 4_65, 1_25], [4_77, 1_16, 5_47, 1_25], [5_62, 1_13, 6_58, 1_25], [6_71, 1_16, 7_48, 1_25], [7_61, 1_13, 8_11, 1_25], [3_69, 1_31, 4_65, 1_43], [4_77, 1_33, 5_48, 1_43], [5_63, 1_30, 6_98, 1_45], [7_10, 1_30, 8_02, 1_46], [3_36, 1_71, 4_12, 1_83], [4_23, 1_71, 5_72, 1_83], [5_82, 1_70, 7_16, 1_84], [7_28, 1_71, 8_17, 1_87], [8_29, 1_71, 8_44, 1_86], [3_38, 1_97, 4_82, 2_12], [5_07, 1_96, 5_57, 2_09], [5_69, 1_96, 5_95, 2_08], [6_10, 1_96, 7_02, 2_09], [5_05, 2_14, 5_83, 2_26], [5_95, 2_14, 6_56, 2_27], [6_70, 2_15, 8_07, 2_27], [3_35, 2_59, 5_43, 2_74], [5_56, 2_59, 7_08, 2_72], [3_72, 2_79, 4_22, 2_91], [4_35, 2_79, 4_60, 2_91], [4_74, 2_79, 5_74, 2_92], [5_87, 2_78, 6_64, 2_91], [6_76, 2_78, 7_38, 2_91], [7_51, 2_79, 8_34, 2_91], [3_72, 2_98, 4_34, 3_10], [3_35, 3_41, 4_83, 3_54], [4_97, 3_41, 6_55, 3_54], [6_67, 3_41, 7_28, 3_54], [7_40, 3_41, 8_25, 3_54], [3_35, 3_60, 4_30, 3_72], [4_42, 3_60, 5_34, 3_72], [5_45, 3_59, 6_87, 3_72], [6_97, 3_60, 7_54, 3_72], [7_65, 3_60, 8_23, 3_73], [3_34, 3_78, 4_28, 3_91], [4_40, 3_78, 5_77, 3_94], [5_90, 3_78, 7_05, 3_91], [7_20, 3_78, 8_01, 3_91], [3_34, 3_97, 4_00, 4_09], [3_70, 4_16, 5_29, 4_29], [5_44, 4_16, 5_76, 4_32], [5_87, 4_16, 6_65, 4_28], [6_77, 4_16, 8_14, 4_29], [3_72, 4_35, 4_52, 4_50], [4_65, 4_34, 4_95, 4_47], [5_11, 4_34, 6_00, 4_47], [6_11, 4_36, 6_37, 4_47], [6_49, 4_36, 6_94, 4_51], [7_05, 4_38, 8_24, 4_47], [3_69, 4_53, 4_52, 4_66], [4_64, 4_54, 5_09, 4_66], [5_22, 4_53, 6_11, 4_69], [6_25, 4_53, 7_92, 4_69], [3_70, 4_72, 5_56, 4_88], [5_70, 4_72, 6_84, 4_87], [6_97, 4_72, 7_18, 4_85], [7_32, 4_72, 8_35, 4_88], [3_69, 4_90, 4_11, 5_03], [4_25, 4_90, 4_84, 5_03], [4_96, 4_90, 6_35, 5_06], [6_45, 4_90, 7_07, 5_03], [7_18, 4_91, 7_61, 5_03], [7_71, 4_90, 8_40, 5_03], [3_36, 5_10, 3_74, 5_21], [3_88, 5_10, 4_47, 5_22], [4_60, 5_10, 4_89, 5_21], [5_03, 5_10, 5_80, 5_22], [5_92, 5_09, 7_36, 5_25], [7_45, 5_09, 7_70, 5_22], [7_81, 5_09, 8_40, 5_22], [3_38, 5_28, 4_34, 5_41], [4_48, 5_28, 5_96, 5_41], [6_09, 5_27, 6_87, 5_40], [7_00, 5_28, 7_92, 5_41], [3_36, 5_46, 3_97, 5_59], [4_07, 5_46, 4_31, 5_59], [4_43, 5_46, 5_25, 5_60], [5_37, 5_46, 6_80, 5_62], [6_88, 5_46, 7_14, 5_59], [7_22, 5_46, 8_37, 5_62], [3_36, 5_65, 4_49, 5_81], [4_61, 5_65, 4_85, 5_77], [4_97, 5_65, 6_65, 5_81], [6_81, 5_65, 7_18, 5_77], [7_32, 5_65, 8_37, 5_80], [3_37, 5_84, 4_38, 5_97], [4_52, 5_83, 5_21, 5_96], [5_35, 5_84, 6_77, 5_99], [6_90, 5_83, 7_87, 5_96], [8_01, 5_83, 8_25, 5_96], [3_38, 6_02, 4_78, 6_15], [4_92, 6_02, 5_30, 6_14], [5_43, 6_02, 6_38, 6_15], [6_50, 6_02, 6_76, 6_14], [6_88, 6_02, 7_88, 6_15], [8_02, 6_02, 8_43, 6_14], [3_37, 6_21, 5_02, 6_33], [5_16, 6_21, 6_15, 6_37], [6_29, 6_21, 7_74, 6_36], [7_89, 6_21, 8_27, 6_33], [3_37, 6_39, 4_18, 6_52], [4_32, 6_40, 5_71, 6_53], [5_87, 6_39, 7_31, 6_55], [7_43, 6_39, 7_69, 6_52], [7_80, 6_39, 8_41, 6_52], [3_38, 6_58, 4_40, 6_73], [4_55, 6_58, 4_91, 6_70], [5_08, 6_58, 6_02, 6_71], [6_16, 6_58, 6_38, 6_70], [6_54, 6_58, 8_35, 6_74], [3_37, 6_77, 4_29, 6_89], [3_37, 7_14, 4_82, 7_26], [4_95, 7_14, 5_48, 7_26], [5_61, 7_14, 6_83, 7_26], [3_38, 7_70, 4_61, 7_82], [4_74, 7_69, 5_54, 7_85], [4_89, 7_88, 5_62, 8_03], [5_76, 7_88, 6_43, 8_01], [6_56, 7_87, 7_51, 8_04], [7_64, 7_88, 8_44, 8_01], [3_34, 8_25, 4_21, 8_38], [4_30, 8_24, 5_74, 8_38], [5_84, 8_24, 7_23, 8_41], [3_35, 8_44, 4_50, 8_57], [4_64, 8_43, 5_83, 8_60], [6_28, 8_62, 7_55, 8_75], [7_69, 8_61, 8_48, 8_78]]]  # noqa: E231
										# fmt: on

										self.assertListEqual(encoding.words,    _lowercase      )
										self.assertListEqual(encoding.boxes,    _lowercase      )

										# with apply_OCR = False
										snake_case       :     int =    LayoutLMvaImageProcessor(apply_ocr=_lowercase      )

										snake_case       :     int =    image_processing(_lowercase,    return_tensors='''pt'''      )

										self.assertEqual(encoding.pixel_values.shape,    (1, 3, 2_24, 2_24)      )



'''simple docstring'''





import argparse
import os

import gluonnlp as nlp
import mxnet as mx
import numpy as np
import torch
from gluonnlp.base import get_home_dir
from gluonnlp.model.bert import BERTEncoder
from gluonnlp.model.utils import _load_vocab
from gluonnlp.vocab import Vocab
from packaging import version
from torch import nn

from transformers import BertConfig, BertForMaskedLM, BertModel, RobertaTokenizer
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(nlp.__version__) != version.parse("0.8.3"):
						raise Exception("requires gluonnlp == 0.8.3")

if version.parse(mx.__version__) != version.parse("1.5.0"):
						raise Exception("requires mxnet == 1.5.0")

logging.set_verbosity_info()
UpperCAmelCase    					=   logging.get_logger(__name__)

UpperCAmelCase    					=   "The Nymphenburg Palace is a beautiful palace in Munich!"

def      A (       A_							:					str    ,							A_							:					str       ):
						snake_case       :     str =    {
						    '''attention_cell''': '''multi_head''',
						    '''num_layers''': 4,
						    '''units''': 1024,
						    '''hidden_size''': 768,
						    '''max_length''': 512,
						    '''num_heads''': 8,
						    '''scaled''': True,
						    '''dropout''': 0.1,
						    '''use_residual''': True,
						    '''embed_size''': 1024,
						    '''embed_dropout''': 0.1,
						    '''word_embed''': None,
						    '''layer_norm_eps''': 1e-5,
						    '''token_type_vocab_size''': 2,
						}

						snake_case       :     int =    bort_4_8_768_1024_hparams

						# Let's construct the original Bort model here
						# Taken from official BERT implementation, see:
						# https://github.com/alexa/bort/blob/master/bort/bort.py
						snake_case       :     int =    BERTEncoder(
						    attention_cell=predefined_args['''attention_cell''']    ,							num_layers=predefined_args['''num_layers''']    ,							units=predefined_args['''units''']    ,							hidden_size=predefined_args['''hidden_size''']    ,							max_length=predefined_args['''max_length''']    ,							num_heads=predefined_args['''num_heads''']    ,							scaled=predefined_args['''scaled''']    ,							dropout=predefined_args['''dropout''']    ,							output_attention=A_    ,							output_all_encodings=A_    ,							use_residual=predefined_args['''use_residual''']    ,							activation=predefined_args.get('''activation'''    ,							'''gelu'''       )    ,							layer_norm_eps=predefined_args.get('''layer_norm_eps'''    ,							A_       )    ,							)

						# Vocab information needs to be fetched first
						# It's the same as RoBERTa, so RobertaTokenizer can be used later
						snake_case       :     List[Any] =    '''openwebtext_ccnews_stories_books_cased'''

						# Specify download folder to Gluonnlp's vocab
						snake_case       :     Optional[Any] =    os.path.join(get_home_dir()    ,							'''models'''       )
						snake_case       :     Dict =    _load_vocab(A_    ,							A_    ,							A_    ,							cls=A_       )

						snake_case       :     Tuple =    nlp.model.BERTModel(
						    A_    ,							len(A_       )    ,							units=predefined_args['''units''']    ,							embed_size=predefined_args['''embed_size''']    ,							embed_dropout=predefined_args['''embed_dropout''']    ,							word_embed=predefined_args['''word_embed''']    ,							use_pooler=A_    ,							use_token_type_embed=A_    ,							token_type_vocab_size=predefined_args['''token_type_vocab_size''']    ,							use_classifier=A_    ,							use_decoder=A_    ,							)

						original_bort.load_parameters(A_    ,							cast_dtype=A_    ,							ignore_extra=A_       )
						snake_case       :     Union[str, Any] =    original_bort._collect_params_with_prefix()

						# Build our config 🤗
						snake_case       :     Optional[int] =    {
						    '''architectures''': ['''BertForMaskedLM'''],
						    '''attention_probs_dropout_prob''': predefined_args['''dropout'''],
						    '''hidden_act''': '''gelu''',
						    '''hidden_dropout_prob''': predefined_args['''dropout'''],
						    '''hidden_size''': predefined_args['''embed_size'''],
						    '''initializer_range''': 0.02,
						    '''intermediate_size''': predefined_args['''hidden_size'''],
						    '''layer_norm_eps''': predefined_args['''layer_norm_eps'''],
						    '''max_position_embeddings''': predefined_args['''max_length'''],
						    '''model_type''': '''bort''',
						    '''num_attention_heads''': predefined_args['''num_heads'''],
						    '''num_hidden_layers''': predefined_args['''num_layers'''],
						    '''pad_token_id''': 1,  # 2 = BERT, 1 = RoBERTa
						    '''type_vocab_size''': 1,  # 2 = BERT, 1 = RoBERTa
						    '''vocab_size''': len(A_       ),
						}

						snake_case       :     Optional[int] =    BertConfig.from_dict(A_       )
						snake_case       :     int =    BertForMaskedLM(A_       )
						hf_bort_model.eval()

						# Parameter mapping table (Gluonnlp to Transformers)
						# * denotes layer index
						#
						# | Gluon Parameter                                                | Transformers Parameter
						# | -------------------------------------------------------------- | ----------------------
						# | `encoder.layer_norm.beta`                                      | `bert.embeddings.LayerNorm.bias`
						# | `encoder.layer_norm.gamma`                                     | `bert.embeddings.LayerNorm.weight`
						# | `encoder.position_weight`                                      | `bert.embeddings.position_embeddings.weight`
						# | `word_embed.0.weight`                                          | `bert.embeddings.word_embeddings.weight`
						# | `encoder.transformer_cells.*.attention_cell.proj_key.bias`     | `bert.encoder.layer.*.attention.self.key.bias`
						# | `encoder.transformer_cells.*.attention_cell.proj_key.weight`   | `bert.encoder.layer.*.attention.self.key.weight`
						# | `encoder.transformer_cells.*.attention_cell.proj_query.bias`   | `bert.encoder.layer.*.attention.self.query.bias`
						# | `encoder.transformer_cells.*.attention_cell.proj_query.weight` | `bert.encoder.layer.*.attention.self.query.weight`
						# | `encoder.transformer_cells.*.attention_cell.proj_value.bias`   | `bert.encoder.layer.*.attention.self.value.bias`
						# | `encoder.transformer_cells.*.attention_cell.proj_value.weight` | `bert.encoder.layer.*.attention.self.value.weight`
						# | `encoder.transformer_cells.*.ffn.ffn_2.bias`                   | `bert.encoder.layer.*.attention.output.dense.bias`
						# | `encoder.transformer_cells.*.ffn.ffn_2.weight`                 | `bert.encoder.layer.*.attention.output.dense.weight`
						# | `encoder.transformer_cells.*.layer_norm.beta`                  | `bert.encoder.layer.*.attention.output.LayerNorm.bias`
						# | `encoder.transformer_cells.*.layer_norm.gamma`                 | `bert.encoder.layer.*.attention.output.LayerNorm.weight`
						# | `encoder.transformer_cells.*.ffn.ffn_1.bias`                   | `bert.encoder.layer.*.intermediate.dense.bias`
						# | `encoder.transformer_cells.*.ffn.ffn_1.weight`                 | `bert.encoder.layer.*.intermediate.dense.weight`
						# | `encoder.transformer_cells.*.ffn.layer_norm.beta`              | `bert.encoder.layer.*.output.LayerNorm.bias`
						# | `encoder.transformer_cells.*.ffn.layer_norm.gamma`             | `bert.encoder.layer.*.output.LayerNorm.weight`
						# | `encoder.transformer_cells.*.proj.bias`                        | `bert.encoder.layer.*.output.dense.bias`
						# | `encoder.transformer_cells.*.proj.weight`                      | `bert.encoder.layer.*.output.dense.weight`

						# Helper function to convert MXNET Arrays to PyTorch
						def to_torch(A_							:					List[str]       ) -> nn.Parameter:
												return nn.Parameter(torch.FloatTensor(mx_array.data().asnumpy()       )       )

						# Check param shapes and map new HF param back
						def check_and_map_params(A_							:					Tuple    ,							A_							:					Dict       ):
												snake_case       :     Tuple =    hf_param.shape

												snake_case       :     List[Any] =    to_torch(params[gluon_param]       )
												snake_case       :     int =    gluon_param.shape

												assert (
												    shape_hf == shape_gluon
												), F"""The gluon parameter {gluon_param} has shape {shape_gluon}, but expects shape {shape_hf} for Transformers"""

												return gluon_param

						snake_case       :     List[str] =    check_and_map_params(
						    hf_bort_model.bert.embeddings.word_embeddings.weight    ,							'''word_embed.0.weight'''       )
						snake_case       :     Optional[Any] =    check_and_map_params(
						    hf_bort_model.bert.embeddings.position_embeddings.weight    ,							'''encoder.position_weight'''       )
						snake_case       :     Dict =    check_and_map_params(
						    hf_bort_model.bert.embeddings.LayerNorm.bias    ,							'''encoder.layer_norm.beta'''       )
						snake_case       :     Tuple =    check_and_map_params(
						    hf_bort_model.bert.embeddings.LayerNorm.weight    ,							'''encoder.layer_norm.gamma'''       )

						# Inspired by RoBERTa conversion script, we just zero them out (Bort does not use them)
						snake_case       :     str =    torch.zeros_like(
						    hf_bort_model.bert.embeddings.token_type_embeddings.weight.data       )

						for i in range(hf_bort_config.num_hidden_layers       ):
												snake_case       :     BertLayer =    hf_bort_model.bert.encoder.layer[i]

												# self attention
												snake_case       :     BertSelfAttention =    layer.attention.self

												snake_case       :     Optional[int] =    check_and_map_params(
												    self_attn.key.bias.data    ,							F"""encoder.transformer_cells.{i}.attention_cell.proj_key.bias"""       )

												snake_case       :     Any =    check_and_map_params(
												    self_attn.key.weight.data    ,							F"""encoder.transformer_cells.{i}.attention_cell.proj_key.weight"""       )
												snake_case       :     Tuple =    check_and_map_params(
												    self_attn.query.bias.data    ,							F"""encoder.transformer_cells.{i}.attention_cell.proj_query.bias"""       )
												snake_case       :     Dict =    check_and_map_params(
												    self_attn.query.weight.data    ,							F"""encoder.transformer_cells.{i}.attention_cell.proj_query.weight"""       )
												snake_case       :     int =    check_and_map_params(
												    self_attn.value.bias.data    ,							F"""encoder.transformer_cells.{i}.attention_cell.proj_value.bias"""       )
												snake_case       :     int =    check_and_map_params(
												    self_attn.value.weight.data    ,							F"""encoder.transformer_cells.{i}.attention_cell.proj_value.weight"""       )

												# self attention output
												snake_case       :     BertSelfOutput =    layer.attention.output

												snake_case       :     Optional[Any] =    check_and_map_params(
												    self_output.dense.bias    ,							F"""encoder.transformer_cells.{i}.proj.bias"""       )
												snake_case       :     Tuple =    check_and_map_params(
												    self_output.dense.weight    ,							F"""encoder.transformer_cells.{i}.proj.weight"""       )
												snake_case       :     int =    check_and_map_params(
												    self_output.LayerNorm.bias    ,							F"""encoder.transformer_cells.{i}.layer_norm.beta"""       )
												snake_case       :     Optional[int] =    check_and_map_params(
												    self_output.LayerNorm.weight    ,							F"""encoder.transformer_cells.{i}.layer_norm.gamma"""       )

												# intermediate
												snake_case       :     BertIntermediate =    layer.intermediate

												snake_case       :     Optional[Any] =    check_and_map_params(
												    intermediate.dense.bias    ,							F"""encoder.transformer_cells.{i}.ffn.ffn_1.bias"""       )
												snake_case       :     Optional[int] =    check_and_map_params(
												    intermediate.dense.weight    ,							F"""encoder.transformer_cells.{i}.ffn.ffn_1.weight"""       )

												# output
												snake_case       :     BertOutput =    layer.output

												snake_case       :     List[Any] =    check_and_map_params(
												    bert_output.dense.bias    ,							F"""encoder.transformer_cells.{i}.ffn.ffn_2.bias"""       )
												snake_case       :     int =    check_and_map_params(
												    bert_output.dense.weight    ,							F"""encoder.transformer_cells.{i}.ffn.ffn_2.weight"""       )
												snake_case       :     List[str] =    check_and_map_params(
												    bert_output.LayerNorm.bias    ,							F"""encoder.transformer_cells.{i}.ffn.layer_norm.beta"""       )
												snake_case       :     List[str] =    check_and_map_params(
												    bert_output.LayerNorm.weight    ,							F"""encoder.transformer_cells.{i}.ffn.layer_norm.gamma"""       )

						# Save space and energy 🎄
						hf_bort_model.half()

						# Compare output of both models
						snake_case       :     Dict =    RobertaTokenizer.from_pretrained('''roberta-base'''       )

						snake_case       :     Any =    tokenizer.encode_plus(A_       )['''input_ids''']

						# Get gluon output
						snake_case       :     Dict =    mx.nd.array([input_ids]       )
						snake_case       :     Optional[int] =    original_bort(inputs=A_    ,							token_types=[]       )

						# Get Transformer output (save and reload model again)
						hf_bort_model.save_pretrained(A_       )
						snake_case       :     Optional[int] =    BertModel.from_pretrained(A_       )
						hf_bort_model.eval()

						snake_case       :     Any =    tokenizer.encode_plus(A_    ,							return_tensors='''pt'''       )
						snake_case       :     Any =    hf_bort_model(**A_       )[0]

						snake_case       :     Optional[Any] =    output_gluon[0].asnumpy()
						snake_case       :     Union[str, Any] =    output_hf[0].detach().numpy()

						snake_case       :     Tuple =    np.max(np.abs(hf_layer - gluon_layer       )       ).item()
						snake_case       :     Tuple =    np.allclose(A_    ,							A_    ,							atol=1e-3       )

						if success:
												print('''✔️ Both model do output the same tensors'''       )
						else:
												print('''❌ Both model do **NOT** output the same tensors'''       )
												print('''Absolute difference is:'''    ,							A_       )


if __name__ == "__main__":
						UpperCAmelCase    					=   argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    "--bort_checkpoint_path", default=None, type=str, required=True, help="Path the official Bort params file."
						)
						parser.add_argument(
						    "--pytorch_dump_folder_path", default=None, type=str, required=True, help="Path to the output PyTorch model."
						)
						UpperCAmelCase    					=   parser.parse_args()
						convert_bort_checkpoint_to_pytorch(args.bort_checkpoint_path, args.pytorch_dump_folder_path)

