
import argparse
import os

import gluonnlp as nlp
import mxnet as mx
import numpy as np
import torch
from gluonnlp.base import get_home_dir
from gluonnlp.model.bert import BERTEncoder
from gluonnlp.model.utils import _load_vocab
from gluonnlp.vocab import Vocab
from packaging import version
from torch import nn

from transformers import BertConfig, BertForMaskedLM, BertModel, RobertaTokenizer
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(nlp.__version__) != version.parse('''0.8.3'''):
		raise Exception('''requires gluonnlp == 0.8.3''')

if version.parse(mx.__version__) != version.parse('''1.5.0'''):
		raise Exception('''requires mxnet == 1.5.0''')

logging.set_verbosity_info()
A		: Optional[int]								=     logging.get_logger(__name__)

A		: Optional[int]								=     '''The Nymphenburg Palace is a beautiful palace in Munich!'''




def 	UpperCamelCase__  (     SCREAMING_SNAKE_CASE_	:	Optional[Any]		,   SCREAMING_SNAKE_CASE_	:	List[str]       )  ->     Optional[int]:
						_lowercase					       =					{
						    """attention_cell""": """multi_head""",
						    """num_layers""": 4,
						    """units""": 10_24,
						    """hidden_size""": 7_68,
						    """max_length""": 5_12,
						    """num_heads""": 8,
						    """scaled""": True,
						    """dropout""": 0.1,
						    """use_residual""": True,
						    """embed_size""": 10_24,
						    """embed_dropout""": 0.1,
						    """word_embed""": None,
						    """layer_norm_eps""": 1E-5,
						    """token_type_vocab_size""": 2,
						}

						_lowercase					       =					bort_4_8_768_1024_hparams

						# Let's construct the original Bort model here
						# Taken from official BERT implementation, see:
						# https://github.com/alexa/bort/blob/master/bort/bort.py
						_lowercase					       =					BERTEncoder(
						    attention_cell=predefined_args["""attention_cell"""]		,   num_layers=predefined_args["""num_layers"""]		,   units=predefined_args["""units"""]		,   hidden_size=predefined_args["""hidden_size"""]		,   max_length=predefined_args["""max_length"""]		,   num_heads=predefined_args["""num_heads"""]		,   scaled=predefined_args["""scaled"""]		,   dropout=predefined_args["""dropout"""]		,   output_attention=SCREAMING_SNAKE_CASE_		,   output_all_encodings=SCREAMING_SNAKE_CASE_		,   use_residual=predefined_args["""use_residual"""]		,   activation=predefined_args.get("""activation"""		,   """gelu"""       )		,   layer_norm_eps=predefined_args.get("""layer_norm_eps"""		,   SCREAMING_SNAKE_CASE_       )		,   )

						# Vocab information needs to be fetched first
						# It's the same as RoBERTa, so RobertaTokenizer can be used later
						_lowercase					       =					"""openwebtext_ccnews_stories_books_cased"""

						# Specify download folder to Gluonnlp's vocab
						_lowercase					       =					os.path.join(get_home_dir()		,   """models"""       )
						_lowercase					       =					_load_vocab(SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_		,   cls=SCREAMING_SNAKE_CASE_       )

						_lowercase					       =					nlp.model.BERTModel(
						    SCREAMING_SNAKE_CASE_		,   len(SCREAMING_SNAKE_CASE_       )		,   units=predefined_args["""units"""]		,   embed_size=predefined_args["""embed_size"""]		,   embed_dropout=predefined_args["""embed_dropout"""]		,   word_embed=predefined_args["""word_embed"""]		,   use_pooler=SCREAMING_SNAKE_CASE_		,   use_token_type_embed=SCREAMING_SNAKE_CASE_		,   token_type_vocab_size=predefined_args["""token_type_vocab_size"""]		,   use_classifier=SCREAMING_SNAKE_CASE_		,   use_decoder=SCREAMING_SNAKE_CASE_		,   )

						original_bort.load_parameters(SCREAMING_SNAKE_CASE_		,   cast_dtype=SCREAMING_SNAKE_CASE_		,   ignore_extra=SCREAMING_SNAKE_CASE_       )
						_lowercase					       =					original_bort._collect_params_with_prefix()

						# Build our config 🤗
						_lowercase					       =					{
						    """architectures""": ["""BertForMaskedLM"""],
						    """attention_probs_dropout_prob""": predefined_args["""dropout"""],
						    """hidden_act""": """gelu""",
						    """hidden_dropout_prob""": predefined_args["""dropout"""],
						    """hidden_size""": predefined_args["""embed_size"""],
						    """initializer_range""": 0.02,
						    """intermediate_size""": predefined_args["""hidden_size"""],
						    """layer_norm_eps""": predefined_args["""layer_norm_eps"""],
						    """max_position_embeddings""": predefined_args["""max_length"""],
						    """model_type""": """bort""",
						    """num_attention_heads""": predefined_args["""num_heads"""],
						    """num_hidden_layers""": predefined_args["""num_layers"""],
						    """pad_token_id""": 1,  # 2 = BERT, 1 = RoBERTa
						    """type_vocab_size""": 1,  # 2 = BERT, 1 = RoBERTa
						    """vocab_size""": len(SCREAMING_SNAKE_CASE_       ),
						}

						_lowercase					       =					BertConfig.from_dict(SCREAMING_SNAKE_CASE_       )
						_lowercase					       =					BertForMaskedLM(SCREAMING_SNAKE_CASE_       )
						hf_bort_model.eval()

						# Parameter mapping table (Gluonnlp to Transformers)
						# * denotes layer index
						#
						# | Gluon Parameter                                                | Transformers Parameter
						# | -------------------------------------------------------------- | ----------------------
						# | `encoder.layer_norm.beta`                                      | `bert.embeddings.LayerNorm.bias`
						# | `encoder.layer_norm.gamma`                                     | `bert.embeddings.LayerNorm.weight`
						# | `encoder.position_weight`                                      | `bert.embeddings.position_embeddings.weight`
						# | `word_embed.0.weight`                                          | `bert.embeddings.word_embeddings.weight`
						# | `encoder.transformer_cells.*.attention_cell.proj_key.bias`     | `bert.encoder.layer.*.attention.self.key.bias`
						# | `encoder.transformer_cells.*.attention_cell.proj_key.weight`   | `bert.encoder.layer.*.attention.self.key.weight`
						# | `encoder.transformer_cells.*.attention_cell.proj_query.bias`   | `bert.encoder.layer.*.attention.self.query.bias`
						# | `encoder.transformer_cells.*.attention_cell.proj_query.weight` | `bert.encoder.layer.*.attention.self.query.weight`
						# | `encoder.transformer_cells.*.attention_cell.proj_value.bias`   | `bert.encoder.layer.*.attention.self.value.bias`
						# | `encoder.transformer_cells.*.attention_cell.proj_value.weight` | `bert.encoder.layer.*.attention.self.value.weight`
						# | `encoder.transformer_cells.*.ffn.ffn_2.bias`                   | `bert.encoder.layer.*.attention.output.dense.bias`
						# | `encoder.transformer_cells.*.ffn.ffn_2.weight`                 | `bert.encoder.layer.*.attention.output.dense.weight`
						# | `encoder.transformer_cells.*.layer_norm.beta`                  | `bert.encoder.layer.*.attention.output.LayerNorm.bias`
						# | `encoder.transformer_cells.*.layer_norm.gamma`                 | `bert.encoder.layer.*.attention.output.LayerNorm.weight`
						# | `encoder.transformer_cells.*.ffn.ffn_1.bias`                   | `bert.encoder.layer.*.intermediate.dense.bias`
						# | `encoder.transformer_cells.*.ffn.ffn_1.weight`                 | `bert.encoder.layer.*.intermediate.dense.weight`
						# | `encoder.transformer_cells.*.ffn.layer_norm.beta`              | `bert.encoder.layer.*.output.LayerNorm.bias`
						# | `encoder.transformer_cells.*.ffn.layer_norm.gamma`             | `bert.encoder.layer.*.output.LayerNorm.weight`
						# | `encoder.transformer_cells.*.proj.bias`                        | `bert.encoder.layer.*.output.dense.bias`
						# | `encoder.transformer_cells.*.proj.weight`                      | `bert.encoder.layer.*.output.dense.weight`

						# Helper function to convert MXNET Arrays to PyTorch
						def to_torch(SCREAMING_SNAKE_CASE_	:	Tuple       ) -> nn.Parameter:
												return nn.Parameter(torch.FloatTensor(mx_array.data().asnumpy()       )       )

						# Check param shapes and map new HF param back
						def check_and_map_params(SCREAMING_SNAKE_CASE_	:	List[Any]		,   SCREAMING_SNAKE_CASE_	:	Dict       ):
												_lowercase					       =					hf_param.shape

												_lowercase					       =					to_torch(params[gluon_param]       )
												_lowercase					       =					gluon_param.shape

												assert (
												    shape_hf == shape_gluon
												), f"""The gluon parameter {gluon_param} has shape {shape_gluon}, but expects shape {shape_hf} for Transformers"""

												return gluon_param

						_lowercase					       =					check_and_map_params(
						    hf_bort_model.bert.embeddings.word_embeddings.weight		,   """word_embed.0.weight"""       )
						_lowercase					       =					check_and_map_params(
						    hf_bort_model.bert.embeddings.position_embeddings.weight		,   """encoder.position_weight"""       )
						_lowercase					       =					check_and_map_params(
						    hf_bort_model.bert.embeddings.LayerNorm.bias		,   """encoder.layer_norm.beta"""       )
						_lowercase					       =					check_and_map_params(
						    hf_bort_model.bert.embeddings.LayerNorm.weight		,   """encoder.layer_norm.gamma"""       )

						# Inspired by RoBERTa conversion script, we just zero them out (Bort does not use them)
						_lowercase					       =					torch.zeros_like(
						    hf_bort_model.bert.embeddings.token_type_embeddings.weight.data       )

						for i in range(hf_bort_config.num_hidden_layers       ):
												_lowercase					       =					hf_bort_model.bert.encoder.layer[i]

												# self attention
												_lowercase					       =					layer.attention.self

												_lowercase					       =					check_and_map_params(
												    self_attn.key.bias.data		,   f"""encoder.transformer_cells.{i}.attention_cell.proj_key.bias"""       )

												_lowercase					       =					check_and_map_params(
												    self_attn.key.weight.data		,   f"""encoder.transformer_cells.{i}.attention_cell.proj_key.weight"""       )
												_lowercase					       =					check_and_map_params(
												    self_attn.query.bias.data		,   f"""encoder.transformer_cells.{i}.attention_cell.proj_query.bias"""       )
												_lowercase					       =					check_and_map_params(
												    self_attn.query.weight.data		,   f"""encoder.transformer_cells.{i}.attention_cell.proj_query.weight"""       )
												_lowercase					       =					check_and_map_params(
												    self_attn.value.bias.data		,   f"""encoder.transformer_cells.{i}.attention_cell.proj_value.bias"""       )
												_lowercase					       =					check_and_map_params(
												    self_attn.value.weight.data		,   f"""encoder.transformer_cells.{i}.attention_cell.proj_value.weight"""       )

												# self attention output
												_lowercase					       =					layer.attention.output

												_lowercase					       =					check_and_map_params(
												    self_output.dense.bias		,   f"""encoder.transformer_cells.{i}.proj.bias"""       )
												_lowercase					       =					check_and_map_params(
												    self_output.dense.weight		,   f"""encoder.transformer_cells.{i}.proj.weight"""       )
												_lowercase					       =					check_and_map_params(
												    self_output.LayerNorm.bias		,   f"""encoder.transformer_cells.{i}.layer_norm.beta"""       )
												_lowercase					       =					check_and_map_params(
												    self_output.LayerNorm.weight		,   f"""encoder.transformer_cells.{i}.layer_norm.gamma"""       )

												# intermediate
												_lowercase					       =					layer.intermediate

												_lowercase					       =					check_and_map_params(
												    intermediate.dense.bias		,   f"""encoder.transformer_cells.{i}.ffn.ffn_1.bias"""       )
												_lowercase					       =					check_and_map_params(
												    intermediate.dense.weight		,   f"""encoder.transformer_cells.{i}.ffn.ffn_1.weight"""       )

												# output
												_lowercase					       =					layer.output

												_lowercase					       =					check_and_map_params(
												    bert_output.dense.bias		,   f"""encoder.transformer_cells.{i}.ffn.ffn_2.bias"""       )
												_lowercase					       =					check_and_map_params(
												    bert_output.dense.weight		,   f"""encoder.transformer_cells.{i}.ffn.ffn_2.weight"""       )
												_lowercase					       =					check_and_map_params(
												    bert_output.LayerNorm.bias		,   f"""encoder.transformer_cells.{i}.ffn.layer_norm.beta"""       )
												_lowercase					       =					check_and_map_params(
												    bert_output.LayerNorm.weight		,   f"""encoder.transformer_cells.{i}.ffn.layer_norm.gamma"""       )

						# Save space and energy 🎄
						hf_bort_model.half()

						# Compare output of both models
						_lowercase					       =					RobertaTokenizer.from_pretrained("""roberta-base"""       )

						_lowercase					       =					tokenizer.encode_plus(SCREAMING_SNAKE_CASE_       )["""input_ids"""]

						# Get gluon output
						_lowercase					       =					mx.nd.array([input_ids]       )
						_lowercase					       =					original_bort(inputs=SCREAMING_SNAKE_CASE_		,   token_types=[]       )

						# Get Transformer output (save and reload model again)
						hf_bort_model.save_pretrained(SCREAMING_SNAKE_CASE_       )
						_lowercase					       =					BertModel.from_pretrained(SCREAMING_SNAKE_CASE_       )
						hf_bort_model.eval()

						_lowercase					       =					tokenizer.encode_plus(SCREAMING_SNAKE_CASE_		,   return_tensors="""pt"""       )
						_lowercase					       =					hf_bort_model(**SCREAMING_SNAKE_CASE_       )[0]

						_lowercase					       =					output_gluon[0].asnumpy()
						_lowercase					       =					output_hf[0].detach().numpy()

						_lowercase					       =					np.max(np.abs(hf_layer - gluon_layer       )       ).item()
						_lowercase					       =					np.allclose(SCREAMING_SNAKE_CASE_		,   SCREAMING_SNAKE_CASE_		,   atol=1E-3       )

						if success:
												print("""✔️ Both model do output the same tensors"""       )
						else:
												print("""❌ Both model do **NOT** output the same tensors"""       )
												print("""Absolute difference is:"""		,   SCREAMING_SNAKE_CASE_       )


if __name__ == "__main__":
		A		: Tuple								=     argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '''--bort_checkpoint_path''', default=None, type=str, required=True, help='''Path the official Bort params file.'''
		)
		parser.add_argument(
		    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
		)
		A		: Any								=     parser.parse_args()
		convert_bort_checkpoint_to_pytorch(args.bort_checkpoint_path, args.pytorch_dump_folder_path)






'''simple docstring'''






import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin




class 					UpperCAmelCase	(     snake_case_				,						unittest.TestCase  ):
      _lowercase:     List[str]			=  DebertaTokenizer
      _lowercase:     Union[str, Any]			=  True
      _lowercase:     Union[str, Any]			=  DebertaTokenizerFast




      def 							lowercase__       (     self     :					Any    )							-> List[Any]:
            super().setUp()

            # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
            _lowerCAmelCase					=				[
                """l""",
                """o""",
                """w""",
                """e""",
                """r""",
                """s""",
                """t""",
                """i""",
                """d""",
                """n""",
                """\u0120""",
                """\u0120l""",
                """\u0120n""",
                """\u0120lo""",
                """\u0120low""",
                """er""",
                """\u0120lowest""",
                """\u0120newer""",
                """\u0120wider""",
                """[UNK]""",
            ]
            _lowerCAmelCase					=				dict(zip(__snake_case     ,     range(len(__snake_case    )    )    )    )
            _lowerCAmelCase					=				["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
            _lowerCAmelCase					=				{"""unk_token""": """[UNK]"""}

            _lowerCAmelCase					=				os.path.join(self.tmpdirname     ,     VOCAB_FILES_NAMES["""vocab_file"""]    )
            _lowerCAmelCase					=				os.path.join(self.tmpdirname     ,     VOCAB_FILES_NAMES["""merges_file"""]    )
            with open(self.vocab_file     ,     """w"""     ,     encoding="""utf-8"""    ) as fp:
                  fp.write(json.dumps(__snake_case    ) + """\n"""    )
            with open(self.merges_file     ,     """w"""     ,     encoding="""utf-8"""    ) as fp:
                  fp.write("""\n""".join(__snake_case    )    )




      def 							lowercase__       (     self     :					List[str]     ,     **__snake_case     :					Any    )							-> Dict:
            kwargs.update(self.special_tokens_map    )
            return self.tokenizer_class.from_pretrained(self.tmpdirname     ,     **__snake_case    )




      def 							lowercase__       (     self     :					Dict     ,     __snake_case     :					Dict    )							-> Optional[Any]:
            _lowerCAmelCase					=				"""lower newer"""
            _lowerCAmelCase					=				"""lower newer"""
            return input_text, output_text




      def 							lowercase__       (     self     :					Union[str, Any]    )							-> List[str]:
            _lowerCAmelCase					=				self.get_tokenizer()
            _lowerCAmelCase					=				"""lower newer"""
            _lowerCAmelCase					=				["""l""", """o""", """w""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
            _lowerCAmelCase					=				tokenizer.tokenize(__snake_case    )
            self.assertListEqual(__snake_case     ,     __snake_case    )

            _lowerCAmelCase					=				tokens + [tokenizer.unk_token]
            _lowerCAmelCase					=				[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case    )     ,     __snake_case    )




      def 							lowercase__       (     self     :					Union[str, Any]    )							-> Tuple:
            _lowerCAmelCase					=				self.get_tokenizer()
            _lowerCAmelCase					=				tokenizer("""Hello"""     ,     """World"""    )
            _lowerCAmelCase					=				[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
            self.assertListEqual(tokd["""token_type_ids"""]     ,     __snake_case    )




      @slow
      def 							lowercase__       (     self     :					int    )							-> Union[str, Any]:
            _lowerCAmelCase					=				self.tokenizer_class.from_pretrained("""microsoft/deberta-base"""    )

            _lowerCAmelCase					=				tokenizer.encode("""sequence builders"""     ,     add_special_tokens=__snake_case    )
            _lowerCAmelCase					=				tokenizer.encode("""multi-sequence build"""     ,     add_special_tokens=__snake_case    )

            _lowerCAmelCase					=				tokenizer.encode(
                """sequence builders"""     ,     add_special_tokens=__snake_case     ,     add_prefix_space=__snake_case    )
            _lowerCAmelCase					=				tokenizer.encode(
                """sequence builders"""     ,     """multi-sequence build"""     ,     add_special_tokens=__snake_case     ,     add_prefix_space=__snake_case    )

            _lowerCAmelCase					=				tokenizer.build_inputs_with_special_tokens(__snake_case    )
            _lowerCAmelCase					=				tokenizer.build_inputs_with_special_tokens(__snake_case     ,     __snake_case    )

            assert encoded_sentence == encoded_text_from_decode
            assert encoded_pair == encoded_pair_from_decode




      @slow
      def 							lowercase__       (     self     :					Any    )							-> str:
            _lowerCAmelCase					=				[self.tokenizer_class]
            if self.test_rust_tokenizer:
                  tokenizer_classes.append(self.rust_tokenizer_class    )

            for tokenizer_class in tokenizer_classes:
                  _lowerCAmelCase					=				tokenizer_class.from_pretrained("""microsoft/deberta-base"""    )

                  _lowerCAmelCase					=				[
                      """ALBERT: A Lite BERT for Self-supervised Learning of Language Representations""",
                      """ALBERT incorporates two parameter reduction techniques""",
                      """The first one is a factorized embedding parameterization. By decomposing the large vocabulary"""
                      """ embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"""
                      """ vocabulary embedding.""",
                  ]

                  _lowerCAmelCase					=				tokenizer(__snake_case     ,     padding=__snake_case    )
                  _lowerCAmelCase					=				[tokenizer.decode(__snake_case     ,     skip_special_tokens=__snake_case    ) for seq in encoding["""input_ids"""]]

                  # fmt: off
                  _lowerCAmelCase					=				{
                      """input_ids""": [
                          [1, 21_18, 1_11_26, 5_65, 35, 83, 2_51_91, 1_63, 1_88_54, 13, 1_21_56, 12, 1_61_01, 2_53_76, 1_38_07, 9, 2_22_05, 2_78_93, 16_35, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                          [1, 21_18, 1_11_26, 5_65, 2_45_36, 80, 4_37_97, 48_78, 73_73, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                          [1, 1_33, 78, 65, 16, 10, 37_24, 15_38, 3_31_83, 1_13_03, 4_37_97, 19_38, 4, 8_70, 2_41_65, 2_91_05, 5, 7_39, 3_26_44, 3_31_83, 1_13_03, 3_61_73, 88, 80, 6_50, 78_21, 4_59_40, 6, 52, 25_59, 5, 18_36, 9, 5, 73_97, 1_31_71, 31, 5, 18_36, 9, 3_26_44, 3_31_83, 1_13_03, 4, 2]
                      ],
                      """token_type_ids""": [
                          [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                          [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                          [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                      ],
                      """attention_mask""": [
                          [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                          [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                          [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
                      ]
                  }
                  # fmt: on

                  _lowerCAmelCase					=				[
                      """ALBERT: A Lite BERT for Self-supervised Learning of Language Representations""",
                      """ALBERT incorporates two parameter reduction techniques""",
                      """The first one is a factorized embedding parameterization. By decomposing the large vocabulary"""
                      """ embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"""
                      """ vocabulary embedding.""",
                  ]

                  self.assertDictEqual(encoding.data     ,     __snake_case    )

                  for expected, decoded in zip(__snake_case     ,     __snake_case    ):
                        self.assertEqual(__snake_case     ,     __snake_case    )






