




import argparse
import json

import gdown
import numpy as np
import torch
from huggingface_hub import hf_hub_download

from transformers import (
    VideoMAEConfig,
    VideoMAEForPreTraining,
    VideoMAEForVideoClassification,
    VideoMAEImageProcessor,
)



def 				_A  (		lowerCAmelCase_	:   Any     ):
     """simple docstring"""

     lowerCAmelCase__            =					VideoMAEConfig()

     set_architecture_configs(lowerCAmelCase_  ,							lowerCAmelCase_     )

     if "finetuned" not in model_name:
          lowerCAmelCase__            =					False

     if "finetuned" in model_name:
          lowerCAmelCase__            =					"huggingface/label-files"
          if "kinetics" in model_name:
               lowerCAmelCase__            =					400
               lowerCAmelCase__            =					"kinetics400-id2label.json"
          elif "ssv2" in model_name:
               lowerCAmelCase__            =					174
               lowerCAmelCase__            =					"something-something-v2-id2label.json"
          else:
               raise ValueError("Model name should either contain 'kinetics' or 'ssv2' in case it's fine-tuned."     )
          lowerCAmelCase__            =					json.load(open(hf_hub_download(lowerCAmelCase_  ,							lowerCAmelCase_  ,							repo_type="dataset"     )  ,							"r"     )     )
          lowerCAmelCase__            =					{int(lowerCAmelCase_     ): v for k, v in idalabel.items()}
          lowerCAmelCase__            =					idalabel
          lowerCAmelCase__            =					{v: k for k, v in idalabel.items()}

     return config



def 				_A  (		lowerCAmelCase_	:   str  ,							lowerCAmelCase_	:   int     ):
     """simple docstring"""

     if "small" in model_name:
          lowerCAmelCase__            =					384
          lowerCAmelCase__            =					1536
          lowerCAmelCase__            =					12
          lowerCAmelCase__            =					16
          lowerCAmelCase__            =					12
          lowerCAmelCase__            =					3
          lowerCAmelCase__            =					192
          lowerCAmelCase__            =					768
     elif "large" in model_name:
          lowerCAmelCase__            =					1024
          lowerCAmelCase__            =					4096
          lowerCAmelCase__            =					24
          lowerCAmelCase__            =					16
          lowerCAmelCase__            =					12
          lowerCAmelCase__            =					8
          lowerCAmelCase__            =					512
          lowerCAmelCase__            =					2048
     elif "huge" in model_name:
          lowerCAmelCase__            =					1280
          lowerCAmelCase__            =					5120
          lowerCAmelCase__            =					32
          lowerCAmelCase__            =					16
          lowerCAmelCase__            =					12
          lowerCAmelCase__            =					8
          lowerCAmelCase__            =					640
          lowerCAmelCase__            =					2560
     elif "base" not in model_name:
          raise ValueError("Model name should include either \"small\", \"base\", \"large\", or \"huge\""     )



def 				_A  (		lowerCAmelCase_	:   Union[str, Any]     ):
     """simple docstring"""

     if "encoder." in name:
          lowerCAmelCase__            =					name.replace("encoder."  ,							""     )
     if "cls_token" in name:
          lowerCAmelCase__            =					name.replace("cls_token"  ,							"videomae.embeddings.cls_token"     )
     if "decoder_pos_embed" in name:
          lowerCAmelCase__            =					name.replace("decoder_pos_embed"  ,							"decoder.decoder_pos_embed"     )
     if "pos_embed" in name and "decoder" not in name:
          lowerCAmelCase__            =					name.replace("pos_embed"  ,							"videomae.embeddings.position_embeddings"     )
     if "patch_embed.proj" in name:
          lowerCAmelCase__            =					name.replace("patch_embed.proj"  ,							"videomae.embeddings.patch_embeddings.projection"     )
     if "patch_embed.norm" in name:
          lowerCAmelCase__            =					name.replace("patch_embed.norm"  ,							"videomae.embeddings.norm"     )
     if "decoder.blocks" in name:
          lowerCAmelCase__            =					name.replace("decoder.blocks"  ,							"decoder.decoder_layers"     )
     if "blocks" in name:
          lowerCAmelCase__            =					name.replace("blocks"  ,							"videomae.encoder.layer"     )
     if "attn.proj" in name:
          lowerCAmelCase__            =					name.replace("attn.proj"  ,							"attention.output.dense"     )
     if "attn" in name and "bias" not in name:
          lowerCAmelCase__            =					name.replace("attn"  ,							"attention.self"     )
     if "attn" in name:
          lowerCAmelCase__            =					name.replace("attn"  ,							"attention.attention"     )
     if "norm1" in name:
          lowerCAmelCase__            =					name.replace("norm1"  ,							"layernorm_before"     )
     if "norm2" in name:
          lowerCAmelCase__            =					name.replace("norm2"  ,							"layernorm_after"     )
     if "mlp.fc1" in name:
          lowerCAmelCase__            =					name.replace("mlp.fc1"  ,							"intermediate.dense"     )
     if "mlp.fc2" in name:
          lowerCAmelCase__            =					name.replace("mlp.fc2"  ,							"output.dense"     )
     if "decoder_embed" in name:
          lowerCAmelCase__            =					name.replace("decoder_embed"  ,							"decoder.decoder_embed"     )
     if "decoder_norm" in name:
          lowerCAmelCase__            =					name.replace("decoder_norm"  ,							"decoder.decoder_norm"     )
     if "decoder_pred" in name:
          lowerCAmelCase__            =					name.replace("decoder_pred"  ,							"decoder.decoder_pred"     )
     if "norm.weight" in name and "decoder" not in name and "fc" not in name:
          lowerCAmelCase__            =					name.replace("norm.weight"  ,							"videomae.layernorm.weight"     )
     if "norm.bias" in name and "decoder" not in name and "fc" not in name:
          lowerCAmelCase__            =					name.replace("norm.bias"  ,							"videomae.layernorm.bias"     )
     if "head" in name and "decoder" not in name:
          lowerCAmelCase__            =					name.replace("head"  ,							"classifier"     )

     return name



def 				_A  (		lowerCAmelCase_	:   Dict  ,							lowerCAmelCase_	:   Dict     ):
     """simple docstring"""

     for key in orig_state_dict.copy().keys():
          lowerCAmelCase__            =					orig_state_dict.pop(lowerCAmelCase_     )

          if key.startswith("encoder."     ):
               lowerCAmelCase__            =					key.replace("encoder."  ,							""     )

          if "qkv" in key:
               lowerCAmelCase__            =					key.split("."     )
               if key.startswith("decoder.blocks"     ):
                    lowerCAmelCase__            =					config.decoder_hidden_size
                    lowerCAmelCase__            =					int(key_split[2]     )
                    lowerCAmelCase__            =					"decoder.decoder_layers."
                    if "weight" in key:
                         lowerCAmelCase__            =					val[:dim, :]
                         lowerCAmelCase__            =					val[dim : dim * 2, :]
                         lowerCAmelCase__            =					val[-dim:, :]
               else:
                    lowerCAmelCase__            =					config.hidden_size
                    lowerCAmelCase__            =					int(key_split[1]     )
                    lowerCAmelCase__            =					"videomae.encoder.layer."
                    if "weight" in key:
                         lowerCAmelCase__            =					val[:dim, :]
                         lowerCAmelCase__            =					val[dim : dim * 2, :]
                         lowerCAmelCase__            =					val[-dim:, :]
          else:
               lowerCAmelCase__            =					val

     return orig_state_dict



def 				_A  (		):
     """simple docstring"""

     lowerCAmelCase__            =					hf_hub_download(
         repo_id="hf-internal-testing/spaghetti-video"  ,							filename="eating_spaghetti.npy"  ,							repo_type="dataset"     )
     lowerCAmelCase__            =					np.load(lowerCAmelCase_     )
     return list(lowerCAmelCase_     )



def 				_A  (		lowerCAmelCase_	:   List[str]  ,							lowerCAmelCase_	:   str  ,							lowerCAmelCase_	:   Dict  ,							lowerCAmelCase_	:   str     ):
     """simple docstring"""

     lowerCAmelCase__            =					get_videomae_config(lowerCAmelCase_     )

     if "finetuned" in model_name:
          lowerCAmelCase__            =					VideoMAEForVideoClassification(lowerCAmelCase_     )
     else:
          lowerCAmelCase__            =					VideoMAEForPreTraining(lowerCAmelCase_     )

     # download original checkpoint, hosted on Google Drive
     lowerCAmelCase__            =					"pytorch_model.bin"
     gdown.cached_download(lowerCAmelCase_  ,							lowerCAmelCase_  ,							quiet=lowerCAmelCase_     )
     lowerCAmelCase__            =					torch.load(lowerCAmelCase_  ,							map_location="cpu"     )
     if "model" in files:
          lowerCAmelCase__            =					files["model"]
     else:
          lowerCAmelCase__            =					files["module"]
     lowerCAmelCase__            =					convert_state_dict(lowerCAmelCase_  ,							lowerCAmelCase_     )

     model.load_state_dict(lowerCAmelCase_     )
     model.eval()

     # verify model on basic input
     lowerCAmelCase__            =					VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5]  ,							image_std=[0.5, 0.5, 0.5]     )
     lowerCAmelCase__            =					prepare_video()
     lowerCAmelCase__            =					image_processor(lowerCAmelCase_  ,							return_tensors="pt"     )

     if "finetuned" not in model_name:
          lowerCAmelCase__            =					hf_hub_download(repo_id="hf-internal-testing/bool-masked-pos"  ,							filename="bool_masked_pos.pt"     )
          lowerCAmelCase__            =					torch.load(lowerCAmelCase_     )

     lowerCAmelCase__            =					model(**lowerCAmelCase_     )
     lowerCAmelCase__            =					outputs.logits

     lowerCAmelCase__            =					[
         "videomae-small-finetuned-kinetics",
         "videomae-small-finetuned-ssv2",
         # Kinetics-400 checkpoints (short = pretrained only for 800 epochs instead of 1600)
         "videomae-base-short",
         "videomae-base-short-finetuned-kinetics",
         "videomae-base",
         "videomae-base-finetuned-kinetics",
         "videomae-large",
         "videomae-large-finetuned-kinetics",
         "videomae-huge-finetuned-kinetics",
         # Something-Something-v2 checkpoints (short = pretrained only for 800 epochs instead of 2400)
         "videomae-base-short-ssv2",
         "videomae-base-short-finetuned-ssv2",
         "videomae-base-ssv2",
         "videomae-base-finetuned-ssv2",
     ]

     # NOTE: logits were tested with image_mean and image_std equal to [0.5, 0.5, 0.5] and [0.5, 0.5, 0.5]
     if model_name == "videomae-small-finetuned-kinetics":
          lowerCAmelCase__            =					torch.Size([1, 400]     )
          lowerCAmelCase__            =					torch.tensor([-0.9291, -0.4061, -0.9307]     )
     elif model_name == "videomae-small-finetuned-ssv2":
          lowerCAmelCase__            =					torch.Size([1, 174]     )
          lowerCAmelCase__            =					torch.tensor([0.2671, -0.4689, -0.8235]     )
     elif model_name == "videomae-base":
          lowerCAmelCase__            =					torch.Size([1, 1408, 1536]     )
          lowerCAmelCase__            =					torch.tensor([[0.7739, 0.7968, 0.7089], [0.6701, 0.7487, 0.6209], [0.4287, 0.5158, 0.4773]]     )
     elif model_name == "videomae-base-short":
          lowerCAmelCase__            =					torch.Size([1, 1408, 1536]     )
          lowerCAmelCase__            =					torch.tensor([[0.7994, 0.9612, 0.8508], [0.7401, 0.8958, 0.8302], [0.5862, 0.7468, 0.7325]]     )
          # we verified the loss both for normalized and unnormalized targets for this one
          lowerCAmelCase__            =					torch.tensor([0.5142]     ) if config.norm_pix_loss else torch.tensor([0.6469]     )
     elif model_name == "videomae-large":
          lowerCAmelCase__            =					torch.Size([1, 1408, 1536]     )
          lowerCAmelCase__            =					torch.tensor([[0.7149, 0.7997, 0.6966], [0.6768, 0.7869, 0.6948], [0.5139, 0.6221, 0.5605]]     )
     elif model_name == "videomae-large-finetuned-kinetics":
          lowerCAmelCase__            =					torch.Size([1, 400]     )
          lowerCAmelCase__            =					torch.tensor([0.0771, 0.0011, -0.3625]     )
     elif model_name == "videomae-huge-finetuned-kinetics":
          lowerCAmelCase__            =					torch.Size([1, 400]     )
          lowerCAmelCase__            =					torch.tensor([0.2433, 0.1632, -0.4894]     )
     elif model_name == "videomae-base-short-finetuned-kinetics":
          lowerCAmelCase__            =					torch.Size([1, 400]     )
          lowerCAmelCase__            =					torch.tensor([0.6588, 0.0990, -0.2493]     )
     elif model_name == "videomae-base-finetuned-kinetics":
          lowerCAmelCase__            =					torch.Size([1, 400]     )
          lowerCAmelCase__            =					torch.tensor([0.3669, -0.0688, -0.2421]     )
     elif model_name == "videomae-base-short-ssv2":
          lowerCAmelCase__            =					torch.Size([1, 1408, 1536]     )
          lowerCAmelCase__            =					torch.tensor([[0.4712, 0.5296, 0.5786], [0.2278, 0.2729, 0.4026], [0.0352, 0.0730, 0.2506]]     )
     elif model_name == "videomae-base-short-finetuned-ssv2":
          lowerCAmelCase__            =					torch.Size([1, 174]     )
          lowerCAmelCase__            =					torch.tensor([-0.0537, -0.1539, -0.3266]     )
     elif model_name == "videomae-base-ssv2":
          lowerCAmelCase__            =					torch.Size([1, 1408, 1536]     )
          lowerCAmelCase__            =					torch.tensor([[0.8131, 0.8727, 0.8546], [0.7366, 0.9377, 0.8870], [0.5935, 0.8874, 0.8564]]     )
     elif model_name == "videomae-base-finetuned-ssv2":
          lowerCAmelCase__            =					torch.Size([1, 174]     )
          lowerCAmelCase__            =					torch.tensor([0.1961, -0.8337, -0.6389]     )
     else:
          raise ValueError(F'Model name not supported. Should be one of {model_names}'     )

     # verify logits
     assert logits.shape == expected_shape
     if "finetuned" in model_name:
          assert torch.allclose(logits[0, :3]  ,							lowerCAmelCase_  ,							atol=1E-4     )
     else:
          print("Logits:"  ,							logits[0, :3, :3]     )
          assert torch.allclose(logits[0, :3, :3]  ,							lowerCAmelCase_  ,							atol=1E-4     )
     print("Logits ok!"     )

     # verify loss, if applicable
     if model_name == "videomae-base-short":
          lowerCAmelCase__            =					outputs.loss
          assert torch.allclose(lowerCAmelCase_  ,							lowerCAmelCase_  ,							atol=1E-4     )
          print("Loss ok!"     )

     if pytorch_dump_folder_path is not None:
          print(F'Saving model and image processor to {pytorch_dump_folder_path}'     )
          image_processor.save_pretrained(lowerCAmelCase_     )
          model.save_pretrained(lowerCAmelCase_     )

     if push_to_hub:
          print("Pushing to the hub..."     )
          model.push_to_hub(lowerCAmelCase_  ,							organization="nielsr"     )


if __name__ == "__main__":
     UpperCamelCase								=	argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         '--checkpoint_url',
         default='https://drive.google.com/u/1/uc?id=1tEhLyskjb755TJ65ptsrafUG2llSwQE1&amp;export=download&amp;confirm=t&amp;uuid=aa3276eb-fb7e-482a-adec-dc7171df14c4',
         type=str,
         help=(
             'URL of the original PyTorch checkpoint (on Google Drive) you\'d like to convert. Should be a direct'
             ' download link.'
         ),
     )
     parser.add_argument(
         '--pytorch_dump_folder_path',
         default='/Users/nielsrogge/Documents/VideoMAE/Test',
         type=str,
         help='Path to the output PyTorch model directory.',
     )
     parser.add_argument('--model_name', default='videomae-base', type=str, help='Name of the model.')
     parser.add_argument(
         '--push_to_hub', action='store_true', help='Whether or not to push the converted model to the 🤗 hub.'
     )

     UpperCamelCase								=	parser.parse_args()
     convert_videomae_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path, args.model_name, args.push_to_hub)





'''simple docstring'''





import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin



class  snake_case__						(    SCREAMING_SNAKE_CASE_  ,     unittest.TestCase				):
 A__				= DebertaTokenizer
 A__				= True
 A__				= DebertaTokenizerFast
 def A_   (   self						:							Optional[int]  )      ->   str:







       '''simple docstring'''
       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       __snake_case :   Union[str, Any]													=    [
           'l',
           'o',
           'w',
           'e',
           'r',
           's',
           't',
           'i',
           'd',
           'n',
           '\u0120',
           '\u0120l',
           '\u0120n',
           '\u0120lo',
           '\u0120low',
           'er',
           '\u0120lowest',
           '\u0120newer',
           '\u0120wider',
           '[UNK]',
       ]
       __snake_case :   List[str]													=    dict(zip(__a      ,     range(len(__a  )  )  )  )
       __snake_case :   int													=    ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
       __snake_case :   Any													=    {'unk_token': '[UNK]'}

       __snake_case :   Union[str, Any]													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['vocab_file']  )
       __snake_case :   Dict													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['merges_file']  )
       with open(self.vocab_file      ,     'w'      ,     encoding='utf-8'  ) as fp:
             fp.write(json.dumps(__a  ) + '\n'  )
       with open(self.merges_file      ,     'w'      ,     encoding='utf-8'  ) as fp:
             fp.write('\n'.join(__a  )  )
 def A_   (   self						:							Optional[int]      ,     **__a						:							List[Any]  )      ->   Optional[Any]:







       '''simple docstring'''
       kwargs.update(self.special_tokens_map  )
       return self.tokenizer_class.from_pretrained(self.tmpdirname      ,     **__a  )
 def A_   (   self						:							Optional[int]      ,     __a						:							Dict  )      ->   List[str]:







       '''simple docstring'''
       __snake_case :   Union[str, Any]													=    'lower newer'
       __snake_case :   List[Any]													=    'lower newer'
       return input_text, output_text
 def A_   (   self						:							Dict  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   str													=    self.get_tokenizer()
       __snake_case :   List[str]													=    'lower newer'
       __snake_case :   Tuple													=    ['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
       __snake_case :   Optional[int]													=    tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   str													=    tokens + [tokenizer.unk_token]
       __snake_case :   Tuple													=    [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__a  )      ,     __a  )
 def A_   (   self						:							List[Any]  )      ->   Dict:







       '''simple docstring'''
       __snake_case :   List[str]													=    self.get_tokenizer()
       __snake_case :   List[str]													=    tokenizer('Hello'      ,     'World'  )
       __snake_case :   int													=    [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
       self.assertListEqual(tokd['token_type_ids']      ,     __a  )
 @slow
 def A_   (   self						:							Dict  )      ->   str:







       '''simple docstring'''
       __snake_case :   str													=    self.tokenizer_class.from_pretrained('microsoft/deberta-base'  )

       __snake_case :   str													=    tokenizer.encode('sequence builders'      ,     add_special_tokens=__a  )
       __snake_case :   Any													=    tokenizer.encode('multi-sequence build'      ,     add_special_tokens=__a  )

       __snake_case :   List[Any]													=    tokenizer.encode(
           'sequence builders'      ,     add_special_tokens=__a      ,     add_prefix_space=__a  )
       __snake_case :   Union[str, Any]													=    tokenizer.encode(
           'sequence builders'      ,     'multi-sequence build'      ,     add_special_tokens=__a      ,     add_prefix_space=__a  )

       __snake_case :   int													=    tokenizer.build_inputs_with_special_tokens(__a  )
       __snake_case :   Tuple													=    tokenizer.build_inputs_with_special_tokens(__a      ,     __a  )

       assert encoded_sentence == encoded_text_from_decode
       assert encoded_pair == encoded_pair_from_decode





 @slow
 def A_   (   self						:							Optional[Any]  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   int													=    [self.tokenizer_class]
       if self.test_rust_tokenizer:
             tokenizer_classes.append(self.rust_tokenizer_class  )

       for tokenizer_class in tokenizer_classes:
             __snake_case :   List[Any]													=    tokenizer_class.from_pretrained('microsoft/deberta-base'  )

             __snake_case :   int													=    [
                 'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
                 'ALBERT incorporates two parameter reduction techniques',
                 'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
                 ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
                 ' vocabulary embedding.',
             ]

             __snake_case :   Union[str, Any]													=    tokenizer(__a      ,     padding=__a  )
             __snake_case :   List[str]													=    [tokenizer.decode(__a      ,     skip_special_tokens=__a  ) for seq in encoding['input_ids']]

             # fmt: off
             __snake_case :   Optional[Any]													=    {
                 'input_ids': [
                     [1, 2118, 11126, 565, 35, 83, 25191, 163, 18854, 13, 12156, 12, 16101, 25376, 13807, 9, 22205, 27893, 1635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 2118, 11126, 565, 24536, 80, 43797, 4878, 7373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 133, 78, 65, 16, 10, 3724, 1538, 33183, 11303, 43797, 1938, 4, 870, 24165, 29105, 5, 739, 32644, 33183, 11303, 36173, 88, 80, 650, 7821, 45940, 6, 52, 2559, 5, 1836, 9, 5, 7397, 13171, 31, 5, 1836, 9, 32644, 33183, 11303, 4, 2]
                 ],
                 'token_type_ids': [
                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                 ],
                 'attention_mask': [
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
                 ]
             }
             # fmt: on

             __snake_case :   int													=    [
                 'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
                 'ALBERT incorporates two parameter reduction techniques',
                 'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
                 ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
                 ' vocabulary embedding.',
             ]

             self.assertDictEqual(encoding.data      ,     __a  )

             for expected, decoded in zip(__a      ,     __a  ):
                   self.assertEqual(__a      ,     __a  )



