


'''simple docstring'''






import argparse
import json

import gdown
import numpy as np
import torch
from huggingface_hub import hf_hub_download

from transformers import (
    VideoMAEConfig,
    VideoMAEForPreTraining,
    VideoMAEForVideoClassification,
    VideoMAEImageProcessor,
)





def lowerCamelCase__     (		__lowerCamelCase			:				Tuple				):







   '''simple docstring'''




   _UpperCAmelCase  :  Union[str, Any]				       =VideoMAEConfig()

   set_architecture_configs(__lowerCamelCase			,       __lowerCamelCase				)

   if "finetuned" not in model_name:
      _UpperCAmelCase  :  Tuple				       =False

   if "finetuned" in model_name:
      _UpperCAmelCase  :  List[str]				       ='huggingface/label-files'
      if "kinetics" in model_name:
         _UpperCAmelCase  :  str				       =4_0_0
         _UpperCAmelCase  :  Optional[Any]				       ='kinetics400-id2label.json'
      elif "ssv2" in model_name:
         _UpperCAmelCase  :  Union[str, Any]				       =1_7_4
         _UpperCAmelCase  :  Dict				       ='something-something-v2-id2label.json'
      else:
         raise ValueError('Model name should either contain \'kinetics\' or \'ssv2\' in case it\'s fine-tuned.'				)
      _UpperCAmelCase  :  Any				       =json.load(open(hf_hub_download(__lowerCamelCase			,       __lowerCamelCase			,       repo_type='dataset'				)			,       'r'				)				)
      _UpperCAmelCase  :  int				       ={int(__lowerCamelCase				): v for k, v in idalabel.items()}
      _UpperCAmelCase  :  List[str]				       =idalabel
      _UpperCAmelCase  :  str				       ={v: k for k, v in idalabel.items()}

   return config





def lowerCamelCase__     (		__lowerCamelCase			:				Optional[int]			,       __lowerCamelCase			:				Optional[Any]				):







   '''simple docstring'''




   if "small" in model_name:
      _UpperCAmelCase  :  Dict				       =3_8_4
      _UpperCAmelCase  :  Optional[Any]				       =1_5_3_6
      _UpperCAmelCase  :  Optional[int]				       =1_2
      _UpperCAmelCase  :  List[str]				       =1_6
      _UpperCAmelCase  :  Dict				       =1_2
      _UpperCAmelCase  :  Tuple				       =3
      _UpperCAmelCase  :  Optional[Any]				       =1_9_2
      _UpperCAmelCase  :  Optional[int]				       =7_6_8
   elif "large" in model_name:
      _UpperCAmelCase  :  Optional[int]				       =1_0_2_4
      _UpperCAmelCase  :  Optional[Any]				       =4_0_9_6
      _UpperCAmelCase  :  Optional[Any]				       =2_4
      _UpperCAmelCase  :  Tuple				       =1_6
      _UpperCAmelCase  :  Any				       =1_2
      _UpperCAmelCase  :  Optional[int]				       =8
      _UpperCAmelCase  :  List[Any]				       =5_1_2
      _UpperCAmelCase  :  Any				       =2_0_4_8
   elif "huge" in model_name:
      _UpperCAmelCase  :  Dict				       =1_2_8_0
      _UpperCAmelCase  :  Union[str, Any]				       =5_1_2_0
      _UpperCAmelCase  :  int				       =3_2
      _UpperCAmelCase  :  List[str]				       =1_6
      _UpperCAmelCase  :  Any				       =1_2
      _UpperCAmelCase  :  Any				       =8
      _UpperCAmelCase  :  Tuple				       =6_4_0
      _UpperCAmelCase  :  Optional[Any]				       =2_5_6_0
   elif "base" not in model_name:
      raise ValueError('Model name should include either "small", "base", "large", or "huge"'				)





def lowerCamelCase__     (		__lowerCamelCase			:				Union[str, Any]				):







   '''simple docstring'''




   if "encoder." in name:
      _UpperCAmelCase  :  List[str]				       =name.replace('encoder.'			,       ''				)
   if "cls_token" in name:
      _UpperCAmelCase  :  List[str]				       =name.replace('cls_token'			,       'videomae.embeddings.cls_token'				)
   if "decoder_pos_embed" in name:
      _UpperCAmelCase  :  Dict				       =name.replace('decoder_pos_embed'			,       'decoder.decoder_pos_embed'				)
   if "pos_embed" in name and "decoder" not in name:
      _UpperCAmelCase  :  Dict				       =name.replace('pos_embed'			,       'videomae.embeddings.position_embeddings'				)
   if "patch_embed.proj" in name:
      _UpperCAmelCase  :  Tuple				       =name.replace('patch_embed.proj'			,       'videomae.embeddings.patch_embeddings.projection'				)
   if "patch_embed.norm" in name:
      _UpperCAmelCase  :  Union[str, Any]				       =name.replace('patch_embed.norm'			,       'videomae.embeddings.norm'				)
   if "decoder.blocks" in name:
      _UpperCAmelCase  :  Tuple				       =name.replace('decoder.blocks'			,       'decoder.decoder_layers'				)
   if "blocks" in name:
      _UpperCAmelCase  :  str				       =name.replace('blocks'			,       'videomae.encoder.layer'				)
   if "attn.proj" in name:
      _UpperCAmelCase  :  int				       =name.replace('attn.proj'			,       'attention.output.dense'				)
   if "attn" in name and "bias" not in name:
      _UpperCAmelCase  :  Optional[Any]				       =name.replace('attn'			,       'attention.self'				)
   if "attn" in name:
      _UpperCAmelCase  :  Optional[Any]				       =name.replace('attn'			,       'attention.attention'				)
   if "norm1" in name:
      _UpperCAmelCase  :  List[Any]				       =name.replace('norm1'			,       'layernorm_before'				)
   if "norm2" in name:
      _UpperCAmelCase  :  Optional[Any]				       =name.replace('norm2'			,       'layernorm_after'				)
   if "mlp.fc1" in name:
      _UpperCAmelCase  :  str				       =name.replace('mlp.fc1'			,       'intermediate.dense'				)
   if "mlp.fc2" in name:
      _UpperCAmelCase  :  str				       =name.replace('mlp.fc2'			,       'output.dense'				)
   if "decoder_embed" in name:
      _UpperCAmelCase  :  Any				       =name.replace('decoder_embed'			,       'decoder.decoder_embed'				)
   if "decoder_norm" in name:
      _UpperCAmelCase  :  Optional[Any]				       =name.replace('decoder_norm'			,       'decoder.decoder_norm'				)
   if "decoder_pred" in name:
      _UpperCAmelCase  :  List[Any]				       =name.replace('decoder_pred'			,       'decoder.decoder_pred'				)
   if "norm.weight" in name and "decoder" not in name and "fc" not in name:
      _UpperCAmelCase  :  Tuple				       =name.replace('norm.weight'			,       'videomae.layernorm.weight'				)
   if "norm.bias" in name and "decoder" not in name and "fc" not in name:
      _UpperCAmelCase  :  Optional[Any]				       =name.replace('norm.bias'			,       'videomae.layernorm.bias'				)
   if "head" in name and "decoder" not in name:
      _UpperCAmelCase  :  Any				       =name.replace('head'			,       'classifier'				)

   return name





def lowerCamelCase__     (		__lowerCamelCase			:				Tuple			,       __lowerCamelCase			:				Any				):







   '''simple docstring'''




   for key in orig_state_dict.copy().keys():
      _UpperCAmelCase  :  Tuple				       =orig_state_dict.pop(__lowerCamelCase				)

      if key.startswith('encoder.'				):
         _UpperCAmelCase  :  List[Any]				       =key.replace('encoder.'			,       ''				)

      if "qkv" in key:
         _UpperCAmelCase  :  int				       =key.split('.'				)
         if key.startswith('decoder.blocks'				):
            _UpperCAmelCase  :  List[Any]				       =config.decoder_hidden_size
            _UpperCAmelCase  :  Any				       =int(key_split[2]				)
            _UpperCAmelCase  :  str				       ='decoder.decoder_layers.'
            if "weight" in key:
               _UpperCAmelCase  :  Any				       =val[:dim, :]
               _UpperCAmelCase  :  Any				       =val[dim : dim * 2, :]
               _UpperCAmelCase  :  int				       =val[-dim:, :]
         else:
            _UpperCAmelCase  :  int				       =config.hidden_size
            _UpperCAmelCase  :  str				       =int(key_split[1]				)
            _UpperCAmelCase  :  Dict				       ='videomae.encoder.layer.'
            if "weight" in key:
               _UpperCAmelCase  :  Tuple				       =val[:dim, :]
               _UpperCAmelCase  :  int				       =val[dim : dim * 2, :]
               _UpperCAmelCase  :  List[str]				       =val[-dim:, :]
      else:
         _UpperCAmelCase  :  List[Any]				       =val

   return orig_state_dict





def lowerCamelCase__     (		):







   '''simple docstring'''




   _UpperCAmelCase  :  str				       =hf_hub_download(
       repo_id='hf-internal-testing/spaghetti-video'			,       filename='eating_spaghetti.npy'			,       repo_type='dataset'				)
   _UpperCAmelCase  :  Dict				       =np.load(__lowerCamelCase				)
   return list(__lowerCamelCase				)





def lowerCamelCase__     (		__lowerCamelCase			:				List[Any]			,       __lowerCamelCase			:				Optional[int]			,       __lowerCamelCase			:				Dict			,       __lowerCamelCase			:				Tuple				):







   '''simple docstring'''




   _UpperCAmelCase  :  List[Any]				       =get_videomae_config(__lowerCamelCase				)

   if "finetuned" in model_name:
      _UpperCAmelCase  :  str				       =VideoMAEForVideoClassification(__lowerCamelCase				)
   else:
      _UpperCAmelCase  :  Any				       =VideoMAEForPreTraining(__lowerCamelCase				)

   # download original checkpoint, hosted on Google Drive
   _UpperCAmelCase  :  Any				       ='pytorch_model.bin'
   gdown.cached_download(__lowerCamelCase			,       __lowerCamelCase			,       quiet=__lowerCamelCase				)
   _UpperCAmelCase  :  Any				       =torch.load(__lowerCamelCase			,       map_location='cpu'				)
   if "model" in files:
      _UpperCAmelCase  :  Union[str, Any]				       =files['model']
   else:
      _UpperCAmelCase  :  Optional[int]				       =files['module']
   _UpperCAmelCase  :  str				       =convert_state_dict(__lowerCamelCase			,       __lowerCamelCase				)

   model.load_state_dict(__lowerCamelCase				)
   model.eval()

   # verify model on basic input
   _UpperCAmelCase  :  Dict				       =VideoMAEImageProcessor(image_mean=[0.5, 0.5, 0.5]			,       image_std=[0.5, 0.5, 0.5]				)
   _UpperCAmelCase  :  int				       =prepare_video()
   _UpperCAmelCase  :  Tuple				       =image_processor(__lowerCamelCase			,       return_tensors='pt'				)

   if "finetuned" not in model_name:
      _UpperCAmelCase  :  List[str]				       =hf_hub_download(repo_id='hf-internal-testing/bool-masked-pos'			,       filename='bool_masked_pos.pt'				)
      _UpperCAmelCase  :  str				       =torch.load(__lowerCamelCase				)

   _UpperCAmelCase  :  List[Any]				       =model(**__lowerCamelCase				)
   _UpperCAmelCase  :  List[Any]				       =outputs.logits

   _UpperCAmelCase  :  Optional[Any]				       =[
       'videomae-small-finetuned-kinetics',
       'videomae-small-finetuned-ssv2',
       # Kinetics-400 checkpoints (short = pretrained only for 800 epochs instead of 1600)
       'videomae-base-short',
       'videomae-base-short-finetuned-kinetics',
       'videomae-base',
       'videomae-base-finetuned-kinetics',
       'videomae-large',
       'videomae-large-finetuned-kinetics',
       'videomae-huge-finetuned-kinetics',
       # Something-Something-v2 checkpoints (short = pretrained only for 800 epochs instead of 2400)
       'videomae-base-short-ssv2',
       'videomae-base-short-finetuned-ssv2',
       'videomae-base-ssv2',
       'videomae-base-finetuned-ssv2',
   ]

   # NOTE: logits were tested with image_mean and image_std equal to [0.5, 0.5, 0.5] and [0.5, 0.5, 0.5]
   if model_name == "videomae-small-finetuned-kinetics":
      _UpperCAmelCase  :  Optional[int]				       =torch.Size([1, 4_0_0]				)
      _UpperCAmelCase  :  Tuple				       =torch.tensor([-0.92_91, -0.40_61, -0.93_07]				)
   elif model_name == "videomae-small-finetuned-ssv2":
      _UpperCAmelCase  :  str				       =torch.Size([1, 1_7_4]				)
      _UpperCAmelCase  :  Union[str, Any]				       =torch.tensor([0.26_71, -0.46_89, -0.82_35]				)
   elif model_name == "videomae-base":
      _UpperCAmelCase  :  List[Any]				       =torch.Size([1, 1_4_0_8, 1_5_3_6]				)
      _UpperCAmelCase  :  Optional[int]				       =torch.tensor([[0.77_39, 0.79_68, 0.70_89], [0.67_01, 0.74_87, 0.62_09], [0.42_87, 0.51_58, 0.47_73]]				)
   elif model_name == "videomae-base-short":
      _UpperCAmelCase  :  List[str]				       =torch.Size([1, 1_4_0_8, 1_5_3_6]				)
      _UpperCAmelCase  :  Dict				       =torch.tensor([[0.79_94, 0.96_12, 0.85_08], [0.74_01, 0.89_58, 0.83_02], [0.58_62, 0.74_68, 0.73_25]]				)
      # we verified the loss both for normalized and unnormalized targets for this one
      _UpperCAmelCase  :  int				       =torch.tensor([0.51_42]				) if config.norm_pix_loss else torch.tensor([0.64_69]				)
   elif model_name == "videomae-large":
      _UpperCAmelCase  :  Dict				       =torch.Size([1, 1_4_0_8, 1_5_3_6]				)
      _UpperCAmelCase  :  int				       =torch.tensor([[0.71_49, 0.79_97, 0.69_66], [0.67_68, 0.78_69, 0.69_48], [0.51_39, 0.62_21, 0.56_05]]				)
   elif model_name == "videomae-large-finetuned-kinetics":
      _UpperCAmelCase  :  Optional[Any]				       =torch.Size([1, 4_0_0]				)
      _UpperCAmelCase  :  Tuple				       =torch.tensor([0.07_71, 0.00_11, -0.36_25]				)
   elif model_name == "videomae-huge-finetuned-kinetics":
      _UpperCAmelCase  :  Optional[int]				       =torch.Size([1, 4_0_0]				)
      _UpperCAmelCase  :  Dict				       =torch.tensor([0.24_33, 0.16_32, -0.48_94]				)
   elif model_name == "videomae-base-short-finetuned-kinetics":
      _UpperCAmelCase  :  Optional[int]				       =torch.Size([1, 4_0_0]				)
      _UpperCAmelCase  :  List[Any]				       =torch.tensor([0.65_88, 0.09_90, -0.24_93]				)
   elif model_name == "videomae-base-finetuned-kinetics":
      _UpperCAmelCase  :  Any				       =torch.Size([1, 4_0_0]				)
      _UpperCAmelCase  :  str				       =torch.tensor([0.36_69, -0.06_88, -0.24_21]				)
   elif model_name == "videomae-base-short-ssv2":
      _UpperCAmelCase  :  Dict				       =torch.Size([1, 1_4_0_8, 1_5_3_6]				)
      _UpperCAmelCase  :  Tuple				       =torch.tensor([[0.47_12, 0.52_96, 0.57_86], [0.22_78, 0.27_29, 0.40_26], [0.03_52, 0.07_30, 0.25_06]]				)
   elif model_name == "videomae-base-short-finetuned-ssv2":
      _UpperCAmelCase  :  str				       =torch.Size([1, 1_7_4]				)
      _UpperCAmelCase  :  List[str]				       =torch.tensor([-0.05_37, -0.15_39, -0.32_66]				)
   elif model_name == "videomae-base-ssv2":
      _UpperCAmelCase  :  List[Any]				       =torch.Size([1, 1_4_0_8, 1_5_3_6]				)
      _UpperCAmelCase  :  List[Any]				       =torch.tensor([[0.81_31, 0.87_27, 0.85_46], [0.73_66, 0.93_77, 0.88_70], [0.59_35, 0.88_74, 0.85_64]]				)
   elif model_name == "videomae-base-finetuned-ssv2":
      _UpperCAmelCase  :  Tuple				       =torch.Size([1, 1_7_4]				)
      _UpperCAmelCase  :  Any				       =torch.tensor([0.19_61, -0.83_37, -0.63_89]				)
   else:
      raise ValueError(f"Model name not supported. Should be one of {model_names}"				)

   # verify logits
   assert logits.shape == expected_shape
   if "finetuned" in model_name:
      assert torch.allclose(logits[0, :3]			,       __lowerCamelCase			,       atol=1e-4				)
   else:
      print('Logits:'			,       logits[0, :3, :3]				)
      assert torch.allclose(logits[0, :3, :3]			,       __lowerCamelCase			,       atol=1e-4				)
   print('Logits ok!'				)

   # verify loss, if applicable
   if model_name == "videomae-base-short":
      _UpperCAmelCase  :  List[Any]				       =outputs.loss
      assert torch.allclose(__lowerCamelCase			,       __lowerCamelCase			,       atol=1e-4				)
      print('Loss ok!'				)

   if pytorch_dump_folder_path is not None:
      print(f"Saving model and image processor to {pytorch_dump_folder_path}"				)
      image_processor.save_pretrained(__lowerCamelCase				)
      model.save_pretrained(__lowerCamelCase				)

   if push_to_hub:
      print('Pushing to the hub...'				)
      model.push_to_hub(__lowerCamelCase			,       organization='nielsr'				)


if __name__ == "__main__":
 lowercase				=argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     '--checkpoint_url',
     default='https://drive.google.com/u/1/uc?id=1tEhLyskjb755TJ65ptsrafUG2llSwQE1&amp;export=download&amp;confirm=t&amp;uuid=aa3276eb-fb7e-482a-adec-dc7171df14c4',
     type=str,
     help=(
         'URL of the original PyTorch checkpoint (on Google Drive) you\'d like to convert. Should be a direct'
         ' download link.'
     ),
 )
 parser.add_argument(
     '--pytorch_dump_folder_path',
     default='/Users/nielsrogge/Documents/VideoMAE/Test',
     type=str,
     help='Path to the output PyTorch model directory.',
 )
 parser.add_argument('--model_name', default='videomae-base', type=str, help='Name of the model.')
 parser.add_argument(
     '--push_to_hub', action='store_true', help='Whether or not to push the converted model to the 🤗 hub.'
 )

 lowercase				=parser.parse_args()
 convert_videomae_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path, args.model_name, args.push_to_hub)





'''simple docstring'''






import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class    __magic_name__				(						lowerCAmelCase   ,unittest.TestCase							):
       UpperCAmelCase		 =CodeGenTokenizer
       UpperCAmelCase		 =CodeGenTokenizerFast
       UpperCAmelCase		 =True
       UpperCAmelCase		 ={"add_prefix_space": True}
       UpperCAmelCase		 =False






       def      lowerCAmelCase						(     self)		->    Optional[int]:




          '''simple docstring'''
          super().setUp()

          # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
          _UpperCAmelCase  :  Optional[Any]				       =[
              'l',
              'o',
              'w',
              'e',
              'r',
              's',
              't',
              'i',
              'd',
              'n',
              '\u0120',
              '\u0120l',
              '\u0120n',
              '\u0120lo',
              '\u0120low',
              'er',
              '\u0120lowest',
              '\u0120newer',
              '\u0120wider',
              '<unk>',
              '<|endoftext|>',
          ]
          _UpperCAmelCase  :  Dict				       =dict(zip(snake_case    ,				range(len(snake_case))))
          _UpperCAmelCase  :  List[Any]				       =['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
          _UpperCAmelCase  :  List[Any]				       ={'unk_token': '<unk>'}

          _UpperCAmelCase  :  List[Any]				       =os.path.join(self.tmpdirname    ,				VOCAB_FILES_NAMES['vocab_file'])
          _UpperCAmelCase  :  List[str]				       =os.path.join(self.tmpdirname    ,				VOCAB_FILES_NAMES['merges_file'])
          with open(self.vocab_file    ,				'w'    ,				encoding='utf-8') as fp:
             fp.write(json.dumps(snake_case) + '\n')
          with open(self.merges_file    ,				'w'    ,				encoding='utf-8') as fp:
             fp.write('\n'.join(snake_case))






       def      lowerCAmelCase						(     self    ,				**snake_case)		->    str:




          '''simple docstring'''
          kwargs.update(self.special_tokens_map)
          return CodeGenTokenizer.from_pretrained(self.tmpdirname    ,				**snake_case)






       def      lowerCAmelCase						(     self    ,				**snake_case)		->    Optional[Any]:




          '''simple docstring'''
          kwargs.update(self.special_tokens_map)
          return CodeGenTokenizerFast.from_pretrained(self.tmpdirname    ,				**snake_case)






       def      lowerCAmelCase						(     self    ,				snake_case)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Union[str, Any]				       ='lower newer'
          _UpperCAmelCase  :  int				       ='lower newer'
          return input_text, output_text






       def      lowerCAmelCase						(     self)		->    int:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[int]				       =CodeGenTokenizer(self.vocab_file    ,				self.merges_file    ,				**self.special_tokens_map)
          _UpperCAmelCase  :  str				       ='lower newer'
          _UpperCAmelCase  :  Union[str, Any]				       =['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
          _UpperCAmelCase  :  Any				       =tokenizer.tokenize(snake_case    ,				add_prefix_space=snake_case)
          self.assertListEqual(snake_case    ,				snake_case)

          _UpperCAmelCase  :  Any				       =tokens + [tokenizer.unk_token]
          _UpperCAmelCase  :  List[str]				       =[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case)    ,				snake_case)






       def      lowerCAmelCase						(     self)		->    Union[str, Any]:




          '''simple docstring'''
          if not self.test_rust_tokenizer:
             return

          _UpperCAmelCase  :  Tuple				       =self.get_tokenizer()
          _UpperCAmelCase  :  Dict				       =self.get_rust_tokenizer(add_prefix_space=snake_case)

          _UpperCAmelCase  :  Union[str, Any]				       ='lower newer'

          # Testing tokenization
          _UpperCAmelCase  :  List[str]				       =tokenizer.tokenize(snake_case    ,				add_prefix_space=snake_case)
          _UpperCAmelCase  :  Optional[int]				       =rust_tokenizer.tokenize(snake_case)
          self.assertListEqual(snake_case    ,				snake_case)

          # Testing conversion to ids without special tokens
          _UpperCAmelCase  :  str				       =tokenizer.encode(snake_case    ,				add_special_tokens=snake_case    ,				add_prefix_space=snake_case)
          _UpperCAmelCase  :  Optional[int]				       =rust_tokenizer.encode(snake_case    ,				add_special_tokens=snake_case)
          self.assertListEqual(snake_case    ,				snake_case)

          # Testing conversion to ids with special tokens
          _UpperCAmelCase  :  Dict				       =self.get_rust_tokenizer(add_prefix_space=snake_case)
          _UpperCAmelCase  :  Tuple				       =tokenizer.encode(snake_case    ,				add_prefix_space=snake_case)
          _UpperCAmelCase  :  int				       =rust_tokenizer.encode(snake_case)
          self.assertListEqual(snake_case    ,				snake_case)

          # Testing the unknown token
          _UpperCAmelCase  :  List[str]				       =tokens + [rust_tokenizer.unk_token]
          _UpperCAmelCase  :  Union[str, Any]				       =[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
          self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(snake_case)    ,				snake_case)






       def      lowerCAmelCase						(     self    ,				*snake_case    ,				**snake_case)		->    Any:




          '''simple docstring'''
          # It's very difficult to mix/test pretokenization with byte-level
          # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
          pass






       def      lowerCAmelCase						(     self    ,				snake_case=1_5)		->    Optional[int]:




          '''simple docstring'''
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                _UpperCAmelCase  :  Optional[Any]				       =self.rust_tokenizer_class.from_pretrained(snake_case    ,				**snake_case)

                # Simple input
                _UpperCAmelCase  :  List[str]				       ='This is a simple input'
                _UpperCAmelCase  :  Optional[Any]				       =['This is a simple input 1', 'This is a simple input 2']
                _UpperCAmelCase  :  int				       =('This is a simple input', 'This is a pair')
                _UpperCAmelCase  :  Union[str, Any]				       =[
                    ('This is a simple input 1', 'This is a simple input 2'),
                    ('This is a simple pair 1', 'This is a simple pair 2'),
                ]

                # Simple input tests
                self.assertRaises(snake_case    ,				tokenizer_r.encode    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length')

                # Simple input
                self.assertRaises(snake_case    ,				tokenizer_r.encode_plus    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length')

                # Simple input
                self.assertRaises(
                    snake_case    ,				tokenizer_r.batch_encode_plus    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length'    ,				)

                # Pair input
                self.assertRaises(snake_case    ,				tokenizer_r.encode    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length')

                # Pair input
                self.assertRaises(snake_case    ,				tokenizer_r.encode_plus    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length')

                # Pair input
                self.assertRaises(
                    snake_case    ,				tokenizer_r.batch_encode_plus    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length'    ,				)






       def      lowerCAmelCase						(     self)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Any				       =CodeGenTokenizer.from_pretrained(self.tmpdirname    ,				pad_token='<pad>')

          # Simple input
          _UpperCAmelCase  :  List[Any]				       ='This is a simple input'
          _UpperCAmelCase  :  Any				       =['This is a simple input looooooooong', 'This is a simple input']
          _UpperCAmelCase  :  List[str]				       =('This is a simple input', 'This is a pair')
          _UpperCAmelCase  :  Optional[int]				       =[
              ('This is a simple input loooooong', 'This is a simple input'),
              ('This is a simple pair loooooong', 'This is a simple pair'),
          ]

          _UpperCAmelCase  :  List[Any]				       =tokenizer.pad_token_id

          _UpperCAmelCase  :  Union[str, Any]				       =tokenizer(snake_case    ,				padding='max_length'    ,				max_length=3_0    ,				return_tensors='np')
          _UpperCAmelCase  :  List[Any]				       =tokenizer(snake_case    ,				padding=snake_case    ,				truncate=snake_case    ,				return_tensors='np')
          _UpperCAmelCase  :  Optional[Any]				       =tokenizer(*snake_case    ,				padding='max_length'    ,				max_length=6_0    ,				return_tensors='np')
          _UpperCAmelCase  :  List[Any]				       =tokenizer(snake_case    ,				padding=snake_case    ,				truncate=snake_case    ,				return_tensors='np')

          # s
          # test single string max_length padding
          self.assertEqual(out_s['input_ids'].shape[-1]    ,				3_0)
          self.assertTrue(pad_token_id in out_s['input_ids'])
          self.assertTrue(0 in out_s['attention_mask'])

          # s2
          # test automatic padding
          self.assertEqual(out_sa['input_ids'].shape[-1]    ,				3_3)
          # long slice doesn't have padding
          self.assertFalse(pad_token_id in out_sa['input_ids'][0])
          self.assertFalse(0 in out_sa['attention_mask'][0])
          # short slice does have padding
          self.assertTrue(pad_token_id in out_sa['input_ids'][1])
          self.assertTrue(0 in out_sa['attention_mask'][1])

          # p
          # test single pair max_length padding
          self.assertEqual(out_p['input_ids'].shape[-1]    ,				6_0)
          self.assertTrue(pad_token_id in out_p['input_ids'])
          self.assertTrue(0 in out_p['attention_mask'])

          # p2
          # test automatic padding pair
          self.assertEqual(out_pa['input_ids'].shape[-1]    ,				5_2)
          # long slice pair doesn't have padding
          self.assertFalse(pad_token_id in out_pa['input_ids'][0])
          self.assertFalse(0 in out_pa['attention_mask'][0])
          # short slice pair does have padding
          self.assertTrue(pad_token_id in out_pa['input_ids'][1])
          self.assertTrue(0 in out_pa['attention_mask'][1])






       def      lowerCAmelCase						(     self)		->    Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase  :  Any				       ='$$$'
          _UpperCAmelCase  :  Any				       =CodeGenTokenizer.from_pretrained(self.tmpdirname    ,				bos_token=snake_case    ,				add_bos_token=snake_case)

          _UpperCAmelCase  :  Optional[Any]				       ='This is a simple input'
          _UpperCAmelCase  :  Any				       =['This is a simple input 1', 'This is a simple input 2']

          _UpperCAmelCase  :  int				       =tokenizer.bos_token_id

          _UpperCAmelCase  :  Optional[Any]				       =tokenizer(snake_case)
          _UpperCAmelCase  :  Tuple				       =tokenizer(snake_case)

          self.assertEqual(out_s.input_ids[0]    ,				snake_case)
          self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

          _UpperCAmelCase  :  Optional[int]				       =tokenizer.decode(out_s.input_ids)
          _UpperCAmelCase  :  Any				       =tokenizer.batch_decode(out_sa.input_ids)

          self.assertEqual(decode_s.split()[0]    ,				snake_case)
          self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))






       @slow
       def      lowerCAmelCase						(     self)		->    Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[Any]				       =CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono')

          _UpperCAmelCase  :  Optional[Any]				       ='\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
          _UpperCAmelCase  :  Tuple				       ='\nif len_a > len_b:      result = a\nelse:      result = b'

          _UpperCAmelCase  :  Optional[int]				       =tokenizer.encode(snake_case)
          _UpperCAmelCase  :  List[str]				       =['^#', re.escape('<|endoftext|>'), '^\'\'\'', '^"""', '\n\n\n']
          _UpperCAmelCase  :  Dict				       =tokenizer.decode(snake_case    ,				truncate_before_pattern=snake_case)
          self.assertEqual(snake_case    ,				snake_case)






       def      lowerCAmelCase						(     self)		->    Optional[Any]:




          '''simple docstring'''
          pass

