


import argparse

import requests
import torch
from PIL import Image

from transformers import ViTMAEConfig, ViTMAEForPreTraining, ViTMAEImageProcessor
def 							__lowerCamelCase		(					UpperCAmelCase_		:						int		):




      """simple docstring"""





      if "cls_token" in name:
            a		:Optional[int]									=			name.replace('''cls_token'''					,    '''vit.embeddings.cls_token'''		)
      if "mask_token" in name:
            a		:Tuple									=			name.replace('''mask_token'''					,    '''decoder.mask_token'''		)
      if "decoder_pos_embed" in name:
            a		:Any									=			name.replace('''decoder_pos_embed'''					,    '''decoder.decoder_pos_embed'''		)
      if "pos_embed" in name and "decoder" not in name:
            a		:Optional[Any]									=			name.replace('''pos_embed'''					,    '''vit.embeddings.position_embeddings'''		)
      if "patch_embed.proj" in name:
            a		:Optional[Any]									=			name.replace('''patch_embed.proj'''					,    '''vit.embeddings.patch_embeddings.projection'''		)
      if "patch_embed.norm" in name:
            a		:Any									=			name.replace('''patch_embed.norm'''					,    '''vit.embeddings.norm'''		)
      if "decoder_blocks" in name:
            a		:Any									=			name.replace('''decoder_blocks'''					,    '''decoder.decoder_layers'''		)
      if "blocks" in name:
            a		:Optional[Any]									=			name.replace('''blocks'''					,    '''vit.encoder.layer'''		)
      if "attn.proj" in name:
            a		:Dict									=			name.replace('''attn.proj'''					,    '''attention.output.dense'''		)
      if "attn" in name:
            a		:int									=			name.replace('''attn'''					,    '''attention.self'''		)
      if "norm1" in name:
            a		:Dict									=			name.replace('''norm1'''					,    '''layernorm_before'''		)
      if "norm2" in name:
            a		:Dict									=			name.replace('''norm2'''					,    '''layernorm_after'''		)
      if "mlp.fc1" in name:
            a		:str									=			name.replace('''mlp.fc1'''					,    '''intermediate.dense'''		)
      if "mlp.fc2" in name:
            a		:List[Any]									=			name.replace('''mlp.fc2'''					,    '''output.dense'''		)
      if "decoder_embed" in name:
            a		:Optional[int]									=			name.replace('''decoder_embed'''					,    '''decoder.decoder_embed'''		)
      if "decoder_norm" in name:
            a		:Any									=			name.replace('''decoder_norm'''					,    '''decoder.decoder_norm'''		)
      if "decoder_pred" in name:
            a		:str									=			name.replace('''decoder_pred'''					,    '''decoder.decoder_pred'''		)
      if "norm.weight" in name and "decoder" not in name:
            a		:int									=			name.replace('''norm.weight'''					,    '''vit.layernorm.weight'''		)
      if "norm.bias" in name and "decoder" not in name:
            a		:Union[str, Any]									=			name.replace('''norm.bias'''					,    '''vit.layernorm.bias'''		)

      return name
def 							__lowerCamelCase		(					UpperCAmelCase_		:						Optional[int]					,    UpperCAmelCase_		:						List[Any]		):




      """simple docstring"""





      for key in orig_state_dict.copy().keys():
            a		:Any									=			orig_state_dict.pop(UpperCAmelCase_		)

            if "qkv" in key:
                  a		:Union[str, Any]									=			key.split('''.'''		)
                  a		:Optional[int]									=			int(key_split[1]		)
                  if "decoder_blocks" in key:
                        a		:List[Any]									=			config.decoder_hidden_size
                        a		:List[Any]									=			'''decoder.decoder_layers.'''
                        if "weight" in key:
                              a		:List[Any]									=			val[:dim, :]
                              a		:Tuple									=			val[dim : dim * 2, :]
                              a		:Dict									=			val[-dim:, :]
                        elif "bias" in key:
                              a		:Union[str, Any]									=			val[:dim]
                              a		:List[Any]									=			val[dim : dim * 2]
                              a		:List[Any]									=			val[-dim:]
                  else:
                        a		:Optional[Any]									=			config.hidden_size
                        a		:List[str]									=			'''vit.encoder.layer.'''
                        if "weight" in key:
                              a		:Optional[int]									=			val[:dim, :]
                              a		:Union[str, Any]									=			val[dim : dim * 2, :]
                              a		:Tuple									=			val[-dim:, :]
                        elif "bias" in key:
                              a		:Tuple									=			val[:dim]
                              a		:Any									=			val[dim : dim * 2]
                              a		:List[str]									=			val[-dim:]

            else:
                  a		:Optional[Any]									=			val

      return orig_state_dict
def 							__lowerCamelCase		(					UpperCAmelCase_		:						str					,    UpperCAmelCase_		:						int		):




      """simple docstring"""





      a		:Optional[int]									=			ViTMAEConfig()
      if "large" in checkpoint_url:
            a		:str									=			1024
            a		:List[Any]									=			4096
            a		:List[Any]									=			24
            a		:Union[str, Any]									=			16
      elif "huge" in checkpoint_url:
            a		:Tuple									=			14
            a		:int									=			1280
            a		:str									=			5120
            a		:List[Any]									=			32
            a		:List[str]									=			16

      a		:str									=			ViTMAEForPreTraining(UpperCAmelCase_		)

      a		:Dict									=			torch.hub.load_state_dict_from_url(UpperCAmelCase_					,    map_location='''cpu'''		)['''model''']

      a		:Any									=			ViTMAEImageProcessor(size=config.image_size		)

      a		:List[Any]									=			convert_state_dict(UpperCAmelCase_					,    UpperCAmelCase_		)

      model.load_state_dict(UpperCAmelCase_		)
      model.eval()

      a		:Any									=			'''https://user-images.githubusercontent.com/11435359/147738734-196fd92f-9260-48d5-ba7e-bf103d29364d.jpg'''

      a		:List[str]									=			Image.open(requests.get(UpperCAmelCase_					,    stream=UpperCAmelCase_		).raw		)
      a		:Optional[int]									=			ViTMAEImageProcessor(size=config.image_size		)
      a		:str									=			image_processor(images=UpperCAmelCase_					,    return_tensors='''pt'''		)

      # forward pass
      torch.manual_seed(2		)
      a		:Any									=			model(**UpperCAmelCase_		)
      a		:str									=			outputs.logits

      if "large" in checkpoint_url:
            a		:int									=			torch.tensor(
                [[-0.7309, -0.7128, -1.0169], [-1.0161, -0.9058, -1.1878], [-1.0478, -0.9411, -1.1911]]		)
      elif "huge" in checkpoint_url:
            a		:Optional[int]									=			torch.tensor(
                [[-1.1599, -0.9199, -1.2221], [-1.1952, -0.9269, -1.2307], [-1.2143, -0.9337, -1.2262]]		)
      else:
            a		:Dict									=			torch.tensor(
                [[-0.9192, -0.8481, -1.1259], [-1.1349, -1.0034, -1.2599], [-1.1757, -1.0429, -1.2726]]		)

      # verify logits
      assert torch.allclose(logits[0, :3, :3]					,    UpperCAmelCase_					,    atol=1E-4		)

      print(F'''Saving model to {pytorch_dump_folder_path}'''		)
      model.save_pretrained(UpperCAmelCase_		)

      print(F'''Saving image processor to {pytorch_dump_folder_path}'''		)
      image_processor.save_pretrained(UpperCAmelCase_		)


if __name__ == "__main__":
     snake_case							:       List[Any]										=	argparse.ArgumentParser()
     # Required parameters
     parser.add_argument(
         '''--checkpoint_url''',
         default='''https://dl.fbaipublicfiles.com/mae/visualize/mae_visualize_vit_base.pth''',
         type=str,
         help='''URL of the checkpoint you\'d like to convert.''',
     )
     parser.add_argument(
         '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model directory.'''
     )

     snake_case							:       Tuple										=	parser.parse_args()
     convert_vit_mae_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)









import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case							:       Optional[int]										=	logging.get_logger(__name__)

snake_case							:       Optional[int]										=	{
    '''microsoft/unispeech-large-1500h-cv''': (
        '''https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json'''
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}







class 						_snake_case    ( _snake_case				):
 SCREAMING_SNAKE_CASE__ =			'unispeech'




 def __init__(						self   ,    _lowerCamelCase=32   ,    _lowerCamelCase=768   ,    _lowerCamelCase=12   ,    _lowerCamelCase=12   ,    _lowerCamelCase=3072   ,    _lowerCamelCase="gelu"   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.0   ,    _lowerCamelCase=0.0   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.02   ,    _lowerCamelCase=1e-5   ,    _lowerCamelCase="group"   ,    _lowerCamelCase="gelu"   ,    _lowerCamelCase=(512, 512, 512, 512, 512, 512, 512)   ,    _lowerCamelCase=(5, 2, 2, 2, 2, 2, 2)   ,    _lowerCamelCase=(10, 3, 3, 3, 3, 2, 2)   ,    _lowerCamelCase=False   ,    _lowerCamelCase=128   ,    _lowerCamelCase=16   ,    _lowerCamelCase=False   ,    _lowerCamelCase=True   ,    _lowerCamelCase=0.05   ,    _lowerCamelCase=10   ,    _lowerCamelCase=2   ,    _lowerCamelCase=0.0   ,    _lowerCamelCase=10   ,    _lowerCamelCase=0   ,    _lowerCamelCase=320   ,    _lowerCamelCase=2   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=100   ,    _lowerCamelCase=256   ,    _lowerCamelCase=256   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase="mean"   ,    _lowerCamelCase=False   ,    _lowerCamelCase=False   ,    _lowerCamelCase=256   ,    _lowerCamelCase=80   ,    _lowerCamelCase=0   ,    _lowerCamelCase=1   ,    _lowerCamelCase=2   ,    _lowerCamelCase=0.5   ,    **_lowerCamelCase   ,    ):
       super().__init__(**_lowerCamelCase   ,    pad_token_id=_lowerCamelCase   ,    bos_token_id=_lowerCamelCase   ,    eos_token_id=_lowerCamelCase       )
       a		:Any									=			hidden_size
       a		:str									=			feat_extract_norm
       a		:List[Any]									=			feat_extract_activation
       a		:Tuple									=			list(_lowerCamelCase       )
       a		:Any									=			list(_lowerCamelCase       )
       a		:List[Any]									=			list(_lowerCamelCase       )
       a		:Union[str, Any]									=			conv_bias
       a		:str									=			num_conv_pos_embeddings
       a		:str									=			num_conv_pos_embedding_groups
       a		:Tuple									=			len(self.conv_dim       )
       a		:int									=			num_hidden_layers
       a		:Any									=			intermediate_size
       a		:Optional[Any]									=			hidden_act
       a		:Tuple									=			num_attention_heads
       a		:Any									=			hidden_dropout
       a		:Any									=			attention_dropout
       a		:Optional[Any]									=			activation_dropout
       a		:Optional[Any]									=			feat_proj_dropout
       a		:Any									=			final_dropout
       a		:int									=			layerdrop
       a		:int									=			layer_norm_eps
       a		:Dict									=			initializer_range
       a		:Dict									=			num_ctc_classes
       a		:Optional[Any]									=			vocab_size
       a		:str									=			do_stable_layer_norm
       a		:Tuple									=			use_weighted_layer_sum
       a		:Any									=			classifier_proj_size

       if (
           (len(self.conv_stride       ) != self.num_feat_extract_layers)
           or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
           or (len(self.conv_dim       ) != self.num_feat_extract_layers)
       ):
             raise ValueError(
                 '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
                 ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
                 F''' {len(self.conv_dim       )}`, `len(config.conv_stride) = {len(self.conv_stride       )}`,'''
                 F''' `len(config.conv_kernel) = {len(self.conv_kernel       )}`.'''       )

       # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
       a		:List[Any]									=			apply_spec_augment
       a		:Any									=			mask_time_prob
       a		:Union[str, Any]									=			mask_time_length
       a		:str									=			mask_time_min_masks
       a		:Tuple									=			mask_feature_prob
       a		:Dict									=			mask_feature_length
       a		:int									=			mask_feature_min_masks

       # parameters for pretraining with codevector quantized representations
       a		:Union[str, Any]									=			num_codevectors_per_group
       a		:Dict									=			num_codevector_groups
       a		:List[Any]									=			contrastive_logits_temperature
       a		:Union[str, Any]									=			feat_quantizer_dropout
       a		:Optional[Any]									=			num_negatives
       a		:Tuple									=			codevector_dim
       a		:Optional[Any]									=			proj_codevector_dim
       a		:Union[str, Any]									=			diversity_loss_weight

       # ctc loss
       a		:List[Any]									=			ctc_loss_reduction
       a		:Union[str, Any]									=			ctc_zero_infinity

       # pretraining loss
       a		:int									=			replace_prob




 @property
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       return functools.reduce(operator.mul   ,    self.conv_stride   ,    1       )





