


import inspect
import unittest

from transformers import ViTConfig
from transformers.testing_utils import (
    require_accelerate,
    require_torch,
    require_torch_gpu,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch
   from torch import nn

   from transformers import ViTForImageClassification, ViTForMaskedImageModeling, ViTModel
   from transformers.models.vit.modeling_vit import VIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
   from PIL import Image

   from transformers import ViTImageProcessor
class 		__magic_name__						:





     '''simple docstring'''



     def __init__(					self:Tuple     ,		_a:int     ,		_a:str=13     ,		_a:Optional[int]=30     ,		_a:List[str]=2     ,		_a:Dict=3     ,		_a:Tuple=True     ,		_a:str=True     ,		_a:Dict=32     ,		_a:str=5     ,		_a:Dict=4     ,		_a:Dict=37     ,		_a:Any="gelu"     ,		_a:Union[str, Any]=0.1     ,		_a:Dict=0.1     ,		_a:Any=10     ,		_a:Union[str, Any]=0.02     ,		_a:Union[str, Any]=None     ,		_a:str=2     ,		):
       snake_case__						= parent
       snake_case__						= batch_size
       snake_case__						= image_size
       snake_case__						= patch_size
       snake_case__						= num_channels
       snake_case__						= is_training
       snake_case__						= use_labels
       snake_case__						= hidden_size
       snake_case__						= num_hidden_layers
       snake_case__						= num_attention_heads
       snake_case__						= intermediate_size
       snake_case__						= hidden_act
       snake_case__						= hidden_dropout_prob
       snake_case__						= attention_probs_dropout_prob
       snake_case__						= type_sequence_label_size
       snake_case__						= initializer_range
       snake_case__						= scope
       snake_case__						= encoder_stride

       # in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
       snake_case__						= (image_size // patch_size) ** 2
       snake_case__						= num_patches + 1

     def 	SCREAMING_SNAKE_CASE__						(					self:int   ):
       snake_case__						= floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]   )

       snake_case__						= None
       if self.use_labels:
         snake_case__						= ids_tensor([self.batch_size]     ,		self.type_sequence_label_size   )

       snake_case__						= self.get_config()

       return config, pixel_values, labels

     def 	SCREAMING_SNAKE_CASE__						(					self:List[str]   ):
       return ViTConfig(
           image_size=self.image_size     ,		patch_size=self.patch_size     ,		num_channels=self.num_channels     ,		hidden_size=self.hidden_size     ,		num_hidden_layers=self.num_hidden_layers     ,		num_attention_heads=self.num_attention_heads     ,		intermediate_size=self.intermediate_size     ,		hidden_act=self.hidden_act     ,		hidden_dropout_prob=self.hidden_dropout_prob     ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,		is_decoder=_a     ,		initializer_range=self.initializer_range     ,		encoder_stride=self.encoder_stride     ,		)

     def 	SCREAMING_SNAKE_CASE__						(					self:Dict     ,		_a:Tuple     ,		_a:Any     ,		_a:Optional[int]   ):
       snake_case__						= ViTModel(config=_a   )
       model.to(_a   )
       model.eval()
       snake_case__						= model(_a   )
       self.parent.assertEqual(result.last_hidden_state.shape     ,		(self.batch_size, self.seq_length, self.hidden_size)   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[Any]     ,		_a:Optional[Any]     ,		_a:Dict     ,		_a:Union[str, Any]   ):
       snake_case__						= ViTForMaskedImageModeling(config=_a   )
       model.to(_a   )
       model.eval()
       snake_case__						= model(_a   )
       self.parent.assertEqual(
           result.reconstruction.shape     ,		(self.batch_size, self.num_channels, self.image_size, self.image_size)   )

       # test greyscale images
       snake_case__						= 1
       snake_case__						= ViTForMaskedImageModeling(_a   )
       model.to(_a   )
       model.eval()

       snake_case__						= floats_tensor([self.batch_size, 1, self.image_size, self.image_size]   )
       snake_case__						= model(_a   )
       self.parent.assertEqual(result.reconstruction.shape     ,		(self.batch_size, 1, self.image_size, self.image_size)   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Dict     ,		_a:Tuple     ,		_a:str     ,		_a:Optional[int]   ):
       snake_case__						= self.type_sequence_label_size
       snake_case__						= ViTForImageClassification(_a   )
       model.to(_a   )
       model.eval()
       snake_case__						= model(_a     ,		labels=_a   )
       self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.type_sequence_label_size)   )

       # test greyscale images
       snake_case__						= 1
       snake_case__						= ViTForImageClassification(_a   )
       model.to(_a   )
       model.eval()

       snake_case__						= floats_tensor([self.batch_size, 1, self.image_size, self.image_size]   )
       snake_case__						= model(_a   )
       self.parent.assertEqual(result.logits.shape     ,		(self.batch_size, self.type_sequence_label_size)   )







     def 	SCREAMING_SNAKE_CASE__						(					self:List[str]   ):
       snake_case__						= self.prepare_config_and_inputs()
       (
           (
           snake_case__
       )				,				(
           snake_case__
       )				,				(
           snake_case__
       )				,				
       )						= config_and_inputs
       snake_case__						= {'''pixel_values''': pixel_values}
       return config, inputs_dict




@require_torch
class 		__magic_name__						(snake_case_   ,snake_case_   ,unittest.TestCase				):





     '''simple docstring'''



     __lowercase				:							Tuple	   =       (
         (
             ViTModel,
             ViTForImageClassification,
             ViTForMaskedImageModeling,
         )
         if is_torch_available()
         else ()
     )
     __lowercase				:							Dict	   =       (
         {'feature-extraction': ViTModel, 'image-classification': ViTForImageClassification}
         if is_torch_available()
         else {}
     )
     __lowercase				:							Any	   =       True

     __lowercase				:							Union[str, Any]	   =       False
     __lowercase				:							int	   =       False
     __lowercase				:							List[str]	   =       False

     def 	SCREAMING_SNAKE_CASE__						(					self:int   ):
       snake_case__						= ViTModelTester(self   )
       snake_case__						= ConfigTester(self     ,		config_class=_a     ,		has_text_modality=_a     ,		hidden_size=37   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Dict   ):
       self.config_tester.run_common_tests()

     @unittest.skip(reason='''ViT does not use inputs_embeds'''   )
     def 	SCREAMING_SNAKE_CASE__						(					self:Any   ):
       pass

     def 	SCREAMING_SNAKE_CASE__						(					self:Any   ):
       snake_case__				,				snake_case__						= self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
         snake_case__						= model_class(_a   )
         self.assertIsInstance(model.get_input_embeddings()     ,		(nn.Module)   )
         snake_case__						= model.get_output_embeddings()
         self.assertTrue(x is None or isinstance(_a     ,		nn.Linear   )   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[Any]   ):
       snake_case__				,				snake_case__						= self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
         snake_case__						= model_class(_a   )
         snake_case__						= inspect.signature(model.forward   )
         # signature.parameters is an OrderedDict => so arg_names order is deterministic
         snake_case__						= [*signature.parameters.keys()]

         snake_case__						= ['''pixel_values''']
         self.assertListEqual(arg_names[:1]     ,		_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:List[Any]   ):
       snake_case__						= self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Union[str, Any]   ):
       snake_case__						= self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_image_modeling(*_a   )

     def 	SCREAMING_SNAKE_CASE__						(					self:Tuple   ):
       snake_case__						= self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_image_classification(*_a   )







     @slow
     def 	SCREAMING_SNAKE_CASE__						(					self:Tuple   ):
       for model_name in VIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
         snake_case__						= ViTModel.from_pretrained(_a   )
         self.assertIsNotNone(_a   )







def SCREAMING_SNAKE_CASE	(			)    ->				Union[str, Any]:
  snake_case__						= Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png'''				)
  return image


@require_torch
@require_vision
class 		__magic_name__						(unittest.TestCase				):





     '''simple docstring'''



     @cached_property
     def 	SCREAMING_SNAKE_CASE__						(					self:List[str]   ):
       return ViTImageProcessor.from_pretrained('''google/vit-base-patch16-224'''   ) if is_vision_available() else None

     @slow
     def 	SCREAMING_SNAKE_CASE__						(					self:int   ):
       snake_case__						= ViTForImageClassification.from_pretrained('''google/vit-base-patch16-224'''   ).to(_a   )

       snake_case__						= self.default_image_processor
       snake_case__						= prepare_img()
       snake_case__						= image_processor(images=_a     ,		return_tensors='''pt'''   ).to(_a   )

       # forward pass
       with torch.no_grad():
         snake_case__						= model(**_a   )

       # verify the logits
       snake_case__						= torch.Size((1, 10_00)   )
       self.assertEqual(outputs.logits.shape     ,		_a   )

       snake_case__						= torch.tensor([-0.2744, 0.8215, -0.0836]   ).to(_a   )

       self.assertTrue(torch.allclose(outputs.logits[0, :3]     ,		_a     ,		atol=1e-4   )   )

     @slow
     def 	SCREAMING_SNAKE_CASE__						(					self:List[Any]   ):
       # ViT models have an `interpolate_pos_encoding` argument in their forward method,
       # allowing to interpolate the pre-trained position embeddings in order to use
       # the model on higher resolutions. The DINO model by Facebook AI leverages this
       # to visualize self-attention on higher resolution images.
       snake_case__						= ViTModel.from_pretrained('''facebook/dino-vits8'''   ).to(_a   )

       snake_case__						= ViTImageProcessor.from_pretrained('''facebook/dino-vits8'''     ,		size=4_80   )
       snake_case__						= prepare_img()
       snake_case__						= image_processor(images=_a     ,		return_tensors='''pt'''   )
       snake_case__						= inputs.pixel_values.to(_a   )

       # forward pass
       with torch.no_grad():
         snake_case__						= model(_a     ,		interpolate_pos_encoding=_a   )

       # verify the logits
       snake_case__						= torch.Size((1, 36_01, 3_84)   )
       self.assertEqual(outputs.last_hidden_state.shape     ,		_a   )

       snake_case__						= torch.tensor(
           [[4.2340, 4.3906, -6.6692], [4.5463, 1.8928, -6.7257], [4.4429, 0.8496, -5.8585]]   ).to(_a   )

       self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]     ,		_a     ,		atol=1e-4   )   )







     @slow
     @require_accelerate
     @require_torch_gpu
     def 	SCREAMING_SNAKE_CASE__						(					self:Optional[Any]   ):
       snake_case__						= ViTModel.from_pretrained('''facebook/dino-vits8'''     ,		torch_dtype=torch.floataa     ,		device_map='''auto'''   )
       snake_case__						= self.default_image_processor

       snake_case__						= prepare_img()
       snake_case__						= image_processor(images=_a     ,		return_tensors='''pt'''   )
       snake_case__						= inputs.pixel_values.to(_a   )

       # forward pass to make sure inference works in fp16
       with torch.no_grad():
         snake_case__						= model(_a   )





import argparse
import json
from pathlib import Path

import requests
import timm
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import DeiTImageProcessor, ViTConfig, ViTForImageClassification, ViTImageProcessor, ViTModel
from transformers.utils import logging


logging.set_verbosity_info()
lowerCamelCase__							:						int							     =   logging.get_logger(__name__)







def SCREAMING_SNAKE_CASE	(			__lowerCAmelCase							,   __lowerCAmelCase=False				)    ->				int:
  snake_case__						= []
  for i in range(config.num_hidden_layers				):
    # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
    rename_keys.append((F"""blocks.{i}.norm1.weight""", F"""vit.encoder.layer.{i}.layernorm_before.weight""")				)
    rename_keys.append((F"""blocks.{i}.norm1.bias""", F"""vit.encoder.layer.{i}.layernorm_before.bias""")				)
    rename_keys.append((F"""blocks.{i}.attn.proj.weight""", F"""vit.encoder.layer.{i}.attention.output.dense.weight""")				)
    rename_keys.append((F"""blocks.{i}.attn.proj.bias""", F"""vit.encoder.layer.{i}.attention.output.dense.bias""")				)
    rename_keys.append((F"""blocks.{i}.norm2.weight""", F"""vit.encoder.layer.{i}.layernorm_after.weight""")				)
    rename_keys.append((F"""blocks.{i}.norm2.bias""", F"""vit.encoder.layer.{i}.layernorm_after.bias""")				)
    rename_keys.append((F"""blocks.{i}.mlp.fc1.weight""", F"""vit.encoder.layer.{i}.intermediate.dense.weight""")				)
    rename_keys.append((F"""blocks.{i}.mlp.fc1.bias""", F"""vit.encoder.layer.{i}.intermediate.dense.bias""")				)
    rename_keys.append((F"""blocks.{i}.mlp.fc2.weight""", F"""vit.encoder.layer.{i}.output.dense.weight""")				)
    rename_keys.append((F"""blocks.{i}.mlp.fc2.bias""", F"""vit.encoder.layer.{i}.output.dense.bias""")				)

  # projection layer + position embeddings
  rename_keys.extend(
      [
          ('''cls_token''', '''vit.embeddings.cls_token'''),
          ('''patch_embed.proj.weight''', '''vit.embeddings.patch_embeddings.projection.weight'''),
          ('''patch_embed.proj.bias''', '''vit.embeddings.patch_embeddings.projection.bias'''),
          ('''pos_embed''', '''vit.embeddings.position_embeddings'''),
      ]				)

  if base_model:
    # layernorm + pooler
    rename_keys.extend(
        [
            ('''norm.weight''', '''layernorm.weight'''),
            ('''norm.bias''', '''layernorm.bias'''),
            ('''pre_logits.fc.weight''', '''pooler.dense.weight'''),
            ('''pre_logits.fc.bias''', '''pooler.dense.bias'''),
        ]				)

    # if just the base model, we should remove "vit" from all keys that start with "vit"
    snake_case__						= [(pair[0], pair[1][4:]) if pair[1].startswith('''vit'''				) else pair for pair in rename_keys]
  else:
    # layernorm + classification head
    rename_keys.extend(
        [
            ('''norm.weight''', '''vit.layernorm.weight'''),
            ('''norm.bias''', '''vit.layernorm.bias'''),
            ('''head.weight''', '''classifier.weight'''),
            ('''head.bias''', '''classifier.bias'''),
        ]				)

  return rename_keys







def SCREAMING_SNAKE_CASE	(			__lowerCAmelCase							,   __lowerCAmelCase							,   __lowerCAmelCase=False				)    ->				Dict:
  for i in range(config.num_hidden_layers				):
    if base_model:
      snake_case__						= ''''''
    else:
      snake_case__						= '''vit.'''
    # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
    snake_case__						= state_dict.pop(F"""blocks.{i}.attn.qkv.weight"""				)
    snake_case__						= state_dict.pop(F"""blocks.{i}.attn.qkv.bias"""				)
    # next, add query, keys and values (in that order) to the state dict
    snake_case__						= in_proj_weight[
        : config.hidden_size, :
    ]
    snake_case__						= in_proj_bias[: config.hidden_size]
    snake_case__						= in_proj_weight[
        config.hidden_size : config.hidden_size * 2, :
    ]
    snake_case__						= in_proj_bias[
        config.hidden_size : config.hidden_size * 2
    ]
    snake_case__						= in_proj_weight[
        -config.hidden_size :, :
    ]
    snake_case__						= in_proj_bias[-config.hidden_size :]







def SCREAMING_SNAKE_CASE	(			__lowerCAmelCase				)    ->				Optional[Any]:
  snake_case__						= ['''head.weight''', '''head.bias''']
  for k in ignore_keys:
    state_dict.pop(__lowerCAmelCase							,   __lowerCAmelCase				)







def SCREAMING_SNAKE_CASE	(			__lowerCAmelCase							,   __lowerCAmelCase							,   __lowerCAmelCase				)    ->				Union[str, Any]:
  snake_case__						= dct.pop(__lowerCAmelCase				)
  snake_case__						= val







def SCREAMING_SNAKE_CASE	(			)    ->				str:
  snake_case__						= '''http://images.cocodataset.org/val2017/000000039769.jpg'''
  snake_case__						= Image.open(requests.get(__lowerCAmelCase							,   stream=__lowerCAmelCase				).raw				)
  return im







@torch.no_grad()
def SCREAMING_SNAKE_CASE	(			__lowerCAmelCase							,   __lowerCAmelCase				)    ->				Dict:
  snake_case__						= ViTConfig()
  snake_case__						= False
  # dataset (ImageNet-21k only or also fine-tuned on ImageNet 2012), patch_size and image_size
  if vit_name[-5:] == "in21k":
    snake_case__						= True
    snake_case__						= int(vit_name[-12:-10]				)
    snake_case__						= int(vit_name[-9:-6]				)
  else:
    snake_case__						= 1000
    snake_case__						= '''huggingface/label-files'''
    snake_case__						= '''imagenet-1k-id2label.json'''
    snake_case__						= json.load(open(hf_hub_download(__lowerCAmelCase							,   __lowerCAmelCase							,   repo_type='''dataset'''				)							,   '''r'''				)				)
    snake_case__						= {int(__lowerCAmelCase				): v for k, v in idalabel.items()}
    snake_case__						= idalabel
    snake_case__						= {v: k for k, v in idalabel.items()}
    snake_case__						= int(vit_name[-6:-4]				)
    snake_case__						= int(vit_name[-3:]				)
  # size of the architecture
  if "deit" in vit_name:
    if vit_name[9:].startswith('''tiny'''				):
      snake_case__						= 192
      snake_case__						= 768
      snake_case__						= 12
      snake_case__						= 3
    elif vit_name[9:].startswith('''small'''				):
      snake_case__						= 384
      snake_case__						= 1536
      snake_case__						= 12
      snake_case__						= 6
    else:
      pass
  else:
    if vit_name[4:].startswith('''small'''				):
      snake_case__						= 768
      snake_case__						= 2304
      snake_case__						= 8
      snake_case__						= 8
    elif vit_name[4:].startswith('''base'''				):
      pass
    elif vit_name[4:].startswith('''large'''				):
      snake_case__						= 1024
      snake_case__						= 4096
      snake_case__						= 24
      snake_case__						= 16
    elif vit_name[4:].startswith('''huge'''				):
      snake_case__						= 1280
      snake_case__						= 5120
      snake_case__						= 32
      snake_case__						= 16

    # load original model from timm
  snake_case__						= timm.create_model(__lowerCAmelCase							,   pretrained=__lowerCAmelCase				)
  timm_model.eval()

  # load state_dict of original model, remove and rename some keys
  snake_case__						= timm_model.state_dict()
  if base_model:
    remove_classification_head_(__lowerCAmelCase				)
  snake_case__						= create_rename_keys(__lowerCAmelCase							,   __lowerCAmelCase				)
  for src, dest in rename_keys:
    rename_key(__lowerCAmelCase							,   __lowerCAmelCase							,   __lowerCAmelCase				)
  read_in_q_k_v(__lowerCAmelCase							,   __lowerCAmelCase							,   __lowerCAmelCase				)

  # load HuggingFace model
  if vit_name[-5:] == "in21k":
    snake_case__						= ViTModel(__lowerCAmelCase				).eval()
  else:
    snake_case__						= ViTForImageClassification(__lowerCAmelCase				).eval()
  model.load_state_dict(__lowerCAmelCase				)

  # Check outputs on an image, prepared by ViTImageProcessor/DeiTImageProcessor
  if "deit" in vit_name:
    snake_case__						= DeiTImageProcessor(size=config.image_size				)
  else:
    snake_case__						= ViTImageProcessor(size=config.image_size				)
  snake_case__						= image_processor(images=prepare_img()							,   return_tensors='''pt'''				)
  snake_case__						= encoding['''pixel_values''']
  snake_case__						= model(__lowerCAmelCase				)

  if base_model:
    snake_case__						= timm_model.forward_features(__lowerCAmelCase				)
    assert timm_pooled_output.shape == outputs.pooler_output.shape
    assert torch.allclose(__lowerCAmelCase							,   outputs.pooler_output							,   atol=1e-3				)
  else:
    snake_case__						= timm_model(__lowerCAmelCase				)
    assert timm_logits.shape == outputs.logits.shape
    assert torch.allclose(__lowerCAmelCase							,   outputs.logits							,   atol=1e-3				)

  Path(__lowerCAmelCase				).mkdir(exist_ok=__lowerCAmelCase				)
  print(F"""Saving model {vit_name} to {pytorch_dump_folder_path}"""				)
  model.save_pretrained(__lowerCAmelCase				)
  print(F"""Saving image processor to {pytorch_dump_folder_path}"""				)
  image_processor.save_pretrained(__lowerCAmelCase				)


if __name__ == "__main__":
   lowerCamelCase__							:						Optional[Any]							     =   argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       """--vit_name""",
       default="""vit_base_patch16_224""",
       type=str,
       help="""Name of the ViT timm model you'd like to convert.""",
   )
   parser.add_argument(
       """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
   )

   lowerCamelCase__							:						str							     =   parser.parse_args()
   convert_vit_checkpoint(args.vit_name, args.pytorch_dump_folder_path)

