




import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ViTImageProcessor, ViTMSNConfig, ViTMSNModel
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD


torch.set_grad_enabled(False)

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase=False     ):
							SCREAMING_SNAKE_CASE_   	=    []
							for i in range(config.num_hidden_layers     ):
														# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
														rename_keys.append((F"module.blocks.{i}.norm1.weight", F"vit.encoder.layer.{i}.layernorm_before.weight")     )
														rename_keys.append((F"module.blocks.{i}.norm1.bias", F"vit.encoder.layer.{i}.layernorm_before.bias")     )
														rename_keys.append(
														    (F"module.blocks.{i}.attn.proj.weight", F"vit.encoder.layer.{i}.attention.output.dense.weight")     )
														rename_keys.append((F"module.blocks.{i}.attn.proj.bias", F"vit.encoder.layer.{i}.attention.output.dense.bias")     )
														rename_keys.append((F"module.blocks.{i}.norm2.weight", F"vit.encoder.layer.{i}.layernorm_after.weight")     )
														rename_keys.append((F"module.blocks.{i}.norm2.bias", F"vit.encoder.layer.{i}.layernorm_after.bias")     )
														rename_keys.append((F"module.blocks.{i}.mlp.fc1.weight", F"vit.encoder.layer.{i}.intermediate.dense.weight")     )
														rename_keys.append((F"module.blocks.{i}.mlp.fc1.bias", F"vit.encoder.layer.{i}.intermediate.dense.bias")     )
														rename_keys.append((F"module.blocks.{i}.mlp.fc2.weight", F"vit.encoder.layer.{i}.output.dense.weight")     )
														rename_keys.append((F"module.blocks.{i}.mlp.fc2.bias", F"vit.encoder.layer.{i}.output.dense.bias")     )

							# projection layer + position embeddings
							rename_keys.extend(
							    [
							        ('''module.cls_token''', '''vit.embeddings.cls_token'''),
							        ('''module.patch_embed.proj.weight''', '''vit.embeddings.patch_embeddings.projection.weight'''),
							        ('''module.patch_embed.proj.bias''', '''vit.embeddings.patch_embeddings.projection.bias'''),
							        ('''module.pos_embed''', '''vit.embeddings.position_embeddings'''),
							    ]     )

							if base_model:
														# layernorm + pooler
														rename_keys.extend(
														    [
														        ('''module.norm.weight''', '''layernorm.weight'''),
														        ('''module.norm.bias''', '''layernorm.bias'''),
														    ]     )

														# if just the base model, we should remove "vit" from all keys that start with "vit"
														SCREAMING_SNAKE_CASE_   	=    [(pair[0], pair[1][4:]) if pair[1].startswith('''vit'''     ) else pair for pair in rename_keys]
							else:
														# layernorm + classification head
														rename_keys.extend(
														    [
														        ('''norm.weight''', '''vit.layernorm.weight'''),
														        ('''norm.bias''', '''vit.layernorm.bias'''),
														        ('''head.weight''', '''classifier.weight'''),
														        ('''head.bias''', '''classifier.bias'''),
														    ]     )

							return rename_keys

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase=False     ):
							for i in range(config.num_hidden_layers     ):
														if base_model:
																					SCREAMING_SNAKE_CASE_   	=    ''''''
														else:
																					SCREAMING_SNAKE_CASE_   	=    '''vit.'''
														# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
														SCREAMING_SNAKE_CASE_   	=    state_dict.pop(F"module.blocks.{i}.attn.qkv.weight"     )
														SCREAMING_SNAKE_CASE_   	=    state_dict.pop(F"module.blocks.{i}.attn.qkv.bias"     )
														# next, add query, keys and values (in that order) to the state dict
														SCREAMING_SNAKE_CASE_   	=    in_proj_weight[
														    : config.hidden_size, :
														]
														SCREAMING_SNAKE_CASE_   	=    in_proj_bias[: config.hidden_size]
														SCREAMING_SNAKE_CASE_   	=    in_proj_weight[
														    config.hidden_size : config.hidden_size * 2, :
														]
														SCREAMING_SNAKE_CASE_   	=    in_proj_bias[
														    config.hidden_size : config.hidden_size * 2
														]
														SCREAMING_SNAKE_CASE_   	=    in_proj_weight[
														    -config.hidden_size :, :
														]
														SCREAMING_SNAKE_CASE_   	=    in_proj_bias[-config.hidden_size :]

def 			a   (_lowerCAmelCase     ):
							SCREAMING_SNAKE_CASE_   	=    ['''head.weight''', '''head.bias''']
							for k in ignore_keys:
														state_dict.pop(_lowerCAmelCase  ,    _lowerCAmelCase     )

def 			a   (_lowerCAmelCase     ):
							# projection head is used in the self-supervised pre-training in MSN,
							# for downstream task it's not needed.
							SCREAMING_SNAKE_CASE_   	=    [
							    '''module.fc.fc1.weight''',
							    '''module.fc.fc1.bias''',
							    '''module.fc.bn1.weight''',
							    '''module.fc.bn1.bias''',
							    '''module.fc.bn1.running_mean''',
							    '''module.fc.bn1.running_var''',
							    '''module.fc.bn1.num_batches_tracked''',
							    '''module.fc.fc2.weight''',
							    '''module.fc.fc2.bias''',
							    '''module.fc.bn2.weight''',
							    '''module.fc.bn2.bias''',
							    '''module.fc.bn2.running_mean''',
							    '''module.fc.bn2.running_var''',
							    '''module.fc.bn2.num_batches_tracked''',
							    '''module.fc.fc3.weight''',
							    '''module.fc.fc3.bias''',
							]
							for k in ignore_keys:
														state_dict.pop(_lowerCAmelCase  ,    _lowerCAmelCase     )

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase     ):
							SCREAMING_SNAKE_CASE_   	=    dct.pop(_lowerCAmelCase     )
							SCREAMING_SNAKE_CASE_   	=    val

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase     ):
							SCREAMING_SNAKE_CASE_   	=    ViTMSNConfig()
							SCREAMING_SNAKE_CASE_   	=    1_0_0_0

							SCREAMING_SNAKE_CASE_   	=    '''datasets/huggingface/label-files'''
							SCREAMING_SNAKE_CASE_   	=    '''imagenet-1k-id2label.json'''
							SCREAMING_SNAKE_CASE_   	=    json.load(open(hf_hub_download(_lowerCAmelCase  ,    _lowerCAmelCase     )  ,    '''r'''     )     )
							SCREAMING_SNAKE_CASE_   	=    {int(_lowerCAmelCase     ): v for k, v in idalabel.items()}
							SCREAMING_SNAKE_CASE_   	=    idalabel
							SCREAMING_SNAKE_CASE_   	=    {v: k for k, v in idalabel.items()}

							if "s16" in checkpoint_url:
														SCREAMING_SNAKE_CASE_   	=    3_8_4
														SCREAMING_SNAKE_CASE_   	=    1_5_3_6
														SCREAMING_SNAKE_CASE_   	=    6
							elif "l16" in checkpoint_url:
														SCREAMING_SNAKE_CASE_   	=    1_0_2_4
														SCREAMING_SNAKE_CASE_   	=    4_0_9_6
														SCREAMING_SNAKE_CASE_   	=    2_4
														SCREAMING_SNAKE_CASE_   	=    1_6
														SCREAMING_SNAKE_CASE_   	=    0.1
							elif "b4" in checkpoint_url:
														SCREAMING_SNAKE_CASE_   	=    4
							elif "l7" in checkpoint_url:
														SCREAMING_SNAKE_CASE_   	=    7
														SCREAMING_SNAKE_CASE_   	=    1_0_2_4
														SCREAMING_SNAKE_CASE_   	=    4_0_9_6
														SCREAMING_SNAKE_CASE_   	=    2_4
														SCREAMING_SNAKE_CASE_   	=    1_6
														SCREAMING_SNAKE_CASE_   	=    0.1

							SCREAMING_SNAKE_CASE_   	=    ViTMSNModel(_lowerCAmelCase     )

							SCREAMING_SNAKE_CASE_   	=    torch.hub.load_state_dict_from_url(_lowerCAmelCase  ,    map_location='''cpu'''     )['''target_encoder''']

							SCREAMING_SNAKE_CASE_   	=    ViTImageProcessor(size=config.image_size     )

							remove_projection_head(_lowerCAmelCase     )
							SCREAMING_SNAKE_CASE_   	=    create_rename_keys(_lowerCAmelCase  ,    base_model=_lowerCAmelCase     )

							for src, dest in rename_keys:
														rename_key(_lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase     )
							read_in_q_k_v(_lowerCAmelCase  ,    _lowerCAmelCase  ,    base_model=_lowerCAmelCase     )

							model.load_state_dict(_lowerCAmelCase     )
							model.eval()

							SCREAMING_SNAKE_CASE_   	=    '''http://images.cocodataset.org/val2017/000000039769.jpg'''

							SCREAMING_SNAKE_CASE_   	=    Image.open(requests.get(_lowerCAmelCase  ,    stream=_lowerCAmelCase     ).raw     )
							SCREAMING_SNAKE_CASE_   	=    ViTImageProcessor(
							    size=config.image_size  ,    image_mean=_lowerCAmelCase  ,    image_std=_lowerCAmelCase     )
							SCREAMING_SNAKE_CASE_   	=    image_processor(images=_lowerCAmelCase  ,    return_tensors='''pt'''     )

							# forward pass
							torch.manual_seed(2     )
							SCREAMING_SNAKE_CASE_   	=    model(**_lowerCAmelCase     )
							SCREAMING_SNAKE_CASE_   	=    outputs.last_hidden_state

							# The following Colab Notebook was used to generate these outputs:
							# https://colab.research.google.com/gist/sayakpaul/3672419a04f5997827503fd84079bdd1/scratchpad.ipynb
							if "s16" in checkpoint_url:
														SCREAMING_SNAKE_CASE_   	=    torch.tensor([[-1.0915, -1.4876, -1.1809]]     )
							elif "b16" in checkpoint_url:
														SCREAMING_SNAKE_CASE_   	=    torch.tensor([[14.2889, -18.9045, 11.7281]]     )
							elif "l16" in checkpoint_url:
														SCREAMING_SNAKE_CASE_   	=    torch.tensor([[41.5028, -22.8681, 45.6475]]     )
							elif "b4" in checkpoint_url:
														SCREAMING_SNAKE_CASE_   	=    torch.tensor([[-4.3868, 5.2932, -0.4137]]     )
							else:
														SCREAMING_SNAKE_CASE_   	=    torch.tensor([[-0.1792, -0.6465, 2.4263]]     )

							# verify logits
							assert torch.allclose(last_hidden_state[:, 0, :3]  ,    _lowerCAmelCase  ,    atol=1e-4     )

							print(F"Saving model to {pytorch_dump_folder_path}"     )
							model.save_pretrained(_lowerCAmelCase     )

							print(F"Saving image processor to {pytorch_dump_folder_path}"     )
							image_processor.save_pretrained(_lowerCAmelCase     )


if __name__ == "__main__":
			__SCREAMING_SNAKE_CASE 						=argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    """--checkpoint_url""",
			    default="""https://dl.fbaipublicfiles.com/msn/vits16_800ep.pth.tar""",
			    type=str,
			    help="""URL of the checkpoint you'd like to convert.""",
			)
			parser.add_argument(
			    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
			)

			__SCREAMING_SNAKE_CASE 						=parser.parse_args()
			convert_vit_msn_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)








import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
			import torch_xla.core.xla_model as xm
			import torch_xla.debug.metrics as met






class  __magic_name__						(  __UpperCAmelCase):
				'''simple docstring'''







				def __init__(							self:   List[Any]		,					*_lowerCamelCase:   Tuple		,					_lowerCamelCase:   Optional[Any]=None		,					_lowerCamelCase:   Dict=None		,					**_lowerCamelCase:   Dict		):
											super().__init__(*_lowerCamelCase		,					**_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    eval_examples
											SCREAMING_SNAKE_CASE_   	=    post_process_function


				def     _A							(							self:   Tuple		,					_lowerCamelCase:   Optional[Dataset] = None		,					_lowerCamelCase:   List[str]=None		,					_lowerCamelCase:   Optional[List[str]] = None		,					_lowerCamelCase:   str = "eval"		,					**_lowerCamelCase:   Union[str, Any]		,					):
											SCREAMING_SNAKE_CASE_   	=    gen_kwargs.copy()
											SCREAMING_SNAKE_CASE_   	=    (
											    gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length'''		) is not None else self.args.generation_max_length
											)
											SCREAMING_SNAKE_CASE_   	=    (
											    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''		) is not None else self.args.generation_num_beams
											)
											SCREAMING_SNAKE_CASE_   	=    gen_kwargs

											SCREAMING_SNAKE_CASE_   	=    self.eval_dataset if eval_dataset is None else eval_dataset
											SCREAMING_SNAKE_CASE_   	=    self.get_eval_dataloader(_lowerCamelCase		)
											SCREAMING_SNAKE_CASE_   	=    self.eval_examples if eval_examples is None else eval_examples

											# Temporarily disable metric computation, we will do it in the loop here.
											SCREAMING_SNAKE_CASE_   	=    self.compute_metrics
											SCREAMING_SNAKE_CASE_   	=    None
											SCREAMING_SNAKE_CASE_   	=    time.time()
											SCREAMING_SNAKE_CASE_   	=    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																		SCREAMING_SNAKE_CASE_   	=    eval_loop(
																		    _lowerCamelCase		,					description='''Evaluation'''		,					prediction_loss_only=True if compute_metrics is None else None		,					ignore_keys=_lowerCamelCase		,					metric_key_prefix=_lowerCamelCase		,					)
											finally:
																		SCREAMING_SNAKE_CASE_   	=    compute_metrics
											SCREAMING_SNAKE_CASE_   	=    self.args.eval_batch_size * self.args.world_size
											if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
																		start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
											output.metrics.update(
											    speed_metrics(
											        _lowerCamelCase		,					_lowerCamelCase		,					num_samples=output.num_samples		,					num_steps=math.ceil(output.num_samples / total_batch_size		)		,					)		)

											if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																		# Only the main node write the results by default
																		SCREAMING_SNAKE_CASE_   	=    self.post_process_function(_lowerCamelCase		,					_lowerCamelCase		,					_lowerCamelCase		)
																		SCREAMING_SNAKE_CASE_   	=    self.compute_metrics(_lowerCamelCase		)

																		# Prefix all keys with metric_key_prefix + '_'
																		for key in list(metrics.keys()		):
																									if not key.startswith(f"{metric_key_prefix}_"		):
																																SCREAMING_SNAKE_CASE_   	=    metrics.pop(_lowerCamelCase		)

																		metrics.update(output.metrics		)
											else:
																		SCREAMING_SNAKE_CASE_   	=    output.metrics

											if self.args.should_log:
																		# Only the main node log the results by default
																		self.log(_lowerCamelCase		)

											if self.args.tpu_metrics_debug or self.args.debug:
																		# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																		xm.master_print(met.metrics_report()		)

											SCREAMING_SNAKE_CASE_   	=    self.callback_handler.on_evaluate(self.args		,					self.state		,					self.control		,					_lowerCamelCase		)
											return metrics





				def     _A							(							self:   List[str]		,					_lowerCamelCase:   int		,					_lowerCamelCase:   List[str]		,					_lowerCamelCase:   Optional[int]=None		,					_lowerCamelCase:   str = "test"		,					**_lowerCamelCase:   str		):
											SCREAMING_SNAKE_CASE_   	=    gen_kwargs.copy()

											SCREAMING_SNAKE_CASE_   	=    self.get_test_dataloader(_lowerCamelCase		)

											# Temporarily disable metric computation, we will do it in the loop here.
											SCREAMING_SNAKE_CASE_   	=    self.compute_metrics
											SCREAMING_SNAKE_CASE_   	=    None
											SCREAMING_SNAKE_CASE_   	=    time.time()
											SCREAMING_SNAKE_CASE_   	=    self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																		SCREAMING_SNAKE_CASE_   	=    eval_loop(
																		    _lowerCamelCase		,					description='''Prediction'''		,					prediction_loss_only=True if compute_metrics is None else None		,					ignore_keys=_lowerCamelCase		,					metric_key_prefix=_lowerCamelCase		,					)
											finally:
																		SCREAMING_SNAKE_CASE_   	=    compute_metrics

											SCREAMING_SNAKE_CASE_   	=    self.args.eval_batch_size * self.args.world_size
											if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
																		start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
											output.metrics.update(
											    speed_metrics(
											        _lowerCamelCase		,					_lowerCamelCase		,					num_samples=output.num_samples		,					num_steps=math.ceil(output.num_samples / total_batch_size		)		,					)		)
											if self.post_process_function is None or self.compute_metrics is None:
																		return output

											SCREAMING_SNAKE_CASE_   	=    self.post_process_function(_lowerCamelCase		,					_lowerCamelCase		,					_lowerCamelCase		,					'''predict'''		)
											SCREAMING_SNAKE_CASE_   	=    self.compute_metrics(_lowerCamelCase		)

											# Prefix all keys with metric_key_prefix + '_'
											for key in list(metrics.keys()		):
																		if not key.startswith(f"{metric_key_prefix}_"		):
																									SCREAMING_SNAKE_CASE_   	=    metrics.pop(_lowerCamelCase		)
											metrics.update(output.metrics		)
											return PredictionOutput(predictions=predictions.predictions		,					label_ids=predictions.label_ids		,					metrics=_lowerCamelCase		)


