






from typing import List, Optional, Union

import numpy as np
import tensorflow as tf

from .utils import logging


A        =			logging.get_logger(__name__)



def 					_lowerCamelCase(			lowerCAmelCase__				:		str      ):





				'''simple docstring'''







				if isinstance(lowerCamelCase__    ,    np.ndarray      ):
								return list(tensor.shape      )

				SCREAMING_SNAKE_CASE_   :	Dict													=   tf.shape(lowerCamelCase__      )

				if tensor.shape == tf.TensorShape(lowerCamelCase__      ):
								return dynamic

				SCREAMING_SNAKE_CASE_   :	List[Any]													=   tensor.shape.as_list()

				return [dynamic[i] if s is None else s for i, s in enumerate(lowerCamelCase__      )]



def 					_lowerCamelCase(			lowerCAmelCase__				:		List[Any]    ,    lowerCAmelCase__				:		List[str] = None    ,    lowerCAmelCase__				:		Optional[int] = None      ):





				'''simple docstring'''







				return tf.nn.softmax(logits=logits + 1E-9    ,    axis=lowerCamelCase__    ,    name=lowerCamelCase__      )



def 					_lowerCamelCase(			lowerCAmelCase__				:		int    ,    lowerCAmelCase__				:		Union[str, Any]    ,    lowerCAmelCase__				:		Union[str, Any]    ,    lowerCAmelCase__				:		Union[str, Any]=1E-5    ,    lowerCAmelCase__				:		Optional[int]=-1      ):





				'''simple docstring'''







				if weight.shape.rank != 1 or bias.shape.rank != 1 or not isinstance(lowerCamelCase__    ,    lowerCamelCase__      ):
								raise NotImplementedError('Only 1D weight and bias tensors are supported for now, with only a single axis.'      )

				# Get mean and variance on the axis to be normalized
				SCREAMING_SNAKE_CASE_   :	Any													=   tf.nn.moments(lowerCamelCase__    ,    axes=[axis]    ,    keepdims=lowerCamelCase__      )

				if axis != -1:
								# Reshape scale and weight to have the same rank as inputs, but with 1 dimensions
								# on every dimension except axis
								SCREAMING_SNAKE_CASE_   :	List[str]													=   [1] * inputs.shape.rank
								SCREAMING_SNAKE_CASE_   :	Dict													=   shape_list(lowerCamelCase__      )[axis]
								SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   tf.reshape(lowerCamelCase__    ,    lowerCamelCase__      )
								SCREAMING_SNAKE_CASE_   :	Optional[int]													=   tf.reshape(lowerCamelCase__    ,    lowerCamelCase__      )

				# Compute layer normalization using the batch_normalization
				# function.
				SCREAMING_SNAKE_CASE_   :	List[str]													=   tf.nn.batch_normalization(
				    lowerCamelCase__    ,    lowerCamelCase__    ,    lowerCamelCase__    ,    offset=lowerCamelCase__    ,    scale=lowerCamelCase__    ,    variance_epsilon=lowerCamelCase__    ,    )
				return outputs



def 					_lowerCamelCase(			lowerCAmelCase__				:		Union[str, Any]    ,    lowerCAmelCase__				:		List[str]=0    ,    lowerCAmelCase__				:		Union[str, Any]=-1      ):





				'''simple docstring'''







				if end_dim < 0:
								end_dim += input.shape.rank
				if start_dim < 0:
								start_dim += input.shape.rank

				if start_dim == end_dim:
								return input

				SCREAMING_SNAKE_CASE_   :	Optional[int]													=   tf.shape(lowerCamelCase__      )
				SCREAMING_SNAKE_CASE_   :	Tuple													=   tf.math.reduce_prod(in_shape[start_dim : end_dim + 1]      )
				SCREAMING_SNAKE_CASE_   :	Optional[int]													=   tf.concat([in_shape[:start_dim], [flattened_dim], in_shape[end_dim + 1 :]]    ,    axis=0      )
				return tf.reshape(lowerCamelCase__    ,    lowerCamelCase__      )



def 					_lowerCamelCase(			lowerCAmelCase__				:		int      ):





				'''simple docstring'''







				if not isinstance(lowerCamelCase__    ,    tf.Tensor      ):
								SCREAMING_SNAKE_CASE_   :	Optional[int]													=   tf.convert_to_tensor(lowerCamelCase__      )  # Catches stray NumPy inputs
				if encoder_attention_mask.shape.rank == 3:
								SCREAMING_SNAKE_CASE_   :	List[str]													=   encoder_attention_mask[:, None, :, :]
				if encoder_attention_mask.shape.rank == 2:
								SCREAMING_SNAKE_CASE_   :	str													=   encoder_attention_mask[:, None, None, :]
				# T5 has a mask that can compare sequence ids, we can simulate this here with this transposition
				# Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow
				# /transformer/transformer_layers.py#L270
				# encoder_extended_attention_mask = (encoder_extended_attention_mask ==
				# encoder_extended_attention_mask.transpose(-1, -2))
				SCREAMING_SNAKE_CASE_   :	Optional[int]													=   (
				    tf.cast(1    ,    encoder_attention_mask.dtype      ) - encoder_extended_attention_mask
				) * encoder_extended_attention_mask.dtype.min

				return encoder_extended_attention_mask



def 					_lowerCamelCase(			lowerCAmelCase__				:		List[Any]    ,    lowerCAmelCase__				:		Dict    ,    lowerCAmelCase__				:		List[Any] = "input_ids"      ):





				'''simple docstring'''







				tf.debugging.assert_less(
				    lowerCamelCase__    ,    tf.cast(lowerCamelCase__    ,    dtype=tensor.dtype      )    ,    message=(
				        F'''The maximum value of {tensor_name} ({tf.math.reduce_max(lowerCamelCase__      )}) must be smaller than the embedding '''
				        F'''layer\'s input dimension ({embed_dim}). The likely cause is some problem at tokenization time.'''
				    )    ,    )



def 					_lowerCamelCase(			lowerCAmelCase__				:		str    ,    lowerCAmelCase__				:		List[Any]    ,    lowerCAmelCase__				:		List[str]      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_   :	List[Any]													=   6_4512
				# Check that no item in `data` is larger than `HDF5_OBJECT_HEADER_LIMIT`
				# because in that case even chunking the array would not make the saving
				# possible.
				SCREAMING_SNAKE_CASE_   :	int													=   [x for x in data if len(lowerCamelCase__      ) > HDF5_OBJECT_HEADER_LIMIT]

				# Expecting this to never be true.
				if bad_attributes:
								raise RuntimeError(
								    'The following attributes cannot be saved to HDF5 file because '
								    F'''they are larger than {HDF5_OBJECT_HEADER_LIMIT} '''
								    F'''bytes: {bad_attributes}'''      )

				SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   np.asarray(lowerCamelCase__      )

				SCREAMING_SNAKE_CASE_   :	Tuple													=   1
				SCREAMING_SNAKE_CASE_   :	Optional[int]													=   np.array_split(lowerCamelCase__    ,    lowerCamelCase__      )

				# This will never loop forever thanks to the test above.
				while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data      ):
								num_chunks += 1
								SCREAMING_SNAKE_CASE_   :	List[Any]													=   np.array_split(lowerCamelCase__    ,    lowerCamelCase__      )

				if num_chunks > 1:
								for chunk_id, chunk_data in enumerate(lowerCamelCase__      ):
												SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   chunk_data
				else:
								SCREAMING_SNAKE_CASE_   :	List[Any]													=   data



def 					_lowerCamelCase(			lowerCAmelCase__				:		Dict    ,    lowerCAmelCase__				:		Dict      ):





				'''simple docstring'''







				if name in group.attrs:
								SCREAMING_SNAKE_CASE_   :	Optional[int]													=   [n.decode('utf8'      ) if hasattr(lowerCamelCase__    ,    'decode'      ) else n for n in group.attrs[name]]
				else:
								SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   []
								SCREAMING_SNAKE_CASE_   :	str													=   0
								while "%s%d" % (name, chunk_id) in group.attrs:
												data.extend(
												    [n.decode('utf8'      ) if hasattr(lowerCamelCase__    ,    'decode'      ) else n for n in group.attrs['%s%d' % (name, chunk_id)]]      )
												chunk_id += 1
				return data



def 					_lowerCamelCase(			lowerCAmelCase__				:		Optional[int]      ):





				'''simple docstring'''







				def _expand_single_ad_tensor(lowerCAmelCase__				:		int      ):
								if isinstance(lowerCamelCase__    ,    tf.Tensor      ) and t.shape.rank == 1:
												return tf.expand_dims(lowerCamelCase__    ,    axis=-1      )
								return t

				return tf.nest.map_structure(_expand_single_ad_tensor    ,    lowerCamelCase__      )







import argparse

import requests
import torch
from PIL import Image

from transformers import ViTMAEConfig, ViTMAEForPreTraining, ViTMAEImageProcessor



def 					_lowerCamelCase(			lowerCAmelCase__				:		Optional[Any]      ):





				'''simple docstring'''







				if "cls_token" in name:
								SCREAMING_SNAKE_CASE_   :	Optional[int]													=   name.replace('cls_token'    ,    'vit.embeddings.cls_token'      )
				if "mask_token" in name:
								SCREAMING_SNAKE_CASE_   :	Optional[int]													=   name.replace('mask_token'    ,    'decoder.mask_token'      )
				if "decoder_pos_embed" in name:
								SCREAMING_SNAKE_CASE_   :	Any													=   name.replace('decoder_pos_embed'    ,    'decoder.decoder_pos_embed'      )
				if "pos_embed" in name and "decoder" not in name:
								SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   name.replace('pos_embed'    ,    'vit.embeddings.position_embeddings'      )
				if "patch_embed.proj" in name:
								SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   name.replace('patch_embed.proj'    ,    'vit.embeddings.patch_embeddings.projection'      )
				if "patch_embed.norm" in name:
								SCREAMING_SNAKE_CASE_   :	Tuple													=   name.replace('patch_embed.norm'    ,    'vit.embeddings.norm'      )
				if "decoder_blocks" in name:
								SCREAMING_SNAKE_CASE_   :	Tuple													=   name.replace('decoder_blocks'    ,    'decoder.decoder_layers'      )
				if "blocks" in name:
								SCREAMING_SNAKE_CASE_   :	Any													=   name.replace('blocks'    ,    'vit.encoder.layer'      )
				if "attn.proj" in name:
								SCREAMING_SNAKE_CASE_   :	Tuple													=   name.replace('attn.proj'    ,    'attention.output.dense'      )
				if "attn" in name:
								SCREAMING_SNAKE_CASE_   :	Any													=   name.replace('attn'    ,    'attention.self'      )
				if "norm1" in name:
								SCREAMING_SNAKE_CASE_   :	List[Any]													=   name.replace('norm1'    ,    'layernorm_before'      )
				if "norm2" in name:
								SCREAMING_SNAKE_CASE_   :	Any													=   name.replace('norm2'    ,    'layernorm_after'      )
				if "mlp.fc1" in name:
								SCREAMING_SNAKE_CASE_   :	Dict													=   name.replace('mlp.fc1'    ,    'intermediate.dense'      )
				if "mlp.fc2" in name:
								SCREAMING_SNAKE_CASE_   :	Tuple													=   name.replace('mlp.fc2'    ,    'output.dense'      )
				if "decoder_embed" in name:
								SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   name.replace('decoder_embed'    ,    'decoder.decoder_embed'      )
				if "decoder_norm" in name:
								SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   name.replace('decoder_norm'    ,    'decoder.decoder_norm'      )
				if "decoder_pred" in name:
								SCREAMING_SNAKE_CASE_   :	str													=   name.replace('decoder_pred'    ,    'decoder.decoder_pred'      )
				if "norm.weight" in name and "decoder" not in name:
								SCREAMING_SNAKE_CASE_   :	Tuple													=   name.replace('norm.weight'    ,    'vit.layernorm.weight'      )
				if "norm.bias" in name and "decoder" not in name:
								SCREAMING_SNAKE_CASE_   :	List[str]													=   name.replace('norm.bias'    ,    'vit.layernorm.bias'      )

				return name



def 					_lowerCamelCase(			lowerCAmelCase__				:		List[str]    ,    lowerCAmelCase__				:		Optional[int]      ):





				'''simple docstring'''







				for key in orig_state_dict.copy().keys():
								SCREAMING_SNAKE_CASE_   :	Tuple													=   orig_state_dict.pop(lowerCAmelCase__      )

								if "qkv" in key:
												SCREAMING_SNAKE_CASE_   :	Optional[int]													=   key.split('.'      )
												SCREAMING_SNAKE_CASE_   :	List[Any]													=   int(key_split[1]      )
												if "decoder_blocks" in key:
																SCREAMING_SNAKE_CASE_   :	Any													=   config.decoder_hidden_size
																SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   'decoder.decoder_layers.'
																if "weight" in key:
																				SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   val[:dim, :]
																				SCREAMING_SNAKE_CASE_   :	List[str]													=   val[dim : dim * 2, :]
																				SCREAMING_SNAKE_CASE_   :	Any													=   val[-dim:, :]
																elif "bias" in key:
																				SCREAMING_SNAKE_CASE_   :	str													=   val[:dim]
																				SCREAMING_SNAKE_CASE_   :	List[Any]													=   val[dim : dim * 2]
																				SCREAMING_SNAKE_CASE_   :	List[Any]													=   val[-dim:]
												else:
																SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   config.hidden_size
																SCREAMING_SNAKE_CASE_   :	str													=   'vit.encoder.layer.'
																if "weight" in key:
																				SCREAMING_SNAKE_CASE_   :	Any													=   val[:dim, :]
																				SCREAMING_SNAKE_CASE_   :	Tuple													=   val[dim : dim * 2, :]
																				SCREAMING_SNAKE_CASE_   :	int													=   val[-dim:, :]
																elif "bias" in key:
																				SCREAMING_SNAKE_CASE_   :	List[Any]													=   val[:dim]
																				SCREAMING_SNAKE_CASE_   :	Dict													=   val[dim : dim * 2]
																				SCREAMING_SNAKE_CASE_   :	str													=   val[-dim:]

								else:
												SCREAMING_SNAKE_CASE_   :	Optional[int]													=   val

				return orig_state_dict



def 					_lowerCamelCase(			lowerCAmelCase__				:		Optional[Any]    ,    lowerCAmelCase__				:		List[str]      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_   :	Dict													=   ViTMAEConfig()
				if "large" in checkpoint_url:
								SCREAMING_SNAKE_CASE_   :	Any													=   1024
								SCREAMING_SNAKE_CASE_   :	Any													=   4096
								SCREAMING_SNAKE_CASE_   :	List[str]													=   24
								SCREAMING_SNAKE_CASE_   :	List[str]													=   16
				elif "huge" in checkpoint_url:
								SCREAMING_SNAKE_CASE_   :	str													=   14
								SCREAMING_SNAKE_CASE_   :	List[str]													=   1280
								SCREAMING_SNAKE_CASE_   :	int													=   5120
								SCREAMING_SNAKE_CASE_   :	Optional[int]													=   32
								SCREAMING_SNAKE_CASE_   :	List[str]													=   16

				SCREAMING_SNAKE_CASE_   :	Optional[int]													=   ViTMAEForPreTraining(lowerCAmelCase__      )

				SCREAMING_SNAKE_CASE_   :	List[str]													=   torch.hub.load_state_dict_from_url(lowerCAmelCase__    ,    map_location='cpu'      )['model']

				SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   ViTMAEImageProcessor(size=config.image_size      )

				SCREAMING_SNAKE_CASE_   :	int													=   convert_state_dict(lowerCAmelCase__    ,    lowerCAmelCase__      )

				model.load_state_dict(lowerCAmelCase__      )
				model.eval()

				SCREAMING_SNAKE_CASE_   :	Any													=   'https://user-images.githubusercontent.com/11435359/147738734-196fd92f-9260-48d5-ba7e-bf103d29364d.jpg'

				SCREAMING_SNAKE_CASE_   :	Dict													=   Image.open(requests.get(lowerCAmelCase__    ,    stream=lowerCAmelCase__      ).raw      )
				SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   ViTMAEImageProcessor(size=config.image_size      )
				SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   image_processor(images=lowerCAmelCase__    ,    return_tensors='pt'      )

				# forward pass
				torch.manual_seed(2      )
				SCREAMING_SNAKE_CASE_   :	Any													=   model(**lowerCAmelCase__      )
				SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   outputs.logits

				if "large" in checkpoint_url:
								SCREAMING_SNAKE_CASE_   :	Any													=   torch.tensor(
								    [[-0.7_309, -0.7_128, -1.0_169], [-1.0_161, -0.9_058, -1.1_878], [-1.0_478, -0.9_411, -1.1_911]]      )
				elif "huge" in checkpoint_url:
								SCREAMING_SNAKE_CASE_   :	Dict													=   torch.tensor(
								    [[-1.1_599, -0.9_199, -1.2_221], [-1.1_952, -0.9_269, -1.2_307], [-1.2_143, -0.9_337, -1.2_262]]      )
				else:
								SCREAMING_SNAKE_CASE_   :	int													=   torch.tensor(
								    [[-0.9_192, -0.8_481, -1.1_259], [-1.1_349, -1.0_034, -1.2_599], [-1.1_757, -1.0_429, -1.2_726]]      )

				# verify logits
				assert torch.allclose(logits[0, :3, :3]    ,    lowerCAmelCase__    ,    atol=1E-4      )

				print(F'''Saving model to {pytorch_dump_folder_path}'''      )
				model.save_pretrained(lowerCAmelCase__      )

				print(F'''Saving image processor to {pytorch_dump_folder_path}'''      )
				image_processor.save_pretrained(lowerCAmelCase__      )


if __name__ == "__main__":
				A        =			argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    '--checkpoint_url',
				    default='https://dl.fbaipublicfiles.com/mae/visualize/mae_visualize_vit_base.pth',
				    type=str,
				    help='URL of the checkpoint you\'d like to convert.',
				)
				parser.add_argument(
				    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
				)

				A        =			parser.parse_args()
				convert_vit_mae_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)