






import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
SCREAMING_SNAKE_CASE__   :      List[str]				=		logging.get_logger(__name__)

def   __lowercase       (			snake_case,	snake_case=False,	snake_case=False,	snake_case=False			):







				"""simple docstring"""






				__magic_name__		:List[str]   =   []
				for i in range(config.num_hidden_layers			):
								# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
								rename_keys.append((f'''transformer.blocks.{i}.norm1.weight''', f'''vilt.encoder.layer.{i}.layernorm_before.weight''')			)
								rename_keys.append((f'''transformer.blocks.{i}.norm1.bias''', f'''vilt.encoder.layer.{i}.layernorm_before.bias''')			)
								rename_keys.append(
								    (f'''transformer.blocks.{i}.attn.proj.weight''', f'''vilt.encoder.layer.{i}.attention.output.dense.weight''')			)
								rename_keys.append(
								    (f'''transformer.blocks.{i}.attn.proj.bias''', f'''vilt.encoder.layer.{i}.attention.output.dense.bias''')			)
								rename_keys.append((f'''transformer.blocks.{i}.norm2.weight''', f'''vilt.encoder.layer.{i}.layernorm_after.weight''')			)
								rename_keys.append((f'''transformer.blocks.{i}.norm2.bias''', f'''vilt.encoder.layer.{i}.layernorm_after.bias''')			)
								rename_keys.append(
								    (f'''transformer.blocks.{i}.mlp.fc1.weight''', f'''vilt.encoder.layer.{i}.intermediate.dense.weight''')			)
								rename_keys.append((f'''transformer.blocks.{i}.mlp.fc1.bias''', f'''vilt.encoder.layer.{i}.intermediate.dense.bias''')			)
								rename_keys.append((f'''transformer.blocks.{i}.mlp.fc2.weight''', f'''vilt.encoder.layer.{i}.output.dense.weight''')			)
								rename_keys.append((f'''transformer.blocks.{i}.mlp.fc2.bias''', f'''vilt.encoder.layer.{i}.output.dense.bias''')			)

				# embeddings
				rename_keys.extend(
				    [
				        # text embeddings
				        ('''text_embeddings.word_embeddings.weight''', '''vilt.embeddings.text_embeddings.word_embeddings.weight'''),
				        (
				            '''text_embeddings.position_embeddings.weight''',
				            '''vilt.embeddings.text_embeddings.position_embeddings.weight''',
				        ),
				        ('''text_embeddings.position_ids''', '''vilt.embeddings.text_embeddings.position_ids'''),
				        (
				            '''text_embeddings.token_type_embeddings.weight''',
				            '''vilt.embeddings.text_embeddings.token_type_embeddings.weight''',
				        ),
				        ('''text_embeddings.LayerNorm.weight''', '''vilt.embeddings.text_embeddings.LayerNorm.weight'''),
				        ('''text_embeddings.LayerNorm.bias''', '''vilt.embeddings.text_embeddings.LayerNorm.bias'''),
				        # patch embeddings
				        ('''transformer.cls_token''', '''vilt.embeddings.cls_token'''),
				        ('''transformer.patch_embed.proj.weight''', '''vilt.embeddings.patch_embeddings.projection.weight'''),
				        ('''transformer.patch_embed.proj.bias''', '''vilt.embeddings.patch_embeddings.projection.bias'''),
				        ('''transformer.pos_embed''', '''vilt.embeddings.position_embeddings'''),
				        # token type embeddings
				        ('''token_type_embeddings.weight''', '''vilt.embeddings.token_type_embeddings.weight'''),
				    ]			)

				# final layernorm + pooler
				rename_keys.extend(
				    [
				        ('''transformer.norm.weight''', '''vilt.layernorm.weight'''),
				        ('''transformer.norm.bias''', '''vilt.layernorm.bias'''),
				        ('''pooler.dense.weight''', '''vilt.pooler.dense.weight'''),
				        ('''pooler.dense.bias''', '''vilt.pooler.dense.bias'''),
				    ]			)

				# classifier head(s)
				if vqa_model:
								# classification head
								rename_keys.extend(
								    [
								        ('''vqa_classifier.0.weight''', '''classifier.0.weight'''),
								        ('''vqa_classifier.0.bias''', '''classifier.0.bias'''),
								        ('''vqa_classifier.1.weight''', '''classifier.1.weight'''),
								        ('''vqa_classifier.1.bias''', '''classifier.1.bias'''),
								        ('''vqa_classifier.3.weight''', '''classifier.3.weight'''),
								        ('''vqa_classifier.3.bias''', '''classifier.3.bias'''),
								    ]			)
				elif nlvr_model:
								# classification head
								rename_keys.extend(
								    [
								        ('''nlvr2_classifier.0.weight''', '''classifier.0.weight'''),
								        ('''nlvr2_classifier.0.bias''', '''classifier.0.bias'''),
								        ('''nlvr2_classifier.1.weight''', '''classifier.1.weight'''),
								        ('''nlvr2_classifier.1.bias''', '''classifier.1.bias'''),
								        ('''nlvr2_classifier.3.weight''', '''classifier.3.weight'''),
								        ('''nlvr2_classifier.3.bias''', '''classifier.3.bias'''),
								    ]			)
				else:
								pass

				return rename_keys

def   __lowercase       (			snake_case,	snake_case			):







				"""simple docstring"""






				for i in range(config.num_hidden_layers			):
								__magic_name__		:int   =   '''vilt.'''
								# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
								__magic_name__		:Tuple   =   state_dict.pop(f'''transformer.blocks.{i}.attn.qkv.weight'''			)
								__magic_name__		:Dict   =   state_dict.pop(f'''transformer.blocks.{i}.attn.qkv.bias'''			)
								# next, add query, keys and values (in that order) to the state dict
								__magic_name__		:Union[str, Any]   =   in_proj_weight[
								    : config.hidden_size, :
								]
								__magic_name__		:Any   =   in_proj_bias[: config.hidden_size]
								__magic_name__		:Optional[int]   =   in_proj_weight[
								    config.hidden_size : config.hidden_size * 2, :
								]
								__magic_name__		:int   =   in_proj_bias[
								    config.hidden_size : config.hidden_size * 2
								]
								__magic_name__		:Optional[int]   =   in_proj_weight[
								    -config.hidden_size :, :
								]
								__magic_name__		:Dict   =   in_proj_bias[-config.hidden_size :]

def   __lowercase       (			snake_case			):







				"""simple docstring"""






				__magic_name__		:str   =   ['''head.weight''', '''head.bias''']
				for k in ignore_keys:
								state_dict.pop(snake_case,	snake_case			)

def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:Optional[Any]   =   dct.pop(snake_case			)
				__magic_name__		:Union[str, Any]   =   val

@torch.no_grad()
def   __lowercase       (			snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:Optional[int]   =   ViltConfig(image_size=3_8_4,	patch_size=3_2,	tie_word_embeddings=snake_case			)
				__magic_name__		:List[str]   =   False
				__magic_name__		:List[str]   =   False
				__magic_name__		:List[Any]   =   False
				__magic_name__		:Union[str, Any]   =   False
				if "vqa" in checkpoint_url:
								__magic_name__		:Union[str, Any]   =   True
								__magic_name__		:List[str]   =   3_1_2_9
								__magic_name__		:List[Any]   =   '''huggingface/label-files'''
								__magic_name__		:List[str]   =   '''vqa2-id2label.json'''
								__magic_name__		:Optional[Any]   =   json.load(open(hf_hub_download(snake_case,	snake_case,	repo_type='''dataset'''			),	'''r'''			)			)
								__magic_name__		:List[Any]   =   {int(snake_case			): v for k, v in idalabel.items()}
								__magic_name__		:Dict   =   idalabel
								__magic_name__		:Tuple   =   {v: k for k, v in idalabel.items()}
								__magic_name__		:List[str]   =   ViltForQuestionAnswering(snake_case			)
				elif "nlvr" in checkpoint_url:
								__magic_name__		:Optional[Any]   =   True
								__magic_name__		:List[Any]   =   2
								__magic_name__		:Tuple   =   {0: '''False''', 1: '''True'''}
								__magic_name__		:List[Any]   =   {v: k for k, v in config.idalabel.items()}
								__magic_name__		:Any   =   3
								__magic_name__		:Optional[int]   =   ViltForImagesAndTextClassification(snake_case			)
				elif "irtr" in checkpoint_url:
								__magic_name__		:List[str]   =   True
								__magic_name__		:Tuple   =   ViltForImageAndTextRetrieval(snake_case			)
				elif "mlm_itm" in checkpoint_url:
								__magic_name__		:Dict   =   True
								__magic_name__		:List[Any]   =   ViltForMaskedLM(snake_case			)
				else:
								raise ValueError('''Unknown model type'''			)

				# load state_dict of original model, remove and rename some keys
				__magic_name__		:Optional[Any]   =   torch.hub.load_state_dict_from_url(snake_case,	map_location='''cpu'''			)['''state_dict''']
				__magic_name__		:str   =   create_rename_keys(snake_case,	snake_case,	snake_case,	snake_case			)
				for src, dest in rename_keys:
								rename_key(snake_case,	snake_case,	snake_case			)
				read_in_q_k_v(snake_case,	snake_case			)
				if mlm_model or irtr_model:
								__magic_name__		:int   =   ['''itm_score.fc.weight''', '''itm_score.fc.bias''']
								for k in ignore_keys:
												state_dict.pop(snake_case,	snake_case			)

    # load state dict into HuggingFace model
				model.eval()
				if mlm_model:
								__magic_name__  ,					__magic_name__		:Union[str, Any]   =   model.load_state_dict(snake_case,	strict=snake_case			)
								assert missing_keys == ["mlm_score.decoder.bias"]
				else:
								model.load_state_dict(snake_case			)

				# Define processor
				__magic_name__		:Optional[int]   =   ViltImageProcessor(size=3_8_4			)
				__magic_name__		:Tuple   =   BertTokenizer.from_pretrained('''bert-base-uncased'''			)
				__magic_name__		:Any   =   ViltProcessor(snake_case,	snake_case			)

				# Forward pass on example inputs (image + text)
				if nlvr_model:
								__magic_name__		:Dict   =   Image.open(requests.get('''https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg''',	stream=snake_case			).raw			)
								__magic_name__		:Tuple   =   Image.open(requests.get('''https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg''',	stream=snake_case			).raw			)
								__magic_name__		:Union[str, Any]   =   (
								    '''The left image contains twice the number of dogs as the right image, and at least two dogs in total are'''
								    ''' standing.'''
								)
								__magic_name__		:str   =   processor(snake_case,	snake_case,	return_tensors='''pt'''			)
								__magic_name__		:str   =   processor(snake_case,	snake_case,	return_tensors='''pt'''			)
								__magic_name__		:Union[str, Any]   =   model(
								    input_ids=encoding_a.input_ids,	pixel_values=encoding_a.pixel_values,	pixel_values_a=encoding_a.pixel_values,	)
				else:
								__magic_name__		:Dict   =   Image.open(requests.get('''http://images.cocodataset.org/val2017/000000039769.jpg''',	stream=snake_case			).raw			)
								if mlm_model:
												__magic_name__		:int   =   '''a bunch of [MASK] laying on a [MASK].'''
								else:
												__magic_name__		:Union[str, Any]   =   '''How many cats are there?'''
								__magic_name__		:Any   =   processor(snake_case,	snake_case,	return_tensors='''pt'''			)
								__magic_name__		:List[str]   =   model(**snake_case			)

				# Verify outputs
				if mlm_model:
								__magic_name__		:Any   =   torch.Size([1, 1_1, 3_0_5_2_2]			)
								__magic_name__		:Any   =   torch.tensor([-12.5061, -12.5123, -12.5174]			)
								assert outputs.logits.shape == expected_shape
								assert torch.allclose(outputs.logits[0, 0, :3],	snake_case,	atol=1E-4			)

								# verify masked token prediction equals "cats"
								__magic_name__		:List[str]   =   outputs.logits[0, 4, :].argmax(-1			).item()
								assert tokenizer.decode([predicted_id]			) == "cats"
				elif vqa_model:
								__magic_name__		:Union[str, Any]   =   torch.Size([1, 3_1_2_9]			)
								__magic_name__		:Union[str, Any]   =   torch.tensor([-15.9495, -18.1472, -10.3041]			)
								assert torch.allclose(outputs.logits[0, :3],	snake_case,	atol=1E-4			)
								assert outputs.logits.shape == expected_shape
								assert torch.allclose(outputs.logits[0, 0, :3],	snake_case,	atol=1E-4			)

								# verify vqa prediction equals "2"
								__magic_name__		:Tuple   =   outputs.logits.argmax(-1			).item()
								assert model.config.idalabel[predicted_idx] == "2"
				elif nlvr_model:
								__magic_name__		:List[Any]   =   torch.Size([1, 2]			)
								__magic_name__		:Optional[Any]   =   torch.tensor([-2.8721, 2.1291]			)
								assert torch.allclose(outputs.logits[0, :3],	snake_case,	atol=1E-4			)
								assert outputs.logits.shape == expected_shape

				Path(snake_case			).mkdir(exist_ok=snake_case			)
				print(f'''Saving model and processor to {pytorch_dump_folder_path}'''			)
				model.save_pretrained(snake_case			)
				processor.save_pretrained(snake_case			)


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE__   :      Union[str, Any]				=		argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--checkpoint_url""",
					    default="""https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt""",
					    type=str,
					    help="""URL of the checkpoint you'd like to convert.""",
					)
					parser.add_argument(
					    """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
					)

					SCREAMING_SNAKE_CASE__   :      List[str]				=		parser.parse_args()
					convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)














import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
class     lowerCamelCase_					(   lowerCamelCase			):
				a__       							=	['''image_processor''', '''tokenizer''']
				a__       							=	'''ChineseCLIPImageProcessor'''
				a__       							=	('''BertTokenizer''', '''BertTokenizerFast''')







				def __init__(    self							,       __lowerCAmelCase=None							,       __lowerCAmelCase=None							,       **__lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:Tuple   =   None
								if "feature_extractor" in kwargs:
												warnings.warn(
												    '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
												    ''' instead.'''							,       __lowerCAmelCase							,       )
												__magic_name__		:Optional[Any]   =   kwargs.pop('''feature_extractor'''  )

								__magic_name__		:Tuple   =   image_processor if image_processor is not None else feature_extractor
								if image_processor is None:
												raise ValueError('''You need to specify an `image_processor`.'''  )
								if tokenizer is None:
												raise ValueError('''You need to specify a `tokenizer`.'''  )

								super().__init__(__lowerCAmelCase							,       __lowerCAmelCase  )
								__magic_name__		:List[Any]   =   self.image_processor







				def __call__(    self							,       __lowerCAmelCase=None							,       __lowerCAmelCase=None							,       __lowerCAmelCase=None							,       **__lowerCAmelCase  ):
								"""simple docstring"""

								if text is None and images is None:
												raise ValueError('''You have to specify either text or images. Both cannot be none.'''  )

								if text is not None:
												__magic_name__		:int   =   self.tokenizer(__lowerCAmelCase							,       return_tensors=__lowerCAmelCase							,       **__lowerCAmelCase  )

								if images is not None:
												__magic_name__		:Dict   =   self.image_processor(__lowerCAmelCase							,       return_tensors=__lowerCAmelCase							,       **__lowerCAmelCase  )

								if text is not None and images is not None:
												__magic_name__		:Union[str, Any]   =   image_features.pixel_values
												return encoding
								elif text is not None:
												return encoding
								else:
												return BatchEncoding(data=dict(**__lowerCAmelCase  )							,       tensor_type=__lowerCAmelCase  )







				def       A						(    self							,       *__lowerCAmelCase							,       **__lowerCAmelCase  ):
								"""simple docstring"""
								return self.tokenizer.batch_decode(*__lowerCAmelCase							,       **__lowerCAmelCase  )







				def       A						(    self							,       *__lowerCAmelCase							,       **__lowerCAmelCase  ):
								"""simple docstring"""
								return self.tokenizer.decode(*__lowerCAmelCase							,       **__lowerCAmelCase  )







				@property
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   self.tokenizer.model_input_names
								__magic_name__		:Any   =   self.image_processor.model_input_names
								return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names  )  )







				@property
				def       A						(    self  ):
								"""simple docstring"""
								warnings.warn(
								    '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''							,       __lowerCAmelCase							,       )
								return self.image_processor_class






