




import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
UpperCAmelCase__				     = logging.get_logger(__name__)







def 		_A(       UpperCamelCase__							:			Any				,       UpperCamelCase__							:			Optional[int]=False				,       UpperCamelCase__							:			Union[str, Any]=False				,       UpperCamelCase__							:			Optional[int]=False				)    -> List[str]:
							'''simple docstring'''


							__lowercase      =    []
							for i in range(config.num_hidden_layers				):
														# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
														rename_keys.append((F'transformer.blocks.{i}.norm1.weight', F'vilt.encoder.layer.{i}.layernorm_before.weight')				)
														rename_keys.append((F'transformer.blocks.{i}.norm1.bias', F'vilt.encoder.layer.{i}.layernorm_before.bias')				)
														rename_keys.append(
														    (F'transformer.blocks.{i}.attn.proj.weight', F'vilt.encoder.layer.{i}.attention.output.dense.weight')				)
														rename_keys.append(
														    (F'transformer.blocks.{i}.attn.proj.bias', F'vilt.encoder.layer.{i}.attention.output.dense.bias')				)
														rename_keys.append((F'transformer.blocks.{i}.norm2.weight', F'vilt.encoder.layer.{i}.layernorm_after.weight')				)
														rename_keys.append((F'transformer.blocks.{i}.norm2.bias', F'vilt.encoder.layer.{i}.layernorm_after.bias')				)
														rename_keys.append(
														    (F'transformer.blocks.{i}.mlp.fc1.weight', F'vilt.encoder.layer.{i}.intermediate.dense.weight')				)
														rename_keys.append((F'transformer.blocks.{i}.mlp.fc1.bias', F'vilt.encoder.layer.{i}.intermediate.dense.bias')				)
														rename_keys.append((F'transformer.blocks.{i}.mlp.fc2.weight', F'vilt.encoder.layer.{i}.output.dense.weight')				)
														rename_keys.append((F'transformer.blocks.{i}.mlp.fc2.bias', F'vilt.encoder.layer.{i}.output.dense.bias')				)

							# embeddings
							rename_keys.extend(
							    [
							        # text embeddings
							        ('''text_embeddings.word_embeddings.weight''', '''vilt.embeddings.text_embeddings.word_embeddings.weight'''),
							        (
							            '''text_embeddings.position_embeddings.weight''',
							            '''vilt.embeddings.text_embeddings.position_embeddings.weight''',
							        ),
							        ('''text_embeddings.position_ids''', '''vilt.embeddings.text_embeddings.position_ids'''),
							        (
							            '''text_embeddings.token_type_embeddings.weight''',
							            '''vilt.embeddings.text_embeddings.token_type_embeddings.weight''',
							        ),
							        ('''text_embeddings.LayerNorm.weight''', '''vilt.embeddings.text_embeddings.LayerNorm.weight'''),
							        ('''text_embeddings.LayerNorm.bias''', '''vilt.embeddings.text_embeddings.LayerNorm.bias'''),
							        # patch embeddings
							        ('''transformer.cls_token''', '''vilt.embeddings.cls_token'''),
							        ('''transformer.patch_embed.proj.weight''', '''vilt.embeddings.patch_embeddings.projection.weight'''),
							        ('''transformer.patch_embed.proj.bias''', '''vilt.embeddings.patch_embeddings.projection.bias'''),
							        ('''transformer.pos_embed''', '''vilt.embeddings.position_embeddings'''),
							        # token type embeddings
							        ('''token_type_embeddings.weight''', '''vilt.embeddings.token_type_embeddings.weight'''),
							    ]				)

							# final layernorm + pooler
							rename_keys.extend(
							    [
							        ('''transformer.norm.weight''', '''vilt.layernorm.weight'''),
							        ('''transformer.norm.bias''', '''vilt.layernorm.bias'''),
							        ('''pooler.dense.weight''', '''vilt.pooler.dense.weight'''),
							        ('''pooler.dense.bias''', '''vilt.pooler.dense.bias'''),
							    ]				)

							# classifier head(s)
							if vqa_model:
														# classification head
														rename_keys.extend(
														    [
														        ('''vqa_classifier.0.weight''', '''classifier.0.weight'''),
														        ('''vqa_classifier.0.bias''', '''classifier.0.bias'''),
														        ('''vqa_classifier.1.weight''', '''classifier.1.weight'''),
														        ('''vqa_classifier.1.bias''', '''classifier.1.bias'''),
														        ('''vqa_classifier.3.weight''', '''classifier.3.weight'''),
														        ('''vqa_classifier.3.bias''', '''classifier.3.bias'''),
														    ]				)
							elif nlvr_model:
														# classification head
														rename_keys.extend(
														    [
														        ('''nlvr2_classifier.0.weight''', '''classifier.0.weight'''),
														        ('''nlvr2_classifier.0.bias''', '''classifier.0.bias'''),
														        ('''nlvr2_classifier.1.weight''', '''classifier.1.weight'''),
														        ('''nlvr2_classifier.1.bias''', '''classifier.1.bias'''),
														        ('''nlvr2_classifier.3.weight''', '''classifier.3.weight'''),
														        ('''nlvr2_classifier.3.bias''', '''classifier.3.bias'''),
														    ]				)
							else:
														pass

							return rename_keys







def 		_A(       UpperCamelCase__							:			Union[str, Any]				,       UpperCamelCase__							:			Union[str, Any]				)    -> Dict:
							'''simple docstring'''


							for i in range(config.num_hidden_layers				):
														__lowercase      =    '''vilt.'''
														# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
														__lowercase      =    state_dict.pop(F'transformer.blocks.{i}.attn.qkv.weight'				)
														__lowercase      =    state_dict.pop(F'transformer.blocks.{i}.attn.qkv.bias'				)
														# next, add query, keys and values (in that order) to the state dict
														__lowercase      =    in_proj_weight[
														    : config.hidden_size, :
														]
														__lowercase      =    in_proj_bias[: config.hidden_size]
														__lowercase      =    in_proj_weight[
														    config.hidden_size : config.hidden_size * 2, :
														]
														__lowercase      =    in_proj_bias[
														    config.hidden_size : config.hidden_size * 2
														]
														__lowercase      =    in_proj_weight[
														    -config.hidden_size :, :
														]
														__lowercase      =    in_proj_bias[-config.hidden_size :]







def 		_A(       UpperCamelCase__							:			List[Any]				)    -> int:
							'''simple docstring'''


							__lowercase      =    ['''head.weight''', '''head.bias''']
							for k in ignore_keys:
														state_dict.pop(UpperCamelCase__				,       UpperCamelCase__				)







def 		_A(       UpperCamelCase__							:			Tuple				,       UpperCamelCase__							:			Dict				,       UpperCamelCase__							:			Optional[Any]				)    -> int:
							'''simple docstring'''


							__lowercase      =    dct.pop(UpperCamelCase__				)
							__lowercase      =    val







@torch.no_grad()
def 		_A(       UpperCamelCase__							:			Dict				,       UpperCamelCase__							:			int				)    -> Any:
							'''simple docstring'''


							__lowercase      =    ViltConfig(image_size=384				,       patch_size=32				,       tie_word_embeddings=UpperCamelCase__				)
							__lowercase      =    False
							__lowercase      =    False
							__lowercase      =    False
							__lowercase      =    False
							if "vqa" in checkpoint_url:
														__lowercase      =    True
														__lowercase      =    3129
														__lowercase      =    '''huggingface/label-files'''
														__lowercase      =    '''vqa2-id2label.json'''
														__lowercase      =    json.load(open(hf_hub_download(UpperCamelCase__				,       UpperCamelCase__				,       repo_type='''dataset'''				)				,       '''r'''				)				)
														__lowercase      =    {int(UpperCamelCase__				): v for k, v in idalabel.items()}
														__lowercase      =    idalabel
														__lowercase      =    {v: k for k, v in idalabel.items()}
														__lowercase      =    ViltForQuestionAnswering(UpperCamelCase__				)
							elif "nlvr" in checkpoint_url:
														__lowercase      =    True
														__lowercase      =    2
														__lowercase      =    {0: '''False''', 1: '''True'''}
														__lowercase      =    {v: k for k, v in config.idalabel.items()}
														__lowercase      =    3
														__lowercase      =    ViltForImagesAndTextClassification(UpperCamelCase__				)
							elif "irtr" in checkpoint_url:
														__lowercase      =    True
														__lowercase      =    ViltForImageAndTextRetrieval(UpperCamelCase__				)
							elif "mlm_itm" in checkpoint_url:
														__lowercase      =    True
														__lowercase      =    ViltForMaskedLM(UpperCamelCase__				)
							else:
														raise ValueError('''Unknown model type'''				)

							# load state_dict of original model, remove and rename some keys
							__lowercase      =    torch.hub.load_state_dict_from_url(UpperCamelCase__				,       map_location='''cpu'''				)['''state_dict''']
							__lowercase      =    create_rename_keys(UpperCamelCase__				,       UpperCamelCase__				,       UpperCamelCase__				,       UpperCamelCase__				)
							for src, dest in rename_keys:
														rename_key(UpperCamelCase__				,       UpperCamelCase__				,       UpperCamelCase__				)
							read_in_q_k_v(UpperCamelCase__				,       UpperCamelCase__				)
							if mlm_model or irtr_model:
														__lowercase      =    ['''itm_score.fc.weight''', '''itm_score.fc.bias''']
														for k in ignore_keys:
																					state_dict.pop(UpperCamelCase__				,       UpperCamelCase__				)

    # load state dict into HuggingFace model
							model.eval()
							if mlm_model:
														__lowercase     ,     __lowercase      =    model.load_state_dict(UpperCamelCase__				,       strict=UpperCamelCase__				)
														assert missing_keys == ["mlm_score.decoder.bias"]
							else:
														model.load_state_dict(UpperCamelCase__				)

							# Define processor
							__lowercase      =    ViltImageProcessor(size=384				)
							__lowercase      =    BertTokenizer.from_pretrained('''bert-base-uncased'''				)
							__lowercase      =    ViltProcessor(UpperCamelCase__				,       UpperCamelCase__				)

							# Forward pass on example inputs (image + text)
							if nlvr_model:
														__lowercase      =    Image.open(requests.get('''https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'''				,       stream=UpperCamelCase__				).raw				)
														__lowercase      =    Image.open(requests.get('''https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg'''				,       stream=UpperCamelCase__				).raw				)
														__lowercase      =    (
														    '''The left image contains twice the number of dogs as the right image, and at least two dogs in total are'''
														    ''' standing.'''
														)
														__lowercase      =    processor(UpperCamelCase__				,       UpperCamelCase__				,       return_tensors='''pt'''				)
														__lowercase      =    processor(UpperCamelCase__				,       UpperCamelCase__				,       return_tensors='''pt'''				)
														__lowercase      =    model(
														    input_ids=encoding_a.input_ids				,       pixel_values=encoding_a.pixel_values				,       pixel_values_a=encoding_a.pixel_values				,       )
							else:
														__lowercase      =    Image.open(requests.get('''http://images.cocodataset.org/val2017/000000039769.jpg'''				,       stream=UpperCamelCase__				).raw				)
														if mlm_model:
																					__lowercase      =    '''a bunch of [MASK] laying on a [MASK].'''
														else:
																					__lowercase      =    '''How many cats are there?'''
														__lowercase      =    processor(UpperCamelCase__				,       UpperCamelCase__				,       return_tensors='''pt'''				)
														__lowercase      =    model(**UpperCamelCase__				)

							# Verify outputs
							if mlm_model:
														__lowercase      =    torch.Size([1, 11, 3_0522]				)
														__lowercase      =    torch.tensor([-12.5061, -12.5123, -12.5174]				)
														assert outputs.logits.shape == expected_shape
														assert torch.allclose(outputs.logits[0, 0, :3]				,       UpperCamelCase__				,       atol=1e-4				)

														# verify masked token prediction equals "cats"
														__lowercase      =    outputs.logits[0, 4, :].argmax(-1				).item()
														assert tokenizer.decode([predicted_id]				) == "cats"
							elif vqa_model:
														__lowercase      =    torch.Size([1, 3129]				)
														__lowercase      =    torch.tensor([-15.9495, -18.1472, -10.3041]				)
														assert torch.allclose(outputs.logits[0, :3]				,       UpperCamelCase__				,       atol=1e-4				)
														assert outputs.logits.shape == expected_shape
														assert torch.allclose(outputs.logits[0, 0, :3]				,       UpperCamelCase__				,       atol=1e-4				)

														# verify vqa prediction equals "2"
														__lowercase      =    outputs.logits.argmax(-1				).item()
														assert model.config.idalabel[predicted_idx] == "2"
							elif nlvr_model:
														__lowercase      =    torch.Size([1, 2]				)
														__lowercase      =    torch.tensor([-2.8721, 2.1291]				)
														assert torch.allclose(outputs.logits[0, :3]				,       UpperCamelCase__				,       atol=1e-4				)
														assert outputs.logits.shape == expected_shape

							Path(UpperCamelCase__				).mkdir(exist_ok=UpperCamelCase__				)
							print(F'Saving model and processor to {pytorch_dump_folder_path}'				)
							model.save_pretrained(UpperCamelCase__				)
							processor.save_pretrained(UpperCamelCase__				)


if __name__ == "__main__":
			UpperCAmelCase__				     = argparse.ArgumentParser()
			# Required parameters
			parser.add_argument(
			    "--checkpoint_url",
			    default="https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt",
			    type=str,
			    help="URL of the checkpoint you'd like to convert.",
			)
			parser.add_argument(
			    "--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model directory."
			)

			UpperCAmelCase__				     = parser.parse_args()
			convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)












import warnings
from typing import List

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import is_flax_available, is_tf_available, is_torch_available







class a (     __SCREAMING_SNAKE_CASE  ):
						"""simple docstring"""

						UpperCamelCase_	:       int       	=		['image_processor', 'tokenizer']
						UpperCamelCase_	:       Dict       	=		'OwlViTImageProcessor'
						UpperCamelCase_	:       List[str]       	=		('CLIPTokenizer', 'CLIPTokenizerFast')






						def __init__( self :     str				,							lowerCamelCase__ :     Any=None				,							lowerCamelCase__ :     List[Any]=None				,							**lowerCamelCase__ :     Union[str, Any]				)    ->						Tuple:
													"""simple docstring"""


													__lowercase      =    None
													if "feature_extractor" in kwargs:
																				warnings.warn(
																				    '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
																				    ''' instead.'''				,							lowerCamelCase__				,							)
																				__lowercase      =    kwargs.pop('''feature_extractor'''				)

													__lowercase      =    image_processor if image_processor is not None else feature_extractor
													if image_processor is None:
																				raise ValueError('''You need to specify an `image_processor`.'''				)
													if tokenizer is None:
																				raise ValueError('''You need to specify a `tokenizer`.'''				)

													super().__init__(lowerCamelCase__				,							lowerCamelCase__				)






						def __call__( self :     List[Any]				,							lowerCamelCase__ :     List[str]=None				,							lowerCamelCase__ :     int=None				,							lowerCamelCase__ :     Any=None				,							lowerCamelCase__ :     str="max_length"				,							lowerCamelCase__ :     Union[str, Any]="np"				,							**lowerCamelCase__ :     Dict				)    ->						Union[str, Any]:
													"""simple docstring"""


													if text is None and query_images is None and images is None:
																				raise ValueError(
																				    '''You have to specify at least one text or query image or image. All three cannot be none.'''				)

													if text is not None:
																				if isinstance(lowerCamelCase__				,							lowerCamelCase__				) or (isinstance(lowerCamelCase__				,							lowerCamelCase__				) and not isinstance(text[0]				,							lowerCamelCase__				)):
																											__lowercase      =    [self.tokenizer(lowerCamelCase__				,							padding=lowerCamelCase__				,							return_tensors=lowerCamelCase__				,							**lowerCamelCase__				)]

																				elif isinstance(lowerCamelCase__				,							lowerCamelCase__				) and isinstance(text[0]				,							lowerCamelCase__				):
																											__lowercase      =    []

																											# Maximum number of queries across batch
																											__lowercase      =    max([len(lowerCamelCase__				) for t in text]				)

																											# Pad all batch samples to max number of text queries
																											for t in text:
																																		if len(lowerCamelCase__				) != max_num_queries:
																																									__lowercase      =    t + [''' '''] * (max_num_queries - len(lowerCamelCase__				))

																																		__lowercase      =    self.tokenizer(lowerCamelCase__				,							padding=lowerCamelCase__				,							return_tensors=lowerCamelCase__				,							**lowerCamelCase__				)
																																		encodings.append(lowerCamelCase__				)
																				else:
																											raise TypeError('''Input text should be a string, a list of strings or a nested list of strings'''				)

																				if return_tensors == "np":
																											__lowercase      =    np.concatenate([encoding['''input_ids'''] for encoding in encodings]				,							axis=0				)
																											__lowercase      =    np.concatenate([encoding['''attention_mask'''] for encoding in encodings]				,							axis=0				)

																				elif return_tensors == "jax" and is_flax_available():
																											import jax.numpy as jnp

																											__lowercase      =    jnp.concatenate([encoding['''input_ids'''] for encoding in encodings]				,							axis=0				)
																											__lowercase      =    jnp.concatenate([encoding['''attention_mask'''] for encoding in encodings]				,							axis=0				)

																				elif return_tensors == "pt" and is_torch_available():
																											import torch

																											__lowercase      =    torch.cat([encoding['''input_ids'''] for encoding in encodings]				,							dim=0				)
																											__lowercase      =    torch.cat([encoding['''attention_mask'''] for encoding in encodings]				,							dim=0				)

																				elif return_tensors == "tf" and is_tf_available():
																											import tensorflow as tf

																											__lowercase      =    tf.stack([encoding['''input_ids'''] for encoding in encodings]				,							axis=0				)
																											__lowercase      =    tf.stack([encoding['''attention_mask'''] for encoding in encodings]				,							axis=0				)

																				else:
																											raise ValueError('''Target return tensor type could not be returned'''				)

																				__lowercase      =    BatchEncoding()
																				__lowercase      =    input_ids
																				__lowercase      =    attention_mask

													if query_images is not None:
																				__lowercase      =    BatchEncoding()
																				__lowercase      =    self.image_processor(
																				    lowerCamelCase__				,							return_tensors=lowerCamelCase__				,							**lowerCamelCase__				).pixel_values
																				__lowercase      =    query_pixel_values

													if images is not None:
																				__lowercase      =    self.image_processor(lowerCamelCase__				,							return_tensors=lowerCamelCase__				,							**lowerCamelCase__				)

													if text is not None and images is not None:
																				__lowercase      =    image_features.pixel_values
																				return encoding
													elif query_images is not None and images is not None:
																				__lowercase      =    image_features.pixel_values
																				return encoding
													elif text is not None or query_images is not None:
																				return encoding
													else:
																				return BatchEncoding(data=dict(**lowerCamelCase__				)				,							tensor_type=lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     int				,							*lowerCamelCase__ :     str				,							**lowerCamelCase__ :     Optional[int]				)    ->						Tuple:
													"""simple docstring"""


													return self.image_processor.post_process(*lowerCamelCase__				,							**lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     Optional[int]				,							*lowerCamelCase__ :     Tuple				,							**lowerCamelCase__ :     Tuple				)    ->						Union[str, Any]:
													"""simple docstring"""


													return self.image_processor.post_process_object_detection(*lowerCamelCase__				,							**lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     List[str]				,							*lowerCamelCase__ :     List[str]				,							**lowerCamelCase__ :     Optional[Any]				)    ->						List[Any]:
													"""simple docstring"""


													return self.image_processor.post_process_image_guided_detection(*lowerCamelCase__				,							**lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     Optional[int]				,							*lowerCamelCase__ :     str				,							**lowerCamelCase__ :     Union[str, Any]				)    ->						Optional[Any]:
													"""simple docstring"""


													return self.tokenizer.batch_decode(*lowerCamelCase__				,							**lowerCamelCase__				)






						def     UpperCAmelCase_      ( self :     str				,							*lowerCamelCase__ :     Optional[int]				,							**lowerCamelCase__ :     Union[str, Any]				)    ->						List[str]:
													"""simple docstring"""


													return self.tokenizer.decode(*lowerCamelCase__				,							**lowerCamelCase__				)






						@property
						def     UpperCAmelCase_      ( self :     Tuple				)    ->						List[str]:
													"""simple docstring"""


													warnings.warn(
													    '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''				,							lowerCamelCase__				,							)
													return self.image_processor_class






						@property
						def     UpperCAmelCase_      ( self :     str				)    ->						Any:
													"""simple docstring"""


													warnings.warn(
													    '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''				,							lowerCamelCase__				,							)
													return self.image_processor






