
import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
lowercase   =	logging.get_logger(__name__)



def   __lowerCAmelCase    (				UpperCAmelCase__					:   Dict      ,					UpperCAmelCase__					:   Optional[Any]=False      ,					UpperCAmelCase__					:   Tuple=False      ,					UpperCAmelCase__					:   Union[str, Any]=False							)	->     Dict:
							lowerCamelCase_					=       []
							for i in range(config.num_hidden_layers							):
														# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
														rename_keys.append((F'''transformer.blocks.{i}.norm1.weight''', F'''vilt.encoder.layer.{i}.layernorm_before.weight''')							)
														rename_keys.append((F'''transformer.blocks.{i}.norm1.bias''', F'''vilt.encoder.layer.{i}.layernorm_before.bias''')							)
														rename_keys.append(
														    (F'''transformer.blocks.{i}.attn.proj.weight''', F'''vilt.encoder.layer.{i}.attention.output.dense.weight''')							)
														rename_keys.append(
														    (F'''transformer.blocks.{i}.attn.proj.bias''', F'''vilt.encoder.layer.{i}.attention.output.dense.bias''')							)
														rename_keys.append((F'''transformer.blocks.{i}.norm2.weight''', F'''vilt.encoder.layer.{i}.layernorm_after.weight''')							)
														rename_keys.append((F'''transformer.blocks.{i}.norm2.bias''', F'''vilt.encoder.layer.{i}.layernorm_after.bias''')							)
														rename_keys.append(
														    (F'''transformer.blocks.{i}.mlp.fc1.weight''', F'''vilt.encoder.layer.{i}.intermediate.dense.weight''')							)
														rename_keys.append((F'''transformer.blocks.{i}.mlp.fc1.bias''', F'''vilt.encoder.layer.{i}.intermediate.dense.bias''')							)
														rename_keys.append((F'''transformer.blocks.{i}.mlp.fc2.weight''', F'''vilt.encoder.layer.{i}.output.dense.weight''')							)
														rename_keys.append((F'''transformer.blocks.{i}.mlp.fc2.bias''', F'''vilt.encoder.layer.{i}.output.dense.bias''')							)

							# embeddings
							rename_keys.extend(
							    [
							        # text embeddings
							        ("""text_embeddings.word_embeddings.weight""", """vilt.embeddings.text_embeddings.word_embeddings.weight"""),
							        (
							            """text_embeddings.position_embeddings.weight""",
							            """vilt.embeddings.text_embeddings.position_embeddings.weight""",
							        ),
							        ("""text_embeddings.position_ids""", """vilt.embeddings.text_embeddings.position_ids"""),
							        (
							            """text_embeddings.token_type_embeddings.weight""",
							            """vilt.embeddings.text_embeddings.token_type_embeddings.weight""",
							        ),
							        ("""text_embeddings.LayerNorm.weight""", """vilt.embeddings.text_embeddings.LayerNorm.weight"""),
							        ("""text_embeddings.LayerNorm.bias""", """vilt.embeddings.text_embeddings.LayerNorm.bias"""),
							        # patch embeddings
							        ("""transformer.cls_token""", """vilt.embeddings.cls_token"""),
							        ("""transformer.patch_embed.proj.weight""", """vilt.embeddings.patch_embeddings.projection.weight"""),
							        ("""transformer.patch_embed.proj.bias""", """vilt.embeddings.patch_embeddings.projection.bias"""),
							        ("""transformer.pos_embed""", """vilt.embeddings.position_embeddings"""),
							        # token type embeddings
							        ("""token_type_embeddings.weight""", """vilt.embeddings.token_type_embeddings.weight"""),
							    ]							)

							# final layernorm + pooler
							rename_keys.extend(
							    [
							        ("""transformer.norm.weight""", """vilt.layernorm.weight"""),
							        ("""transformer.norm.bias""", """vilt.layernorm.bias"""),
							        ("""pooler.dense.weight""", """vilt.pooler.dense.weight"""),
							        ("""pooler.dense.bias""", """vilt.pooler.dense.bias"""),
							    ]							)

							# classifier head(s)
							if vqa_model:
														# classification head
														rename_keys.extend(
														    [
														        ("""vqa_classifier.0.weight""", """classifier.0.weight"""),
														        ("""vqa_classifier.0.bias""", """classifier.0.bias"""),
														        ("""vqa_classifier.1.weight""", """classifier.1.weight"""),
														        ("""vqa_classifier.1.bias""", """classifier.1.bias"""),
														        ("""vqa_classifier.3.weight""", """classifier.3.weight"""),
														        ("""vqa_classifier.3.bias""", """classifier.3.bias"""),
														    ]							)
							elif nlvr_model:
														# classification head
														rename_keys.extend(
														    [
														        ("""nlvr2_classifier.0.weight""", """classifier.0.weight"""),
														        ("""nlvr2_classifier.0.bias""", """classifier.0.bias"""),
														        ("""nlvr2_classifier.1.weight""", """classifier.1.weight"""),
														        ("""nlvr2_classifier.1.bias""", """classifier.1.bias"""),
														        ("""nlvr2_classifier.3.weight""", """classifier.3.weight"""),
														        ("""nlvr2_classifier.3.bias""", """classifier.3.bias"""),
														    ]							)
							else:
														pass

							return rename_keys



def   __lowerCAmelCase    (				UpperCAmelCase__					:   Tuple      ,					UpperCAmelCase__					:   List[str]							)	->     Optional[Any]:
							for i in range(config.num_hidden_layers							):
														lowerCamelCase_					=       """vilt."""
														# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
														lowerCamelCase_					=       state_dict.pop(F'''transformer.blocks.{i}.attn.qkv.weight'''							)
														lowerCamelCase_					=       state_dict.pop(F'''transformer.blocks.{i}.attn.qkv.bias'''							)
														# next, add query, keys and values (in that order) to the state dict
														lowerCamelCase_					=       in_proj_weight[
														    : config.hidden_size, :
														]
														lowerCamelCase_					=       in_proj_bias[: config.hidden_size]
														lowerCamelCase_					=       in_proj_weight[
														    config.hidden_size : config.hidden_size * 2, :
														]
														lowerCamelCase_					=       in_proj_bias[
														    config.hidden_size : config.hidden_size * 2
														]
														lowerCamelCase_					=       in_proj_weight[
														    -config.hidden_size :, :
														]
														lowerCamelCase_					=       in_proj_bias[-config.hidden_size :]



def   __lowerCAmelCase    (				UpperCAmelCase__					:   Any							)	->     Optional[int]:
							lowerCamelCase_					=       ["""head.weight""", """head.bias"""]
							for k in ignore_keys:
														state_dict.pop(UpperCAmelCase__      ,					UpperCAmelCase__							)



def   __lowerCAmelCase    (				UpperCAmelCase__					:   str      ,					UpperCAmelCase__					:   Any      ,					UpperCAmelCase__					:   str							)	->     List[Any]:
							lowerCamelCase_					=       dct.pop(UpperCAmelCase__							)
							lowerCamelCase_					=       val



@torch.no_grad()
def   __lowerCAmelCase    (				UpperCAmelCase__					:   List[Any]      ,					UpperCAmelCase__					:   str							)	->     List[str]:
							lowerCamelCase_					=       ViltConfig(image_size=3_8_4      ,					patch_size=3_2      ,					tie_word_embeddings=UpperCAmelCase__							)
							lowerCamelCase_					=       False
							lowerCamelCase_					=       False
							lowerCamelCase_					=       False
							lowerCamelCase_					=       False
							if "vqa" in checkpoint_url:
														lowerCamelCase_					=       True
														lowerCamelCase_					=       3_1_2_9
														lowerCamelCase_					=       """huggingface/label-files"""
														lowerCamelCase_					=       """vqa2-id2label.json"""
														lowerCamelCase_					=       json.load(open(hf_hub_download(UpperCAmelCase__      ,					UpperCAmelCase__      ,					repo_type="""dataset"""							)      ,					"""r"""							)							)
														lowerCamelCase_					=       {int(UpperCAmelCase__							): v for k, v in idalabel.items()}
														lowerCamelCase_					=       idalabel
														lowerCamelCase_					=       {v: k for k, v in idalabel.items()}
														lowerCamelCase_					=       ViltForQuestionAnswering(UpperCAmelCase__							)
							elif "nlvr" in checkpoint_url:
														lowerCamelCase_					=       True
														lowerCamelCase_					=       2
														lowerCamelCase_					=       {0: """False""", 1: """True"""}
														lowerCamelCase_					=       {v: k for k, v in config.idalabel.items()}
														lowerCamelCase_					=       3
														lowerCamelCase_					=       ViltForImagesAndTextClassification(UpperCAmelCase__							)
							elif "irtr" in checkpoint_url:
														lowerCamelCase_					=       True
														lowerCamelCase_					=       ViltForImageAndTextRetrieval(UpperCAmelCase__							)
							elif "mlm_itm" in checkpoint_url:
														lowerCamelCase_					=       True
														lowerCamelCase_					=       ViltForMaskedLM(UpperCAmelCase__							)
							else:
														raise ValueError("""Unknown model type"""							)

							# load state_dict of original model, remove and rename some keys
							lowerCamelCase_					=       torch.hub.load_state_dict_from_url(UpperCAmelCase__      ,					map_location="""cpu"""							)["""state_dict"""]
							lowerCamelCase_					=       create_rename_keys(UpperCAmelCase__      ,					UpperCAmelCase__      ,					UpperCAmelCase__      ,					UpperCAmelCase__							)
							for src, dest in rename_keys:
														rename_key(UpperCAmelCase__      ,					UpperCAmelCase__      ,					UpperCAmelCase__							)
							read_in_q_k_v(UpperCAmelCase__      ,					UpperCAmelCase__							)
							if mlm_model or irtr_model:
														lowerCamelCase_					=       ["""itm_score.fc.weight""", """itm_score.fc.bias"""]
														for k in ignore_keys:
																					state_dict.pop(UpperCAmelCase__      ,					UpperCAmelCase__							)

    # load state dict into HuggingFace model
							model.eval()
							if mlm_model:
														lowerCamelCase_	, lowerCamelCase_					=       model.load_state_dict(UpperCAmelCase__      ,					strict=UpperCAmelCase__							)
														assert missing_keys == ["mlm_score.decoder.bias"]
							else:
														model.load_state_dict(UpperCAmelCase__							)

							# Define processor
							lowerCamelCase_					=       ViltImageProcessor(size=3_8_4							)
							lowerCamelCase_					=       BertTokenizer.from_pretrained("""bert-base-uncased"""							)
							lowerCamelCase_					=       ViltProcessor(UpperCAmelCase__      ,					UpperCAmelCase__							)

							# Forward pass on example inputs (image + text)
							if nlvr_model:
														lowerCamelCase_					=       Image.open(requests.get("""https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg"""      ,					stream=UpperCAmelCase__							).raw							)
														lowerCamelCase_					=       Image.open(requests.get("""https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg"""      ,					stream=UpperCAmelCase__							).raw							)
														lowerCamelCase_					=       (
														    """The left image contains twice the number of dogs as the right image, and at least two dogs in total are"""
														    """ standing."""
														)
														lowerCamelCase_					=       processor(UpperCAmelCase__      ,					UpperCAmelCase__      ,					return_tensors="""pt"""							)
														lowerCamelCase_					=       processor(UpperCAmelCase__      ,					UpperCAmelCase__      ,					return_tensors="""pt"""							)
														lowerCamelCase_					=       model(
														    input_ids=encoding_a.input_ids      ,					pixel_values=encoding_a.pixel_values      ,					pixel_values_a=encoding_a.pixel_values      ,					)
							else:
														lowerCamelCase_					=       Image.open(requests.get("""http://images.cocodataset.org/val2017/000000039769.jpg"""      ,					stream=UpperCAmelCase__							).raw							)
														if mlm_model:
																					lowerCamelCase_					=       """a bunch of [MASK] laying on a [MASK]."""
														else:
																					lowerCamelCase_					=       """How many cats are there?"""
														lowerCamelCase_					=       processor(UpperCAmelCase__      ,					UpperCAmelCase__      ,					return_tensors="""pt"""							)
														lowerCamelCase_					=       model(**UpperCAmelCase__							)

							# Verify outputs
							if mlm_model:
														lowerCamelCase_					=       torch.Size([1, 1_1, 3_0_5_2_2]							)
														lowerCamelCase_					=       torch.tensor([-1_2.5_0_6_1, -1_2.5_1_2_3, -1_2.5_1_7_4]							)
														assert outputs.logits.shape == expected_shape
														assert torch.allclose(outputs.logits[0, 0, :3]      ,					UpperCAmelCase__      ,					atol=1e-4							)

														# verify masked token prediction equals "cats"
														lowerCamelCase_					=       outputs.logits[0, 4, :].argmax(-1							).item()
														assert tokenizer.decode([predicted_id]							) == "cats"
							elif vqa_model:
														lowerCamelCase_					=       torch.Size([1, 3_1_2_9]							)
														lowerCamelCase_					=       torch.tensor([-1_5.9_4_9_5, -1_8.1_4_7_2, -1_0.3_0_4_1]							)
														assert torch.allclose(outputs.logits[0, :3]      ,					UpperCAmelCase__      ,					atol=1e-4							)
														assert outputs.logits.shape == expected_shape
														assert torch.allclose(outputs.logits[0, 0, :3]      ,					UpperCAmelCase__      ,					atol=1e-4							)

														# verify vqa prediction equals "2"
														lowerCamelCase_					=       outputs.logits.argmax(-1							).item()
														assert model.config.idalabel[predicted_idx] == "2"
							elif nlvr_model:
														lowerCamelCase_					=       torch.Size([1, 2]							)
														lowerCamelCase_					=       torch.tensor([-2.8_7_2_1, 2.1_2_9_1]							)
														assert torch.allclose(outputs.logits[0, :3]      ,					UpperCAmelCase__      ,					atol=1e-4							)
														assert outputs.logits.shape == expected_shape

							Path(UpperCAmelCase__							).mkdir(exist_ok=UpperCAmelCase__							)
							print(F'''Saving model and processor to {pytorch_dump_folder_path}'''							)
							model.save_pretrained(UpperCAmelCase__							)
							processor.save_pretrained(UpperCAmelCase__							)


if __name__ == "__main__":
		lowercase   =	argparse.ArgumentParser()
		# Required parameters
		parser.add_argument(
		    '''--checkpoint_url''',
		    default='''https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt''',
		    type=str,
		    help='''URL of the checkpoint you\'d like to convert.''',
		)
		parser.add_argument(
		    '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model directory.'''
		)

		lowercase   =	parser.parse_args()
		convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)







import os
import sys

lowercase   =	os.path.join(os.path.dirname(__file__), '''src''')
sys.path.append(SRC_DIR)


from transformers import (
    AutoConfig,
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForMaskedLM,
    AutoModelForQuestionAnswering,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    add_start_docstrings,
)


lowercase   =	[
    '''torch''',
    '''numpy''',
    '''tokenizers''',
    '''filelock''',
    '''requests''',
    '''tqdm''',
    '''regex''',
    '''sentencepiece''',
    '''sacremoses''',
    '''importlib_metadata''',
    '''huggingface_hub''',
]



@add_start_docstrings(AutoConfig.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   List[Any]      ,					**UpperCAmelCase__					:   List[str]							)	->     List[str]:
							return AutoConfig.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)



@add_start_docstrings(AutoTokenizer.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   List[str]      ,					**UpperCAmelCase__					:   List[Any]							)	->     Any:
							return AutoTokenizer.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)



@add_start_docstrings(AutoModel.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   Union[str, Any]      ,					**UpperCAmelCase__					:   Dict							)	->     Union[str, Any]:
							return AutoModel.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)



@add_start_docstrings(AutoModelForCausalLM.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   List[str]      ,					**UpperCAmelCase__					:   List[Any]							)	->     Optional[Any]:
							return AutoModelForCausalLM.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)



@add_start_docstrings(AutoModelForMaskedLM.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   Union[str, Any]      ,					**UpperCAmelCase__					:   Dict							)	->     int:
							return AutoModelForMaskedLM.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)



@add_start_docstrings(AutoModelForSequenceClassification.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   List[Any]      ,					**UpperCAmelCase__					:   Dict							)	->     Dict:
							return AutoModelForSequenceClassification.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)



@add_start_docstrings(AutoModelForQuestionAnswering.__doc__							)
def   __lowerCAmelCase    (				*UpperCAmelCase__					:   Any      ,					**UpperCAmelCase__					:   List[str]							)	->     Union[str, Any]:
							return AutoModelForQuestionAnswering.from_pretrained(*UpperCAmelCase__      ,					**UpperCAmelCase__							)





