



import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    BertTokenizer,
    ViltConfig,
    ViltForImageAndTextRetrieval,
    ViltForImagesAndTextClassification,
    ViltForMaskedLM,
    ViltForQuestionAnswering,
    ViltImageProcessor,
    ViltProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
lowercase_													=      logging.get_logger(__name__)






def        __lowerCAmelCase		(	__lowerCamelCase						:     Dict					, __lowerCamelCase						:     Optional[Any]=False					, __lowerCamelCase						:     str=False					, __lowerCamelCase						:     Optional[Any]=False						)	->    List[str]:
					__lowerCAmelCase						 =[]
					for i in range(config.num_hidden_layers						):
										# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
										rename_keys.append((f"""transformer.blocks.{i}.norm1.weight""", f"""vilt.encoder.layer.{i}.layernorm_before.weight""")						)
										rename_keys.append((f"""transformer.blocks.{i}.norm1.bias""", f"""vilt.encoder.layer.{i}.layernorm_before.bias""")						)
										rename_keys.append(
										    (f"""transformer.blocks.{i}.attn.proj.weight""", f"""vilt.encoder.layer.{i}.attention.output.dense.weight""")						)
										rename_keys.append(
										    (f"""transformer.blocks.{i}.attn.proj.bias""", f"""vilt.encoder.layer.{i}.attention.output.dense.bias""")						)
										rename_keys.append((f"""transformer.blocks.{i}.norm2.weight""", f"""vilt.encoder.layer.{i}.layernorm_after.weight""")						)
										rename_keys.append((f"""transformer.blocks.{i}.norm2.bias""", f"""vilt.encoder.layer.{i}.layernorm_after.bias""")						)
										rename_keys.append(
										    (f"""transformer.blocks.{i}.mlp.fc1.weight""", f"""vilt.encoder.layer.{i}.intermediate.dense.weight""")						)
										rename_keys.append((f"""transformer.blocks.{i}.mlp.fc1.bias""", f"""vilt.encoder.layer.{i}.intermediate.dense.bias""")						)
										rename_keys.append((f"""transformer.blocks.{i}.mlp.fc2.weight""", f"""vilt.encoder.layer.{i}.output.dense.weight""")						)
										rename_keys.append((f"""transformer.blocks.{i}.mlp.fc2.bias""", f"""vilt.encoder.layer.{i}.output.dense.bias""")						)

					# embeddings
					rename_keys.extend(
					    [
					        # text embeddings
					        ("""text_embeddings.word_embeddings.weight""", """vilt.embeddings.text_embeddings.word_embeddings.weight"""),
					        (
					            """text_embeddings.position_embeddings.weight""",
					            """vilt.embeddings.text_embeddings.position_embeddings.weight""",
					        ),
					        ("""text_embeddings.position_ids""", """vilt.embeddings.text_embeddings.position_ids"""),
					        (
					            """text_embeddings.token_type_embeddings.weight""",
					            """vilt.embeddings.text_embeddings.token_type_embeddings.weight""",
					        ),
					        ("""text_embeddings.LayerNorm.weight""", """vilt.embeddings.text_embeddings.LayerNorm.weight"""),
					        ("""text_embeddings.LayerNorm.bias""", """vilt.embeddings.text_embeddings.LayerNorm.bias"""),
					        # patch embeddings
					        ("""transformer.cls_token""", """vilt.embeddings.cls_token"""),
					        ("""transformer.patch_embed.proj.weight""", """vilt.embeddings.patch_embeddings.projection.weight"""),
					        ("""transformer.patch_embed.proj.bias""", """vilt.embeddings.patch_embeddings.projection.bias"""),
					        ("""transformer.pos_embed""", """vilt.embeddings.position_embeddings"""),
					        # token type embeddings
					        ("""token_type_embeddings.weight""", """vilt.embeddings.token_type_embeddings.weight"""),
					    ]						)

					# final layernorm + pooler
					rename_keys.extend(
					    [
					        ("""transformer.norm.weight""", """vilt.layernorm.weight"""),
					        ("""transformer.norm.bias""", """vilt.layernorm.bias"""),
					        ("""pooler.dense.weight""", """vilt.pooler.dense.weight"""),
					        ("""pooler.dense.bias""", """vilt.pooler.dense.bias"""),
					    ]						)

					# classifier head(s)
					if vqa_model:
										# classification head
										rename_keys.extend(
										    [
										        ("""vqa_classifier.0.weight""", """classifier.0.weight"""),
										        ("""vqa_classifier.0.bias""", """classifier.0.bias"""),
										        ("""vqa_classifier.1.weight""", """classifier.1.weight"""),
										        ("""vqa_classifier.1.bias""", """classifier.1.bias"""),
										        ("""vqa_classifier.3.weight""", """classifier.3.weight"""),
										        ("""vqa_classifier.3.bias""", """classifier.3.bias"""),
										    ]						)
					elif nlvr_model:
										# classification head
										rename_keys.extend(
										    [
										        ("""nlvr2_classifier.0.weight""", """classifier.0.weight"""),
										        ("""nlvr2_classifier.0.bias""", """classifier.0.bias"""),
										        ("""nlvr2_classifier.1.weight""", """classifier.1.weight"""),
										        ("""nlvr2_classifier.1.bias""", """classifier.1.bias"""),
										        ("""nlvr2_classifier.3.weight""", """classifier.3.weight"""),
										        ("""nlvr2_classifier.3.bias""", """classifier.3.bias"""),
										    ]						)
					else:
										pass

					return rename_keys






def        __lowerCAmelCase		(	__lowerCamelCase						:     List[str]					, __lowerCamelCase						:     Dict						)	->    int:
					for i in range(config.num_hidden_layers						):
										__lowerCAmelCase						 ="""vilt."""
										# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
										__lowerCAmelCase						 =state_dict.pop(f"""transformer.blocks.{i}.attn.qkv.weight"""						)
										__lowerCAmelCase						 =state_dict.pop(f"""transformer.blocks.{i}.attn.qkv.bias"""						)
										# next, add query, keys and values (in that order) to the state dict
										__lowerCAmelCase						 =in_proj_weight[
										    : config.hidden_size, :
										]
										__lowerCAmelCase						 =in_proj_bias[: config.hidden_size]
										__lowerCAmelCase						 =in_proj_weight[
										    config.hidden_size : config.hidden_size * 2, :
										]
										__lowerCAmelCase						 =in_proj_bias[
										    config.hidden_size : config.hidden_size * 2
										]
										__lowerCAmelCase						 =in_proj_weight[
										    -config.hidden_size :, :
										]
										__lowerCAmelCase						 =in_proj_bias[-config.hidden_size :]






def        __lowerCAmelCase		(	__lowerCamelCase						:     List[str]						)	->    Optional[int]:
					__lowerCAmelCase						 =["""head.weight""", """head.bias"""]
					for k in ignore_keys:
										state_dict.pop(__lowerCamelCase					, __lowerCamelCase						)






def        __lowerCAmelCase		(	__lowerCamelCase						:     Optional[Any]					, __lowerCamelCase						:     Optional[int]					, __lowerCamelCase						:     List[str]						)	->    List[Any]:
					__lowerCAmelCase						 =dct.pop(__lowerCamelCase						)
					__lowerCAmelCase						 =val






@torch.no_grad()
def        __lowerCAmelCase		(	__lowerCamelCase						:     str					, __lowerCamelCase						:     int						)	->    Tuple:
					__lowerCAmelCase						 =ViltConfig(image_size=384					, patch_size=32					, tie_word_embeddings=__lowerCamelCase						)
					__lowerCAmelCase						 =False
					__lowerCAmelCase						 =False
					__lowerCAmelCase						 =False
					__lowerCAmelCase						 =False
					if "vqa" in checkpoint_url:
										__lowerCAmelCase						 =True
										__lowerCAmelCase						 =3129
										__lowerCAmelCase						 ="""huggingface/label-files"""
										__lowerCAmelCase						 ="""vqa2-id2label.json"""
										__lowerCAmelCase						 =json.load(open(hf_hub_download(__lowerCamelCase					, __lowerCamelCase					, repo_type="""dataset"""						)					, """r"""						)						)
										__lowerCAmelCase						 ={int(__lowerCamelCase						): v for k, v in idalabel.items()}
										__lowerCAmelCase						 =idalabel
										__lowerCAmelCase						 ={v: k for k, v in idalabel.items()}
										__lowerCAmelCase						 =ViltForQuestionAnswering(__lowerCamelCase						)
					elif "nlvr" in checkpoint_url:
										__lowerCAmelCase						 =True
										__lowerCAmelCase						 =2
										__lowerCAmelCase						 ={0: """False""", 1: """True"""}
										__lowerCAmelCase						 ={v: k for k, v in config.idalabel.items()}
										__lowerCAmelCase						 =3
										__lowerCAmelCase						 =ViltForImagesAndTextClassification(__lowerCamelCase						)
					elif "irtr" in checkpoint_url:
										__lowerCAmelCase						 =True
										__lowerCAmelCase						 =ViltForImageAndTextRetrieval(__lowerCamelCase						)
					elif "mlm_itm" in checkpoint_url:
										__lowerCAmelCase						 =True
										__lowerCAmelCase						 =ViltForMaskedLM(__lowerCamelCase						)
					else:
										raise ValueError("""Unknown model type"""						)

					# load state_dict of original model, remove and rename some keys
					__lowerCAmelCase						 =torch.hub.load_state_dict_from_url(__lowerCamelCase					, map_location="""cpu"""						)["""state_dict"""]
					__lowerCAmelCase						 =create_rename_keys(__lowerCamelCase					, __lowerCamelCase					, __lowerCamelCase					, __lowerCamelCase						)
					for src, dest in rename_keys:
										rename_key(__lowerCamelCase					, __lowerCamelCase					, __lowerCamelCase						)
					read_in_q_k_v(__lowerCamelCase					, __lowerCamelCase						)
					if mlm_model or irtr_model:
										__lowerCAmelCase						 =["""itm_score.fc.weight""", """itm_score.fc.bias"""]
										for k in ignore_keys:
															state_dict.pop(__lowerCamelCase					, __lowerCamelCase						)

    # load state dict into HuggingFace model
					model.eval()
					if mlm_model:
										__lowerCAmelCase ,						__lowerCAmelCase						 =model.load_state_dict(__lowerCamelCase					, strict=__lowerCamelCase						)
										assert missing_keys == ["mlm_score.decoder.bias"]
					else:
										model.load_state_dict(__lowerCamelCase						)

					# Define processor
					__lowerCAmelCase						 =ViltImageProcessor(size=384						)
					__lowerCAmelCase						 =BertTokenizer.from_pretrained("""bert-base-uncased"""						)
					__lowerCAmelCase						 =ViltProcessor(__lowerCamelCase					, __lowerCamelCase						)

					# Forward pass on example inputs (image + text)
					if nlvr_model:
										__lowerCAmelCase						 =Image.open(requests.get("""https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg"""					, stream=__lowerCamelCase						).raw						)
										__lowerCAmelCase						 =Image.open(requests.get("""https://lil.nlp.cornell.edu/nlvr/exs/ex0_0.jpg"""					, stream=__lowerCamelCase						).raw						)
										__lowerCAmelCase						 =(
										    """The left image contains twice the number of dogs as the right image, and at least two dogs in total are"""
										    """ standing."""
										)
										__lowerCAmelCase						 =processor(__lowerCamelCase					, __lowerCamelCase					, return_tensors="""pt"""						)
										__lowerCAmelCase						 =processor(__lowerCamelCase					, __lowerCamelCase					, return_tensors="""pt"""						)
										__lowerCAmelCase						 =model(
										    input_ids=encoding_a.input_ids					, pixel_values=encoding_a.pixel_values					, pixel_values_a=encoding_a.pixel_values					, )
					else:
										__lowerCAmelCase						 =Image.open(requests.get("""http://images.cocodataset.org/val2017/000000039769.jpg"""					, stream=__lowerCamelCase						).raw						)
										if mlm_model:
															__lowerCAmelCase						 ="""a bunch of [MASK] laying on a [MASK]."""
										else:
															__lowerCAmelCase						 ="""How many cats are there?"""
										__lowerCAmelCase						 =processor(__lowerCamelCase					, __lowerCamelCase					, return_tensors="""pt"""						)
										__lowerCAmelCase						 =model(**__lowerCamelCase						)

					# Verify outputs
					if mlm_model:
										__lowerCAmelCase						 =torch.Size([1, 11, 30522]						)
										__lowerCAmelCase						 =torch.tensor([-12.5061, -12.5123, -12.5174]						)
										assert outputs.logits.shape == expected_shape
										assert torch.allclose(outputs.logits[0, 0, :3]					, __lowerCamelCase					, atol=1E-4						)

										# verify masked token prediction equals "cats"
										__lowerCAmelCase						 =outputs.logits[0, 4, :].argmax(-1						).item()
										assert tokenizer.decode([predicted_id]						) == "cats"
					elif vqa_model:
										__lowerCAmelCase						 =torch.Size([1, 3129]						)
										__lowerCAmelCase						 =torch.tensor([-15.9495, -18.1472, -10.3041]						)
										assert torch.allclose(outputs.logits[0, :3]					, __lowerCamelCase					, atol=1E-4						)
										assert outputs.logits.shape == expected_shape
										assert torch.allclose(outputs.logits[0, 0, :3]					, __lowerCamelCase					, atol=1E-4						)

										# verify vqa prediction equals "2"
										__lowerCAmelCase						 =outputs.logits.argmax(-1						).item()
										assert model.config.idalabel[predicted_idx] == "2"
					elif nlvr_model:
										__lowerCAmelCase						 =torch.Size([1, 2]						)
										__lowerCAmelCase						 =torch.tensor([-2.8721, 2.1291]						)
										assert torch.allclose(outputs.logits[0, :3]					, __lowerCamelCase					, atol=1E-4						)
										assert outputs.logits.shape == expected_shape

					Path(__lowerCamelCase						).mkdir(exist_ok=__lowerCamelCase						)
					print(f"""Saving model and processor to {pytorch_dump_folder_path}"""						)
					model.save_pretrained(__lowerCamelCase						)
					processor.save_pretrained(__lowerCamelCase						)


if __name__ == "__main__":
				lowercase_													=      argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    '''--checkpoint_url''',
				    default='''https://github.com/dandelin/ViLT/releases/download/200k/vilt_200k_mlm_itm.ckpt''',
				    type=str,
				    help='''URL of the checkpoint you\'d like to convert.''',
				)
				parser.add_argument(
				    '''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model directory.'''
				)

				lowercase_													=      parser.parse_args()
				convert_vilt_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)









import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.text import TextDatasetReader

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases






def        __lowerCAmelCase		(	__lowerCamelCase						:     List[Any]					, __lowerCamelCase						:     Any						)	->    Dict:
					assert isinstance(__lowerCamelCase					, __lowerCamelCase						)
					assert dataset.num_rows == 4
					assert dataset.num_columns == 1
					assert dataset.column_names == ["text"]
					for feature, expected_dtype in expected_features.items():
										assert dataset.features[feature].dtype == expected_dtype






@pytest.mark.parametrize("""keep_in_memory"""					, [False, True]						)
def        __lowerCAmelCase		(	__lowerCamelCase						:     Tuple					, __lowerCamelCase						:     str					, __lowerCamelCase						:     Optional[Any]						)	->    List[Any]:
					__lowerCAmelCase						 =tmp_path / """cache"""
					__lowerCAmelCase						 ={"""text""": """string"""}
					with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
										__lowerCAmelCase						 =TextDatasetReader(__lowerCamelCase					, cache_dir=__lowerCamelCase					, keep_in_memory=__lowerCamelCase						).read()
					_check_text_dataset(__lowerCamelCase					, __lowerCamelCase						)






@pytest.mark.parametrize(
    """features"""					, [
        None,
        {"""text""": """string"""},
        {"""text""": """int32"""},
        {"""text""": """float32"""},
    ]					, )
def        __lowerCAmelCase		(	__lowerCamelCase						:     Tuple					, __lowerCamelCase						:     Any					, __lowerCamelCase						:     Tuple						)	->    List[Any]:
					__lowerCAmelCase						 =tmp_path / """cache"""
					__lowerCAmelCase						 ={"""text""": """string"""}
					__lowerCAmelCase						 =features.copy() if features else default_expected_features
					__lowerCAmelCase						 =(
					    Features({feature: Value(__lowerCamelCase						) for feature, dtype in features.items()}						) if features is not None else None
					)
					__lowerCAmelCase						 =TextDatasetReader(__lowerCamelCase					, features=__lowerCamelCase					, cache_dir=__lowerCamelCase						).read()
					_check_text_dataset(__lowerCamelCase					, __lowerCamelCase						)






@pytest.mark.parametrize("""split"""					, [None, NamedSplit("""train"""						), """train""", """test"""]						)
def        __lowerCAmelCase		(	__lowerCamelCase						:     Union[str, Any]					, __lowerCamelCase						:     List[str]					, __lowerCamelCase						:     Union[str, Any]						)	->    Any:
					__lowerCAmelCase						 =tmp_path / """cache"""
					__lowerCAmelCase						 ={"""text""": """string"""}
					__lowerCAmelCase						 =TextDatasetReader(__lowerCamelCase					, cache_dir=__lowerCamelCase					, split=__lowerCamelCase						).read()
					_check_text_dataset(__lowerCamelCase					, __lowerCamelCase						)
					assert dataset.split == split if split else "train"






@pytest.mark.parametrize("""path_type"""					, [str, list]						)
def        __lowerCAmelCase		(	__lowerCamelCase						:     List[Any]					, __lowerCamelCase						:     int					, __lowerCamelCase						:     List[str]						)	->    Any:
					if issubclass(__lowerCamelCase					, __lowerCamelCase						):
										__lowerCAmelCase						 =text_path
					elif issubclass(__lowerCamelCase					, __lowerCamelCase						):
										__lowerCAmelCase						 =[text_path]
					__lowerCAmelCase						 =tmp_path / """cache"""
					__lowerCAmelCase						 ={"""text""": """string"""}
					__lowerCAmelCase						 =TextDatasetReader(__lowerCamelCase					, cache_dir=__lowerCamelCase						).read()
					_check_text_dataset(__lowerCamelCase					, __lowerCamelCase						)






def        __lowerCAmelCase		(	__lowerCamelCase						:     Tuple					, __lowerCamelCase						:     List[str]					, __lowerCamelCase						:     str=("train",)						)	->    Dict:
					assert isinstance(__lowerCamelCase					, __lowerCamelCase						)
					for split in splits:
										__lowerCAmelCase						 =dataset_dict[split]
										assert dataset.num_rows == 4
										assert dataset.num_columns == 1
										assert dataset.column_names == ["text"]
										for feature, expected_dtype in expected_features.items():
															assert dataset.features[feature].dtype == expected_dtype






@pytest.mark.parametrize("""keep_in_memory"""					, [False, True]						)
def        __lowerCAmelCase		(	__lowerCamelCase						:     Optional[int]					, __lowerCamelCase						:     List[Any]					, __lowerCamelCase						:     List[Any]						)	->    Dict:
					__lowerCAmelCase						 =tmp_path / """cache"""
					__lowerCAmelCase						 ={"""text""": """string"""}
					with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
										__lowerCAmelCase						 =TextDatasetReader({"""train""": text_path}					, cache_dir=__lowerCamelCase					, keep_in_memory=__lowerCamelCase						).read()
					_check_text_datasetdict(__lowerCamelCase					, __lowerCamelCase						)






@pytest.mark.parametrize(
    """features"""					, [
        None,
        {"""text""": """string"""},
        {"""text""": """int32"""},
        {"""text""": """float32"""},
    ]					, )
def        __lowerCAmelCase		(	__lowerCamelCase						:     Any					, __lowerCamelCase						:     Optional[int]					, __lowerCamelCase						:     Union[str, Any]						)	->    Tuple:
					__lowerCAmelCase						 =tmp_path / """cache"""
					# CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
					__lowerCAmelCase						 ={"""text""": """string"""}
					__lowerCAmelCase						 =features.copy() if features else default_expected_features
					__lowerCAmelCase						 =(
					    Features({feature: Value(__lowerCamelCase						) for feature, dtype in features.items()}						) if features is not None else None
					)
					__lowerCAmelCase						 =TextDatasetReader({"""train""": text_path}					, features=__lowerCamelCase					, cache_dir=__lowerCamelCase						).read()
					_check_text_datasetdict(__lowerCamelCase					, __lowerCamelCase						)






@pytest.mark.parametrize("""split"""					, [None, NamedSplit("""train"""						), """train""", """test"""]						)
def        __lowerCAmelCase		(	__lowerCamelCase						:     int					, __lowerCamelCase						:     List[str]					, __lowerCamelCase						:     Any						)	->    int:
					if split:
										__lowerCAmelCase						 ={split: text_path}
					else:
										__lowerCAmelCase						 ="""train"""
										__lowerCAmelCase						 ={"""train""": text_path, """test""": text_path}
					__lowerCAmelCase						 =tmp_path / """cache"""
					__lowerCAmelCase						 ={"""text""": """string"""}
					__lowerCAmelCase						 =TextDatasetReader(__lowerCamelCase					, cache_dir=__lowerCamelCase						).read()
					_check_text_datasetdict(__lowerCamelCase					, __lowerCamelCase					, splits=list(path.keys()						)						)
					assert all(dataset[split].split == split for split in path.keys()						)




