



import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis_float32 (there's also the fix_lavis branch)
# also note: to convert Vicuna checkpoints, we had to include /home/niels/python_projects/checkpoints/FastChat/vicuna-7b in lavis/configs/models/blip2/blip2_instruct_vicuna7b.yaml
# same for Vicuna-13b
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipImageProcessor,
    InstructBlipConfig,
    InstructBlipForConditionalGeneration,
    InstructBlipProcessor,
    InstructBlipQFormerConfig,
    InstructBlipVisionConfig,
    LlamaConfig,
    LlamaTokenizerFast,
    TaConfig,
    TaTokenizerFast,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD



def __lowerCAmelCase			(  ):
			lowercase__       	=					"https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg"
			lowercase__       	=					Image.open(requests.get(snake_case__    ,	stream=snake_case__	).raw	).convert("RGB"	)

			return image



def __lowerCAmelCase			(  SCREAMING_SNAKE_CASE_	):
			lowercase__       	=					[]
			# fmt: off

			# vision encoder
			rename_keys.append(("visual_encoder.cls_token", "vision_model.embeddings.class_embedding")	)
			rename_keys.append(("visual_encoder.pos_embed", "vision_model.embeddings.position_embedding")	)
			rename_keys.append(("visual_encoder.patch_embed.proj.weight", "vision_model.embeddings.patch_embedding.weight")	)
			rename_keys.append(("visual_encoder.patch_embed.proj.bias", "vision_model.embeddings.patch_embedding.bias")	)
			rename_keys.append(("ln_vision.weight", "vision_model.post_layernorm.weight")	)
			rename_keys.append(("ln_vision.bias", "vision_model.post_layernorm.bias")	)

			for i in range(config.vision_config.num_hidden_layers	):
						rename_keys.append((f'''visual_encoder.blocks.{i}.norm1.weight''', f'''vision_model.encoder.layers.{i}.layer_norm1.weight''')	)
						rename_keys.append((f'''visual_encoder.blocks.{i}.norm1.bias''', f'''vision_model.encoder.layers.{i}.layer_norm1.bias''')	)
						rename_keys.append((f'''visual_encoder.blocks.{i}.norm2.weight''', f'''vision_model.encoder.layers.{i}.layer_norm2.weight''')	)
						rename_keys.append((f'''visual_encoder.blocks.{i}.norm2.bias''', f'''vision_model.encoder.layers.{i}.layer_norm2.bias''')	)
						rename_keys.append((f'''visual_encoder.blocks.{i}.attn.qkv.weight''', f'''vision_model.encoder.layers.{i}.self_attn.qkv.weight''')	)
						rename_keys.append((f'''visual_encoder.blocks.{i}.attn.proj.weight''', f'''vision_model.encoder.layers.{i}.self_attn.projection.weight''',)	)
						rename_keys.append((f'''visual_encoder.blocks.{i}.attn.proj.bias''', f'''vision_model.encoder.layers.{i}.self_attn.projection.bias''')	)
						rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc1.weight''', f'''vision_model.encoder.layers.{i}.mlp.fc1.weight''')	)
						rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc1.bias''', f'''vision_model.encoder.layers.{i}.mlp.fc1.bias''')	)
						rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc2.weight''', f'''vision_model.encoder.layers.{i}.mlp.fc2.weight''')	)
						rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc2.bias''', f'''vision_model.encoder.layers.{i}.mlp.fc2.bias''')	)

			# QFormer
			rename_keys.append(("Qformer.bert.embeddings.LayerNorm.weight", "qformer.embeddings.layernorm.weight")	)
			rename_keys.append(("Qformer.bert.embeddings.LayerNorm.bias", "qformer.embeddings.layernorm.bias")	)

			# fmt: on
			return rename_keys



def __lowerCAmelCase			(  SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_	):
			lowercase__       	=					dct.pop(snake_case__	)
			lowercase__       	=					val



def __lowerCAmelCase			(  SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_	):





			for i in range(config.vision_config.num_hidden_layers	):
						# read in original q and v biases
						lowercase__       	=					state_dict.pop(f'''visual_encoder.blocks.{i}.attn.q_bias'''	)
						lowercase__       	=					state_dict.pop(f'''visual_encoder.blocks.{i}.attn.v_bias'''	)

						# next, set bias in the state dict
						lowercase__       	=					torch.cat((q_bias, torch.zeros_like(snake_case__    ,	requires_grad=snake_case__	), v_bias)	)
						lowercase__       	=					qkv_bias



def __lowerCAmelCase			(  SCREAMING_SNAKE_CASE_	):
			lowercase__       	=					364 if "coco" in model_name else 224
			lowercase__       	=					InstructBlipVisionConfig(image_size=snake_case__	).to_dict()

			# make sure the models have proper bos_token_id and eos_token_id set (important for generation)
			# seems like flan-T5 models don't have bos_token_id properly set?
			if "t5-xl" in model_name:
						lowercase__       	=					TaConfig.from_pretrained("google/flan-t5-xl"    ,	dense_act_fn="gelu"    ,	bos_token_id=1	).to_dict()
			elif "t5-xxl" in model_name:
						lowercase__       	=					TaConfig.from_pretrained("google/flan-t5-xxl"    ,	dense_act_fn="gelu"    ,	bos_token_id=1	).to_dict()
			elif "vicuna-7b" in model_name:
						lowercase__       	=					LlamaConfig.from_pretrained("decapoda-research/llama-7b-hf"    ,	vocab_size=3_2001	).to_dict()
			elif "vicuna-13b" in model_name:
						lowercase__       	=					LlamaConfig.from_pretrained("decapoda-research/llama-13b-hf"    ,	vocab_size=3_2001	).to_dict()
			else:
						raise ValueError("Model name not supported"	)

			# the authors add one special "[DEC]" token to the vocab of Q-Former, hence vocab size = 30522 + 1
			lowercase__       	=					InstructBlipQFormerConfig(vocab_size=3_0523	).to_dict()
			lowercase__       	=					InstructBlipConfig(vision_config=snake_case__    ,	text_config=snake_case__    ,	qformer_config=snake_case__	)

			return config, image_size



@torch.no_grad()
def __lowerCAmelCase			(  SCREAMING_SNAKE_CASE_    ,	SCREAMING_SNAKE_CASE_=None    ,	SCREAMING_SNAKE_CASE_=False	):
			lowercase__       	=					AutoTokenizer.from_pretrained("bert-base-uncased"    ,	truncation_side="left"	)
			qformer_tokenizer.add_special_tokens({"bos_token": "[DEC]"}	)

			if "t5" in model_name:
						lowercase__       	=					TaTokenizerFast.from_pretrained("google/flan-t5-xl"    ,	truncation_side="left"	)
			elif "vicuna" in model_name:
						# the following was used in the original implementation:
						# tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b", use_fast=False, truncation_side="left")
						# tokenizer.add_special_tokens({"pad_token": "[PAD]"})
						# tokenizer.add_special_tokens({"bos_token": "</s>"})
						# tokenizer.add_special_tokens({"eos_token": "</s>"})
						# tokenizer.add_special_tokens({"unk_token": "</s>"})
						lowercase__       	=					LlamaTokenizerFast.from_pretrained(
						    "huggyllama/llama-7b"    ,	truncation_side="left"    ,	bos_token="</s>"    ,	unk_token="</s>"	)
						tokenizer.add_special_tokens({"pad_token": "[PAD]"}	)

			lowercase__	,							lowercase__       	=					get_blipa_config(snake_case__	)
			lowercase__       	=					InstructBlipForConditionalGeneration(snake_case__	).eval()

			lowercase__       	=					{
			    "instructblip-vicuna-7b": ("blip2_vicuna_instruct", "vicuna7b"),
			    "instructblip-vicuna-13b": ("blip2_vicuna_instruct", "vicuna13b"),
			    "instructblip-flan-t5-xl": ("blip2_t5_instruct", "flant5xl"),
			    "instructblip-flan-t5-xxl": ("blip2_t5_instruct", "flant5xxl"),
			}

			lowercase__	,							lowercase__       	=					model_name_to_original[model_name]

			# load original model
			print("Loading original model..."	)
			lowercase__       	=					"cuda:1" if torch.cuda.is_available() else "cpu"
			lowercase__       	=					"cuda:2" if torch.cuda.is_available() else "cpu"
			lowercase__	,							lowercase__	,							lowercase__       	=					load_model_and_preprocess(
			    name=snake_case__    ,	model_type=snake_case__    ,	is_eval=snake_case__    ,	device=snake_case__	)
			original_model.eval()
			print("Done!"	)

			# update state dict keys
			lowercase__       	=					original_model.state_dict()
			lowercase__       	=					create_rename_keys(snake_case__	)
			for src, dest in rename_keys:
						rename_key(snake_case__    ,	snake_case__    ,	snake_case__	)

			# some keys can be renamed efficiently
			for key, val in state_dict.copy().items():
						lowercase__       	=					state_dict.pop(snake_case__	)
						if key.startswith("Qformer.bert"	):
									lowercase__       	=					key.replace("Qformer.bert"    ,	"qformer"	)
						if "attention.self" in key:
									lowercase__       	=					key.replace("self"    ,	"attention"	)
						if "llm_proj" in key:
									lowercase__       	=					key.replace("llm_proj"    ,	"language_projection"	)
						if "t5_proj" in key:
									lowercase__       	=					key.replace("t5_proj"    ,	"language_projection"	)
						if key.startswith("llm_model"	):
									lowercase__       	=					key.replace("llm_model"    ,	"language_model"	)
						if key.startswith("t5"	):
									lowercase__       	=					key.replace("t5"    ,	"language"	)
						lowercase__       	=					val

			# read in qv biases
			read_in_q_v_bias(snake_case__    ,	snake_case__	)

			# note: weights get loaded in torch.float32 by default
			hf_model.load_state_dict(snake_case__    ,	strict=snake_case__	)

			lowercase__       	=					load_demo_image()
			lowercase__       	=					"What is unusual about this image?"

			# create processor
			lowercase__       	=					BlipImageProcessor(
			    size={"height": image_size, "width": image_size}    ,	image_mean=snake_case__    ,	image_std=snake_case__	)
			lowercase__       	=					InstructBlipProcessor(
			    image_processor=snake_case__    ,	tokenizer=snake_case__    ,	qformer_tokenizer=snake_case__    ,	)
			lowercase__       	=					processor(images=snake_case__    ,	text=snake_case__    ,	return_tensors="pt"	).to(snake_case__	)

			# make sure processor creates exact same pixel values
			lowercase__       	=					vis_processors["eval"](snake_case__	).unsqueeze(0	).to(snake_case__	)
			lowercase__       	=					inputs.pixel_values
			assert torch.allclose(original_pixel_values.to(pixel_values.device	)    ,	snake_case__	)

			original_model.to(snake_case__	)
			hf_model.to(snake_case__	)
			with torch.no_grad():
						if "vicuna" in model_name:
									lowercase__       	=					original_model({"image": original_pixel_values, "text_input": [prompt]}	).logits
									lowercase__       	=					hf_model(**snake_case__	).logits
						else:
									lowercase__       	=					original_model(
									    {"image": original_pixel_values, "text_input": [prompt], "text_output": ["\n"]}	).logits
									lowercase__       	=					tokenizer("\n"    ,	return_tensors="pt"	).input_ids.to(snake_case__	)
									lowercase__       	=					label_input_ids.masked_fill(label_input_ids == tokenizer.pad_token_id    ,	-100	)
									lowercase__       	=					hf_model(**snake_case__    ,	labels=snake_case__	).logits

			print("First values of original logits:"    ,	original_logits[0, :3, :3]	)
			print("First values of HF logits:"    ,	logits[0, :3, :3]	)

			# assert values
			assert original_logits.shape == logits.shape
			lowercase__       	=					1e-4 if "vicuna" in model_name else 1e-5
			assert torch.allclose(original_logits.to(logits.device	)    ,	snake_case__    ,	atol=snake_case__	)
			print("Looks ok!"	)

			print("Generating with original model..."	)
			lowercase__       	=					original_model.generate({"image": original_pixel_values, "prompt": prompt}    ,	num_beams=5	)

			# important: we need to cast the weights of the HF model to the appropriate type
			print("Generating with HF model..."	)
			lowercase__       	=					hf_model.generate(
			    **snake_case__    ,	do_sample=snake_case__    ,	num_beams=5    ,	max_length=256    ,	min_length=1    ,	top_p=0.9    ,	repetition_penalty=1.5    ,	length_penalty=1.0    ,	temperature=1    ,	)
			if "vicuna" in model_name:
						# convert output id 0 to 2 (eos_token_id)
						# TODO add this in the generate method?
						lowercase__       	=					2
			print("Original generation:"    ,	snake_case__	)
			lowercase__       	=					processor.batch_decode(snake_case__    ,	skip_special_tokens=snake_case__	)
			lowercase__       	=					[text.strip() for text in output_text]
			print("HF generation:"    ,	snake_case__	)

			if pytorch_dump_folder_path is not None:
						processor.save_pretrained(snake_case__	)
						hf_model.save_pretrained(snake_case__	)

			if push_to_hub:
						processor.push_to_hub(f'''Salesforce/{model_name}'''	)
						hf_model.push_to_hub(f'''Salesforce/{model_name}'''	)


if __name__ == "__main__":
							lowercase_										=  argparse.ArgumentParser()
							lowercase_										=  [
							    "instructblip-vicuna-7b",
							    "instructblip-vicuna-13b",
							    "instructblip-flan-t5-xl",
							    "instructblip-flan-t5-xxl",
							]
							parser.add_argument(
							    """--model_name""",
							    default="""instructblip-flan-t5-xl""",
							    choices=choices,
							    type=str,
							    help="""Path to hf config.json of model to convert""",
							)
							parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
							parser.add_argument(
							    """--push_to_hub""",
							    action="""store_true""",
							    help="""Whether to push the model and processor to the hub after converting""",
							)

							lowercase_										=  parser.parse_args()

							convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_lowerCAmelCase	:   Any    =   {
    "configuration_chinese_clip": [
        "CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "ChineseCLIPConfig",
        "ChineseCLIPOnnxConfig",
        "ChineseCLIPTextConfig",
        "ChineseCLIPVisionConfig",
    ],
    "processing_chinese_clip": ["ChineseCLIPProcessor"],
}

try:
	if not is_vision_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowerCAmelCase	:   str    =   ["ChineseCLIPFeatureExtractor"]
	_lowerCAmelCase	:   Any    =   ["ChineseCLIPImageProcessor"]

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_lowerCAmelCase	:   Optional[int]    =   [
	    "CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST",
	    "ChineseCLIPModel",
	    "ChineseCLIPPreTrainedModel",
	    "ChineseCLIPTextModel",
	    "ChineseCLIPVisionModel",
	]

if TYPE_CHECKING:
	from .configuration_chinese_clip import (
	    CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
	    ChineseCLIPConfig,
	    ChineseCLIPOnnxConfig,
	    ChineseCLIPTextConfig,
	    ChineseCLIPVisionConfig,
	)
	from .processing_chinese_clip import ChineseCLIPProcessor

	try:
		if not is_vision_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .feature_extraction_chinese_clip import ChineseCLIPFeatureExtractor, ChineseCLIPImageProcessor

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_chinese_clip import (
		    CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
		    ChineseCLIPModel,
		    ChineseCLIPPreTrainedModel,
		    ChineseCLIPTextModel,
		    ChineseCLIPVisionModel,
		)

else:
	import sys

	_lowerCAmelCase	:   Any    =   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
