


import argparse
import re

import requests
import torch

# git clone https://github.com/salesforce/BLIP.git
from models.blip import blip_decoder
from models.blip_itm import blip_itm
from models.blip_vqa import blip_vqa
from PIL import Image
from torchvision import transforms
from torchvision.transforms.functional import InterpolationMode

from transformers import (
    BertTokenizer,
    BlipConfig,
    BlipForConditionalGeneration,
    BlipForImageTextRetrieval,
    BlipForQuestionAnswering,
)






def 							_lowercase    (		SCREAMING_SNAKE_CASE_   :		Tuple       ,  SCREAMING_SNAKE_CASE_   :		Dict	):





			"""simple docstring"""




			UpperCamelCase         =					"""https://storage.googleapis.com/sfr-vision-language-research/BLIP/demo.jpg"""
			UpperCamelCase         =					Image.open(requests.get(SCREAMING_SNAKE_CASE_       ,  stream=SCREAMING_SNAKE_CASE_	).raw	).convert("""RGB"""	)

			UpperCamelCase         =					transforms.Compose(
			    [
			        transforms.Resize((image_size, image_size)       ,  interpolation=InterpolationMode.BICUBIC	),
			        transforms.ToTensor(),
			        transforms.Normalize((0.4814_5466, 0.457_8275, 0.4082_1073)       ,  (0.2686_2954, 0.2613_0258, 0.2757_7711)	),
			    ]	)
			UpperCamelCase         =					transform(SCREAMING_SNAKE_CASE_	).unsqueeze(0	).to(SCREAMING_SNAKE_CASE_	)
			return image






def 							_lowercase    (		SCREAMING_SNAKE_CASE_   :		Dict	):





			"""simple docstring"""




			if "visual_encoder" in key:
						UpperCamelCase         =					re.sub("""visual_encoder*"""       ,  """vision_model.encoder"""       ,  SCREAMING_SNAKE_CASE_	)
			if "blocks" in key:
						UpperCamelCase         =					re.sub(r"""blocks"""       ,  """layers"""       ,  SCREAMING_SNAKE_CASE_	)
			if "attn" in key:
						UpperCamelCase         =					re.sub(r"""attn"""       ,  """self_attn"""       ,  SCREAMING_SNAKE_CASE_	)
			if "norm1" in key:
						UpperCamelCase         =					re.sub(r"""norm1"""       ,  """layer_norm1"""       ,  SCREAMING_SNAKE_CASE_	)
			if "norm2" in key:
						UpperCamelCase         =					re.sub(r"""norm2"""       ,  """layer_norm2"""       ,  SCREAMING_SNAKE_CASE_	)
			if "encoder.norm" in key:
						UpperCamelCase         =					re.sub(r"""encoder.norm"""       ,  """post_layernorm"""       ,  SCREAMING_SNAKE_CASE_	)
			if "encoder.patch_embed.proj" in key:
						UpperCamelCase         =					re.sub(r"""encoder.patch_embed.proj"""       ,  """embeddings.patch_embedding"""       ,  SCREAMING_SNAKE_CASE_	)

			if "encoder.pos_embed" in key:
						UpperCamelCase         =					re.sub(r"""encoder.pos_embed"""       ,  """embeddings.position_embedding"""       ,  SCREAMING_SNAKE_CASE_	)
			if "encoder.cls_token" in key:
						UpperCamelCase         =					re.sub(r"""encoder.cls_token"""       ,  """embeddings.class_embedding"""       ,  SCREAMING_SNAKE_CASE_	)

			if "self_attn" in key:
						UpperCamelCase         =					re.sub(r"""self_attn.proj"""       ,  """self_attn.projection"""       ,  SCREAMING_SNAKE_CASE_	)

			return key






@torch.no_grad()
def 							_lowercase    (		SCREAMING_SNAKE_CASE_   :		List[Any]       ,  SCREAMING_SNAKE_CASE_   :		Tuple=None	):





			"""simple docstring"""




			if config_path is not None:
						UpperCamelCase         =					BlipConfig.from_pretrained(SCREAMING_SNAKE_CASE_	)
			else:
						UpperCamelCase         =					BlipConfig(projection_dim=512       ,  text_config={}       ,  vision_config={}	)

			UpperCamelCase         =					BlipForConditionalGeneration(SCREAMING_SNAKE_CASE_	).eval()

			UpperCamelCase         =					"""https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_capfilt_large.pth"""

			UpperCamelCase         =					blip_decoder(pretrained=SCREAMING_SNAKE_CASE_       ,  image_size=384       ,  vit="""base"""	)
			UpperCamelCase         =					pt_model.eval()

			UpperCamelCase         =					pt_model.state_dict()
			for key in modified_state_dict.copy():
						UpperCamelCase         =					modified_state_dict.pop(SCREAMING_SNAKE_CASE_	)
						UpperCamelCase         =					rename_key(SCREAMING_SNAKE_CASE_	)
						UpperCamelCase         =					value

			hf_model.load_state_dict(SCREAMING_SNAKE_CASE_	)

			UpperCamelCase         =					384
			UpperCamelCase         =					load_demo_image(image_size=SCREAMING_SNAKE_CASE_       ,  device="""cpu"""	)
			UpperCamelCase         =					BertTokenizer.from_pretrained("""bert-base-uncased"""	)
			UpperCamelCase         =					tokenizer(["""a picture of"""]	).input_ids

			UpperCamelCase         =					hf_model.generate(SCREAMING_SNAKE_CASE_       ,  SCREAMING_SNAKE_CASE_	)

			assert out[0].tolist() == [30_522, 1_037, 3_861, 1_997, 1_037, 2_450, 3_564, 2_006, 1_996, 3_509, 2_007, 2_014, 3_899, 102]

			UpperCamelCase         =					hf_model.generate(SCREAMING_SNAKE_CASE_	)

			assert out[0].tolist() == [30_522, 1_037, 2_450, 3_564, 2_006, 1_996, 3_509, 2_007, 2_014, 3_899, 102]

			if pytorch_dump_folder_path is not None:
						hf_model.save_pretrained(SCREAMING_SNAKE_CASE_	)

			# model_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_vqa.pth'
			UpperCamelCase         =					(
			    """https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_vqa_capfilt_large.pth"""
			)

			UpperCamelCase         =					blip_vqa(pretrained=SCREAMING_SNAKE_CASE_       ,  image_size=SCREAMING_SNAKE_CASE_       ,  vit="""base"""	)
			vqa_model.eval()

			UpperCamelCase         =					vqa_model.state_dict()
			for key in modified_state_dict.copy():
						UpperCamelCase         =					modified_state_dict.pop(SCREAMING_SNAKE_CASE_	)
						UpperCamelCase         =					rename_key(SCREAMING_SNAKE_CASE_	)
						UpperCamelCase         =					value

			UpperCamelCase         =					BlipForQuestionAnswering(SCREAMING_SNAKE_CASE_	)

			hf_vqa_model.load_state_dict(SCREAMING_SNAKE_CASE_	)

			UpperCamelCase         =					["""How many dogs are in this image?"""]
			UpperCamelCase         =					tokenizer(SCREAMING_SNAKE_CASE_       ,  return_tensors="""pt"""	).input_ids

			UpperCamelCase         =					hf_vqa_model.generate(SCREAMING_SNAKE_CASE_       ,  SCREAMING_SNAKE_CASE_	)
			print(tokenizer.decode(answer[0]	)	)

			assert tokenizer.decode(answer[0]	) == "[UNK] 1 [SEP]"
			if pytorch_dump_folder_path is not None:
						hf_vqa_model.save_pretrained(pytorch_dump_folder_path + """_vqa"""	)

			UpperCamelCase         =					"""https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_retrieval_coco.pth"""

			UpperCamelCase         =					blip_itm(pretrained=SCREAMING_SNAKE_CASE_       ,  image_size=SCREAMING_SNAKE_CASE_       ,  vit="""base"""	)
			itm_model.eval()

			UpperCamelCase         =					itm_model.state_dict()
			for key in modified_state_dict.copy():
						UpperCamelCase         =					modified_state_dict.pop(SCREAMING_SNAKE_CASE_	)
						UpperCamelCase         =					rename_key(SCREAMING_SNAKE_CASE_	)
						UpperCamelCase         =					value

			UpperCamelCase         =					BlipForImageTextRetrieval(SCREAMING_SNAKE_CASE_	)

			UpperCamelCase         =					["""A picture of a woman with a dog sitting in a beach"""]
			UpperCamelCase         =					tokenizer(
			    SCREAMING_SNAKE_CASE_       ,  return_tensors="""pt"""       ,  padding="""max_length"""       ,  truncation=SCREAMING_SNAKE_CASE_       ,  max_length=35       ,  ).input_ids

			hf_itm_model.load_state_dict(SCREAMING_SNAKE_CASE_	)
			hf_itm_model.eval()

			UpperCamelCase         =					hf_itm_model(SCREAMING_SNAKE_CASE_       ,  SCREAMING_SNAKE_CASE_       ,  use_itm_head=SCREAMING_SNAKE_CASE_	)
			UpperCamelCase         =					hf_itm_model(SCREAMING_SNAKE_CASE_       ,  SCREAMING_SNAKE_CASE_       ,  use_itm_head=SCREAMING_SNAKE_CASE_	)

			assert out[0].item() == 0.2110_6874_9427_7954
			assert torch.nn.functional.softmax(out_itm[0]       ,  dim=1	)[:, 1].item() == 0.4_5698_8453_8650_5127

			if pytorch_dump_folder_path is not None:
						hf_itm_model.save_pretrained(pytorch_dump_folder_path + """_itm"""	)


if __name__ == "__main__":
			__snake_case            =   argparse.ArgumentParser()
			parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
			parser.add_argument("--config_path", default=None, type=str, help="Path to hf config.json of model to convert")
			__snake_case            =   parser.parse_args()

			convert_blip_checkpoint(args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path)








import logging
import os

import quant_trainer
import torch
from torch.utils.data import DataLoader

from transformers import Trainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput


__snake_case            =   logging.getLogger(__name__)

if is_torch_tpu_available(check_device=False):
			import torch_xla.core.xla_model as xm
			import torch_xla.debug.metrics as met




class    UpperCAmelCase					(       __snake_case    ):




			def __init__(						self							:		Dict       ,       *__magic_name__							:		Union[str, Any]       ,       __magic_name__							:		Any=None       ,       __magic_name__							:		Union[str, Any]=None       ,       __magic_name__							:		Any=None       ,       **__magic_name__							:		Optional[Any]						):







						"""simple docstring"""







						super().__init__(*__magic_name__       ,       **__magic_name__						)
						UpperCamelCase         =					eval_examples
						UpperCamelCase         =					post_process_function
						UpperCamelCase         =					quant_trainer_args
						UpperCamelCase         =					1_2_8  # default number of calibration samples




			def   lowerCamelCase_					(						self							:		str       ,       __magic_name__							:		List[Any]=None						):







						"""simple docstring"""







						if calib_dataset is None and self.calib_dataset is None:
									raise ValueError("""Trainer: calibration requires an calib_dataset."""						)
						UpperCamelCase         =					calib_dataset if calib_dataset is not None else self.calib_dataset

						UpperCamelCase         =					self._remove_unused_columns(__magic_name__       ,       description="""Calibration"""						)

						return DataLoader(
						    __magic_name__       ,       batch_size=self.args.eval_batch_size       ,       collate_fn=self.data_collator       ,       drop_last=self.args.dataloader_drop_last       ,       num_workers=self.args.dataloader_num_workers       ,       pin_memory=self.args.dataloader_pin_memory       ,       shuffle=__magic_name__       ,       )




			def   lowerCamelCase_					(						self							:		Tuple       ,       __magic_name__							:		List[str]=None						):







						"""simple docstring"""







						UpperCamelCase         =					self.train_dataset if calib_dataset is None else calib_dataset
						UpperCamelCase         =					self.get_calib_dataloader(__magic_name__						)

						UpperCamelCase         =					self.model
						quant_trainer.configure_model(__magic_name__       ,       self.quant_trainer_args       ,       calib=__magic_name__						)
						model.eval()
						quant_trainer.enable_calibration(__magic_name__						)

						logger.info("""***** Running calibration *****"""						)
						logger.info(F'  Num examples = {self.calib_num}'						)
						logger.info(F'  Batch size = {calib_dataloader.batch_size}'						)

						for step, inputs in enumerate(__magic_name__						):
									# Prediction step
									UpperCamelCase  ,  UpperCamelCase  ,  UpperCamelCase         =					self.prediction_step(__magic_name__       ,       __magic_name__       ,       prediction_loss_only=__magic_name__						)
									if (step + 1) * calib_dataloader.batch_size >= self.calib_num:
												break

						quant_trainer.finish_calibration(__magic_name__       ,       self.quant_trainer_args						)
						UpperCamelCase         =					model




			def   lowerCamelCase_					(						self							:		Any       ,       __magic_name__							:		Optional[int]=None       ,       __magic_name__							:		Optional[Any]=None       ,       __magic_name__							:		Union[str, Any]=None       ,       __magic_name__							:		str = "eval"						):







						"""simple docstring"""







						UpperCamelCase         =					self.eval_dataset if eval_dataset is None else eval_dataset
						UpperCamelCase         =					self.get_eval_dataloader(__magic_name__						)
						UpperCamelCase         =					self.eval_examples if eval_examples is None else eval_examples

						# Temporarily disable metric computation, we will do it in the loop here.
						UpperCamelCase         =					self.compute_metrics
						UpperCamelCase         =					None
						UpperCamelCase         =					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
						try:
									UpperCamelCase         =					eval_loop(
									    __magic_name__       ,       description="""Evaluation"""       ,       prediction_loss_only=True if compute_metrics is None else None       ,       ignore_keys=__magic_name__       ,       )
						finally:
									UpperCamelCase         =					compute_metrics

						if self.post_process_function is not None and self.compute_metrics is not None:
									UpperCamelCase         =					self.post_process_function(__magic_name__       ,       __magic_name__       ,       output.predictions						)
									UpperCamelCase         =					self.compute_metrics(__magic_name__						)

									# Prefix all keys with metric_key_prefix + '_'
									for key in list(metrics.keys()						):
												if not key.startswith(F'{metric_key_prefix}_'						):
															UpperCamelCase         =					metrics.pop(__magic_name__						)

									self.log(__magic_name__						)
						else:
									UpperCamelCase         =					{}

						if self.args.tpu_metrics_debug or self.args.debug:
									# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
									xm.master_print(met.metrics_report()						)

						UpperCamelCase         =					self.callback_handler.on_evaluate(self.args       ,       self.state       ,       self.control       ,       __magic_name__						)
						return metrics




			def   lowerCamelCase_					(						self							:		Dict       ,       __magic_name__							:		List[str]       ,       __magic_name__							:		List[Any]       ,       __magic_name__							:		List[Any]=None       ,       __magic_name__							:		str = "test"						):







						"""simple docstring"""







						UpperCamelCase         =					self.get_test_dataloader(__magic_name__						)

						# Temporarily disable metric computation, we will do it in the loop here.
						UpperCamelCase         =					self.compute_metrics
						UpperCamelCase         =					None
						UpperCamelCase         =					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
						try:
									UpperCamelCase         =					eval_loop(
									    __magic_name__       ,       description="""Prediction"""       ,       prediction_loss_only=True if compute_metrics is None else None       ,       ignore_keys=__magic_name__       ,       )
						finally:
									UpperCamelCase         =					compute_metrics

						if self.post_process_function is None or self.compute_metrics is None:
									return output

						UpperCamelCase         =					self.post_process_function(__magic_name__       ,       __magic_name__       ,       output.predictions       ,       """predict"""						)
						UpperCamelCase         =					self.compute_metrics(__magic_name__						)

						# Prefix all keys with metric_key_prefix + '_'
						for key in list(metrics.keys()						):
									if not key.startswith(F'{metric_key_prefix}_'						):
												UpperCamelCase         =					metrics.pop(__magic_name__						)

						return PredictionOutput(predictions=predictions.predictions       ,       label_ids=predictions.label_ids       ,       metrics=__magic_name__						)





			def   lowerCamelCase_					(						self							:		List[str]       ,       __magic_name__							:		Optional[Any]="./"						):







						"""simple docstring"""







						UpperCamelCase         =					self.eval_dataset
						UpperCamelCase         =					self.get_eval_dataloader(__magic_name__						)

						UpperCamelCase         =					next(iter(__magic_name__						)						)

						# saving device - to make it consistent
						UpperCamelCase         =					torch.device("""cuda""" if torch.cuda.is_available() else """cpu"""						)

						# convert to tuple
						UpperCamelCase         =					tuple(v.to(__magic_name__						) for k, v in batch.items()						)

						logger.info("""Converting model to be onnx compatible"""						)
						from pytorch_quantization.nn import TensorQuantizer

						UpperCamelCase         =					True

						UpperCamelCase         =					self.model.to(__magic_name__						)

						model.eval()
						model.float()

						UpperCamelCase         =					model.module if hasattr(__magic_name__       ,       """module"""						) else model
						quant_trainer.configure_model(__magic_name__       ,       self.quant_trainer_args						)

						UpperCamelCase         =					os.path.join(__magic_name__       ,       """model.onnx"""						)
						logger.info(F'exporting model to {output_model_file}'						)

						UpperCamelCase         =					{0: """batch_size""", 1: """seq_len"""}

						torch.onnx.export(
						    __magic_name__       ,       __magic_name__       ,       __magic_name__       ,       export_params=__magic_name__       ,       opset_version=1_3       ,       do_constant_folding=__magic_name__       ,       input_names=["""input_ids""", """attention_mask""", """token_type_ids"""]       ,       output_names=["""output_start_logits""", """output_end_logits"""]       ,       dynamic_axes={
						        """input_ids""": axes,
						        """attention_mask""": axes,
						        """token_type_ids""": axes,
						        """output_start_logits""": axes,
						        """output_end_logits""": axes,
						    }       ,       verbose=__magic_name__       ,       )
						logger.info("""onnx export finished"""						)




