






'''simple docstring'''





import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis_float32 (there's also the fix_lavis branch)
# also note: to convert Vicuna checkpoints, we had to include /home/niels/python_projects/checkpoints/FastChat/vicuna-7b in lavis/configs/models/blip2/blip2_instruct_vicuna7b.yaml
# same for Vicuna-13b
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipImageProcessor,
    InstructBlipConfig,
    InstructBlipForConditionalGeneration,
    InstructBlipProcessor,
    InstructBlipQFormerConfig,
    InstructBlipVisionConfig,
    LlamaConfig,
    LlamaTokenizerFast,
    TaConfig,
    TaTokenizerFast,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD






def    lowerCamelCase_     (    ):
 lowerCamelCase__  =     "https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg"
 lowerCamelCase__  =     Image.open(requests.get(lowercase__				,     stream=lowercase__).raw).convert("RGB")

 return image






def    lowerCamelCase_     (    lowercase__):
 lowerCamelCase__  =     []
 # fmt: off

 # vision encoder
 rename_keys.append(("visual_encoder.cls_token", "vision_model.embeddings.class_embedding"))
 rename_keys.append(("visual_encoder.pos_embed", "vision_model.embeddings.position_embedding"))
 rename_keys.append(("visual_encoder.patch_embed.proj.weight", "vision_model.embeddings.patch_embedding.weight"))
 rename_keys.append(("visual_encoder.patch_embed.proj.bias", "vision_model.embeddings.patch_embedding.bias"))
 rename_keys.append(("ln_vision.weight", "vision_model.post_layernorm.weight"))
 rename_keys.append(("ln_vision.bias", "vision_model.post_layernorm.bias"))

 for i in range(config.vision_config.num_hidden_layers):
  rename_keys.append((F'''visual_encoder.blocks.{i}.norm1.weight''', F'''vision_model.encoder.layers.{i}.layer_norm1.weight'''))
  rename_keys.append((F'''visual_encoder.blocks.{i}.norm1.bias''', F'''vision_model.encoder.layers.{i}.layer_norm1.bias'''))
  rename_keys.append((F'''visual_encoder.blocks.{i}.norm2.weight''', F'''vision_model.encoder.layers.{i}.layer_norm2.weight'''))
  rename_keys.append((F'''visual_encoder.blocks.{i}.norm2.bias''', F'''vision_model.encoder.layers.{i}.layer_norm2.bias'''))
  rename_keys.append((F'''visual_encoder.blocks.{i}.attn.qkv.weight''', F'''vision_model.encoder.layers.{i}.self_attn.qkv.weight'''))
  rename_keys.append((F'''visual_encoder.blocks.{i}.attn.proj.weight''', F'''vision_model.encoder.layers.{i}.self_attn.projection.weight''',))
  rename_keys.append((F'''visual_encoder.blocks.{i}.attn.proj.bias''', F'''vision_model.encoder.layers.{i}.self_attn.projection.bias'''))
  rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc1.weight''', F'''vision_model.encoder.layers.{i}.mlp.fc1.weight'''))
  rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc1.bias''', F'''vision_model.encoder.layers.{i}.mlp.fc1.bias'''))
  rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc2.weight''', F'''vision_model.encoder.layers.{i}.mlp.fc2.weight'''))
  rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc2.bias''', F'''vision_model.encoder.layers.{i}.mlp.fc2.bias'''))

 # QFormer
 rename_keys.append(("Qformer.bert.embeddings.LayerNorm.weight", "qformer.embeddings.layernorm.weight"))
 rename_keys.append(("Qformer.bert.embeddings.LayerNorm.bias", "qformer.embeddings.layernorm.bias"))

 # fmt: on
 return rename_keys






def    lowerCamelCase_     (    lowercase__				,     lowercase__				,     lowercase__):
 lowerCamelCase__  =     dct.pop(lowercase__)
 lowerCamelCase__  =     val






def    lowerCamelCase_     (    lowercase__				,     lowercase__):
 for i in range(config.vision_config.num_hidden_layers):
  # read in original q and v biases
  lowerCamelCase__  =     state_dict.pop(F'''visual_encoder.blocks.{i}.attn.q_bias''')
  lowerCamelCase__  =     state_dict.pop(F'''visual_encoder.blocks.{i}.attn.v_bias''')

  # next, set bias in the state dict
  lowerCamelCase__  =     torch.cat((q_bias, torch.zeros_like(lowercase__				,     requires_grad=lowercase__), v_bias))
  lowerCamelCase__  =     qkv_bias






def    lowerCamelCase_     (    lowercase__):
 lowerCamelCase__  =     364 if "coco" in model_name else 224
 lowerCamelCase__  =     InstructBlipVisionConfig(image_size=lowercase__).to_dict()

 # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
 # seems like flan-T5 models don't have bos_token_id properly set?
 if "t5-xl" in model_name:
  lowerCamelCase__  =     TaConfig.from_pretrained("google/flan-t5-xl"				,     dense_act_fn="gelu"				,     bos_token_id=1).to_dict()
 elif "t5-xxl" in model_name:
  lowerCamelCase__  =     TaConfig.from_pretrained("google/flan-t5-xxl"				,     dense_act_fn="gelu"				,     bos_token_id=1).to_dict()
 elif "vicuna-7b" in model_name:
  lowerCamelCase__  =     LlamaConfig.from_pretrained("decapoda-research/llama-7b-hf"				,     vocab_size=32001).to_dict()
 elif "vicuna-13b" in model_name:
  lowerCamelCase__  =     LlamaConfig.from_pretrained("decapoda-research/llama-13b-hf"				,     vocab_size=32001).to_dict()
 else:
  raise ValueError("Model name not supported")

 # the authors add one special "[DEC]" token to the vocab of Q-Former, hence vocab size = 30522 + 1
 lowerCamelCase__  =     InstructBlipQFormerConfig(vocab_size=30523).to_dict()
 lowerCamelCase__  =     InstructBlipConfig(vision_config=lowercase__				,     text_config=lowercase__				,     qformer_config=lowercase__)

 return config, image_size






@torch.no_grad()
def    lowerCamelCase_     (    lowercase__				,     lowercase__=None				,     lowercase__=False):
 lowerCamelCase__  =     AutoTokenizer.from_pretrained("bert-base-uncased"				,     truncation_side="left")
 qformer_tokenizer.add_special_tokens({"bos_token": "[DEC]"})

 if "t5" in model_name:
  lowerCamelCase__  =     TaTokenizerFast.from_pretrained("google/flan-t5-xl"				,     truncation_side="left")
 elif "vicuna" in model_name:
  # the following was used in the original implementation:
  # tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b", use_fast=False, truncation_side="left")
  # tokenizer.add_special_tokens({"pad_token": "[PAD]"})
  # tokenizer.add_special_tokens({"bos_token": "</s>"})
  # tokenizer.add_special_tokens({"eos_token": "</s>"})
  # tokenizer.add_special_tokens({"unk_token": "</s>"})
  lowerCamelCase__  =     LlamaTokenizerFast.from_pretrained(
      "huggyllama/llama-7b"				,     truncation_side="left"				,     bos_token="</s>"				,     unk_token="</s>")
  tokenizer.add_special_tokens({"pad_token": "[PAD]"})

 lowerCamelCase__  ,	lowerCamelCase__  =     get_blipa_config(lowercase__)
 lowerCamelCase__  =     InstructBlipForConditionalGeneration(lowercase__).eval()

 lowerCamelCase__  =     {
     "instructblip-vicuna-7b": ("blip2_vicuna_instruct", "vicuna7b"),
     "instructblip-vicuna-13b": ("blip2_vicuna_instruct", "vicuna13b"),
     "instructblip-flan-t5-xl": ("blip2_t5_instruct", "flant5xl"),
     "instructblip-flan-t5-xxl": ("blip2_t5_instruct", "flant5xxl"),
 }

 lowerCamelCase__  ,	lowerCamelCase__  =     model_name_to_original[model_name]

 # load original model
 print("Loading original model...")
 lowerCamelCase__  =     "cuda:1" if torch.cuda.is_available() else "cpu"
 lowerCamelCase__  =     "cuda:2" if torch.cuda.is_available() else "cpu"
 lowerCamelCase__  ,	lowerCamelCase__  ,	lowerCamelCase__  =     load_model_and_preprocess(
     name=lowercase__				,     model_type=lowercase__				,     is_eval=lowercase__				,     device=lowercase__)
 original_model.eval()
 print("Done!")

 # update state dict keys
 lowerCamelCase__  =     original_model.state_dict()
 lowerCamelCase__  =     create_rename_keys(lowercase__)
 for src, dest in rename_keys:
  rename_key(lowercase__				,     lowercase__				,     lowercase__)

 # some keys can be renamed efficiently
 for key, val in state_dict.copy().items():
  lowerCamelCase__  =     state_dict.pop(lowercase__)
  if key.startswith("Qformer.bert"):
   lowerCamelCase__  =     key.replace("Qformer.bert"				,     "qformer")
  if "attention.self" in key:
   lowerCamelCase__  =     key.replace("self"				,     "attention")
  if "llm_proj" in key:
   lowerCamelCase__  =     key.replace("llm_proj"				,     "language_projection")
  if "t5_proj" in key:
   lowerCamelCase__  =     key.replace("t5_proj"				,     "language_projection")
  if key.startswith("llm_model"):
   lowerCamelCase__  =     key.replace("llm_model"				,     "language_model")
  if key.startswith("t5"):
   lowerCamelCase__  =     key.replace("t5"				,     "language")
  lowerCamelCase__  =     val

 # read in qv biases
 read_in_q_v_bias(lowercase__				,     lowercase__)

 # note: weights get loaded in torch.float32 by default
 hf_model.load_state_dict(lowercase__				,     strict=lowercase__)

 lowerCamelCase__  =     load_demo_image()
 lowerCamelCase__  =     "What is unusual about this image?"

 # create processor
 lowerCamelCase__  =     BlipImageProcessor(
     size={"height": image_size, "width": image_size}				,     image_mean=lowercase__				,     image_std=lowercase__)
 lowerCamelCase__  =     InstructBlipProcessor(
     image_processor=lowercase__				,     tokenizer=lowercase__				,     qformer_tokenizer=lowercase__				,     )
 lowerCamelCase__  =     processor(images=lowercase__				,     text=lowercase__				,     return_tensors="pt").to(lowercase__)

 # make sure processor creates exact same pixel values
 lowerCamelCase__  =     vis_processors["eval"](lowercase__).unsqueeze(0).to(lowercase__)
 lowerCamelCase__  =     inputs.pixel_values
 assert torch.allclose(original_pixel_values.to(pixel_values.device)				,     lowercase__)

 original_model.to(lowercase__)
 hf_model.to(lowercase__)
 with torch.no_grad():
  if "vicuna" in model_name:
   lowerCamelCase__  =     original_model({"image": original_pixel_values, "text_input": [prompt]}).logits
   lowerCamelCase__  =     hf_model(**lowercase__).logits
  else:
   lowerCamelCase__  =     original_model(
       {"image": original_pixel_values, "text_input": [prompt], "text_output": ["\n"]}).logits
   lowerCamelCase__  =     tokenizer("\n"				,     return_tensors="pt").input_ids.to(lowercase__)
   lowerCamelCase__  =     label_input_ids.masked_fill(label_input_ids == tokenizer.pad_token_id				,     -100)
   lowerCamelCase__  =     hf_model(**lowercase__				,     labels=lowercase__).logits

 print("First values of original logits:"				,     original_logits[0, :3, :3])
 print("First values of HF logits:"				,     logits[0, :3, :3])

 # assert values
 assert original_logits.shape == logits.shape
 lowerCamelCase__  =     1E-4 if "vicuna" in model_name else 1E-5
 assert torch.allclose(original_logits.to(logits.device)				,     lowercase__				,     atol=lowercase__)
 print("Looks ok!")

 print("Generating with original model...")
 lowerCamelCase__  =     original_model.generate({"image": original_pixel_values, "prompt": prompt}				,     num_beams=5)

 # important: we need to cast the weights of the HF model to the appropriate type
 print("Generating with HF model...")
 lowerCamelCase__  =     hf_model.generate(
     **lowercase__				,     do_sample=lowercase__				,     num_beams=5				,     max_length=256				,     min_length=1				,     top_p=0.9				,     repetition_penalty=1.5				,     length_penalty=1.0				,     temperature=1				,     )
 if "vicuna" in model_name:
  # convert output id 0 to 2 (eos_token_id)
  # TODO add this in the generate method?
  lowerCamelCase__  =     2
 print("Original generation:"				,     lowercase__)
 lowerCamelCase__  =     processor.batch_decode(lowercase__				,     skip_special_tokens=lowercase__)
 lowerCamelCase__  =     [text.strip() for text in output_text]
 print("HF generation:"				,     lowercase__)

 if pytorch_dump_folder_path is not None:
  processor.save_pretrained(lowercase__)
  hf_model.save_pretrained(lowercase__)

 if push_to_hub:
  processor.push_to_hub(F'''Salesforce/{model_name}''')
  hf_model.push_to_hub(F'''Salesforce/{model_name}''')


if __name__ == "__main__":
 __A							:							Union[str, Any]  							=		argparse.ArgumentParser()
 __A							:							Union[str, Any]  							=		[
     """instructblip-vicuna-7b""",
     """instructblip-vicuna-13b""",
     """instructblip-flan-t5-xl""",
     """instructblip-flan-t5-xxl""",
 ]
 parser.add_argument(
     """--model_name""",
     default="""instructblip-flan-t5-xl""",
     choices=choices,
     type=str,
     help="""Path to hf config.json of model to convert""",
 )
 parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
 parser.add_argument(
     """--push_to_hub""",
     action="""store_true""",
     help="""Whether to push the model and processor to the hub after converting""",
 )

 __A							:							str  							=		parser.parse_args()

 convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)









'''simple docstring'''





from pathlib import Path

import cva
import numpy as np
from matplotlib import pyplot as plt






def    lowerCamelCase_     (    lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__):
 lowerCamelCase__  =     cva.getAffineTransform(lowercase__				,     lowercase__)
 return cva.warpAffine(lowercase__				,     lowercase__				,     (rows, cols))


if __name__ == "__main__":
 # read original image
 __A							:							Any  							=		cva.imread(
     str(Path(__file__).resolve().parent.parent / """image_data""" / """lena.jpg""")
 )
 # turn image in gray scale value
 __A							:							int  							=		cva.cvtColor(image, cva.COLOR_BGR2GRAY)
 # get image shape
 __A							,			__A							:							Dict  							=		gray_img.shape

 # set different points to rotate image
 __A							:							int  							=		np.array([[50, 50], [2_00, 50], [50, 2_00]], np.floataa)
 __A							:							Any  							=		np.array([[10, 1_00], [2_00, 50], [1_00, 2_50]], np.floataa)
 __A							:							str  							=		np.array([[50, 50], [1_50, 50], [1_20, 2_00]], np.floataa)
 __A							:							Optional[int]  							=		np.array([[10, 1_00], [80, 50], [1_80, 2_50]], np.floataa)

 # add all rotated images in a list
 __A							:							List[Any]  							=		[
     gray_img,
     get_rotation(gray_img, ptsa, ptsa, img_rows, img_cols),
     get_rotation(gray_img, ptsa, ptsa, img_rows, img_cols),
     get_rotation(gray_img, ptsa, ptsa, img_rows, img_cols),
 ]

 # plot different image rotations
 __A							:							int  							=		plt.figure(1)
 __A							:							int  							=		["""Original""", """Rotation 1""", """Rotation 2""", """Rotation 3"""]
 for i, image in enumerate(images):
  plt.subplot(2, 2, i + 1), plt.imshow(image, """gray""")
  plt.title(titles[i])
  plt.axis("""off""")
  plt.subplots_adjust(left=0.0, bottom=0.05, right=1.0, top=0.95)
 plt.show()

