





import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis_float32 (there's also the fix_lavis branch)
# also note: to convert Vicuna checkpoints, we had to include /home/niels/python_projects/checkpoints/FastChat/vicuna-7b in lavis/configs/models/blip2/blip2_instruct_vicuna7b.yaml
# same for Vicuna-13b
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipImageProcessor,
    InstructBlipConfig,
    InstructBlipForConditionalGeneration,
    InstructBlipProcessor,
    InstructBlipQFormerConfig,
    InstructBlipVisionConfig,
    LlamaConfig,
    LlamaTokenizerFast,
    TaConfig,
    TaTokenizerFast,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD



def 				lowerCAmelCase_ (							):
    UpperCamelCase__:	Optional[int]					=     "https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg"
    UpperCamelCase__:	Tuple					=     Image.open(requests.get(A_    ,stream=A_).raw).convert("RGB")

    return image



def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	int					=     []
    # fmt: off

    # vision encoder
    rename_keys.append(("visual_encoder.cls_token", "vision_model.embeddings.class_embedding"))
    rename_keys.append(("visual_encoder.pos_embed", "vision_model.embeddings.position_embedding"))
    rename_keys.append(("visual_encoder.patch_embed.proj.weight", "vision_model.embeddings.patch_embedding.weight"))
    rename_keys.append(("visual_encoder.patch_embed.proj.bias", "vision_model.embeddings.patch_embedding.bias"))
    rename_keys.append(("ln_vision.weight", "vision_model.post_layernorm.weight"))
    rename_keys.append(("ln_vision.bias", "vision_model.post_layernorm.bias"))

    for i in range(config.vision_config.num_hidden_layers):
        rename_keys.append((F"visual_encoder.blocks.{i}.norm1.weight", F"vision_model.encoder.layers.{i}.layer_norm1.weight"))
        rename_keys.append((F"visual_encoder.blocks.{i}.norm1.bias", F"vision_model.encoder.layers.{i}.layer_norm1.bias"))
        rename_keys.append((F"visual_encoder.blocks.{i}.norm2.weight", F"vision_model.encoder.layers.{i}.layer_norm2.weight"))
        rename_keys.append((F"visual_encoder.blocks.{i}.norm2.bias", F"vision_model.encoder.layers.{i}.layer_norm2.bias"))
        rename_keys.append((F"visual_encoder.blocks.{i}.attn.qkv.weight", F"vision_model.encoder.layers.{i}.self_attn.qkv.weight"))
        rename_keys.append((F"visual_encoder.blocks.{i}.attn.proj.weight", F"vision_model.encoder.layers.{i}.self_attn.projection.weight",))
        rename_keys.append((F"visual_encoder.blocks.{i}.attn.proj.bias", F"vision_model.encoder.layers.{i}.self_attn.projection.bias"))
        rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc1.weight", F"vision_model.encoder.layers.{i}.mlp.fc1.weight"))
        rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc1.bias", F"vision_model.encoder.layers.{i}.mlp.fc1.bias"))
        rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc2.weight", F"vision_model.encoder.layers.{i}.mlp.fc2.weight"))
        rename_keys.append((F"visual_encoder.blocks.{i}.mlp.fc2.bias", F"vision_model.encoder.layers.{i}.mlp.fc2.bias"))

    # QFormer
    rename_keys.append(("Qformer.bert.embeddings.LayerNorm.weight", "qformer.embeddings.layernorm.weight"))
    rename_keys.append(("Qformer.bert.embeddings.LayerNorm.bias", "qformer.embeddings.layernorm.bias"))

    # fmt: on
    return rename_keys



def 				lowerCAmelCase_ (							A_    ,A_    ,A_):
    UpperCamelCase__:	str					=     dct.pop(A_)
    UpperCamelCase__:	str					=     val



def 				lowerCAmelCase_ (							A_    ,A_):
    for i in range(config.vision_config.num_hidden_layers):
        # read in original q and v biases
        UpperCamelCase__:	int					=     state_dict.pop(F"visual_encoder.blocks.{i}.attn.q_bias")
        UpperCamelCase__:	Tuple					=     state_dict.pop(F"visual_encoder.blocks.{i}.attn.v_bias")

        # next, set bias in the state dict
        UpperCamelCase__:	Union[str, Any]					=     torch.cat((q_bias, torch.zeros_like(A_    ,requires_grad=A_), v_bias))
        UpperCamelCase__:	List[Any]					=     qkv_bias



def 				lowerCAmelCase_ (							A_):
    UpperCamelCase__:	Union[str, Any]					=     3_64 if "coco" in model_name else 2_24
    UpperCamelCase__:	Union[str, Any]					=     InstructBlipVisionConfig(image_size=A_).to_dict()

    # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
    # seems like flan-T5 models don't have bos_token_id properly set?
    if "t5-xl" in model_name:
        UpperCamelCase__:	List[str]					=     TaConfig.from_pretrained("google/flan-t5-xl"    ,dense_act_fn="gelu"    ,bos_token_id=1).to_dict()
    elif "t5-xxl" in model_name:
        UpperCamelCase__:	List[Any]					=     TaConfig.from_pretrained("google/flan-t5-xxl"    ,dense_act_fn="gelu"    ,bos_token_id=1).to_dict()
    elif "vicuna-7b" in model_name:
        UpperCamelCase__:	Tuple					=     LlamaConfig.from_pretrained("decapoda-research/llama-7b-hf"    ,vocab_size=3_20_01).to_dict()
    elif "vicuna-13b" in model_name:
        UpperCamelCase__:	Union[str, Any]					=     LlamaConfig.from_pretrained("decapoda-research/llama-13b-hf"    ,vocab_size=3_20_01).to_dict()
    else:
        raise ValueError("Model name not supported")

    # the authors add one special "[DEC]" token to the vocab of Q-Former, hence vocab size = 30522 + 1
    UpperCamelCase__:	List[Any]					=     InstructBlipQFormerConfig(vocab_size=3_05_23).to_dict()
    UpperCamelCase__:	List[Any]					=     InstructBlipConfig(vision_config=A_    ,text_config=A_    ,qformer_config=A_)

    return config, image_size



@torch.no_grad()
def 				lowerCAmelCase_ (							A_    ,A_=None    ,A_=False):
    UpperCamelCase__:	Optional[Any]					=     AutoTokenizer.from_pretrained("bert-base-uncased"    ,truncation_side="left")
    qformer_tokenizer.add_special_tokens({"bos_token": "[DEC]"})

    if "t5" in model_name:
        UpperCamelCase__:	Any					=     TaTokenizerFast.from_pretrained("google/flan-t5-xl"    ,truncation_side="left")
    elif "vicuna" in model_name:
        # the following was used in the original implementation:
        # tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b", use_fast=False, truncation_side="left")
        # tokenizer.add_special_tokens({"pad_token": "[PAD]"})
        # tokenizer.add_special_tokens({"bos_token": "</s>"})
        # tokenizer.add_special_tokens({"eos_token": "</s>"})
        # tokenizer.add_special_tokens({"unk_token": "</s>"})
        UpperCamelCase__:	int					=     LlamaTokenizerFast.from_pretrained(
            "huggyllama/llama-7b"    ,truncation_side="left"    ,bos_token="</s>"    ,unk_token="</s>")
        tokenizer.add_special_tokens({"pad_token": "[PAD]"})

    UpperCamelCase__				, UpperCamelCase__:	Optional[Any]					=     get_blipa_config(A_)
    UpperCamelCase__:	Tuple					=     InstructBlipForConditionalGeneration(A_).eval()

    UpperCamelCase__:	int					=     {
        "instructblip-vicuna-7b": ("blip2_vicuna_instruct", "vicuna7b"),
        "instructblip-vicuna-13b": ("blip2_vicuna_instruct", "vicuna13b"),
        "instructblip-flan-t5-xl": ("blip2_t5_instruct", "flant5xl"),
        "instructblip-flan-t5-xxl": ("blip2_t5_instruct", "flant5xxl"),
    }

    UpperCamelCase__				, UpperCamelCase__:	Optional[Any]					=     model_name_to_original[model_name]

    # load original model
    print("Loading original model...")
    UpperCamelCase__:	Optional[Any]					=     "cuda:1" if torch.cuda.is_available() else "cpu"
    UpperCamelCase__:	Optional[Any]					=     "cuda:2" if torch.cuda.is_available() else "cpu"
    UpperCamelCase__				, UpperCamelCase__				, UpperCamelCase__:	List[Any]					=     load_model_and_preprocess(
        name=A_    ,model_type=A_    ,is_eval=A_    ,device=A_)
    original_model.eval()
    print("Done!")

    # update state dict keys
    UpperCamelCase__:	List[Any]					=     original_model.state_dict()
    UpperCamelCase__:	str					=     create_rename_keys(A_)
    for src, dest in rename_keys:
        rename_key(A_    ,A_    ,A_)

    # some keys can be renamed efficiently
    for key, val in state_dict.copy().items():
        UpperCamelCase__:	List[Any]					=     state_dict.pop(A_)
        if key.startswith("Qformer.bert"):
            UpperCamelCase__:	str					=     key.replace("Qformer.bert"    ,"qformer")
        if "attention.self" in key:
            UpperCamelCase__:	Any					=     key.replace("self"    ,"attention")
        if "llm_proj" in key:
            UpperCamelCase__:	str					=     key.replace("llm_proj"    ,"language_projection")
        if "t5_proj" in key:
            UpperCamelCase__:	List[Any]					=     key.replace("t5_proj"    ,"language_projection")
        if key.startswith("llm_model"):
            UpperCamelCase__:	str					=     key.replace("llm_model"    ,"language_model")
        if key.startswith("t5"):
            UpperCamelCase__:	Tuple					=     key.replace("t5"    ,"language")
        UpperCamelCase__:	Dict					=     val

    # read in qv biases
    read_in_q_v_bias(A_    ,A_)

    # note: weights get loaded in torch.float32 by default
    hf_model.load_state_dict(A_    ,strict=A_)

    UpperCamelCase__:	List[Any]					=     load_demo_image()
    UpperCamelCase__:	List[str]					=     "What is unusual about this image?"

    # create processor
    UpperCamelCase__:	List[str]					=     BlipImageProcessor(
        size={"height": image_size, "width": image_size}    ,image_mean=A_    ,image_std=A_)
    UpperCamelCase__:	Tuple					=     InstructBlipProcessor(
        image_processor=A_    ,tokenizer=A_    ,qformer_tokenizer=A_    ,)
    UpperCamelCase__:	str					=     processor(images=A_    ,text=A_    ,return_tensors="pt").to(A_)

    # make sure processor creates exact same pixel values
    UpperCamelCase__:	Dict					=     vis_processors["eval"](A_).unsqueeze(0).to(A_)
    UpperCamelCase__:	str					=     inputs.pixel_values
    assert torch.allclose(original_pixel_values.to(pixel_values.device)    ,A_)

    original_model.to(A_)
    hf_model.to(A_)
    with torch.no_grad():
        if "vicuna" in model_name:
            UpperCamelCase__:	str					=     original_model({"image": original_pixel_values, "text_input": [prompt]}).logits
            UpperCamelCase__:	str					=     hf_model(**A_).logits
        else:
            UpperCamelCase__:	List[Any]					=     original_model(
                {"image": original_pixel_values, "text_input": [prompt], "text_output": ["\n"]}).logits
            UpperCamelCase__:	str					=     tokenizer("\n"    ,return_tensors="pt").input_ids.to(A_)
            UpperCamelCase__:	Optional[Any]					=     label_input_ids.masked_fill(label_input_ids == tokenizer.pad_token_id    ,-1_00)
            UpperCamelCase__:	Optional[int]					=     hf_model(**A_    ,labels=A_).logits

    print("First values of original logits:"    ,original_logits[0, :3, :3])
    print("First values of HF logits:"    ,logits[0, :3, :3])

    # assert values
    assert original_logits.shape == logits.shape
    UpperCamelCase__:	List[str]					=     1e-4 if "vicuna" in model_name else 1e-5
    assert torch.allclose(original_logits.to(logits.device)    ,A_    ,atol=A_)
    print("Looks ok!")

    print("Generating with original model...")
    UpperCamelCase__:	str					=     original_model.generate({"image": original_pixel_values, "prompt": prompt}    ,num_beams=5)

    # important: we need to cast the weights of the HF model to the appropriate type
    print("Generating with HF model...")
    UpperCamelCase__:	Optional[Any]					=     hf_model.generate(
        **A_    ,do_sample=A_    ,num_beams=5    ,max_length=2_56    ,min_length=1    ,top_p=0.9    ,repetition_penalty=1.5    ,length_penalty=1.0    ,temperature=1    ,)
    if "vicuna" in model_name:
        # convert output id 0 to 2 (eos_token_id)
        # TODO add this in the generate method?
        UpperCamelCase__:	Optional[int]					=     2
    print("Original generation:"    ,A_)
    UpperCamelCase__:	List[str]					=     processor.batch_decode(A_    ,skip_special_tokens=A_)
    UpperCamelCase__:	Tuple					=     [text.strip() for text in output_text]
    print("HF generation:"    ,A_)

    if pytorch_dump_folder_path is not None:
        processor.save_pretrained(A_)
        hf_model.save_pretrained(A_)

    if push_to_hub:
        processor.push_to_hub(F"Salesforce/{model_name}")
        hf_model.push_to_hub(F"Salesforce/{model_name}")


if __name__ == "__main__":
   A__:    Union[str, Any]  	=	argparse.ArgumentParser()
   A__:    Dict  	=	[
       '''instructblip-vicuna-7b''',
       '''instructblip-vicuna-13b''',
       '''instructblip-flan-t5-xl''',
       '''instructblip-flan-t5-xxl''',
   ]
   parser.add_argument(
       '''--model_name''',
       default='''instructblip-flan-t5-xl''',
       choices=choices,
       type=str,
       help='''Path to hf config.json of model to convert''',
   )
   parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
   parser.add_argument(
       '''--push_to_hub''',
       action='''store_true''',
       help='''Whether to push the model and processor to the hub after converting''',
   )

   A__:    str  	=	parser.parse_args()

   convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)








import unittest

from knapsack import knapsack as k




class        _a    (    unittest.TestCase):

      """simple docstring"""


      def 			UpperCAmelCase_       (		self:			Union[str, Any]					):
          '''simple docstring'''


          UpperCamelCase__:	Tuple					=     0
          UpperCamelCase__:	Optional[int]					=     [0]
          UpperCamelCase__:	Dict					=     [0]
          UpperCamelCase__:	str					=     len(__lowerCamelCase					)
          self.assertEqual(k.knapsack(__lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase					)	,    0					)

          UpperCamelCase__:	Tuple					=     [60]
          UpperCamelCase__:	List[Any]					=     [10]
          UpperCamelCase__:	Any					=     len(__lowerCamelCase					)
          self.assertEqual(k.knapsack(__lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase					)	,    0					)






      def 			UpperCAmelCase_       (		self:			Any					):
          '''simple docstring'''


          UpperCamelCase__:	List[str]					=     3
          UpperCamelCase__:	Any					=     [1, 2, 3]
          UpperCamelCase__:	Dict					=     [3, 2, 1]
          UpperCamelCase__:	List[str]					=     len(__lowerCamelCase					)
          self.assertEqual(k.knapsack(__lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase					)	,    5					)






      def 			UpperCAmelCase_       (		self:			List[Any]					):
          '''simple docstring'''


          UpperCamelCase__:	List[str]					=     50
          UpperCamelCase__:	Optional[Any]					=     [60, 100, 120]
          UpperCamelCase__:	Optional[Any]					=     [10, 20, 30]
          UpperCamelCase__:	Any					=     len(__lowerCamelCase					)
          self.assertEqual(k.knapsack(__lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase	,    __lowerCamelCase					)	,    220					)


if __name__ == "__main__":
   unittest.main()

