


'''simple docstring'''




import argparse

import gdown
import numpy as np
import torch
from huggingface_hub import hf_hub_download

from transformers import (
    CLIPTokenizer,
    CLIPTokenizerFast,
    VideoMAEImageProcessor,
    XCLIPConfig,
    XCLIPModel,
    XCLIPProcessor,
    XCLIPTextConfig,
    XCLIPVisionConfig,
)


def    UpperCAmelCase_  (       A		,	A      ):
   '''simple docstring'''





   _a					:  int						=  XCLIPTextConfig()

   # derive patch size from model name
   _a					:  int						=  model_name.find('patch'      )
   _a					:  List[Any]						=  int(model_name[start_idx + len('patch'      ) : start_idx + len('patch'      ) + 2]      )
   _a					:  Any						=  XCLIPVisionConfig(patch_size=A		,	num_frames=A      )

   if "large" in model_name:
      _a					:  Tuple						=  7_6_8
      _a					:  Dict						=  3_0_7_2
      _a					:  Optional[Any]						=  1_2

      _a					:  Optional[int]						=  1_0_2_4
      _a					:  Optional[int]						=  4_0_9_6
      _a					:  List[Any]						=  1_6
      _a					:  int						=  2_4
      _a					:  List[str]						=  7_6_8
      _a					:  str						=  3_0_7_2

   if model_name == "xclip-large-patch14-16-frames":
      _a					:  List[Any]						=  3_3_6

   _a					:  Dict						=  XCLIPConfig.from_text_vision_configs(A		,	A      )

   if "large" in model_name:
      _a					:  List[Any]						=  7_6_8

   return config


def    UpperCAmelCase_  (       A      ):
   '''simple docstring'''





   if name == "token_embedding.weight":
      _a					:  Optional[Any]						=  name.replace('token_embedding.weight'		,	'text_model.embeddings.token_embedding.weight'      )
   if name == "positional_embedding":
      _a					:  Any						=  name.replace('positional_embedding'		,	'text_model.embeddings.position_embedding.weight'      )
   if "ln_1" in name:
      _a					:  List[Any]						=  name.replace('ln_1'		,	'layer_norm1'      )
   if "ln_2" in name:
      _a					:  Optional[Any]						=  name.replace('ln_2'		,	'layer_norm2'      )
   if "c_fc" in name:
      _a					:  Union[str, Any]						=  name.replace('c_fc'		,	'fc1'      )
   if "c_proj" in name:
      _a					:  Optional[Any]						=  name.replace('c_proj'		,	'fc2'      )
   if name.startswith('transformer.resblocks'      ):
      _a					:  Dict						=  name.replace('transformer.resblocks'		,	'text_model.encoder.layers'      )
   if "attn.out_proj" in name and "message" not in name:
      _a					:  Dict						=  name.replace('attn.out_proj'		,	'self_attn.out_proj'      )
   if "ln_final" in name:
      _a					:  int						=  name.replace('ln_final'		,	'text_model.final_layer_norm'      )
   # visual encoder
   if name == "visual.class_embedding":
      _a					:  List[str]						=  name.replace('visual.class_embedding'		,	'vision_model.embeddings.class_embedding'      )
   if name == "visual.positional_embedding":
      _a					:  Optional[int]						=  name.replace('visual.positional_embedding'		,	'vision_model.embeddings.position_embedding.weight'      )
   if name.startswith('visual.transformer.resblocks'      ):
      _a					:  Any						=  name.replace('visual.transformer.resblocks'		,	'vision_model.encoder.layers'      )
   if "visual.conv1" in name:
      _a					:  Tuple						=  name.replace('visual.conv1'		,	'vision_model.embeddings.patch_embedding'      )
   if "visual.ln_pre" in name:
      _a					:  Optional[int]						=  name.replace('visual.ln_pre'		,	'vision_model.pre_layernorm'      )
   if "visual.ln_post" in name:
      _a					:  Union[str, Any]						=  name.replace('visual.ln_post'		,	'vision_model.post_layernorm'      )
   if "visual.proj" in name:
      _a					:  Optional[int]						=  name.replace('visual.proj'		,	'visual_projection.weight'      )
   if "text_projection" in name:
      _a					:  Any						=  name.replace('text_projection'		,	'text_projection.weight'      )
   # things on top
   if "prompts_visual_proj" in name:
      _a					:  Dict						=  name.replace('prompts_visual_proj'		,	'prompts_visual_projection'      )
   if "prompts_visual_ln" in name:
      _a					:  str						=  name.replace('prompts_visual_ln'		,	'prompts_visual_layernorm'      )
   # mit
   if name == "mit.positional_embedding":
      _a					:  Optional[Any]						=  name.replace('positional'		,	'position'      )
   if name.startswith('mit.resblocks'      ):
      _a					:  List[str]						=  name.replace('mit.resblocks'		,	'mit.encoder.layers'      )
   # prompts generator
   if name.startswith('prompts_generator.norm'      ):
      _a					:  str						=  name.replace('prompts_generator.norm'		,	'prompts_generator.layernorm'      )

   return name


def    UpperCAmelCase_  (       A		,	A      ):
   '''simple docstring'''





   for key in orig_state_dict.copy().keys():
      _a					:  Optional[int]						=  orig_state_dict.pop(A      )

      if "attn.in_proj" in key:
         _a					:  Optional[int]						=  key.split('.'      )
         if key.startswith('visual'      ):
            _a					:  Any						=  key_split[3]
            _a					:  Optional[int]						=  config.vision_config.hidden_size
            if "message_attn" in key:
               if "weight" in key:
                  _a					:  Dict						=  val[
                      :dim, :
                  ]
                  _a					:  str						=  val[
                      dim : dim * 2, :
                  ]
                  _a					:  List[str]						=  val[
                      -dim:, :
                  ]
               else:
                  _a					:  Optional[int]						=  val[
                      :dim
                  ]
                  _a					:  Union[str, Any]						=  val[
                      dim : dim * 2
                  ]
                  _a					:  Tuple						=  val[
                      -dim:
                  ]
            else:
               if "weight" in key:
                  _a					:  Optional[int]						=  val[
                      :dim, :
                  ]
                  _a					:  Optional[int]						=  val[
                      dim : dim * 2, :
                  ]
                  _a					:  Dict						=  val[
                      -dim:, :
                  ]
               else:
                  _a					:  Dict						=  val[:dim]
                  _a					:  Dict						=  val[
                      dim : dim * 2
                  ]
                  _a					:  List[str]						=  val[-dim:]
         elif key.startswith('mit'      ):
            _a					:  List[str]						=  key_split[2]
            _a					:  Any						=  config.vision_config.mit_hidden_size
            if "weight" in key:
               _a					:  str						=  val[:dim, :]
               _a					:  Union[str, Any]						=  val[dim : dim * 2, :]
               _a					:  Any						=  val[-dim:, :]
            else:
               _a					:  Dict						=  val[:dim]
               _a					:  List[str]						=  val[dim : dim * 2]
               _a					:  int						=  val[-dim:]
         else:
            _a					:  Union[str, Any]						=  key_split[2]
            _a					:  Tuple						=  config.text_config.hidden_size
            if "weight" in key:
               _a					:  Tuple						=  val[:dim, :]
               _a					:  str						=  val[
                   dim : dim * 2, :
               ]
               _a					:  str						=  val[-dim:, :]
            else:
               _a					:  Tuple						=  val[:dim]
               _a					:  Optional[Any]						=  val[
                   dim : dim * 2
               ]
               _a					:  Tuple						=  val[-dim:]
      else:
         _a					:  str						=  rename_key(A      )
         if new_key_name in ["visual_projection.weight", "text_projection.weight"]:
            _a					:  Optional[int]						=  val.T
         _a					:  Dict						=  val

   return orig_state_dict


def    UpperCAmelCase_  (       A      ):
   '''simple docstring'''





   if num_frames == 8:
      _a					:  Any						=  'eating_spaghetti_8_frames.npy'
   elif num_frames == 1_6:
      _a					:  Optional[Any]						=  'eating_spaghetti.npy'
   elif num_frames == 3_2:
      _a					:  str						=  'eating_spaghetti_32_frames.npy'
   _a					:  Optional[int]						=  hf_hub_download(
       repo_id='hf-internal-testing/spaghetti-video'		,	filename=A		,	repo_type='dataset'		,	)
   _a					:  str						=  np.load(A      )
   return list(A      )


def    UpperCAmelCase_  (       A		,	A=None		,	A=False      ):
   '''simple docstring'''





   _a					:  Any						=  {
       # fully supervised kinetics-400 checkpoints
       'xclip-base-patch32': 'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_32_8.pth',
       'xclip-base-patch32-16-frames': (
           'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_32_16.pth'
       ),
       'xclip-base-patch16': 'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_16_8.pth',
       'xclip-base-patch16-16-frames': (
           'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k400_16_16.pth'
       ),
       'xclip-large-patch14': 'https://drive.google.com/u/0/uc?id=1NUOImq0o5DlQTST17iIP3vG7DgmHQuCx&amp;export=download&amp;confirm=t&amp;uuid=b26caedc-88e2-473e-830a-9d158b653cdb',
       'xclip-large-patch14-16-frames': 'https://drive.google.com/u/0/uc?id=1FOYgnJc097OJ4lGwtRCCydQyVPJEOH7d&amp;export=download&amp;confirm=t&amp;uuid=538fa810-e671-4050-b385-9a623f89804f',
       # fully supervised kinetics-600 checkpoints
       'xclip-base-patch16-kinetics-600': (
           'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k600_16_8.pth'
       ),
       'xclip-base-patch16-kinetics-600-16-frames': (
           'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/k600_16_16.pth'
       ),
       'xclip-large-patch14-kinetics-600': 'https://drive.google.com/u/0/uc?id=1FV8C1INuM91sLAN4ImjzePLIlpMSihwV&amp;export=download&amp;confirm=t&amp;uuid=141d4977-4a65-44ae-864f-4b0c19f838be',
       # few shot
       'xclip-base-patch16-hmdb-2-shot': (
           'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_2.pth'
       ),
       'xclip-base-patch16-hmdb-4-shot': (
           'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_4.pth'
       ),
       'xclip-base-patch16-hmdb-8-shot': (
           'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_8.pth'
       ),
       'xclip-base-patch16-hmdb-16-shot': (
           'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_hmdb_16.pth'
       ),
       'xclip-base-patch16-ucf-2-shot': (
           'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_2.pth'
       ),
       'xclip-base-patch16-ucf-4-shot': (
           'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_4.pth'
       ),
       'xclip-base-patch16-ucf-8-shot': (
           'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_8.pth'
       ),
       'xclip-base-patch16-ucf-16-shot': (
           'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/few_ucf_16.pth'
       ),
       # zero shot
       'xclip-base-patch16-zero-shot': 'https://github.com/nbl97/X-CLIP_Model_Zoo/releases/download/v1.0/zero.pth',
   }

   _a					:  Optional[Any]						=  model_to_url[model_name]
   _a					:  str						=  8
   if "16-frames" in model_name:
      _a					:  Optional[Any]						=  1_6
   elif "shot" in model_name:
      _a					:  List[Any]						=  3_2

   _a					:  str						=  get_xclip_config(A		,	A      )
   _a					:  Optional[int]						=  XCLIPModel(A      )
   model.eval()

   if "drive" in checkpoint_url:
      _a					:  Optional[int]						=  'pytorch_model.bin'
      gdown.cached_download(A		,	A		,	quiet=A      )
      _a					:  str						=  torch.load(A		,	map_location='cpu'      )['model']
   else:
      _a					:  Any						=  torch.hub.load_state_dict_from_url(A      )['model']

   _a					:  str						=  convert_state_dict(A		,	A      )

   _a					:  Optional[Any]						=  XCLIPModel(A      )
   _a		,						_a					:  Optional[Any]						=  model.load_state_dict(A		,	strict=A      )
   assert missing_keys == ["text_model.embeddings.position_ids", "vision_model.embeddings.position_ids"]
   model.eval()

   _a					:  Optional[Any]						=  3_3_6 if model_name == 'xclip-large-patch14-16-frames' else 2_2_4
   _a					:  Any						=  VideoMAEImageProcessor(size=A      )
   _a					:  Dict						=  CLIPTokenizer.from_pretrained('openai/clip-vit-base-patch32'      )
   _a					:  Union[str, Any]						=  CLIPTokenizerFast.from_pretrained('openai/clip-vit-base-patch32'      )
   _a					:  Optional[Any]						=  XCLIPProcessor(image_processor=A		,	tokenizer=A      )

   _a					:  List[str]						=  prepare_video(A      )
   _a					:  List[Any]						=  processor(
       text=['playing sports', 'eating spaghetti', 'go shopping']		,	videos=A		,	return_tensors='pt'		,	padding=A      )

   print('Shape of pixel values:'		,	inputs.pixel_values.shape      )

   with torch.no_grad():
      _a					:  int						=  model(**A      )

   # Verify outputs
   _a					:  Tuple						=  outputs.logits_per_video
   _a					:  List[Any]						=  logits_per_video.softmax(dim=1      )
   print('Probs:'		,	A      )
   # kinetics-400
   if model_name == "xclip-base-patch32":
      _a					:  Union[str, Any]						=  torch.tensor([[0.00_19, 0.99_51, 0.00_30]]      )
   elif model_name == "xclip-base-patch32-16-frames":
      _a					:  Tuple						=  torch.tensor([[7.0_999E-04, 9.9_883E-01, 4.5_580E-04]]      )
   elif model_name == "xclip-base-patch16":
      _a					:  List[str]						=  torch.tensor([[0.00_83, 0.96_81, 0.02_36]]      )
   elif model_name == "xclip-base-patch16-16-frames":
      _a					:  Tuple						=  torch.tensor([[7.6_937E-04, 9.9_728E-01, 1.9_473E-03]]      )
   elif model_name == "xclip-large-patch14":
      _a					:  Optional[Any]						=  torch.tensor([[0.00_62, 0.98_64, 0.00_75]]      )
   elif model_name == "xclip-large-patch14-16-frames":
      _a					:  int						=  torch.tensor([[3.3_877E-04, 9.9_937E-01, 2.8_888E-04]]      )
   # kinetics-600
   elif model_name == "xclip-base-patch16-kinetics-600":
      _a					:  List[Any]						=  torch.tensor([[0.05_55, 0.89_14, 0.05_31]]      )
   elif model_name == "xclip-base-patch16-kinetics-600-16-frames":
      _a					:  Optional[Any]						=  torch.tensor([[3.8_554E-04, 9.9_929E-01, 3.2_754E-04]]      )
   elif model_name == "xclip-large-patch14-kinetics-600":
      _a					:  List[Any]						=  torch.tensor([[0.00_36, 0.99_20, 0.00_45]]      )
   # few shot
   elif model_name == "xclip-base-patch16-hmdb-2-shot":
      _a					:  str						=  torch.tensor([[7.1_890E-06, 9.9_994E-01, 5.6_559E-05]]      )
   elif model_name == "xclip-base-patch16-hmdb-4-shot":
      _a					:  int						=  torch.tensor([[1.0_320E-05, 9.9_993E-01, 6.2_435E-05]]      )
   elif model_name == "xclip-base-patch16-hmdb-8-shot":
      _a					:  str						=  torch.tensor([[4.1_377E-06, 9.9_990E-01, 9.8_386E-05]]      )
   elif model_name == "xclip-base-patch16-hmdb-16-shot":
      _a					:  List[str]						=  torch.tensor([[4.1_347E-05, 9.9_962E-01, 3.3_411E-04]]      )
   elif model_name == "xclip-base-patch16-ucf-2-shot":
      _a					:  Any						=  torch.tensor([[8.5_857E-05, 9.9_928E-01, 6.3_291E-04]]      )
   elif model_name == "xclip-base-patch16-ucf-4-shot":
      _a					:  Union[str, Any]						=  torch.tensor([[8.5_857E-05, 9.9_928E-01, 6.3_291E-04]]      )
   elif model_name == "xclip-base-patch16-ucf-8-shot":
      _a					:  str						=  torch.tensor([[0.00_27, 0.99_04, 0.00_70]]      )
   elif model_name == "xclip-base-patch16-ucf-16-shot":
      _a					:  str						=  torch.tensor([[9.8_219E-04, 9.9_593E-01, 3.0_863E-03]]      )
   # zero shot
   elif model_name == "xclip-base-patch16-zero-shot":
      _a					:  List[Any]						=  torch.tensor([[3.5_082E-04, 9.9_785E-01, 1.7_966E-03]]      )
   else:
      raise ValueError(f'''Model name {model_name} not supported'''      )
   assert torch.allclose(A		,	A		,	atol=1E-3      )
   print('Looks ok!'      )

   if pytorch_dump_folder_path is not None:
      print(f'''Saving model {model_name} to {pytorch_dump_folder_path}'''      )
      model.save_pretrained(A      )

   if push_to_hub:
      print('Pushing model, processor and slow tokenizer files to the hub...'      )
      model.push_to_hub(A		,	organization='nielsr'      )
      processor.push_to_hub(A		,	organization='nielsr'      )
      slow_tokenizer.push_to_hub(A		,	organization='nielsr'      )


if __name__ == "__main__":
 UpperCAmelCase_      :		List[str]          =		argparse.ArgumentParser()
 # Required parameters
 parser.add_argument(
     "--model_name",
     default="xclip-base-patch32",
     type=str,
     help="Name of the model.",
 )
 parser.add_argument(
     "--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model directory."
 )
 parser.add_argument(
     "--push_to_hub", action="store_true", help="Whether or not to push the converted model to the 🤗 hub."
 )

 UpperCAmelCase_      :		Optional[int]          =		parser.parse_args()
 convert_xclip_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)






'''simple docstring'''




import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis_float32 (there's also the fix_lavis branch)
# also note: to convert Vicuna checkpoints, we had to include /home/niels/python_projects/checkpoints/FastChat/vicuna-7b in lavis/configs/models/blip2/blip2_instruct_vicuna7b.yaml
# same for Vicuna-13b
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipImageProcessor,
    InstructBlipConfig,
    InstructBlipForConditionalGeneration,
    InstructBlipProcessor,
    InstructBlipQFormerConfig,
    InstructBlipVisionConfig,
    LlamaConfig,
    LlamaTokenizerFast,
    TaConfig,
    TaTokenizerFast,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD


def    UpperCAmelCase_  (       ):
   '''simple docstring'''





   _a					:  str						=  'https://raw.githubusercontent.com/salesforce/LAVIS/main/docs/_static/Confusing-Pictures.jpg'
   _a					:  Any						=  Image.open(requests.get(A		,	stream=A      ).raw      ).convert('RGB'      )

   return image


def    UpperCAmelCase_  (       A      ):
   '''simple docstring'''





   _a					:  Optional[int]						=  []
   # fmt: off

   # vision encoder
   rename_keys.append(('visual_encoder.cls_token', 'vision_model.embeddings.class_embedding')      )
   rename_keys.append(('visual_encoder.pos_embed', 'vision_model.embeddings.position_embedding')      )
   rename_keys.append(('visual_encoder.patch_embed.proj.weight', 'vision_model.embeddings.patch_embedding.weight')      )
   rename_keys.append(('visual_encoder.patch_embed.proj.bias', 'vision_model.embeddings.patch_embedding.bias')      )
   rename_keys.append(('ln_vision.weight', 'vision_model.post_layernorm.weight')      )
   rename_keys.append(('ln_vision.bias', 'vision_model.post_layernorm.bias')      )

   for i in range(config.vision_config.num_hidden_layers      ):
      rename_keys.append((f'''visual_encoder.blocks.{i}.norm1.weight''', f'''vision_model.encoder.layers.{i}.layer_norm1.weight''')      )
      rename_keys.append((f'''visual_encoder.blocks.{i}.norm1.bias''', f'''vision_model.encoder.layers.{i}.layer_norm1.bias''')      )
      rename_keys.append((f'''visual_encoder.blocks.{i}.norm2.weight''', f'''vision_model.encoder.layers.{i}.layer_norm2.weight''')      )
      rename_keys.append((f'''visual_encoder.blocks.{i}.norm2.bias''', f'''vision_model.encoder.layers.{i}.layer_norm2.bias''')      )
      rename_keys.append((f'''visual_encoder.blocks.{i}.attn.qkv.weight''', f'''vision_model.encoder.layers.{i}.self_attn.qkv.weight''')      )
      rename_keys.append((f'''visual_encoder.blocks.{i}.attn.proj.weight''', f'''vision_model.encoder.layers.{i}.self_attn.projection.weight''',)      )
      rename_keys.append((f'''visual_encoder.blocks.{i}.attn.proj.bias''', f'''vision_model.encoder.layers.{i}.self_attn.projection.bias''')      )
      rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc1.weight''', f'''vision_model.encoder.layers.{i}.mlp.fc1.weight''')      )
      rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc1.bias''', f'''vision_model.encoder.layers.{i}.mlp.fc1.bias''')      )
      rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc2.weight''', f'''vision_model.encoder.layers.{i}.mlp.fc2.weight''')      )
      rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc2.bias''', f'''vision_model.encoder.layers.{i}.mlp.fc2.bias''')      )

   # QFormer
   rename_keys.append(('Qformer.bert.embeddings.LayerNorm.weight', 'qformer.embeddings.layernorm.weight')      )
   rename_keys.append(('Qformer.bert.embeddings.LayerNorm.bias', 'qformer.embeddings.layernorm.bias')      )

   # fmt: on
   return rename_keys


def    UpperCAmelCase_  (       A		,	A		,	A      ):
   '''simple docstring'''





   _a					:  Dict						=  dct.pop(A      )
   _a					:  Tuple						=  val


def    UpperCAmelCase_  (       A		,	A      ):
   '''simple docstring'''





   for i in range(config.vision_config.num_hidden_layers      ):
      # read in original q and v biases
      _a					:  Optional[Any]						=  state_dict.pop(f'''visual_encoder.blocks.{i}.attn.q_bias'''      )
      _a					:  str						=  state_dict.pop(f'''visual_encoder.blocks.{i}.attn.v_bias'''      )

      # next, set bias in the state dict
      _a					:  Any						=  torch.cat((q_bias, torch.zeros_like(A		,	requires_grad=A      ), v_bias)      )
      _a					:  Tuple						=  qkv_bias


def    UpperCAmelCase_  (       A      ):
   '''simple docstring'''





   _a					:  int						=  3_6_4 if 'coco' in model_name else 2_2_4
   _a					:  List[Any]						=  InstructBlipVisionConfig(image_size=A      ).to_dict()

   # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
   # seems like flan-T5 models don't have bos_token_id properly set?
   if "t5-xl" in model_name:
      _a					:  Optional[Any]						=  TaConfig.from_pretrained('google/flan-t5-xl'		,	dense_act_fn='gelu'		,	bos_token_id=1      ).to_dict()
   elif "t5-xxl" in model_name:
      _a					:  Tuple						=  TaConfig.from_pretrained('google/flan-t5-xxl'		,	dense_act_fn='gelu'		,	bos_token_id=1      ).to_dict()
   elif "vicuna-7b" in model_name:
      _a					:  int						=  LlamaConfig.from_pretrained('decapoda-research/llama-7b-hf'		,	vocab_size=3_2_0_0_1      ).to_dict()
   elif "vicuna-13b" in model_name:
      _a					:  int						=  LlamaConfig.from_pretrained('decapoda-research/llama-13b-hf'		,	vocab_size=3_2_0_0_1      ).to_dict()
   else:
      raise ValueError('Model name not supported'      )

   # the authors add one special "[DEC]" token to the vocab of Q-Former, hence vocab size = 30522 + 1
   _a					:  Tuple						=  InstructBlipQFormerConfig(vocab_size=3_0_5_2_3      ).to_dict()
   _a					:  Optional[Any]						=  InstructBlipConfig(vision_config=A		,	text_config=A		,	qformer_config=A      )

   return config, image_size


@torch.no_grad()
def    UpperCAmelCase_  (       A		,	A=None		,	A=False      ):
   '''simple docstring'''





   _a					:  Tuple						=  AutoTokenizer.from_pretrained('bert-base-uncased'		,	truncation_side='left'      )
   qformer_tokenizer.add_special_tokens({'bos_token': '[DEC]'}      )

   if "t5" in model_name:
      _a					:  int						=  TaTokenizerFast.from_pretrained('google/flan-t5-xl'		,	truncation_side='left'      )
   elif "vicuna" in model_name:
      # the following was used in the original implementation:
      # tokenizer = LlamaTokenizer.from_pretrained("huggyllama/llama-7b", use_fast=False, truncation_side="left")
      # tokenizer.add_special_tokens({"pad_token": "[PAD]"})
      # tokenizer.add_special_tokens({"bos_token": "</s>"})
      # tokenizer.add_special_tokens({"eos_token": "</s>"})
      # tokenizer.add_special_tokens({"unk_token": "</s>"})
      _a					:  Optional[Any]						=  LlamaTokenizerFast.from_pretrained(
          'huggyllama/llama-7b'		,	truncation_side='left'		,	bos_token='</s>'		,	unk_token='</s>'      )
      tokenizer.add_special_tokens({'pad_token': '[PAD]'}      )

   _a		,						_a					:  Tuple						=  get_blipa_config(A      )
   _a					:  Optional[Any]						=  InstructBlipForConditionalGeneration(A      ).eval()

   _a					:  Union[str, Any]						=  {
       'instructblip-vicuna-7b': ('blip2_vicuna_instruct', 'vicuna7b'),
       'instructblip-vicuna-13b': ('blip2_vicuna_instruct', 'vicuna13b'),
       'instructblip-flan-t5-xl': ('blip2_t5_instruct', 'flant5xl'),
       'instructblip-flan-t5-xxl': ('blip2_t5_instruct', 'flant5xxl'),
   }

   _a		,						_a					:  Dict						=  model_name_to_original[model_name]

   # load original model
   print('Loading original model...'      )
   _a					:  int						=  'cuda:1' if torch.cuda.is_available() else 'cpu'
   _a					:  Any						=  'cuda:2' if torch.cuda.is_available() else 'cpu'
   _a		,						_a		,						_a					:  Optional[int]						=  load_model_and_preprocess(
       name=A		,	model_type=A		,	is_eval=A		,	device=A      )
   original_model.eval()
   print('Done!'      )

   # update state dict keys
   _a					:  Tuple						=  original_model.state_dict()
   _a					:  List[Any]						=  create_rename_keys(A      )
   for src, dest in rename_keys:
      rename_key(A		,	A		,	A      )

   # some keys can be renamed efficiently
   for key, val in state_dict.copy().items():
      _a					:  Union[str, Any]						=  state_dict.pop(A      )
      if key.startswith('Qformer.bert'      ):
         _a					:  Union[str, Any]						=  key.replace('Qformer.bert'		,	'qformer'      )
      if "attention.self" in key:
         _a					:  List[Any]						=  key.replace('self'		,	'attention'      )
      if "llm_proj" in key:
         _a					:  int						=  key.replace('llm_proj'		,	'language_projection'      )
      if "t5_proj" in key:
         _a					:  str						=  key.replace('t5_proj'		,	'language_projection'      )
      if key.startswith('llm_model'      ):
         _a					:  int						=  key.replace('llm_model'		,	'language_model'      )
      if key.startswith('t5'      ):
         _a					:  Tuple						=  key.replace('t5'		,	'language'      )
      _a					:  Optional[Any]						=  val

   # read in qv biases
   read_in_q_v_bias(A		,	A      )

   # note: weights get loaded in torch.float32 by default
   hf_model.load_state_dict(A		,	strict=A      )

   _a					:  Dict						=  load_demo_image()
   _a					:  Any						=  'What is unusual about this image?'

   # create processor
   _a					:  Any						=  BlipImageProcessor(
       size={'height': image_size, 'width': image_size}		,	image_mean=A		,	image_std=A      )
   _a					:  Dict						=  InstructBlipProcessor(
       image_processor=A		,	tokenizer=A		,	qformer_tokenizer=A		,	)
   _a					:  str						=  processor(images=A		,	text=A		,	return_tensors='pt'      ).to(A      )

   # make sure processor creates exact same pixel values
   _a					:  str						=  vis_processors['eval'](A      ).unsqueeze(0      ).to(A      )
   _a					:  Optional[int]						=  inputs.pixel_values
   assert torch.allclose(original_pixel_values.to(pixel_values.device      )		,	A      )

   original_model.to(A      )
   hf_model.to(A      )
   with torch.no_grad():
      if "vicuna" in model_name:
         _a					:  Any						=  original_model({'image': original_pixel_values, 'text_input': [prompt]}      ).logits
         _a					:  str						=  hf_model(**A      ).logits
      else:
         _a					:  Optional[Any]						=  original_model(
             {'image': original_pixel_values, 'text_input': [prompt], 'text_output': ['\n']}      ).logits
         _a					:  List[Any]						=  tokenizer('\n'		,	return_tensors='pt'      ).input_ids.to(A      )
         _a					:  str						=  label_input_ids.masked_fill(label_input_ids == tokenizer.pad_token_id		,	-1_0_0      )
         _a					:  List[str]						=  hf_model(**A		,	labels=A      ).logits

   print('First values of original logits:'		,	original_logits[0, :3, :3]      )
   print('First values of HF logits:'		,	logits[0, :3, :3]      )

   # assert values
   assert original_logits.shape == logits.shape
   _a					:  str						=  1E-4 if 'vicuna' in model_name else 1E-5
   assert torch.allclose(original_logits.to(logits.device      )		,	A		,	atol=A      )
   print('Looks ok!'      )

   print('Generating with original model...'      )
   _a					:  Optional[int]						=  original_model.generate({'image': original_pixel_values, 'prompt': prompt}		,	num_beams=5      )

   # important: we need to cast the weights of the HF model to the appropriate type
   print('Generating with HF model...'      )
   _a					:  Dict						=  hf_model.generate(
       **A		,	do_sample=A		,	num_beams=5		,	max_length=2_5_6		,	min_length=1		,	top_p=0.9		,	repetition_penalty=1.5		,	length_penalty=1.0		,	temperature=1		,	)
   if "vicuna" in model_name:
      # convert output id 0 to 2 (eos_token_id)
      # TODO add this in the generate method?
      _a					:  Tuple						=  2
   print('Original generation:'		,	A      )
   _a					:  Dict						=  processor.batch_decode(A		,	skip_special_tokens=A      )
   _a					:  Tuple						=  [text.strip() for text in output_text]
   print('HF generation:'		,	A      )

   if pytorch_dump_folder_path is not None:
      processor.save_pretrained(A      )
      hf_model.save_pretrained(A      )

   if push_to_hub:
      processor.push_to_hub(f'''Salesforce/{model_name}'''      )
      hf_model.push_to_hub(f'''Salesforce/{model_name}'''      )


if __name__ == "__main__":
 UpperCAmelCase_      :		Optional[int]          =		argparse.ArgumentParser()
 UpperCAmelCase_      :		List[str]          =		[
     "instructblip-vicuna-7b",
     "instructblip-vicuna-13b",
     "instructblip-flan-t5-xl",
     "instructblip-flan-t5-xxl",
 ]
 parser.add_argument(
     "--model_name",
     default="instructblip-flan-t5-xl",
     choices=choices,
     type=str,
     help="Path to hf config.json of model to convert",
 )
 parser.add_argument("--pytorch_dump_folder_path", default=None, type=str, help="Path to the output PyTorch model.")
 parser.add_argument(
     "--push_to_hub",
     action="store_true",
     help="Whether to push the model and processor to the hub after converting",
 )

 UpperCAmelCase_      :		Optional[int]          =		parser.parse_args()

 convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)


