
import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD







def 						_snake_case(  )			->   int:
       lowercase							:					Optional[int]	     =				"""https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png"""
       lowercase							:					str	     =				Image.open(requests.get(SCREAMING_SNAKE_CASE__ ,     stream=SCREAMING_SNAKE_CASE__    ).raw    ).convert("""RGB"""    )

       return image







def 						_snake_case(  SCREAMING_SNAKE_CASE__    )			->   Any:
       lowercase							:					List[str]	     =				[]
       # fmt: off

       # vision encoder
       rename_keys.append(("""visual_encoder.cls_token""", """vision_model.embeddings.class_embedding""")    )
       rename_keys.append(("""visual_encoder.pos_embed""", """vision_model.embeddings.position_embedding""")    )
       rename_keys.append(("""visual_encoder.patch_embed.proj.weight""", """vision_model.embeddings.patch_embedding.weight""")    )
       rename_keys.append(("""visual_encoder.patch_embed.proj.bias""", """vision_model.embeddings.patch_embedding.bias""")    )
       rename_keys.append(("""ln_vision.weight""", """vision_model.post_layernorm.weight""")    )
       rename_keys.append(("""ln_vision.bias""", """vision_model.post_layernorm.bias""")    )

       for i in range(config.vision_config.num_hidden_layers    ):
              rename_keys.append((f"visual_encoder.blocks.{i}.norm1.weight", f"vision_model.encoder.layers.{i}.layer_norm1.weight")    )
              rename_keys.append((f"visual_encoder.blocks.{i}.norm1.bias", f"vision_model.encoder.layers.{i}.layer_norm1.bias")    )
              rename_keys.append((f"visual_encoder.blocks.{i}.norm2.weight", f"vision_model.encoder.layers.{i}.layer_norm2.weight")    )
              rename_keys.append((f"visual_encoder.blocks.{i}.norm2.bias", f"vision_model.encoder.layers.{i}.layer_norm2.bias")    )
              rename_keys.append((f"visual_encoder.blocks.{i}.attn.qkv.weight", f"vision_model.encoder.layers.{i}.self_attn.qkv.weight")    )
              rename_keys.append((f"visual_encoder.blocks.{i}.attn.proj.weight", f"vision_model.encoder.layers.{i}.self_attn.projection.weight",)    )
              rename_keys.append((f"visual_encoder.blocks.{i}.attn.proj.bias", f"vision_model.encoder.layers.{i}.self_attn.projection.bias")    )
              rename_keys.append((f"visual_encoder.blocks.{i}.mlp.fc1.weight", f"vision_model.encoder.layers.{i}.mlp.fc1.weight")    )
              rename_keys.append((f"visual_encoder.blocks.{i}.mlp.fc1.bias", f"vision_model.encoder.layers.{i}.mlp.fc1.bias")    )
              rename_keys.append((f"visual_encoder.blocks.{i}.mlp.fc2.weight", f"vision_model.encoder.layers.{i}.mlp.fc2.weight")    )
              rename_keys.append((f"visual_encoder.blocks.{i}.mlp.fc2.bias", f"vision_model.encoder.layers.{i}.mlp.fc2.bias")    )

       # QFormer
       rename_keys.append(("""Qformer.bert.embeddings.LayerNorm.weight""", """qformer.layernorm.weight""")    )
       rename_keys.append(("""Qformer.bert.embeddings.LayerNorm.bias""", """qformer.layernorm.bias""")    )

       # fmt: on
       return rename_keys







def 						_snake_case(  SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__    )			->   List[Any]:
       lowercase							:					str	     =				dct.pop(SCREAMING_SNAKE_CASE__    )
       lowercase							:					Any	     =				val







def 						_snake_case(  SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__    )			->   Optional[int]:
       for i in range(config.vision_config.num_hidden_layers    ):
              # read in original q and v biases
              lowercase							:					List[str]	     =				state_dict.pop(f"visual_encoder.blocks.{i}.attn.q_bias"    )
              lowercase							:					int	     =				state_dict.pop(f"visual_encoder.blocks.{i}.attn.v_bias"    )

              # next, set bias in the state dict
              lowercase							:					List[str]	     =				torch.cat((q_bias, torch.zeros_like(SCREAMING_SNAKE_CASE__ ,     requires_grad=SCREAMING_SNAKE_CASE__    ), v_bias)    )
              lowercase							:					Any	     =				qkv_bias







def 						_snake_case(  SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__    )			->   Union[str, Any]:
       lowercase							:					str	     =				364 if """coco""" in model_name else 224
       lowercase							:					Any	     =				BlipaVisionConfig(image_size=SCREAMING_SNAKE_CASE__    ).to_dict()

       # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
       # seems like flan-T5 models don't have bos_token_id properly set?
       if "opt-2.7b" in model_name:
              lowercase							:					Dict	     =				OPTConfig.from_pretrained("""facebook/opt-2.7b""" ,     eos_token_id=SCREAMING_SNAKE_CASE__    ).to_dict()
       elif "opt-6.7b" in model_name:
              lowercase							:					Any	     =				OPTConfig.from_pretrained("""facebook/opt-6.7b""" ,     eos_token_id=SCREAMING_SNAKE_CASE__    ).to_dict()
       elif "t5-xl" in model_name:
              lowercase							:					int	     =				TaConfig.from_pretrained("""google/flan-t5-xl""" ,     dense_act_fn="""gelu""" ,     bos_token_id=1    ).to_dict()
       elif "t5-xxl" in model_name:
              lowercase							:					Union[str, Any]	     =				TaConfig.from_pretrained("""google/flan-t5-xxl""" ,     dense_act_fn="""gelu""" ,     bos_token_id=1    ).to_dict()

       lowercase							:					Optional[int]	     =				BlipaConfig(vision_config=SCREAMING_SNAKE_CASE__ ,     text_config=SCREAMING_SNAKE_CASE__    )

       return config, image_size







@torch.no_grad()
def 						_snake_case(  SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__=None ,     SCREAMING_SNAKE_CASE__=False    )			->   Any:
       lowercase							:					Dict	     =				(
           AutoTokenizer.from_pretrained("""facebook/opt-2.7b"""    )
           if """opt""" in model_name
           else AutoTokenizer.from_pretrained("""google/flan-t5-xl"""    )
       )
       lowercase							:					List[str]	     =				tokenizer("""\n""" ,     add_special_tokens=SCREAMING_SNAKE_CASE__    ).input_ids[0]
       lowercase							:					Optional[int]	     =				get_blipa_config(SCREAMING_SNAKE_CASE__ ,     eos_token_id=SCREAMING_SNAKE_CASE__    )

       lowercase							:					Tuple	     =				BlipaForConditionalGeneration(SCREAMING_SNAKE_CASE__    ).eval()

       lowercase							:					str	     =				{
           """blip2-opt-2.7b""": ("""blip2_opt""", """pretrain_opt2.7b"""),
           """blip2-opt-6.7b""": ("""blip2_opt""", """pretrain_opt6.7b"""),
           """blip2-opt-2.7b-coco""": ("""blip2_opt""", """caption_coco_opt2.7b"""),
           """blip2-opt-6.7b-coco""": ("""blip2_opt""", """caption_coco_opt6.7b"""),
           """blip2-flan-t5-xl""": ("""blip2_t5""", """pretrain_flant5xl"""),
           """blip2-flan-t5-xl-coco""": ("""blip2_t5""", """caption_coco_flant5xl"""),
           """blip2-flan-t5-xxl""": ("""blip2_t5""", """pretrain_flant5xxl"""),
       }

       lowercase							:					int	     =				model_name_to_original[model_name]

       # load original model
       print("""Loading original model..."""    )
       lowercase							:					Tuple	     =				"""cuda""" if torch.cuda.is_available() else """cpu"""
       lowercase							:					Tuple	     =				load_model_and_preprocess(
           name=SCREAMING_SNAKE_CASE__ ,     model_type=SCREAMING_SNAKE_CASE__ ,     is_eval=SCREAMING_SNAKE_CASE__ ,     device=SCREAMING_SNAKE_CASE__    )
       original_model.eval()
       print("""Done!"""    )

       # update state dict keys
       lowercase							:					List[Any]	     =				original_model.state_dict()
       lowercase							:					Dict	     =				create_rename_keys(SCREAMING_SNAKE_CASE__    )
       for src, dest in rename_keys:
              rename_key(SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__    )

       # some keys can be renamed efficiently
       for key, val in state_dict.copy().items():
              lowercase							:					List[str]	     =				state_dict.pop(SCREAMING_SNAKE_CASE__    )
              if key.startswith("""Qformer.bert"""    ):
                     lowercase							:					Union[str, Any]	     =				key.replace("""Qformer.bert""" ,     """qformer"""    )
              if "attention.self" in key:
                     lowercase							:					Any	     =				key.replace("""self""" ,     """attention"""    )
              if "opt_proj" in key:
                     lowercase							:					Optional[int]	     =				key.replace("""opt_proj""" ,     """language_projection"""    )
              if "t5_proj" in key:
                     lowercase							:					Any	     =				key.replace("""t5_proj""" ,     """language_projection"""    )
              if key.startswith("""opt"""    ):
                     lowercase							:					Optional[int]	     =				key.replace("""opt""" ,     """language"""    )
              if key.startswith("""t5"""    ):
                     lowercase							:					str	     =				key.replace("""t5""" ,     """language"""    )
              lowercase							:					Union[str, Any]	     =				val

       # read in qv biases
       read_in_q_v_bias(SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__    )

       lowercase							:					Optional[Any]	     =				hf_model.load_state_dict(SCREAMING_SNAKE_CASE__ ,     strict=SCREAMING_SNAKE_CASE__    )
       assert len(SCREAMING_SNAKE_CASE__    ) == 0
       assert unexpected_keys == ["qformer.embeddings.position_ids"]

       lowercase							:					Optional[Any]	     =				load_demo_image()
       lowercase							:					Optional[Any]	     =				vis_processors["""eval"""](SCREAMING_SNAKE_CASE__    ).unsqueeze(0    ).to(SCREAMING_SNAKE_CASE__    )
       lowercase							:					Tuple	     =				tokenizer(["""\n"""] ,     return_tensors="""pt"""    ).input_ids.to(SCREAMING_SNAKE_CASE__    )

       # create processor
       lowercase							:					Union[str, Any]	     =				BlipImageProcessor(
           size={"""height""": image_size, """width""": image_size} ,     image_mean=SCREAMING_SNAKE_CASE__ ,     image_std=SCREAMING_SNAKE_CASE__    )
       lowercase							:					Union[str, Any]	     =				BlipaProcessor(image_processor=SCREAMING_SNAKE_CASE__ ,     tokenizer=SCREAMING_SNAKE_CASE__    )
       lowercase							:					Optional[int]	     =				processor(images=SCREAMING_SNAKE_CASE__ ,     return_tensors="""pt"""    ).pixel_values.to(SCREAMING_SNAKE_CASE__    )

       # make sure processor creates exact same pixel values
       assert torch.allclose(SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__    )

       original_model.to(SCREAMING_SNAKE_CASE__    )
       hf_model.to(SCREAMING_SNAKE_CASE__    )
       with torch.no_grad():
              if "opt" in model_name:
                     lowercase							:					Union[str, Any]	     =				original_model({"""image""": original_pixel_values, """text_input""": [""""""]}    ).logits
                     lowercase							:					Optional[Any]	     =				hf_model(SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__    ).logits
              else:
                     lowercase							:					Union[str, Any]	     =				original_model(
                         {"""image""": original_pixel_values, """text_input""": ["""\n"""], """text_output""": ["""\n"""]}    ).logits
                     lowercase							:					Any	     =				input_ids.masked_fill(input_ids == tokenizer.pad_token_id ,     -100    )
                     lowercase							:					Union[str, Any]	     =				hf_model(SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__ ,     labels=SCREAMING_SNAKE_CASE__    ).logits

       assert original_logits.shape == logits.shape
       print("""First values of original logits:""" ,     original_logits[0, :3, :3]    )
       print("""First values of HF logits:""" ,     logits[0, :3, :3]    )

       # assert values
       if model_name == "blip2-flan-t5-xl":
              lowercase							:					List[str]	     =				torch.tensor(
                  [[-41.5850, -4.4440, -8.9922], [-47.4322, -5.9143, -1.7340]] ,     device=SCREAMING_SNAKE_CASE__    )
              assert torch.allclose(logits[0, :3, :3] ,     SCREAMING_SNAKE_CASE__ ,     atol=1e-4    )
       elif model_name == "blip2-flan-t5-xl-coco":
              lowercase							:					int	     =				torch.tensor(
                  [[-57.0109, -9.8967, -12.6280], [-68.6578, -12.7191, -10.5065]] ,     device=SCREAMING_SNAKE_CASE__    )
       else:
              # cast to same type
              lowercase							:					str	     =				logits.dtype
              assert torch.allclose(original_logits.to(SCREAMING_SNAKE_CASE__    ) ,     SCREAMING_SNAKE_CASE__ ,     atol=1e-2    )
       print("""Looks ok!"""    )

       print("""Generating a caption..."""    )
       lowercase							:					str	     =				""""""
       lowercase							:					Optional[int]	     =				tokenizer(SCREAMING_SNAKE_CASE__ ,     return_tensors="""pt"""    ).input_ids.to(SCREAMING_SNAKE_CASE__    )

       lowercase							:					Tuple	     =				original_model.generate({"""image""": original_pixel_values}    )
       lowercase							:					Any	     =				hf_model.generate(
           SCREAMING_SNAKE_CASE__ ,     SCREAMING_SNAKE_CASE__ ,     do_sample=SCREAMING_SNAKE_CASE__ ,     num_beams=5 ,     max_length=30 ,     min_length=1 ,     top_p=0.9 ,     repetition_penalty=1.0 ,     length_penalty=1.0 ,     temperature=1 ,     )
       print("""Original generation:""" ,     SCREAMING_SNAKE_CASE__    )
       lowercase							:					Optional[Any]	     =				input_ids.shape[1]
       lowercase							:					Optional[int]	     =				processor.batch_decode(outputs[:, prompt_length:] ,     skip_special_tokens=SCREAMING_SNAKE_CASE__    )
       lowercase							:					List[Any]	     =				[text.strip() for text in output_text]
       print("""HF generation:""" ,     SCREAMING_SNAKE_CASE__    )

       if pytorch_dump_folder_path is not None:
              processor.save_pretrained(SCREAMING_SNAKE_CASE__    )
              hf_model.save_pretrained(SCREAMING_SNAKE_CASE__    )

       if push_to_hub:
              processor.push_to_hub(f"nielsr/{model_name}"    )
              hf_model.push_to_hub(f"nielsr/{model_name}"    )


if __name__ == "__main__":
  lowercase  : List[str]						   =							argparse.ArgumentParser()
  lowercase  : str						   =							[
      """blip2-opt-2.7b""",
      """blip2-opt-6.7b""",
      """blip2-opt-2.7b-coco""",
      """blip2-opt-6.7b-coco""",
      """blip2-flan-t5-xl""",
      """blip2-flan-t5-xl-coco""",
      """blip2-flan-t5-xxl""",
  ]
  parser.add_argument(
      """--model_name""",
      default="""blip2-opt-2.7b""",
      choices=choices,
      type=str,
      help="""Path to hf config.json of model to convert""",
  )
  parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
  parser.add_argument(
      """--push_to_hub""",
      action="""store_true""",
      help="""Whether to push the model and processor to the hub after converting""",
  )

  lowercase  : Union[str, Any]						   =							parser.parse_args()

  convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)


from typing import List, Optional, Union

import numpy as np

from ....audio_utils import mel_filter_bank, optimal_fft_length, spectrogram, window_function
from ....feature_extraction_sequence_utils import SequenceFeatureExtractor
from ....feature_extraction_utils import BatchFeature
from ....file_utils import PaddingStrategy, TensorType
from ....utils import logging


lowercase  : Tuple						   =							logging.get_logger(__name__)

class __snake_case		(				lowerCAmelCase					):
    _a		:						Optional[Any]=    ["input_features", "attention_mask"]
    def __init__(  self		,snake_case=80		,snake_case=16000		,snake_case=0.0		,snake_case=10		,snake_case=25		,snake_case="hamming_window"		,snake_case=32_768.0		,snake_case=0.97		,snake_case=1.0		,snake_case=True		,snake_case=True		,snake_case=False		,**snake_case		,):



           '''simple docstring'''







           super().__init__(feature_size=snake_case		,sampling_rate=snake_case		,padding_value=snake_case		,**snake_case				)

           lowercase							:					Optional[Any]	     =				feature_size
           lowercase							:					List[Any]	     =				sampling_rate
           lowercase							:					int	     =				padding_value
           lowercase							:					Dict	     =				hop_length
           lowercase							:					List[str]	     =				win_length
           lowercase							:					List[Any]	     =				frame_signal_scale
           lowercase							:					List[Any]	     =				preemphasis_coeff
           lowercase							:					str	     =				mel_floor
           lowercase							:					int	     =				normalize_means
           lowercase							:					List[Any]	     =				normalize_vars
           lowercase							:					List[Any]	     =				win_function
           lowercase							:					int	     =				return_attention_mask

           lowercase							:					Any	     =				win_length * sampling_rate // 1000
           lowercase							:					Tuple	     =				hop_length * sampling_rate // 1000

           lowercase							:					Tuple	     =				optimal_fft_length(self.sample_size				)
           lowercase							:					Dict	     =				(self.n_fft // 2) + 1
    def 				_SCREAMING_SNAKE_CASE	(  self		,snake_case				):



           '''simple docstring'''







           if self.win_function == "hamming_window":
                  lowercase							:					Optional[Any]	     =				window_function(window_length=self.sample_size		,name=self.win_function		,periodic=snake_case				)
           else:
                  lowercase							:					Optional[Any]	     =				window_function(window_length=self.sample_size		,name=self.win_function				)

           lowercase							:					int	     =				mel_filter_bank(
               num_frequency_bins=self.n_freqs		,num_mel_filters=self.feature_size		,min_frequency=0.0		,max_frequency=self.sampling_rate / 2.0		,sampling_rate=self.sampling_rate		,)

           lowercase							:					Dict	     =				spectrogram(
               one_waveform * self.frame_signal_scale		,window=snake_case		,frame_length=self.sample_size		,hop_length=self.sample_stride		,fft_length=self.n_fft		,center=snake_case		,preemphasis=self.preemphasis_coeff		,mel_filters=snake_case		,mel_floor=self.mel_floor		,log_mel="""log"""		,)
           return msfc_features.T
    def 				_SCREAMING_SNAKE_CASE	(  self		,snake_case		,snake_case		,snake_case				):



           '''simple docstring'''







           if self.normalize_means:
                  lowercase							:					List[Any]	     =				x[:input_length].mean(axis=0				)
                  lowercase							:					Dict	     =				np.subtract(snake_case		,snake_case				)
           if self.normalize_vars:
                  lowercase							:					List[Any]	     =				x[:input_length].std(axis=0				)
                  lowercase							:					List[Any]	     =				np.divide(snake_case		,snake_case				)

           if input_length < x.shape[0]:
                  lowercase							:					Any	     =				padding_value

           # make sure array is in float32
           lowercase							:					Tuple	     =				x.astype(np.floataa				)

           return x
    def 				_SCREAMING_SNAKE_CASE	(  self		,snake_case		,snake_case = None				):



           '''simple docstring'''







           lowercase							:					Optional[Any]	     =				attention_mask.sum(-1				) if attention_mask is not None else [x.shape[0] for x in input_features]
           return [self._normalize_one(snake_case		,snake_case		,self.padding_value				) for x, n in zip(snake_case		,snake_case				)]
    def __call__(  self		,snake_case		,snake_case = False		,snake_case = None		,snake_case = False		,snake_case = None		,snake_case = None		,snake_case = None		,snake_case = None		,**snake_case		,):



           '''simple docstring'''







           if sampling_rate is not None:
                  if sampling_rate != self.sampling_rate:
                         raise ValueError(
                             f"The model corresponding to this feature extractor: {self} was trained using a sampling rate of"
                             f" {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with"
                             f" {self.sampling_rate} and not {sampling_rate}."				)
           else:
                  logger.warning(
                      """It is strongly recommended to pass the ``sampling_rate`` argument to this function. """
                      """Failing to do so can result in silent errors that might be hard to debug."""				)

           lowercase							:					List[Any]	     =				isinstance(snake_case		,np.ndarray				) and len(raw_speech.shape				) > 1
           if is_batched_numpy and len(raw_speech.shape				) > 2:
                  raise ValueError(f"Only mono-channel audio is supported for input to {self}"				)
           lowercase							:					str	     =				is_batched_numpy or (
               isinstance(snake_case		,(list, tuple)				) and (isinstance(raw_speech[0]		,(np.ndarray, tuple, list)				))
           )

           if is_batched:
                  lowercase							:					List[str]	     =				[np.asarray(snake_case		,dtype=np.floataa				) for speech in raw_speech]
           elif not is_batched and not isinstance(snake_case		,np.ndarray				):
                  lowercase							:					int	     =				np.asarray(snake_case		,dtype=np.floataa				)
           elif isinstance(snake_case		,np.ndarray				) and raw_speech.dtype is np.dtype(np.floataa				):
                  lowercase							:					Any	     =				raw_speech.astype(np.floataa				)

           # always return batch
           if not is_batched:
                  lowercase							:					Dict	     =				[raw_speech]

           # extract fbank features
           lowercase							:					Tuple	     =				[self._extract_mfsc_features(snake_case				) for one_waveform in raw_speech]

           # convert into correct format for padding
           lowercase							:					Union[str, Any]	     =				BatchFeature({"""input_features""": features}				)

           lowercase							:					Optional[int]	     =				self.pad(
               snake_case		,padding=snake_case		,max_length=snake_case		,truncation=snake_case		,pad_to_multiple_of=snake_case		,return_attention_mask=snake_case		,**snake_case		,)
           # make sure list is in array format
           lowercase							:					Tuple	     =				padded_inputs.get("""input_features"""				)
           if isinstance(input_features[0]		,snake_case				):
                  lowercase							:					List[Any]	     =				[np.asarray(snake_case		,dtype=np.floataa				) for feature in input_features]

           lowercase							:					int	     =				padded_inputs.get("""attention_mask"""				)
           if attention_mask is not None:
                  lowercase							:					Any	     =				[np.asarray(snake_case		,dtype=np.intaa				) for array in attention_mask]

           if self.normalize_means or self.normalize_vars:
                  lowercase							:					List[str]	     =				(
                      np.array(snake_case		,dtype=np.intaa				)
                      if self._get_padding_strategies(snake_case		,max_length=snake_case				) is not PaddingStrategy.DO_NOT_PAD
                      and padding
                      else None
                  )
                  lowercase							:					List[str]	     =				self.normalize(
                      padded_inputs["""input_features"""]		,attention_mask=snake_case				)

           if return_tensors is not None:
                  lowercase							:					str	     =				padded_inputs.convert_to_tensors(snake_case				)

           return padded_inputs
