


import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
  from .features import FeatureType


__a					,   __a					,   __a 					=   False, False, False







@dataclass
class 					__SCREAMING_SNAKE_CASE :
      A					:				Dict     				=  None
      A					:				Tuple     				=  True
      A					:				str     				=  True
      A					:				Dict     				=  None
      # Automatically constructed
      A					:				Dict     				=  'dict'
      A					:				List[str]     				=  pa.struct({'bytes': pa.binary(), 'path': pa.string()} )
      A					:				List[Any]     				=  field(default='Audio'	,    init=_a	,    repr=_a )


      def __call__(				self		):
            return self.pa_type


      def 							__lowerCamelCase      (				self				,    SCREAMING_SNAKE_CASE__		):




            try:
                  import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
            except ImportError as err:
                  raise ImportError('''To support encoding audio data, please install \'soundfile\'.'''		) from err
            if isinstance(A_				,    A_		):
                  return {"bytes": None, "path": value}
            elif isinstance(A_				,    A_		):
                  return {"bytes": value, "path": None}
            elif "array" in value:
                  # convert the audio array to wav bytes
                  lowercase      : Any        =		BytesIO()
                  sf.write(A_				,    value['''array''']				,    value['''sampling_rate''']				,    format='''wav'''		)
                  return {"bytes": buffer.getvalue(), "path": None}
            elif value.get('''path'''		) is not None and os.path.isfile(value['''path''']		):
                  # we set "bytes": None to not duplicate the data if they're already available locally
                  if value["path"].endswith('''pcm'''		):
                        # "PCM" only has raw audio bytes
                        if value.get('''sampling_rate'''		) is None:
                              # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
                              raise KeyError('''To use PCM files, please specify a \'sampling_rate\' in Audio object'''		)
                        if value.get('''bytes'''		):
                              # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
                              lowercase      : Tuple        =		np.frombuffer(value['''bytes''']				,    dtype=np.intaa		).astype(np.floataa		) / 32767
                        else:
                              lowercase      : List[Any]        =		np.memmap(value['''path''']				,    dtype='''h'''				,    mode='''r'''		).astype(np.floataa		) / 32767

                        lowercase      : Tuple        =		BytesIO(bytes()		)
                        sf.write(A_				,    A_				,    value['''sampling_rate''']				,    format='''wav'''		)
                        return {"bytes": buffer.getvalue(), "path": None}
                  else:
                        return {"bytes": None, "path": value.get('''path'''		)}
            elif value.get('''bytes'''		) is not None or value.get('''path'''		) is not None:
                  # store the audio bytes, and path is used to infer the audio format using the file extension
                  return {"bytes": value.get('''bytes'''		), "path": value.get('''path'''		)}
            else:
                  raise ValueError(
                      f"""An audio sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}."""		)


      def 							__lowerCamelCase      (				self				,    SCREAMING_SNAKE_CASE__				,    SCREAMING_SNAKE_CASE__ = None		):




            if not self.decode:
                  raise RuntimeError('''Decoding is disabled for this feature. Please use Audio(decode=True) instead.'''		)

            lowercase ,      lowercase      : str        =		(value['''path'''], BytesIO(value['''bytes''']		)) if value['''bytes'''] is not None else (value['''path'''], None)
            if path is None and file is None:
                  raise ValueError(f"""An audio sample should have one of \'path\' or \'bytes\' but both are None in {value}."""		)

            try:
                  import librosa
                  import soundfile as sf
            except ImportError as err:
                  raise ImportError('''To support decoding audio files, please install \'librosa\' and \'soundfile\'.'''		) from err

            lowercase      : Dict        =		xsplitext(A_		)[1][1:].lower() if path is not None else None
            if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
                  raise RuntimeError(
                      '''Decoding \'opus\' files requires system library \'libsndfile\'>=1.0.31, '''
                      '''You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. '''		)
            elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
                  raise RuntimeError(
                      '''Decoding \'mp3\' files requires system library \'libsndfile\'>=1.1.0, '''
                      '''You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. '''		)

            if file is None:
                  lowercase      : str        =		token_per_repo_id or {}
                  lowercase      : Dict        =		path.split('''::'''		)[-1]
                  try:
                        lowercase      : List[Any]        =		string_to_dict(A_				,    config.HUB_DATASETS_URL		)['''repo_id''']
                        lowercase      : List[Any]        =		token_per_repo_id[repo_id]
                  except (ValueError, KeyError):
                        lowercase      : int        =		None

                  with xopen(A_				,    '''rb'''				,    use_auth_token=A_		) as f:
                        lowercase ,      lowercase      : Dict        =		sf.read(A_		)

            else:
                  lowercase ,      lowercase      : Union[str, Any]        =		sf.read(A_		)

            lowercase      : Any        =		array.T
            if self.mono:
                  lowercase      : Optional[int]        =		librosa.to_mono(A_		)
            if self.sampling_rate and self.sampling_rate != sampling_rate:
                  lowercase      : List[str]        =		librosa.resample(A_				,    orig_sr=A_				,    target_sr=self.sampling_rate		)
                  lowercase      : List[Any]        =		self.sampling_rate

            return {"path": path, "array": array, "sampling_rate": sampling_rate}


      def 							__lowerCamelCase      (				self		):
            from .features import Value

            if self.decode:
                  raise ValueError('''Cannot flatten a decoded Audio feature.'''		)
            return {
                "bytes": Value('''binary'''		),
                "path": Value('''string'''		),
            }


      def 							__lowerCamelCase      (				self				,    SCREAMING_SNAKE_CASE__		):




            if pa.types.is_string(storage.type		):
                  lowercase      : str        =		pa.array([None] * len(A_		)				,    type=pa.binary()		)
                  lowercase      : str        =		pa.StructArray.from_arrays([bytes_array, storage]				,    ['''bytes''', '''path''']				,    mask=storage.is_null()		)
            elif pa.types.is_binary(storage.type		):
                  lowercase      : Tuple        =		pa.array([None] * len(A_		)				,    type=pa.string()		)
                  lowercase      : int        =		pa.StructArray.from_arrays([storage, path_array]				,    ['''bytes''', '''path''']				,    mask=storage.is_null()		)
            elif pa.types.is_struct(storage.type		) and storage.type.get_all_field_indices('''array'''		):
                  lowercase      : Any        =		pa.array([Audio().encode_example(A_		) if x is not None else None for x in storage.to_pylist()]		)
            elif pa.types.is_struct(storage.type		):
                  if storage.type.get_field_index('''bytes'''		) >= 0:
                        lowercase      : Optional[int]        =		storage.field('''bytes'''		)
                  else:
                        lowercase      : Any        =		pa.array([None] * len(A_		)				,    type=pa.binary()		)
                  if storage.type.get_field_index('''path'''		) >= 0:
                        lowercase      : Union[str, Any]        =		storage.field('''path'''		)
                  else:
                        lowercase      : Optional[int]        =		pa.array([None] * len(A_		)				,    type=pa.string()		)
                  lowercase      : Union[str, Any]        =		pa.StructArray.from_arrays([bytes_array, path_array]				,    ['''bytes''', '''path''']				,    mask=storage.is_null()		)
            return array_cast(A_				,    self.pa_type		)






      def 							__lowerCamelCase      (				self				,    SCREAMING_SNAKE_CASE__		):




            @no_op_if_value_is_null
            def path_to_bytes(SCREAMING_SNAKE_CASE__		):
                  with xopen(A_				,    '''rb'''		) as f:
                        lowercase      : Any        =		f.read()
                  return bytes_

            lowercase      : List[str]        =		pa.array(
                [
                    (path_to_bytes(x['''path''']		) if x['''bytes'''] is None else x['''bytes''']) if x is not None else None
                    for x in storage.to_pylist()
                ]				,    type=pa.binary()				,    )
            lowercase      : Tuple        =		pa.array(
                [os.path.basename(A_		) if path is not None else None for path in storage.field('''path'''		).to_pylist()]				,    type=pa.string()				,    )
            lowercase      : List[str]        =		pa.StructArray.from_arrays([bytes_array, path_array]				,    ['''bytes''', '''path''']				,    mask=bytes_array.is_null()		)
            return array_cast(A_				,    self.pa_type		)





import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD






def 		_A		(		)					-> int:


     """simple docstring"""


     __UpperCamelCase =			'https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png'
     __UpperCamelCase =			Image.open(requests.get(_lowercase					,  stream=_lowercase ).raw ).convert('RGB' )

     return image






def 		_A		(		_lowercase )					-> int:


     """simple docstring"""


     __UpperCamelCase =			[]
     # fmt: off

     # vision encoder
     rename_keys.append(('visual_encoder.cls_token', 'vision_model.embeddings.class_embedding') )
     rename_keys.append(('visual_encoder.pos_embed', 'vision_model.embeddings.position_embedding') )
     rename_keys.append(('visual_encoder.patch_embed.proj.weight', 'vision_model.embeddings.patch_embedding.weight') )
     rename_keys.append(('visual_encoder.patch_embed.proj.bias', 'vision_model.embeddings.patch_embedding.bias') )
     rename_keys.append(('ln_vision.weight', 'vision_model.post_layernorm.weight') )
     rename_keys.append(('ln_vision.bias', 'vision_model.post_layernorm.bias') )

     for i in range(config.vision_config.num_hidden_layers ):
          rename_keys.append((f'''visual_encoder.blocks.{i}.norm1.weight''', f'''vision_model.encoder.layers.{i}.layer_norm1.weight''') )
          rename_keys.append((f'''visual_encoder.blocks.{i}.norm1.bias''', f'''vision_model.encoder.layers.{i}.layer_norm1.bias''') )
          rename_keys.append((f'''visual_encoder.blocks.{i}.norm2.weight''', f'''vision_model.encoder.layers.{i}.layer_norm2.weight''') )
          rename_keys.append((f'''visual_encoder.blocks.{i}.norm2.bias''', f'''vision_model.encoder.layers.{i}.layer_norm2.bias''') )
          rename_keys.append((f'''visual_encoder.blocks.{i}.attn.qkv.weight''', f'''vision_model.encoder.layers.{i}.self_attn.qkv.weight''') )
          rename_keys.append((f'''visual_encoder.blocks.{i}.attn.proj.weight''', f'''vision_model.encoder.layers.{i}.self_attn.projection.weight''',) )
          rename_keys.append((f'''visual_encoder.blocks.{i}.attn.proj.bias''', f'''vision_model.encoder.layers.{i}.self_attn.projection.bias''') )
          rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc1.weight''', f'''vision_model.encoder.layers.{i}.mlp.fc1.weight''') )
          rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc1.bias''', f'''vision_model.encoder.layers.{i}.mlp.fc1.bias''') )
          rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc2.weight''', f'''vision_model.encoder.layers.{i}.mlp.fc2.weight''') )
          rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc2.bias''', f'''vision_model.encoder.layers.{i}.mlp.fc2.bias''') )

     # QFormer
     rename_keys.append(('Qformer.bert.embeddings.LayerNorm.weight', 'qformer.layernorm.weight') )
     rename_keys.append(('Qformer.bert.embeddings.LayerNorm.bias', 'qformer.layernorm.bias') )

     # fmt: on
     return rename_keys






def 		_A		(		_lowercase					,  _lowercase					,  _lowercase )					-> Optional[int]:


     """simple docstring"""


     __UpperCamelCase =			dct.pop(_lowercase )
     __UpperCamelCase =			val






def 		_A		(		_lowercase					,  _lowercase )					-> int:


     """simple docstring"""


     for i in range(config.vision_config.num_hidden_layers ):
          # read in original q and v biases
          __UpperCamelCase =			state_dict.pop(f'''visual_encoder.blocks.{i}.attn.q_bias''' )
          __UpperCamelCase =			state_dict.pop(f'''visual_encoder.blocks.{i}.attn.v_bias''' )

          # next, set bias in the state dict
          __UpperCamelCase =			torch.cat((q_bias, torch.zeros_like(_lowercase					,  requires_grad=_lowercase ), v_bias) )
          __UpperCamelCase =			qkv_bias






def 		_A		(		_lowercase					,  _lowercase )					-> Any:


     """simple docstring"""


     __UpperCamelCase =			3_64 if 'coco' in model_name else 2_24
     __UpperCamelCase =			BlipaVisionConfig(image_size=_lowercase ).to_dict()

     # make sure the models have proper bos_token_id and eos_token_id set (important for generation)
     # seems like flan-T5 models don't have bos_token_id properly set?
     if "opt-2.7b" in model_name:
          __UpperCamelCase =			OPTConfig.from_pretrained('facebook/opt-2.7b'					,  eos_token_id=_lowercase ).to_dict()
     elif "opt-6.7b" in model_name:
          __UpperCamelCase =			OPTConfig.from_pretrained('facebook/opt-6.7b'					,  eos_token_id=_lowercase ).to_dict()
     elif "t5-xl" in model_name:
          __UpperCamelCase =			TaConfig.from_pretrained('google/flan-t5-xl'					,  dense_act_fn='gelu'					,  bos_token_id=1 ).to_dict()
     elif "t5-xxl" in model_name:
          __UpperCamelCase =			TaConfig.from_pretrained('google/flan-t5-xxl'					,  dense_act_fn='gelu'					,  bos_token_id=1 ).to_dict()

     __UpperCamelCase =			BlipaConfig(vision_config=_lowercase					,  text_config=_lowercase )

     return config, image_size






@torch.no_grad()
def 		_A		(		_lowercase					,  _lowercase=None					,  _lowercase=False )					-> Union[str, Any]:


     """simple docstring"""


     __UpperCamelCase =			(
         AutoTokenizer.from_pretrained('facebook/opt-2.7b' )
         if 'opt' in model_name
         else AutoTokenizer.from_pretrained('google/flan-t5-xl' )
     )
     __UpperCamelCase =			tokenizer('\n'					,  add_special_tokens=_lowercase ).input_ids[0]
     __UpperCamelCase,    __UpperCamelCase =			get_blipa_config(_lowercase					,  eos_token_id=_lowercase )

     __UpperCamelCase =			BlipaForConditionalGeneration(_lowercase ).eval()

     __UpperCamelCase =			{
         'blip2-opt-2.7b': ('blip2_opt', 'pretrain_opt2.7b'),
         'blip2-opt-6.7b': ('blip2_opt', 'pretrain_opt6.7b'),
         'blip2-opt-2.7b-coco': ('blip2_opt', 'caption_coco_opt2.7b'),
         'blip2-opt-6.7b-coco': ('blip2_opt', 'caption_coco_opt6.7b'),
         'blip2-flan-t5-xl': ('blip2_t5', 'pretrain_flant5xl'),
         'blip2-flan-t5-xl-coco': ('blip2_t5', 'caption_coco_flant5xl'),
         'blip2-flan-t5-xxl': ('blip2_t5', 'pretrain_flant5xxl'),
     }

     __UpperCamelCase,    __UpperCamelCase =			model_name_to_original[model_name]

     # load original model
     print('Loading original model...' )
     __UpperCamelCase =			'cuda' if torch.cuda.is_available() else 'cpu'
     __UpperCamelCase,    __UpperCamelCase,    __UpperCamelCase =			load_model_and_preprocess(
         name=_lowercase					,  model_type=_lowercase					,  is_eval=_lowercase					,  device=_lowercase )
     original_model.eval()
     print('Done!' )

     # update state dict keys
     __UpperCamelCase =			original_model.state_dict()
     __UpperCamelCase =			create_rename_keys(_lowercase )
     for src, dest in rename_keys:
          rename_key(_lowercase					,  _lowercase					,  _lowercase )

     # some keys can be renamed efficiently
     for key, val in state_dict.copy().items():
          __UpperCamelCase =			state_dict.pop(_lowercase )
          if key.startswith('Qformer.bert' ):
               __UpperCamelCase =			key.replace('Qformer.bert'					,  'qformer' )
          if "attention.self" in key:
               __UpperCamelCase =			key.replace('self'					,  'attention' )
          if "opt_proj" in key:
               __UpperCamelCase =			key.replace('opt_proj'					,  'language_projection' )
          if "t5_proj" in key:
               __UpperCamelCase =			key.replace('t5_proj'					,  'language_projection' )
          if key.startswith('opt' ):
               __UpperCamelCase =			key.replace('opt'					,  'language' )
          if key.startswith('t5' ):
               __UpperCamelCase =			key.replace('t5'					,  'language' )
          __UpperCamelCase =			val

     # read in qv biases
     read_in_q_v_bias(_lowercase					,  _lowercase )

     __UpperCamelCase,    __UpperCamelCase =			hf_model.load_state_dict(_lowercase					,  strict=_lowercase )
     assert len(_lowercase ) == 0
     assert unexpected_keys == ["qformer.embeddings.position_ids"]

     __UpperCamelCase =			load_demo_image()
     __UpperCamelCase =			vis_processors['eval'](_lowercase ).unsqueeze(0 ).to(_lowercase )
     __UpperCamelCase =			tokenizer(['\n']					,  return_tensors='pt' ).input_ids.to(_lowercase )

     # create processor
     __UpperCamelCase =			BlipImageProcessor(
         size={'height': image_size, 'width': image_size}					,  image_mean=_lowercase					,  image_std=_lowercase )
     __UpperCamelCase =			BlipaProcessor(image_processor=_lowercase					,  tokenizer=_lowercase )
     __UpperCamelCase =			processor(images=_lowercase					,  return_tensors='pt' ).pixel_values.to(_lowercase )

     # make sure processor creates exact same pixel values
     assert torch.allclose(_lowercase					,  _lowercase )

     original_model.to(_lowercase )
     hf_model.to(_lowercase )
     with torch.no_grad():
          if "opt" in model_name:
               __UpperCamelCase =			original_model({'image': original_pixel_values, 'text_input': ['']} ).logits
               __UpperCamelCase =			hf_model(_lowercase					,  _lowercase ).logits
          else:
               __UpperCamelCase =			original_model(
                   {'image': original_pixel_values, 'text_input': ['\n'], 'text_output': ['\n']} ).logits
               __UpperCamelCase =			input_ids.masked_fill(input_ids == tokenizer.pad_token_id					,  -1_00 )
               __UpperCamelCase =			hf_model(_lowercase					,  _lowercase					,  labels=_lowercase ).logits

     assert original_logits.shape == logits.shape
     print('First values of original logits:'					,  original_logits[0, :3, :3] )
     print('First values of HF logits:'					,  logits[0, :3, :3] )

     # assert values
     if model_name == "blip2-flan-t5-xl":
          __UpperCamelCase =			torch.tensor(
              [[-41.58_50, -4.44_40, -8.99_22], [-47.43_22, -5.91_43, -1.73_40]]					,  device=_lowercase )
          assert torch.allclose(logits[0, :3, :3]					,  _lowercase					,  atol=1e-4 )
     elif model_name == "blip2-flan-t5-xl-coco":
          __UpperCamelCase =			torch.tensor(
              [[-57.01_09, -9.89_67, -12.62_80], [-68.65_78, -12.71_91, -10.50_65]]					,  device=_lowercase )
     else:
          # cast to same type
          __UpperCamelCase =			logits.dtype
          assert torch.allclose(original_logits.to(_lowercase )					,  _lowercase					,  atol=1e-2 )
     print('Looks ok!' )

     print('Generating a caption...' )
     __UpperCamelCase =			''
     __UpperCamelCase =			tokenizer(_lowercase					,  return_tensors='pt' ).input_ids.to(_lowercase )

     __UpperCamelCase =			original_model.generate({'image': original_pixel_values} )
     __UpperCamelCase =			hf_model.generate(
         _lowercase					,  _lowercase					,  do_sample=_lowercase					,  num_beams=5					,  max_length=30					,  min_length=1					,  top_p=0.9					,  repetition_penalty=1.0					,  length_penalty=1.0					,  temperature=1					,  )
     print('Original generation:'					,  _lowercase )
     __UpperCamelCase =			input_ids.shape[1]
     __UpperCamelCase =			processor.batch_decode(outputs[:, prompt_length:]					,  skip_special_tokens=_lowercase )
     __UpperCamelCase =			[text.strip() for text in output_text]
     print('HF generation:'					,  _lowercase )

     if pytorch_dump_folder_path is not None:
          processor.save_pretrained(_lowercase )
          hf_model.save_pretrained(_lowercase )

     if push_to_hub:
          processor.push_to_hub(f'''nielsr/{model_name}''' )
          hf_model.push_to_hub(f'''nielsr/{model_name}''' )


if __name__ == "__main__":
    __snake_case   							=						argparse.ArgumentParser()
    __snake_case   							=						[
        '''blip2-opt-2.7b''',
        '''blip2-opt-6.7b''',
        '''blip2-opt-2.7b-coco''',
        '''blip2-opt-6.7b-coco''',
        '''blip2-flan-t5-xl''',
        '''blip2-flan-t5-xl-coco''',
        '''blip2-flan-t5-xxl''',
    ]
    parser.add_argument(
        '''--model_name''',
        default='''blip2-opt-2.7b''',
        choices=choices,
        type=str,
        help='''Path to hf config.json of model to convert''',
    )
    parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
    parser.add_argument(
        '''--push_to_hub''',
        action='''store_true''',
        help='''Whether to push the model and processor to the hub after converting''',
    )

    __snake_case   							=						parser.parse_args()

    convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)


