




"""simple docstring"""







import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD





def 							lowercase					()     ->  Tuple:


				'''simple docstring'''






				lowerCAmelCase  =				"""https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png"""
				lowerCAmelCase  =				Image.open(requests.get(snake_case__      ,       stream=snake_case__ ).raw ).convert("""RGB""" )

				return image





def 							lowercase					(snake_case__   : Dict )     ->  Optional[int]:


				'''simple docstring'''






				lowerCAmelCase  =				[]
				# fmt: off

				# vision encoder
				rename_keys.append(("""visual_encoder.cls_token""", """vision_model.embeddings.class_embedding""") )
				rename_keys.append(("""visual_encoder.pos_embed""", """vision_model.embeddings.position_embedding""") )
				rename_keys.append(("""visual_encoder.patch_embed.proj.weight""", """vision_model.embeddings.patch_embedding.weight""") )
				rename_keys.append(("""visual_encoder.patch_embed.proj.bias""", """vision_model.embeddings.patch_embedding.bias""") )
				rename_keys.append(("""ln_vision.weight""", """vision_model.post_layernorm.weight""") )
				rename_keys.append(("""ln_vision.bias""", """vision_model.post_layernorm.bias""") )

				for i in range(config.vision_config.num_hidden_layers ):
								rename_keys.append((f'''visual_encoder.blocks.{i}.norm1.weight''', f'''vision_model.encoder.layers.{i}.layer_norm1.weight''') )
								rename_keys.append((f'''visual_encoder.blocks.{i}.norm1.bias''', f'''vision_model.encoder.layers.{i}.layer_norm1.bias''') )
								rename_keys.append((f'''visual_encoder.blocks.{i}.norm2.weight''', f'''vision_model.encoder.layers.{i}.layer_norm2.weight''') )
								rename_keys.append((f'''visual_encoder.blocks.{i}.norm2.bias''', f'''vision_model.encoder.layers.{i}.layer_norm2.bias''') )
								rename_keys.append((f'''visual_encoder.blocks.{i}.attn.qkv.weight''', f'''vision_model.encoder.layers.{i}.self_attn.qkv.weight''') )
								rename_keys.append((f'''visual_encoder.blocks.{i}.attn.proj.weight''', f'''vision_model.encoder.layers.{i}.self_attn.projection.weight''',) )
								rename_keys.append((f'''visual_encoder.blocks.{i}.attn.proj.bias''', f'''vision_model.encoder.layers.{i}.self_attn.projection.bias''') )
								rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc1.weight''', f'''vision_model.encoder.layers.{i}.mlp.fc1.weight''') )
								rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc1.bias''', f'''vision_model.encoder.layers.{i}.mlp.fc1.bias''') )
								rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc2.weight''', f'''vision_model.encoder.layers.{i}.mlp.fc2.weight''') )
								rename_keys.append((f'''visual_encoder.blocks.{i}.mlp.fc2.bias''', f'''vision_model.encoder.layers.{i}.mlp.fc2.bias''') )

				# QFormer
				rename_keys.append(("""Qformer.bert.embeddings.LayerNorm.weight""", """qformer.layernorm.weight""") )
				rename_keys.append(("""Qformer.bert.embeddings.LayerNorm.bias""", """qformer.layernorm.bias""") )

				# fmt: on
				return rename_keys





def 							lowercase					(snake_case__   : Optional[int]      ,       snake_case__   : Optional[Any]      ,       snake_case__   : int )     ->  Tuple:


				'''simple docstring'''






				lowerCAmelCase  =				dct.pop(snake_case__ )
				lowerCAmelCase  =				val





def 							lowercase					(snake_case__   : Optional[int]      ,       snake_case__   : Union[str, Any] )     ->  List[Any]:


				'''simple docstring'''






				for i in range(config.vision_config.num_hidden_layers ):
								# read in original q and v biases
								lowerCAmelCase  =				state_dict.pop(f'''visual_encoder.blocks.{i}.attn.q_bias''' )
								lowerCAmelCase  =				state_dict.pop(f'''visual_encoder.blocks.{i}.attn.v_bias''' )

								# next, set bias in the state dict
								lowerCAmelCase  =				torch.cat((q_bias, torch.zeros_like(snake_case__      ,       requires_grad=snake_case__ ), v_bias) )
								lowerCAmelCase  =				qkv_bias





def 							lowercase					(snake_case__   : Any      ,       snake_case__   : Union[str, Any] )     ->  List[str]:


				'''simple docstring'''






				lowerCAmelCase  =				364 if """coco""" in model_name else 224
				lowerCAmelCase  =				BlipaVisionConfig(image_size=snake_case__ ).to_dict()

				# make sure the models have proper bos_token_id and eos_token_id set (important for generation)
				# seems like flan-T5 models don't have bos_token_id properly set?
				if "opt-2.7b" in model_name:
								lowerCAmelCase  =				OPTConfig.from_pretrained("""facebook/opt-2.7b"""      ,       eos_token_id=snake_case__ ).to_dict()
				elif "opt-6.7b" in model_name:
								lowerCAmelCase  =				OPTConfig.from_pretrained("""facebook/opt-6.7b"""      ,       eos_token_id=snake_case__ ).to_dict()
				elif "t5-xl" in model_name:
								lowerCAmelCase  =				TaConfig.from_pretrained("""google/flan-t5-xl"""      ,       dense_act_fn="""gelu"""      ,       bos_token_id=1 ).to_dict()
				elif "t5-xxl" in model_name:
								lowerCAmelCase  =				TaConfig.from_pretrained("""google/flan-t5-xxl"""      ,       dense_act_fn="""gelu"""      ,       bos_token_id=1 ).to_dict()

				lowerCAmelCase  =				BlipaConfig(vision_config=snake_case__      ,       text_config=snake_case__ )

				return config, image_size





@torch.no_grad()
def 							lowercase					(snake_case__   : Dict      ,       snake_case__   : List[str]=None      ,       snake_case__   : int=False )     ->  Optional[Any]:


				'''simple docstring'''






				lowerCAmelCase  =				(
				    AutoTokenizer.from_pretrained("""facebook/opt-2.7b""" )
				    if """opt""" in model_name
				    else AutoTokenizer.from_pretrained("""google/flan-t5-xl""" )
				)
				lowerCAmelCase  =				tokenizer("""\n"""      ,       add_special_tokens=snake_case__ ).input_ids[0]
				lowerCAmelCase ,						lowerCAmelCase  =				get_blipa_config(snake_case__      ,       eos_token_id=snake_case__ )

				lowerCAmelCase  =				BlipaForConditionalGeneration(snake_case__ ).eval()

				lowerCAmelCase  =				{
				    """blip2-opt-2.7b""": ("""blip2_opt""", """pretrain_opt2.7b"""),
				    """blip2-opt-6.7b""": ("""blip2_opt""", """pretrain_opt6.7b"""),
				    """blip2-opt-2.7b-coco""": ("""blip2_opt""", """caption_coco_opt2.7b"""),
				    """blip2-opt-6.7b-coco""": ("""blip2_opt""", """caption_coco_opt6.7b"""),
				    """blip2-flan-t5-xl""": ("""blip2_t5""", """pretrain_flant5xl"""),
				    """blip2-flan-t5-xl-coco""": ("""blip2_t5""", """caption_coco_flant5xl"""),
				    """blip2-flan-t5-xxl""": ("""blip2_t5""", """pretrain_flant5xxl"""),
				}

				lowerCAmelCase ,						lowerCAmelCase  =				model_name_to_original[model_name]

				# load original model
				print("""Loading original model...""" )
				lowerCAmelCase  =				"""cuda""" if torch.cuda.is_available() else """cpu"""
				lowerCAmelCase ,						lowerCAmelCase ,						lowerCAmelCase  =				load_model_and_preprocess(
				    name=snake_case__      ,       model_type=snake_case__      ,       is_eval=snake_case__      ,       device=snake_case__ )
				original_model.eval()
				print("""Done!""" )

				# update state dict keys
				lowerCAmelCase  =				original_model.state_dict()
				lowerCAmelCase  =				create_rename_keys(snake_case__ )
				for src, dest in rename_keys:
								rename_key(snake_case__      ,       snake_case__      ,       snake_case__ )

				# some keys can be renamed efficiently
				for key, val in state_dict.copy().items():
								lowerCAmelCase  =				state_dict.pop(snake_case__ )
								if key.startswith("""Qformer.bert""" ):
												lowerCAmelCase  =				key.replace("""Qformer.bert"""      ,       """qformer""" )
								if "attention.self" in key:
												lowerCAmelCase  =				key.replace("""self"""      ,       """attention""" )
								if "opt_proj" in key:
												lowerCAmelCase  =				key.replace("""opt_proj"""      ,       """language_projection""" )
								if "t5_proj" in key:
												lowerCAmelCase  =				key.replace("""t5_proj"""      ,       """language_projection""" )
								if key.startswith("""opt""" ):
												lowerCAmelCase  =				key.replace("""opt"""      ,       """language""" )
								if key.startswith("""t5""" ):
												lowerCAmelCase  =				key.replace("""t5"""      ,       """language""" )
								lowerCAmelCase  =				val

				# read in qv biases
				read_in_q_v_bias(snake_case__      ,       snake_case__ )

				lowerCAmelCase ,						lowerCAmelCase  =				hf_model.load_state_dict(snake_case__      ,       strict=snake_case__ )
				assert len(snake_case__ ) == 0
				assert unexpected_keys == ["qformer.embeddings.position_ids"]

				lowerCAmelCase  =				load_demo_image()
				lowerCAmelCase  =				vis_processors["""eval"""](snake_case__ ).unsqueeze(0 ).to(snake_case__ )
				lowerCAmelCase  =				tokenizer(["""\n"""]      ,       return_tensors="""pt""" ).input_ids.to(snake_case__ )

				# create processor
				lowerCAmelCase  =				BlipImageProcessor(
				    size={"""height""": image_size, """width""": image_size}      ,       image_mean=snake_case__      ,       image_std=snake_case__ )
				lowerCAmelCase  =				BlipaProcessor(image_processor=snake_case__      ,       tokenizer=snake_case__ )
				lowerCAmelCase  =				processor(images=snake_case__      ,       return_tensors="""pt""" ).pixel_values.to(snake_case__ )

				# make sure processor creates exact same pixel values
				assert torch.allclose(snake_case__      ,       snake_case__ )

				original_model.to(snake_case__ )
				hf_model.to(snake_case__ )
				with torch.no_grad():
								if "opt" in model_name:
												lowerCAmelCase  =				original_model({"""image""": original_pixel_values, """text_input""": [""""""]} ).logits
												lowerCAmelCase  =				hf_model(snake_case__      ,       snake_case__ ).logits
								else:
												lowerCAmelCase  =				original_model(
												    {"""image""": original_pixel_values, """text_input""": ["""\n"""], """text_output""": ["""\n"""]} ).logits
												lowerCAmelCase  =				input_ids.masked_fill(input_ids == tokenizer.pad_token_id      ,       -100 )
												lowerCAmelCase  =				hf_model(snake_case__      ,       snake_case__      ,       labels=snake_case__ ).logits

				assert original_logits.shape == logits.shape
				print("""First values of original logits:"""      ,       original_logits[0, :3, :3] )
				print("""First values of HF logits:"""      ,       logits[0, :3, :3] )

				# assert values
				if model_name == "blip2-flan-t5-xl":
								lowerCAmelCase  =				torch.tensor(
								    [[-41.5_850, -4.4_440, -8.9_922], [-47.4_322, -5.9_143, -1.7_340]]      ,       device=snake_case__ )
								assert torch.allclose(logits[0, :3, :3]      ,       snake_case__      ,       atol=1e-4 )
				elif model_name == "blip2-flan-t5-xl-coco":
								lowerCAmelCase  =				torch.tensor(
								    [[-57.0_109, -9.8_967, -12.6_280], [-68.6_578, -12.7_191, -10.5_065]]      ,       device=snake_case__ )
				else:
								# cast to same type
								lowerCAmelCase  =				logits.dtype
								assert torch.allclose(original_logits.to(snake_case__ )      ,       snake_case__      ,       atol=1e-2 )
				print("""Looks ok!""" )

				print("""Generating a caption...""" )
				lowerCAmelCase  =				""""""
				lowerCAmelCase  =				tokenizer(snake_case__      ,       return_tensors="""pt""" ).input_ids.to(snake_case__ )

				lowerCAmelCase  =				original_model.generate({"""image""": original_pixel_values} )
				lowerCAmelCase  =				hf_model.generate(
				    snake_case__      ,       snake_case__      ,       do_sample=snake_case__      ,       num_beams=5      ,       max_length=30      ,       min_length=1      ,       top_p=0.9      ,       repetition_penalty=1.0      ,       length_penalty=1.0      ,       temperature=1      ,       )
				print("""Original generation:"""      ,       snake_case__ )
				lowerCAmelCase  =				input_ids.shape[1]
				lowerCAmelCase  =				processor.batch_decode(outputs[:, prompt_length:]      ,       skip_special_tokens=snake_case__ )
				lowerCAmelCase  =				[text.strip() for text in output_text]
				print("""HF generation:"""      ,       snake_case__ )

				if pytorch_dump_folder_path is not None:
								processor.save_pretrained(snake_case__ )
								hf_model.save_pretrained(snake_case__ )

				if push_to_hub:
								processor.push_to_hub(f'''nielsr/{model_name}''' )
								hf_model.push_to_hub(f'''nielsr/{model_name}''' )


if __name__ == "__main__":
		a      =							argparse.ArgumentParser()
		a      =							[
		    'blip2-opt-2.7b',
		    'blip2-opt-6.7b',
		    'blip2-opt-2.7b-coco',
		    'blip2-opt-6.7b-coco',
		    'blip2-flan-t5-xl',
		    'blip2-flan-t5-xl-coco',
		    'blip2-flan-t5-xxl',
		]
		parser.add_argument(
		    '--model_name',
		    default='blip2-opt-2.7b',
		    choices=choices,
		    type=str,
		    help='Path to hf config.json of model to convert',
		)
		parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
		parser.add_argument(
		    '--push_to_hub',
		    action='store_true',
		    help='Whether to push the model and processor to the hub after converting',
		)

		a      =							parser.parse_args()

		convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)











"""simple docstring"""







import argparse
import collections
import os
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
a      =							'src/transformers'
a      =							'docs/source/en'
a      =							'.'





def 							lowercase					(snake_case__   : int      ,       snake_case__   : Optional[int]      ,       snake_case__   : Tuple )     ->  Any:


				'''simple docstring'''






				with open(snake_case__      ,       """r"""      ,       encoding="""utf-8"""      ,       newline="""\n""" ) as f:
								lowerCAmelCase  =				f.readlines()
				# Find the start prompt.
				lowerCAmelCase  =				0
				while not lines[start_index].startswith(snake_case__ ):
								start_index += 1
				start_index += 1

				lowerCAmelCase  =				start_index
				while not lines[end_index].startswith(snake_case__ ):
								end_index += 1
				end_index -= 1

				while len(lines[start_index] ) <= 1:
								start_index += 1
				while len(lines[end_index] ) <= 1:
								end_index -= 1
				end_index += 1
				return "".join(lines[start_index:end_index] ), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
a      =							'Model|Encoder|Decoder|ForConditionalGeneration'
# Regexes that match TF/Flax/PT model names.
a      =							re.compile(R'TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
a      =							re.compile(R'Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
a      =							re.compile(R'(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')


# This is to make sure the transformers module imported is the one in the repo.
a      =							direct_transformers_import(TRANSFORMERS_PATH)





def 							lowercase					(snake_case__   : int )     ->  Any:


				'''simple docstring'''






				lowerCAmelCase  =				re.finditer(""".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"""      ,       snake_case__ )
				return [m.group(0 ) for m in matches]





def 							lowercase					(snake_case__   : str      ,       snake_case__   : Tuple )     ->  int:


				'''simple docstring'''






				lowerCAmelCase  =				2 if text == """✅""" or text == """❌""" else len(snake_case__ )
				lowerCAmelCase  =				(width - text_length) // 2
				lowerCAmelCase  =				width - text_length - left_indent
				return " " * left_indent + text + " " * right_indent





def 							lowercase					()     ->  str:


				'''simple docstring'''






				lowerCAmelCase  =				transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
				lowerCAmelCase  =				{
				    name: config_maping_names[code]
				    for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
				    if code in config_maping_names
				}
				lowerCAmelCase  =				{name: config.replace("""Config"""      ,       """""" ) for name, config in model_name_to_config.items()}

				# Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
				lowerCAmelCase  =				collections.defaultdict(snake_case__ )
				lowerCAmelCase  =				collections.defaultdict(snake_case__ )
				lowerCAmelCase  =				collections.defaultdict(snake_case__ )
				lowerCAmelCase  =				collections.defaultdict(snake_case__ )
				lowerCAmelCase  =				collections.defaultdict(snake_case__ )

				# Let's lookup through all transformers object (once).
				for attr_name in dir(snake_case__ ):
								lowerCAmelCase  =				None
								if attr_name.endswith("""Tokenizer""" ):
												lowerCAmelCase  =				slow_tokenizers
												lowerCAmelCase  =				attr_name[:-9]
								elif attr_name.endswith("""TokenizerFast""" ):
												lowerCAmelCase  =				fast_tokenizers
												lowerCAmelCase  =				attr_name[:-13]
								elif _re_tf_models.match(snake_case__ ) is not None:
												lowerCAmelCase  =				tf_models
												lowerCAmelCase  =				_re_tf_models.match(snake_case__ ).groups()[0]
								elif _re_flax_models.match(snake_case__ ) is not None:
												lowerCAmelCase  =				flax_models
												lowerCAmelCase  =				_re_flax_models.match(snake_case__ ).groups()[0]
								elif _re_pt_models.match(snake_case__ ) is not None:
												lowerCAmelCase  =				pt_models
												lowerCAmelCase  =				_re_pt_models.match(snake_case__ ).groups()[0]

								if lookup_dict is not None:
												while len(snake_case__ ) > 0:
																if attr_name in model_name_to_prefix.values():
																				lowerCAmelCase  =				True
																				break
																# Try again after removing the last word in the name
																lowerCAmelCase  =				"""""".join(camel_case_split(snake_case__ )[:-1] )

    # Let's build that table!
				lowerCAmelCase  =				list(model_name_to_config.keys() )
				model_names.sort(key=str.lower )
				lowerCAmelCase  =				["""Model""", """Tokenizer slow""", """Tokenizer fast""", """PyTorch support""", """TensorFlow support""", """Flax Support"""]
				# We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
				lowerCAmelCase  =				[len(snake_case__ ) + 2 for c in columns]
				lowerCAmelCase  =				max([len(snake_case__ ) for name in model_names] ) + 2

				# Build the table per se
				lowerCAmelCase  =				"""|""" + """|""".join([_center_text(snake_case__      ,       snake_case__ ) for c, w in zip(snake_case__      ,       snake_case__ )] ) + """|\n"""
				# Use ":-----:" format to center-aligned table cell texts
				table += "|" + "|".join([""":""" + """-""" * (w - 2) + """:""" for w in widths] ) + "|\n"

				lowerCAmelCase  =				{True: """✅""", False: """❌"""}
				for name in model_names:
								lowerCAmelCase  =				model_name_to_prefix[name]
								lowerCAmelCase  =				[
								    name,
								    check[slow_tokenizers[prefix]],
								    check[fast_tokenizers[prefix]],
								    check[pt_models[prefix]],
								    check[tf_models[prefix]],
								    check[flax_models[prefix]],
								]
								table += "|" + "|".join([_center_text(snake_case__      ,       snake_case__ ) for l, w in zip(snake_case__      ,       snake_case__ )] ) + "|\n"
				return table





def 							lowercase					(snake_case__   : Dict=False )     ->  Any:


				'''simple docstring'''






				lowerCAmelCase ,						lowerCAmelCase ,						lowerCAmelCase ,						lowerCAmelCase  =				_find_text_in_file(
				    filename=os.path.join(snake_case__      ,       """index.md""" )      ,       start_prompt="""<!--This table is updated automatically from the auto modules"""      ,       end_prompt="""<!-- End table-->"""      ,       )
				lowerCAmelCase  =				get_model_table_from_auto_modules()

				if current_table != new_table:
								if overwrite:
												with open(os.path.join(snake_case__      ,       """index.md""" )      ,       """w"""      ,       encoding="""utf-8"""      ,       newline="""\n""" ) as f:
																f.writelines(lines[:start_index] + [new_table] + lines[end_index:] )
								else:
												raise ValueError(
												    """The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this.""" )


if __name__ == "__main__":
		a      =							argparse.ArgumentParser()
		parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
		a      =							parser.parse_args()

		check_model_table(args.fix_and_overwrite)





