






import argparse

import requests
import torch

# pip3 install salesforce-lavis
# I'm actually installing a slightly modified version: pip3 install git+https://github.com/nielsrogge/LAVIS.git@fix_lavis
from lavis.models import load_model_and_preprocess
from PIL import Image

from transformers import (
    AutoTokenizer,
    BlipaConfig,
    BlipaForConditionalGeneration,
    BlipaProcessor,
    BlipaVisionConfig,
    BlipImageProcessor,
    OPTConfig,
    TaConfig,
)
from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD



def 					_lowerCamelCase(			):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_   :	List[Any]													=   '''https://storage.googleapis.com/sfr-vision-language-research/LAVIS/assets/merlion.png'''
				SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   Image.open(requests.get(lowerCAmelCase__    ,    stream=lowerCAmelCase__      ).raw      ).convert('RGB'      )

				return image



def 					_lowerCamelCase(			lowerCAmelCase__				:		Union[str, Any]      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   []
				# fmt: off

				# vision encoder
				rename_keys.append(('visual_encoder.cls_token', 'vision_model.embeddings.class_embedding')      )
				rename_keys.append(('visual_encoder.pos_embed', 'vision_model.embeddings.position_embedding')      )
				rename_keys.append(('visual_encoder.patch_embed.proj.weight', 'vision_model.embeddings.patch_embedding.weight')      )
				rename_keys.append(('visual_encoder.patch_embed.proj.bias', 'vision_model.embeddings.patch_embedding.bias')      )
				rename_keys.append(('ln_vision.weight', 'vision_model.post_layernorm.weight')      )
				rename_keys.append(('ln_vision.bias', 'vision_model.post_layernorm.bias')      )

				for i in range(config.vision_config.num_hidden_layers      ):
								rename_keys.append((F'''visual_encoder.blocks.{i}.norm1.weight''', F'''vision_model.encoder.layers.{i}.layer_norm1.weight''')      )
								rename_keys.append((F'''visual_encoder.blocks.{i}.norm1.bias''', F'''vision_model.encoder.layers.{i}.layer_norm1.bias''')      )
								rename_keys.append((F'''visual_encoder.blocks.{i}.norm2.weight''', F'''vision_model.encoder.layers.{i}.layer_norm2.weight''')      )
								rename_keys.append((F'''visual_encoder.blocks.{i}.norm2.bias''', F'''vision_model.encoder.layers.{i}.layer_norm2.bias''')      )
								rename_keys.append((F'''visual_encoder.blocks.{i}.attn.qkv.weight''', F'''vision_model.encoder.layers.{i}.self_attn.qkv.weight''')      )
								rename_keys.append((F'''visual_encoder.blocks.{i}.attn.proj.weight''', F'''vision_model.encoder.layers.{i}.self_attn.projection.weight''',)      )
								rename_keys.append((F'''visual_encoder.blocks.{i}.attn.proj.bias''', F'''vision_model.encoder.layers.{i}.self_attn.projection.bias''')      )
								rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc1.weight''', F'''vision_model.encoder.layers.{i}.mlp.fc1.weight''')      )
								rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc1.bias''', F'''vision_model.encoder.layers.{i}.mlp.fc1.bias''')      )
								rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc2.weight''', F'''vision_model.encoder.layers.{i}.mlp.fc2.weight''')      )
								rename_keys.append((F'''visual_encoder.blocks.{i}.mlp.fc2.bias''', F'''vision_model.encoder.layers.{i}.mlp.fc2.bias''')      )

				# QFormer
				rename_keys.append(('Qformer.bert.embeddings.LayerNorm.weight', 'qformer.layernorm.weight')      )
				rename_keys.append(('Qformer.bert.embeddings.LayerNorm.bias', 'qformer.layernorm.bias')      )

				# fmt: on
				return rename_keys



def 					_lowerCamelCase(			lowerCAmelCase__				:		int    ,    lowerCAmelCase__				:		List[Any]    ,    lowerCAmelCase__				:		int      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_   :	Any													=   dct.pop(lowerCAmelCase__      )
				SCREAMING_SNAKE_CASE_   :	List[Any]													=   val



def 					_lowerCamelCase(			lowerCAmelCase__				:		Dict    ,    lowerCAmelCase__				:		Any      ):





				'''simple docstring'''







				for i in range(config.vision_config.num_hidden_layers      ):
								# read in original q and v biases
								SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   state_dict.pop(F'''visual_encoder.blocks.{i}.attn.q_bias'''      )
								SCREAMING_SNAKE_CASE_   :	List[str]													=   state_dict.pop(F'''visual_encoder.blocks.{i}.attn.v_bias'''      )

								# next, set bias in the state dict
								SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   torch.cat((q_bias, torch.zeros_like(lowerCAmelCase__    ,    requires_grad=lowerCAmelCase__      ), v_bias)      )
								SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   qkv_bias



def 					_lowerCamelCase(			lowerCAmelCase__				:		Optional[Any]    ,    lowerCAmelCase__				:		List[str]      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_   :	Tuple													=   364 if '''coco''' in model_name else 224
				SCREAMING_SNAKE_CASE_   :	Dict													=   BlipaVisionConfig(image_size=lowerCAmelCase__      ).to_dict()

				# make sure the models have proper bos_token_id and eos_token_id set (important for generation)
				# seems like flan-T5 models don't have bos_token_id properly set?
				if "opt-2.7b" in model_name:
								SCREAMING_SNAKE_CASE_   :	Dict													=   OPTConfig.from_pretrained('facebook/opt-2.7b'    ,    eos_token_id=lowerCAmelCase__      ).to_dict()
				elif "opt-6.7b" in model_name:
								SCREAMING_SNAKE_CASE_   :	Optional[int]													=   OPTConfig.from_pretrained('facebook/opt-6.7b'    ,    eos_token_id=lowerCAmelCase__      ).to_dict()
				elif "t5-xl" in model_name:
								SCREAMING_SNAKE_CASE_   :	Dict													=   TaConfig.from_pretrained('google/flan-t5-xl'    ,    dense_act_fn='gelu'    ,    bos_token_id=1      ).to_dict()
				elif "t5-xxl" in model_name:
								SCREAMING_SNAKE_CASE_   :	str													=   TaConfig.from_pretrained('google/flan-t5-xxl'    ,    dense_act_fn='gelu'    ,    bos_token_id=1      ).to_dict()

				SCREAMING_SNAKE_CASE_   :	Dict													=   BlipaConfig(vision_config=lowerCAmelCase__    ,    text_config=lowerCAmelCase__      )

				return config, image_size



@torch.no_grad()
def 					_lowerCamelCase(			lowerCAmelCase__				:		List[str]    ,    lowerCAmelCase__				:		Union[str, Any]=None    ,    lowerCAmelCase__				:		str=False      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_   :	str													=   (
				    AutoTokenizer.from_pretrained('facebook/opt-2.7b'      )
				    if '''opt''' in model_name
				    else AutoTokenizer.from_pretrained('google/flan-t5-xl'      )
				)
				SCREAMING_SNAKE_CASE_   :	int													=   tokenizer('\n'    ,    add_special_tokens=lowerCAmelCase__      ).input_ids[0]
				SCREAMING_SNAKE_CASE_   :	List[Any]													=   get_blipa_config(lowerCAmelCase__    ,    eos_token_id=lowerCAmelCase__      )

				SCREAMING_SNAKE_CASE_   :	List[Any]													=   BlipaForConditionalGeneration(lowerCAmelCase__      ).eval()

				SCREAMING_SNAKE_CASE_   :	Dict													=   {
				    '''blip2-opt-2.7b''': ('''blip2_opt''', '''pretrain_opt2.7b'''),
				    '''blip2-opt-6.7b''': ('''blip2_opt''', '''pretrain_opt6.7b'''),
				    '''blip2-opt-2.7b-coco''': ('''blip2_opt''', '''caption_coco_opt2.7b'''),
				    '''blip2-opt-6.7b-coco''': ('''blip2_opt''', '''caption_coco_opt6.7b'''),
				    '''blip2-flan-t5-xl''': ('''blip2_t5''', '''pretrain_flant5xl'''),
				    '''blip2-flan-t5-xl-coco''': ('''blip2_t5''', '''caption_coco_flant5xl'''),
				    '''blip2-flan-t5-xxl''': ('''blip2_t5''', '''pretrain_flant5xxl'''),
				}

				SCREAMING_SNAKE_CASE_   :	int													=   model_name_to_original[model_name]

				# load original model
				print('Loading original model...'      )
				SCREAMING_SNAKE_CASE_   :	List[str]													=   '''cuda''' if torch.cuda.is_available() else '''cpu'''
				SCREAMING_SNAKE_CASE_   :	List[Any]													=   load_model_and_preprocess(
				    name=lowerCAmelCase__    ,    model_type=lowerCAmelCase__    ,    is_eval=lowerCAmelCase__    ,    device=lowerCAmelCase__      )
				original_model.eval()
				print('Done!'      )

				# update state dict keys
				SCREAMING_SNAKE_CASE_   :	List[Any]													=   original_model.state_dict()
				SCREAMING_SNAKE_CASE_   :	List[str]													=   create_rename_keys(lowerCAmelCase__      )
				for src, dest in rename_keys:
								rename_key(lowerCAmelCase__    ,    lowerCAmelCase__    ,    lowerCAmelCase__      )

				# some keys can be renamed efficiently
				for key, val in state_dict.copy().items():
								SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   state_dict.pop(lowerCAmelCase__      )
								if key.startswith('Qformer.bert'      ):
												SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   key.replace('Qformer.bert'    ,    'qformer'      )
								if "attention.self" in key:
												SCREAMING_SNAKE_CASE_   :	str													=   key.replace('self'    ,    'attention'      )
								if "opt_proj" in key:
												SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   key.replace('opt_proj'    ,    'language_projection'      )
								if "t5_proj" in key:
												SCREAMING_SNAKE_CASE_   :	Dict													=   key.replace('t5_proj'    ,    'language_projection'      )
								if key.startswith('opt'      ):
												SCREAMING_SNAKE_CASE_   :	str													=   key.replace('opt'    ,    'language'      )
								if key.startswith('t5'      ):
												SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   key.replace('t5'    ,    'language'      )
								SCREAMING_SNAKE_CASE_   :	List[Any]													=   val

				# read in qv biases
				read_in_q_v_bias(lowerCAmelCase__    ,    lowerCAmelCase__      )

				SCREAMING_SNAKE_CASE_   :	Any													=   hf_model.load_state_dict(lowerCAmelCase__    ,    strict=lowerCAmelCase__      )
				assert len(lowerCAmelCase__      ) == 0
				assert unexpected_keys == ["qformer.embeddings.position_ids"]

				SCREAMING_SNAKE_CASE_   :	str													=   load_demo_image()
				SCREAMING_SNAKE_CASE_   :	Any													=   vis_processors['''eval'''](lowerCAmelCase__      ).unsqueeze(0      ).to(lowerCAmelCase__      )
				SCREAMING_SNAKE_CASE_   :	Tuple													=   tokenizer(['\n']    ,    return_tensors='pt'      ).input_ids.to(lowerCAmelCase__      )

				# create processor
				SCREAMING_SNAKE_CASE_   :	Dict													=   BlipImageProcessor(
				    size={'height': image_size, 'width': image_size}    ,    image_mean=lowerCAmelCase__    ,    image_std=lowerCAmelCase__      )
				SCREAMING_SNAKE_CASE_   :	str													=   BlipaProcessor(image_processor=lowerCAmelCase__    ,    tokenizer=lowerCAmelCase__      )
				SCREAMING_SNAKE_CASE_   :	List[str]													=   processor(images=lowerCAmelCase__    ,    return_tensors='pt'      ).pixel_values.to(lowerCAmelCase__      )

				# make sure processor creates exact same pixel values
				assert torch.allclose(lowerCAmelCase__    ,    lowerCAmelCase__      )

				original_model.to(lowerCAmelCase__      )
				hf_model.to(lowerCAmelCase__      )
				with torch.no_grad():
								if "opt" in model_name:
												SCREAMING_SNAKE_CASE_   :	Dict													=   original_model({'image': original_pixel_values, 'text_input': ['']}      ).logits
												SCREAMING_SNAKE_CASE_   :	Any													=   hf_model(lowerCAmelCase__    ,    lowerCAmelCase__      ).logits
								else:
												SCREAMING_SNAKE_CASE_   :	List[Any]													=   original_model(
												    {'image': original_pixel_values, 'text_input': ['\n'], 'text_output': ['\n']}      ).logits
												SCREAMING_SNAKE_CASE_   :	str													=   input_ids.masked_fill(input_ids == tokenizer.pad_token_id    ,    -100      )
												SCREAMING_SNAKE_CASE_   :	List[Any]													=   hf_model(lowerCAmelCase__    ,    lowerCAmelCase__    ,    labels=lowerCAmelCase__      ).logits

				assert original_logits.shape == logits.shape
				print('First values of original logits:'    ,    original_logits[0, :3, :3]      )
				print('First values of HF logits:'    ,    logits[0, :3, :3]      )

				# assert values
				if model_name == "blip2-flan-t5-xl":
								SCREAMING_SNAKE_CASE_   :	List[str]													=   torch.tensor(
								    [[-41.5_850, -4.4_440, -8.9_922], [-47.4_322, -5.9_143, -1.7_340]]    ,    device=lowerCAmelCase__      )
								assert torch.allclose(logits[0, :3, :3]    ,    lowerCAmelCase__    ,    atol=1E-4      )
				elif model_name == "blip2-flan-t5-xl-coco":
								SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   torch.tensor(
								    [[-57.0_109, -9.8_967, -12.6_280], [-68.6_578, -12.7_191, -10.5_065]]    ,    device=lowerCAmelCase__      )
				else:
								# cast to same type
								SCREAMING_SNAKE_CASE_   :	List[str]													=   logits.dtype
								assert torch.allclose(original_logits.to(lowerCAmelCase__      )    ,    lowerCAmelCase__    ,    atol=1E-2      )
				print('Looks ok!'      )

				print('Generating a caption...'      )
				SCREAMING_SNAKE_CASE_   :	List[str]													=   ''''''
				SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   tokenizer(lowerCAmelCase__    ,    return_tensors='pt'      ).input_ids.to(lowerCAmelCase__      )

				SCREAMING_SNAKE_CASE_   :	Optional[int]													=   original_model.generate({'image': original_pixel_values}      )
				SCREAMING_SNAKE_CASE_   :	Dict													=   hf_model.generate(
				    lowerCAmelCase__    ,    lowerCAmelCase__    ,    do_sample=lowerCAmelCase__    ,    num_beams=5    ,    max_length=30    ,    min_length=1    ,    top_p=0.9    ,    repetition_penalty=1.0    ,    length_penalty=1.0    ,    temperature=1    ,    )
				print('Original generation:'    ,    lowerCAmelCase__      )
				SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   input_ids.shape[1]
				SCREAMING_SNAKE_CASE_   :	int													=   processor.batch_decode(outputs[:, prompt_length:]    ,    skip_special_tokens=lowerCAmelCase__      )
				SCREAMING_SNAKE_CASE_   :	Optional[Any]													=   [text.strip() for text in output_text]
				print('HF generation:'    ,    lowerCAmelCase__      )

				if pytorch_dump_folder_path is not None:
								processor.save_pretrained(lowerCAmelCase__      )
								hf_model.save_pretrained(lowerCAmelCase__      )

				if push_to_hub:
								processor.push_to_hub(F'''nielsr/{model_name}'''      )
								hf_model.push_to_hub(F'''nielsr/{model_name}'''      )


if __name__ == "__main__":
				A        =			argparse.ArgumentParser()
				A        =			[
				    "blip2-opt-2.7b",
				    "blip2-opt-6.7b",
				    "blip2-opt-2.7b-coco",
				    "blip2-opt-6.7b-coco",
				    "blip2-flan-t5-xl",
				    "blip2-flan-t5-xl-coco",
				    "blip2-flan-t5-xxl",
				]
				parser.add_argument(
				    '--model_name',
				    default='blip2-opt-2.7b',
				    choices=choices,
				    type=str,
				    help='Path to hf config.json of model to convert',
				)
				parser.add_argument('--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model.')
				parser.add_argument(
				    '--push_to_hub',
				    action='store_true',
				    help='Whether to push the model and processor to the hub after converting',
				)

				A        =			parser.parse_args()

				convert_blipa_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)







import argparse
import os

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_task_guides.py
A        =			'src/transformers'
A        =			'docs/source/en/tasks'



def 					_lowerCamelCase(			lowerCAmelCase__				:		Tuple    ,    lowerCAmelCase__				:		Optional[Any]    ,    lowerCAmelCase__				:		Union[str, Any]      ):





				'''simple docstring'''







				with open(lowerCAmelCase__    ,    'r'    ,    encoding='utf-8'    ,    newline='\n'      ) as f:
								SCREAMING_SNAKE_CASE_   :	Optional[int]													=   f.readlines()
				# Find the start prompt.
				SCREAMING_SNAKE_CASE_   :	Optional[int]													=   0
				while not lines[start_index].startswith(lowerCAmelCase__      ):
								start_index += 1
				start_index += 1

				SCREAMING_SNAKE_CASE_   :	int													=   start_index
				while not lines[end_index].startswith(lowerCAmelCase__      ):
								end_index += 1
				end_index -= 1

				while len(lines[start_index]      ) <= 1:
								start_index += 1
				while len(lines[end_index]      ) <= 1:
								end_index -= 1
				end_index += 1
				return "".join(lines[start_index:end_index]      ), start_index, end_index, lines


# This is to make sure the transformers module imported is the one in the repo.
A        =			direct_transformers_import(TRANSFORMERS_PATH)

A        =			{
    'asr.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,
    'audio_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,
    'language_modeling.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    'image_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,
    'masked_language_modeling.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_MASKED_LM_MAPPING_NAMES,
    'multiple_choice.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES,
    'object_detection.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,
    'question_answering.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,
    'semantic_segmentation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES,
    'sequence_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,
    'summarization.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    'token_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,
    'translation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    'video_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES,
    'document_question_answering.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES,
    'monocular_depth_estimation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES,
}

# This list contains model types used in some task guides that are not in `CONFIG_MAPPING_NAMES` (therefore not in any
# `MODEL_MAPPING_NAMES` or any `MODEL_FOR_XXX_MAPPING_NAMES`).
A        =			{
    'summarization.md': ('nllb',),
    'translation.md': ('nllb',),
}



def 					_lowerCamelCase(			lowerCAmelCase__				:		Dict      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_   :	List[str]													=   TASK_GUIDE_TO_MODELS[task_guide]
				SCREAMING_SNAKE_CASE_   :	Union[str, Any]													=   SPECIAL_TASK_GUIDE_TO_MODEL_TYPES.get(lowerCAmelCase__    ,    set()      )
				SCREAMING_SNAKE_CASE_   :	Optional[int]													=   {
				    code: name
				    for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
				    if (code in model_maping_names or code in special_model_types)
				}
				return ", ".join([F'''[{name}](../model_doc/{code})''' for code, name in model_names.items()]      ) + "\n"



def 					_lowerCamelCase(			lowerCAmelCase__				:		List[Any]    ,    lowerCAmelCase__				:		List[str]=False      ):





				'''simple docstring'''







				SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_   :	Dict													=   _find_text_in_file(
				    filename=os.path.join(lowerCAmelCase__    ,    lowerCAmelCase__      )    ,    start_prompt='<!--This tip is automatically generated by `make fix-copies`, do not fill manually!-->'    ,    end_prompt='<!--End of the generated tip-->'    ,    )

				SCREAMING_SNAKE_CASE_   :	Dict													=   get_model_list_for_task(lowerCAmelCase__      )

				if current_list != new_list:
								if overwrite:
												with open(os.path.join(lowerCAmelCase__    ,    lowerCAmelCase__      )    ,    'w'    ,    encoding='utf-8'    ,    newline='\n'      ) as f:
																f.writelines(lines[:start_index] + [new_list] + lines[end_index:]      )
								else:
												raise ValueError(
												    F'''The list of models that can be used in the {task_guide} guide needs an update. Run `make fix-copies`'''
												    ' to fix this.'      )


if __name__ == "__main__":
				A        =			argparse.ArgumentParser()
				parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
				A        =			parser.parse_args()

				for task_guide in TASK_GUIDE_TO_MODELS.keys():
								check_model_list_for_task(task_guide, args.fix_and_overwrite)