






from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    get_torch_dist_unique_port,
    require_torch_multi_gpu,
    require_torch_neuroncore,
)
from transformers.training_args import ParallelMode
from transformers.utils import logging


lowerCamelCase			:					Optional[int]         =	logging.get_logger(__name__)


if is_torch_available():
 import torch
 from torch import nn
 from torch.utils.data import Dataset

 from transformers import Trainer



 class 							_UpperCamelCase			(a_			):


       def __init__(	self					,						__UpperCamelCase = 1_0_1	)->	Optional[Any]:
            __lowerCAmelCase								=					length


       def __len__(	self	)->	Dict:
            return self.length


       def __getitem__(	self					,						__UpperCamelCase	)->	int:
            return i



 class 							_UpperCamelCase			:


       def __call__(	self					,						__UpperCamelCase	)->	str:
            return {"input_ids": torch.tensor(__UpperCamelCase	), "labels": torch.tensor(__UpperCamelCase	)}



 class 							_UpperCamelCase			(nn.Module			):


       def __init__(	self	)->	Optional[int]:
            super().__init__()
            # Add some (unused) params otherwise DDP will complain.
            __lowerCAmelCase								=					nn.Linear(1_2_0					,						8_0	)


       def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase=None	)->	Any:
            if labels is not None:
                 return torch.tensor(0.0					,						device=input_ids.device	), input_ids
            else:
                 return input_ids



class 							_UpperCamelCase			(a_			):


      @require_torch_neuroncore
      def  __UpperCAmelCase		(	self	)->	Optional[int]:
           __lowerCAmelCase								=					F"""--nproc_per_node=2
            --master_port={get_torch_dist_unique_port()}
            {self.test_file_dir}/test_trainer_distributed.py
        """.split()
           __lowerCAmelCase								=					self.get_auto_remove_tmp_dir()
           __lowerCAmelCase								=					F"""--output_dir {output_dir}""".split()
           __lowerCAmelCase								=					["torchrun"] + distributed_args + args
           execute_subprocess_async(__UpperCamelCase					,						env=self.get_env()	)
           # successful return here == success - any errors would have caused an error in the sub-call



class 							_UpperCamelCase			(a_			):


      @require_torch_multi_gpu
      def  __UpperCAmelCase		(	self	)->	Union[str, Any]:
           __lowerCAmelCase								=					F"""--nproc_per_node={torch.cuda.device_count()}
            --master_port={get_torch_dist_unique_port()}
            {self.test_file_dir}/test_trainer_distributed.py
        """.split()
           __lowerCAmelCase								=					self.get_auto_remove_tmp_dir()
           __lowerCAmelCase								=					F"""--output_dir {output_dir}""".split()
           __lowerCAmelCase								=					["torchrun"] + distributed_args + args
           execute_subprocess_async(__UpperCamelCase					,						env=self.get_env()	)
           # successful return here == success - any errors would have caused an error in the sub-call


if __name__ == "__main__":
 # The script below is meant to be run under torch.distributed, on a machine with multiple GPUs:
 #
 # PYTHONPATH="src" python -m torch.distributed.run --nproc_per_node 2 --output_dir output_dir ./tests/test_trainer_distributed.py

 lowerCamelCase			:					List[Any]         =	HfArgumentParser((TrainingArguments,))
 lowerCamelCase			:					Union[str, Any]         =	parser.parse_args_into_dataclasses()[0]

 logger.warning(
     F'''Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}, '''
     F'''distributed training: {training_args.parallel_mode != ParallelMode.NOT_DISTRIBUTED}'''
 )

 # Essentially, what we want to verify in the distributed case is that we get all samples back,
 # in the right order. (this is crucial for prediction for instance)
 for dataset_length in [101, 40, 7]:
  lowerCamelCase			:					str         =	DummyDataset(dataset_length)





  def       __lowerCAmelCase      (						__snake_case       ):
       __lowerCAmelCase								=					list(range(len(__snake_case       )       )       )
       __lowerCAmelCase								=					p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
       if not success and training_args.local_rank == 0:
            logger.warning(
                "Predictions and/or labels do not match expected results:\n  - predictions: "
                F"""{p.predictions.tolist()}\n  - labels: {p.label_ids.tolist()}\n  - expected: {sequential}"""       )
       return {"success": success}

  lowerCamelCase			:					Optional[int]         =	Trainer(
      model=DummyModel(),
      args=training_args,
      data_collator=DummyDataCollator(),
      eval_dataset=dataset,
      compute_metrics=compute_metrics,
  )
  lowerCamelCase			:					Optional[Any]         =	trainer.evaluate()
  logger.info(metrics)
  if metrics["eval_success"] is not True:
   logger.error(metrics)
   exit(1)

  lowerCamelCase			:					List[Any]         =	trainer.predict(dataset)
  logger.info(p.metrics)
  if p.metrics["test_success"] is not True:
   logger.error(p.metrics)
   exit(1)

  lowerCamelCase			:					Dict         =	2

  lowerCamelCase			:					str         =	trainer.evaluate()
  logger.info(metrics)
  if metrics["eval_success"] is not True:
   logger.error(metrics)
   exit(1)

  lowerCamelCase			:					Tuple         =	trainer.predict(dataset)
  logger.info(p.metrics)
  if p.metrics["test_success"] is not True:
   logger.error(p.metrics)
   exit(1)



  lowerCamelCase			:					Tuple         =	None














import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
from ...utils import logging
from ..auto import CONFIG_MAPPING


lowerCamelCase			:					Union[str, Any]         =	logging.get_logger(__name__)

lowerCamelCase			:					Any         =	{
    '''Salesforce/instruct-blip-flan-t5''': '''https://huggingface.co/Salesforce/instruct-blip-flan-t5/resolve/main/config.json''',
}



class 							_UpperCamelCase			(a_			):
      snake_case_  =							"""instructblip_vision_model"""


      def __init__(	self					,						__UpperCamelCase=1_4_0_8					,						__UpperCamelCase=6_1_4_4					,						__UpperCamelCase=3_9					,						__UpperCamelCase=1_6					,						__UpperCamelCase=2_2_4					,						__UpperCamelCase=1_4					,						__UpperCamelCase="gelu"					,						__UpperCamelCase=1e-6					,						__UpperCamelCase=0.0					,						__UpperCamelCase=1e-10					,						__UpperCamelCase=True					,						**__UpperCamelCase					,						)->	Union[str, Any]:
           super().__init__(**__UpperCamelCase	)

           __lowerCAmelCase								=					hidden_size
           __lowerCAmelCase								=					intermediate_size
           __lowerCAmelCase								=					num_hidden_layers
           __lowerCAmelCase								=					num_attention_heads
           __lowerCAmelCase								=					patch_size
           __lowerCAmelCase								=					image_size
           __lowerCAmelCase								=					initializer_range
           __lowerCAmelCase								=					attention_dropout
           __lowerCAmelCase								=					layer_norm_eps
           __lowerCAmelCase								=					hidden_act
           __lowerCAmelCase								=					qkv_bias


      @classmethod
      def  __UpperCAmelCase		(	cls					,						__UpperCamelCase					,						**__UpperCamelCase	)->	"PretrainedConfig":
           cls._set_token_in_kwargs(__UpperCamelCase	)

           __lowerCAmelCase			, __lowerCAmelCase								=					cls.get_config_dict(__UpperCamelCase					,						**__UpperCamelCase	)

           # get the vision config dict if we are loading from InstructBlipConfig
           if config_dict.get("model_type"	) == "instructblip":
                __lowerCAmelCase								=					config_dict["vision_config"]

           if "model_type" in config_dict and hasattr(cls					,						"model_type"	) and config_dict["model_type"] != cls.model_type:
                logger.warning(
                    F"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
                    F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""	)

           return cls.from_dict(__UpperCamelCase					,						**__UpperCamelCase	)



class 							_UpperCamelCase			(a_			):
      snake_case_  =							"""instructblip_qformer"""


      def __init__(	self					,						__UpperCamelCase=3_0_5_2_2					,						__UpperCamelCase=7_6_8					,						__UpperCamelCase=1_2					,						__UpperCamelCase=1_2					,						__UpperCamelCase=3_0_7_2					,						__UpperCamelCase="gelu"					,						__UpperCamelCase=0.1					,						__UpperCamelCase=0.1					,						__UpperCamelCase=5_1_2					,						__UpperCamelCase=0.0_2					,						__UpperCamelCase=1e-12					,						__UpperCamelCase=0					,						__UpperCamelCase="absolute"					,						__UpperCamelCase=2					,						__UpperCamelCase=1_4_0_8					,						**__UpperCamelCase					,						)->	Any:
           super().__init__(pad_token_id=__UpperCamelCase					,						**__UpperCamelCase	)

           __lowerCAmelCase								=					vocab_size
           __lowerCAmelCase								=					hidden_size
           __lowerCAmelCase								=					num_hidden_layers
           __lowerCAmelCase								=					num_attention_heads
           __lowerCAmelCase								=					hidden_act
           __lowerCAmelCase								=					intermediate_size
           __lowerCAmelCase								=					hidden_dropout_prob
           __lowerCAmelCase								=					attention_probs_dropout_prob
           __lowerCAmelCase								=					max_position_embeddings
           __lowerCAmelCase								=					initializer_range
           __lowerCAmelCase								=					layer_norm_eps
           __lowerCAmelCase								=					position_embedding_type
           __lowerCAmelCase								=					cross_attention_frequency
           __lowerCAmelCase								=					encoder_hidden_size


      @classmethod
      def  __UpperCAmelCase		(	cls					,						__UpperCamelCase					,						**__UpperCamelCase	)->	"PretrainedConfig":
           cls._set_token_in_kwargs(__UpperCamelCase	)

           __lowerCAmelCase			, __lowerCAmelCase								=					cls.get_config_dict(__UpperCamelCase					,						**__UpperCamelCase	)

           # get the qformer config dict if we are loading from InstructBlipConfig
           if config_dict.get("model_type"	) == "instructblip":
                __lowerCAmelCase								=					config_dict["qformer_config"]

           if "model_type" in config_dict and hasattr(cls					,						"model_type"	) and config_dict["model_type"] != cls.model_type:
                logger.warning(
                    F"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
                    F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""	)

           return cls.from_dict(__UpperCamelCase					,						**__UpperCamelCase	)



class 							_UpperCamelCase			(a_			):
      snake_case_  =							"""instructblip"""
      snake_case_  =							True


      def __init__(	self					,						__UpperCamelCase=None					,						__UpperCamelCase=None					,						__UpperCamelCase=None					,						__UpperCamelCase=3_2					,						**__UpperCamelCase	)->	Tuple:
           super().__init__(**__UpperCamelCase	)

           if vision_config is None:
                __lowerCAmelCase								=					{}
                logger.info("vision_config is None. initializing the InstructBlipVisionConfig with default values."	)

           if qformer_config is None:
                __lowerCAmelCase								=					{}
                logger.info("qformer_config is None. Initializing the InstructBlipQFormerConfig with default values."	)

           if text_config is None:
                __lowerCAmelCase								=					{}
                logger.info("text_config is None. Initializing the text config with default values (`OPTConfig`)."	)

           __lowerCAmelCase								=					InstructBlipVisionConfig(**__UpperCamelCase	)
           __lowerCAmelCase								=					InstructBlipQFormerConfig(**__UpperCamelCase	)
           __lowerCAmelCase								=					text_config["model_type"] if "model_type" in text_config else "opt"
           __lowerCAmelCase								=					CONFIG_MAPPING[text_model_type](**__UpperCamelCase	)

           __lowerCAmelCase								=					self.text_config.tie_word_embeddings
           __lowerCAmelCase								=					self.text_config.is_encoder_decoder

           __lowerCAmelCase								=					num_query_tokens
           __lowerCAmelCase								=					self.vision_config.hidden_size
           __lowerCAmelCase								=					self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
           __lowerCAmelCase								=					1.0
           __lowerCAmelCase								=					0.0_2


      @classmethod
      def  __UpperCAmelCase		(	cls					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase					,						**__UpperCamelCase					,						)->	int:
           return cls(
               vision_config=vision_config.to_dict()					,						qformer_config=qformer_config.to_dict()					,						text_config=text_config.to_dict()					,						**__UpperCamelCase					,						)


      def  __UpperCAmelCase		(	self	)->	Dict:
           __lowerCAmelCase								=					copy.deepcopy(self.__dict__	)
           __lowerCAmelCase								=					self.vision_config.to_dict()
           __lowerCAmelCase								=					self.qformer_config.to_dict()
           __lowerCAmelCase								=					self.text_config.to_dict()
           __lowerCAmelCase								=					self.__class__.model_type
           return output






