






import os
from tempfile import TemporaryDirectory
from unittest import TestCase

import pytest
from absl.testing import parameterized

from datasets import config
from datasets.arrow_reader import HF_GCP_BASE_URL
from datasets.builder import DatasetBuilder
from datasets.dataset_dict import IterableDatasetDict
from datasets.iterable_dataset import IterableDataset
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.file_utils import cached_path


lowerCamelCase			:					Tuple         =	[
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.de'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.en'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.fr'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.frr'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.it'''},
    {'''dataset''': '''wikipedia''', '''config_name''': '''20220301.simple'''},
    {'''dataset''': '''snli''', '''config_name''': '''plain_text'''},
    {'''dataset''': '''eli5''', '''config_name''': '''LFQA_reddit'''},
    {'''dataset''': '''wiki40b''', '''config_name''': '''en'''},
    {'''dataset''': '''wiki_dpr''', '''config_name''': '''psgs_w100.nq.compressed'''},
    {'''dataset''': '''wiki_dpr''', '''config_name''': '''psgs_w100.nq.no_index'''},
    {'''dataset''': '''wiki_dpr''', '''config_name''': '''psgs_w100.multiset.no_index'''},
    {'''dataset''': '''natural_questions''', '''config_name''': '''default'''},
]





def       __lowerCAmelCase      (						__snake_case=True       ):







     if with_config:
          return [
              {
                  "testcase_name": d["dataset"] + "/" + d["config_name"],
                  "dataset": d["dataset"],
                  "config_name": d["config_name"],
              }
              for d in DATASETS_ON_HF_GCP
          ]
     else:
          return [
              {"testcase_name": dataset, "dataset": dataset} for dataset in {d["dataset"] for d in DATASETS_ON_HF_GCP}
          ]



@parameterized.named_parameters(list_datasets_on_hf_gcp_parameters(with_config=a_			)			)
class 							_UpperCamelCase			(a_			):
      snake_case_  =							None
      snake_case_  =							None


      def  __UpperCAmelCase		(	self					,						__UpperCamelCase					,						__UpperCamelCase	)->	Any:
           with TemporaryDirectory() as tmp_dir:
                __lowerCAmelCase								=					dataset_module_factory(__A					,						cache_dir=__A	)

                __lowerCAmelCase								=					import_main_class(dataset_module.module_path					,						dataset=__A	)

                __lowerCAmelCase								=					builder_cls(
                    cache_dir=__A					,						config_name=__A					,						hash=dataset_module.hash					,						)

                __lowerCAmelCase								=					"/".join(
                    [
                        HF_GCP_BASE_URL,
                        builder_instance._relative_data_dir(with_hash=__A	).replace(os.sep					,						"/"	),
                        config.DATASET_INFO_FILENAME,
                    ]	)
                __lowerCAmelCase								=					cached_path(__A					,						cache_dir=__A	)
                self.assertTrue(os.path.exists(__A	)	)





@pytest.mark.integration
def       __lowerCAmelCase      (						__snake_case       ):
     __lowerCAmelCase								=					tmp_path_factory.mktemp("test_hf_gcp"       ) / "test_wikipedia_simple"
     __lowerCAmelCase								=					dataset_module_factory("wikipedia"					, cache_dir=_lowercase       )
     __lowerCAmelCase								=					import_main_class(dataset_module.module_path       )
     __lowerCAmelCase								=					builder_cls(
         cache_dir=_lowercase					, config_name="20220301.frr"					, hash=dataset_module.hash					, )
     # use the HF cloud storage, not the original download_and_prepare that uses apache-beam
     __lowerCAmelCase								=					None
     builder_instance.download_and_prepare()
     __lowerCAmelCase								=					builder_instance.as_dataset()
     assert ds





@pytest.mark.integration
def       __lowerCAmelCase      (						__snake_case       ):
     __lowerCAmelCase								=					dataset_module_factory("wikipedia"					, cache_dir=_lowercase       )
     __lowerCAmelCase								=					import_main_class(dataset_module.module_path					, dataset=_lowercase       )
     __lowerCAmelCase								=					builder_cls(
         cache_dir=_lowercase					, config_name="20220301.frr"					, hash=dataset_module.hash					, )
     __lowerCAmelCase								=					builder_instance.as_streaming_dataset()
     assert ds
     assert isinstance(_lowercase					, _lowercase       )
     assert "train" in ds
     assert isinstance(ds["train"]					, _lowercase       )
     assert next(iter(ds["train"]       )       )














import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
from ...utils import logging
from ..auto import CONFIG_MAPPING


lowerCamelCase			:					Union[str, Any]         =	logging.get_logger(__name__)

lowerCamelCase			:					Any         =	{
    '''Salesforce/instruct-blip-flan-t5''': '''https://huggingface.co/Salesforce/instruct-blip-flan-t5/resolve/main/config.json''',
}



class 							_UpperCamelCase			(a_			):
      snake_case_  =							"""instructblip_vision_model"""


      def __init__(	self					,						__UpperCamelCase=1_4_0_8					,						__UpperCamelCase=6_1_4_4					,						__UpperCamelCase=3_9					,						__UpperCamelCase=1_6					,						__UpperCamelCase=2_2_4					,						__UpperCamelCase=1_4					,						__UpperCamelCase="gelu"					,						__UpperCamelCase=1e-6					,						__UpperCamelCase=0.0					,						__UpperCamelCase=1e-10					,						__UpperCamelCase=True					,						**__UpperCamelCase					,						)->	Union[str, Any]:
           super().__init__(**__UpperCamelCase	)

           __lowerCAmelCase								=					hidden_size
           __lowerCAmelCase								=					intermediate_size
           __lowerCAmelCase								=					num_hidden_layers
           __lowerCAmelCase								=					num_attention_heads
           __lowerCAmelCase								=					patch_size
           __lowerCAmelCase								=					image_size
           __lowerCAmelCase								=					initializer_range
           __lowerCAmelCase								=					attention_dropout
           __lowerCAmelCase								=					layer_norm_eps
           __lowerCAmelCase								=					hidden_act
           __lowerCAmelCase								=					qkv_bias


      @classmethod
      def  __UpperCAmelCase		(	cls					,						__UpperCamelCase					,						**__UpperCamelCase	)->	"PretrainedConfig":
           cls._set_token_in_kwargs(__UpperCamelCase	)

           __lowerCAmelCase			, __lowerCAmelCase								=					cls.get_config_dict(__UpperCamelCase					,						**__UpperCamelCase	)

           # get the vision config dict if we are loading from InstructBlipConfig
           if config_dict.get("model_type"	) == "instructblip":
                __lowerCAmelCase								=					config_dict["vision_config"]

           if "model_type" in config_dict and hasattr(cls					,						"model_type"	) and config_dict["model_type"] != cls.model_type:
                logger.warning(
                    F"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
                    F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""	)

           return cls.from_dict(__UpperCamelCase					,						**__UpperCamelCase	)



class 							_UpperCamelCase			(a_			):
      snake_case_  =							"""instructblip_qformer"""


      def __init__(	self					,						__UpperCamelCase=3_0_5_2_2					,						__UpperCamelCase=7_6_8					,						__UpperCamelCase=1_2					,						__UpperCamelCase=1_2					,						__UpperCamelCase=3_0_7_2					,						__UpperCamelCase="gelu"					,						__UpperCamelCase=0.1					,						__UpperCamelCase=0.1					,						__UpperCamelCase=5_1_2					,						__UpperCamelCase=0.0_2					,						__UpperCamelCase=1e-12					,						__UpperCamelCase=0					,						__UpperCamelCase="absolute"					,						__UpperCamelCase=2					,						__UpperCamelCase=1_4_0_8					,						**__UpperCamelCase					,						)->	Any:
           super().__init__(pad_token_id=__UpperCamelCase					,						**__UpperCamelCase	)

           __lowerCAmelCase								=					vocab_size
           __lowerCAmelCase								=					hidden_size
           __lowerCAmelCase								=					num_hidden_layers
           __lowerCAmelCase								=					num_attention_heads
           __lowerCAmelCase								=					hidden_act
           __lowerCAmelCase								=					intermediate_size
           __lowerCAmelCase								=					hidden_dropout_prob
           __lowerCAmelCase								=					attention_probs_dropout_prob
           __lowerCAmelCase								=					max_position_embeddings
           __lowerCAmelCase								=					initializer_range
           __lowerCAmelCase								=					layer_norm_eps
           __lowerCAmelCase								=					position_embedding_type
           __lowerCAmelCase								=					cross_attention_frequency
           __lowerCAmelCase								=					encoder_hidden_size


      @classmethod
      def  __UpperCAmelCase		(	cls					,						__UpperCamelCase					,						**__UpperCamelCase	)->	"PretrainedConfig":
           cls._set_token_in_kwargs(__UpperCamelCase	)

           __lowerCAmelCase			, __lowerCAmelCase								=					cls.get_config_dict(__UpperCamelCase					,						**__UpperCamelCase	)

           # get the qformer config dict if we are loading from InstructBlipConfig
           if config_dict.get("model_type"	) == "instructblip":
                __lowerCAmelCase								=					config_dict["qformer_config"]

           if "model_type" in config_dict and hasattr(cls					,						"model_type"	) and config_dict["model_type"] != cls.model_type:
                logger.warning(
                    F"""You are using a model of type {config_dict["model_type"]} to instantiate a model of type """
                    F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""	)

           return cls.from_dict(__UpperCamelCase					,						**__UpperCamelCase	)



class 							_UpperCamelCase			(a_			):
      snake_case_  =							"""instructblip"""
      snake_case_  =							True


      def __init__(	self					,						__UpperCamelCase=None					,						__UpperCamelCase=None					,						__UpperCamelCase=None					,						__UpperCamelCase=3_2					,						**__UpperCamelCase	)->	Tuple:
           super().__init__(**__UpperCamelCase	)

           if vision_config is None:
                __lowerCAmelCase								=					{}
                logger.info("vision_config is None. initializing the InstructBlipVisionConfig with default values."	)

           if qformer_config is None:
                __lowerCAmelCase								=					{}
                logger.info("qformer_config is None. Initializing the InstructBlipQFormerConfig with default values."	)

           if text_config is None:
                __lowerCAmelCase								=					{}
                logger.info("text_config is None. Initializing the text config with default values (`OPTConfig`)."	)

           __lowerCAmelCase								=					InstructBlipVisionConfig(**__UpperCamelCase	)
           __lowerCAmelCase								=					InstructBlipQFormerConfig(**__UpperCamelCase	)
           __lowerCAmelCase								=					text_config["model_type"] if "model_type" in text_config else "opt"
           __lowerCAmelCase								=					CONFIG_MAPPING[text_model_type](**__UpperCamelCase	)

           __lowerCAmelCase								=					self.text_config.tie_word_embeddings
           __lowerCAmelCase								=					self.text_config.is_encoder_decoder

           __lowerCAmelCase								=					num_query_tokens
           __lowerCAmelCase								=					self.vision_config.hidden_size
           __lowerCAmelCase								=					self.text_config.model_type in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
           __lowerCAmelCase								=					1.0
           __lowerCAmelCase								=					0.0_2


      @classmethod
      def  __UpperCAmelCase		(	cls					,						__UpperCamelCase					,						__UpperCamelCase					,						__UpperCamelCase					,						**__UpperCamelCase					,						)->	int:
           return cls(
               vision_config=vision_config.to_dict()					,						qformer_config=qformer_config.to_dict()					,						text_config=text_config.to_dict()					,						**__UpperCamelCase					,						)


      def  __UpperCAmelCase		(	self	)->	Dict:
           __lowerCAmelCase								=					copy.deepcopy(self.__dict__	)
           __lowerCAmelCase								=					self.vision_config.to_dict()
           __lowerCAmelCase								=					self.qformer_config.to_dict()
           __lowerCAmelCase								=					self.text_config.to_dict()
           __lowerCAmelCase								=					self.__class__.model_type
           return output






