






import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases




def 				A		(					lowercase__  :  Optional[int] ,		lowercase__  :  Any  )		-> Any:
	assert isinstance(lowercase__ ,		lowercase__  )
	assert dataset.num_rows == 4
	assert dataset.num_columns == 3
	assert dataset.column_names == ["col_1", "col_2", "col_3"]
	for feature, expected_dtype in expected_features.items():
		assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("""keep_in_memory""" ,		[False, True]  )
def 				A		(					lowercase__  :  Any ,		lowercase__  :  Union[str, Any] ,		lowercase__  :  List[str]  )		-> Tuple:
	UpperCamelCase__ :List[str]						=      tmp_path / """cache"""
	UpperCamelCase__ :Union[str, Any]						=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
		UpperCamelCase__ :int						=      JsonDatasetReader(lowercase__ ,		cache_dir=lowercase__ ,		keep_in_memory=lowercase__  ).read()
	_check_json_dataset(lowercase__ ,		lowercase__  )




@pytest.mark.parametrize(
    """features""" ,		[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ] ,		)
def 				A		(					lowercase__  :  str ,		lowercase__  :  Union[str, Any] ,		lowercase__  :  Union[str, Any]  )		-> Tuple:
	UpperCamelCase__ :Tuple						=      tmp_path / """cache"""
	UpperCamelCase__ :Any						=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	UpperCamelCase__ :Union[str, Any]						=      features.copy() if features else default_expected_features
	UpperCamelCase__ :Union[str, Any]						=      (
	    Features({feature: Value(lowercase__  ) for feature, dtype in features.items()}  ) if features is not None else None
	)
	UpperCamelCase__ :Any						=      JsonDatasetReader(lowercase__ ,		features=lowercase__ ,		cache_dir=lowercase__  ).read()
	_check_json_dataset(lowercase__ ,		lowercase__  )




@pytest.mark.parametrize(
    """features""" ,		[
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ] ,		)
def 				A		(					lowercase__  :  Tuple ,		lowercase__  :  Union[str, Any] ,		lowercase__  :  Optional[Any]  )		-> List[str]:
	UpperCamelCase__ :Union[str, Any]						=      tmp_path / """cache"""
	UpperCamelCase__ :List[str]						=      {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""}
	UpperCamelCase__ :Optional[int]						=      features.copy() if features else default_expected_features
	UpperCamelCase__ :Dict						=      (
	    Features({feature: Value(lowercase__  ) for feature, dtype in features.items()}  ) if features is not None else None
	)
	UpperCamelCase__ :str						=      JsonDatasetReader(lowercase__ ,		features=lowercase__ ,		cache_dir=lowercase__  ).read()
	assert isinstance(lowercase__ ,		lowercase__  )
	assert dataset.num_rows == 2
	assert dataset.num_columns == 3
	assert dataset.column_names == ["col_3", "col_1", "col_2"]
	for feature, expected_dtype in expected_features.items():
		assert dataset.features[feature].dtype == expected_dtype




def 				A		(					lowercase__  :  Optional[Any] ,		lowercase__  :  int  )		-> Dict:
	# jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
	UpperCamelCase__ :int						=      {"""col_2""": """int64""", """col_3""": """float64""", """col_1""": """string"""}
	UpperCamelCase__ :Tuple						=      features.copy()
	UpperCamelCase__ :Union[str, Any]						=      (
	    Features({feature: Value(lowercase__  ) for feature, dtype in features.items()}  ) if features is not None else None
	)
	UpperCamelCase__ :Any						=      tmp_path / """cache"""
	UpperCamelCase__ :Any						=      JsonDatasetReader(lowercase__ ,		features=lowercase__ ,		cache_dir=lowercase__  ).read()
	assert isinstance(lowercase__ ,		lowercase__  )
	assert dataset.num_rows == 2
	assert dataset.num_columns == 3
	assert dataset.column_names == ["col_2", "col_3", "col_1"]
	for feature, expected_dtype in expected_features.items():
		assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("""split""" ,		[None, NamedSplit("""train"""  ), """train""", """test"""]  )
def 				A		(					lowercase__  :  str ,		lowercase__  :  Any ,		lowercase__  :  Optional[Any]  )		-> Any:
	UpperCamelCase__ :Tuple						=      tmp_path / """cache"""
	UpperCamelCase__ :Optional[Any]						=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	UpperCamelCase__ :Any						=      JsonDatasetReader(lowercase__ ,		cache_dir=lowercase__ ,		split=lowercase__  ).read()
	_check_json_dataset(lowercase__ ,		lowercase__  )
	assert dataset.split == split if split else "train"




@pytest.mark.parametrize("""path_type""" ,		[str, list]  )
def 				A		(					lowercase__  :  List[str] ,		lowercase__  :  List[Any] ,		lowercase__  :  Union[str, Any]  )		-> Tuple:
	if issubclass(lowercase__ ,		lowercase__  ):
		UpperCamelCase__ :Any						=      jsonl_path
	elif issubclass(lowercase__ ,		lowercase__  ):
		UpperCamelCase__ :List[str]						=      [jsonl_path]
	UpperCamelCase__ :Optional[int]						=      tmp_path / """cache"""
	UpperCamelCase__ :str						=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	UpperCamelCase__ :List[Any]						=      JsonDatasetReader(lowercase__ ,		cache_dir=lowercase__  ).read()
	_check_json_dataset(lowercase__ ,		lowercase__  )




def 				A		(					lowercase__  :  Optional[int] ,		lowercase__  :  List[Any] ,		lowercase__  :  Tuple=("train",)  )		-> Optional[Any]:
	assert isinstance(lowercase__ ,		lowercase__  )
	for split in splits:
		UpperCamelCase__ :Union[str, Any]						=      dataset_dict[split]
		assert dataset.num_rows == 4
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_1", "col_2", "col_3"]
		for feature, expected_dtype in expected_features.items():
			assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("""keep_in_memory""" ,		[False, True]  )
def 				A		(					lowercase__  :  Optional[int] ,		lowercase__  :  Optional[Any] ,		lowercase__  :  Tuple  )		-> Optional[Any]:
	UpperCamelCase__ :int						=      tmp_path / """cache"""
	UpperCamelCase__ :List[str]						=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
		UpperCamelCase__ :List[Any]						=      JsonDatasetReader({"""train""": jsonl_path} ,		cache_dir=lowercase__ ,		keep_in_memory=lowercase__  ).read()
	_check_json_datasetdict(lowercase__ ,		lowercase__  )




@pytest.mark.parametrize(
    """features""" ,		[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ] ,		)
def 				A		(					lowercase__  :  Optional[Any] ,		lowercase__  :  Any ,		lowercase__  :  List[Any]  )		-> List[Any]:
	UpperCamelCase__ :int						=      tmp_path / """cache"""
	UpperCamelCase__ :Tuple						=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	UpperCamelCase__ :str						=      features.copy() if features else default_expected_features
	UpperCamelCase__ :Optional[int]						=      (
	    Features({feature: Value(lowercase__  ) for feature, dtype in features.items()}  ) if features is not None else None
	)
	UpperCamelCase__ :List[Any]						=      JsonDatasetReader({"""train""": jsonl_path} ,		features=lowercase__ ,		cache_dir=lowercase__  ).read()
	_check_json_datasetdict(lowercase__ ,		lowercase__  )




@pytest.mark.parametrize("""split""" ,		[None, NamedSplit("""train"""  ), """train""", """test"""]  )
def 				A		(					lowercase__  :  Tuple ,		lowercase__  :  str ,		lowercase__  :  int  )		-> List[Any]:
	if split:
		UpperCamelCase__ :Dict						=      {split: jsonl_path}
	else:
		UpperCamelCase__ :Tuple						=      """train"""
		UpperCamelCase__ :Any						=      {"""train""": jsonl_path, """test""": jsonl_path}
	UpperCamelCase__ :Union[str, Any]						=      tmp_path / """cache"""
	UpperCamelCase__ :Tuple						=      {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
	UpperCamelCase__ :int						=      JsonDatasetReader(lowercase__ ,		cache_dir=lowercase__  ).read()
	_check_json_datasetdict(lowercase__ ,		lowercase__ ,		splits=list(path.keys()  )  )
	assert all(dataset[split].split == split for split in path.keys()  )




def 				A		(					lowercase__  :  List[str]  )		-> List[str]:
	return json.load(lowercase__  )




def 				A		(					lowercase__  :  Optional[Any]  )		-> Any:
	return [json.loads(lowercase__  ) for line in buffer]




class        lowerCAmelCase_				:







				"""simple docstring"""




				@pytest.mark.parametrize("""lines, load_json_function"""  ,     [(True, load_json_lines), (False, load_json)]     )
				def 				__a   ( self     :str  ,     lowerCamelCase__     :int  ,     lowerCamelCase__     :int  ,     lowerCamelCase__     :Optional[Any]     ):
					with io.BytesIO() as buffer:
						JsonDatasetWriter(lowerCamelCase__  ,     lowerCamelCase__  ,     lines=lowerCamelCase__     ).write()
						buffer.seek(0     )
						UpperCamelCase__ :str						=      load_json_function(lowerCamelCase__     )
					assert isinstance(lowerCamelCase__  ,     lowerCamelCase__     )
					assert isinstance(exported_content[0]  ,     lowerCamelCase__     )
					assert len(lowerCamelCase__     ) == 10







				@pytest.mark.parametrize(
				    """orient, container, keys, len_at"""  ,     [
				        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
				        ("""split""", dict, {"""columns""", """data"""}, """data"""),
				        ("""index""", dict, set("""0123456789"""     ), None),
				        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
				        ("""values""", list, None, None),
				        ("""table""", dict, {"""schema""", """data"""}, """data"""),
				    ]  ,     )
				def 				__a   ( self     :Any  ,     lowerCamelCase__     :Any  ,     lowerCamelCase__     :Any  ,     lowerCamelCase__     :Tuple  ,     lowerCamelCase__     :List[str]  ,     lowerCamelCase__     :Union[str, Any]     ):
					with io.BytesIO() as buffer:
						JsonDatasetWriter(lowerCamelCase__  ,     lowerCamelCase__  ,     lines=lowerCamelCase__  ,     orient=lowerCamelCase__     ).write()
						buffer.seek(0     )
						UpperCamelCase__ :Optional[int]						=      load_json(lowerCamelCase__     )
					assert isinstance(lowerCamelCase__  ,     lowerCamelCase__     )
					if keys:
						if container is dict:
							assert exported_content.keys() == keys
						else:
							assert exported_content[0].keys() == keys
					else:
						assert not hasattr(lowerCamelCase__  ,     """keys"""     ) and not hasattr(exported_content[0]  ,     """keys"""     )
					if len_at:
						assert len(exported_content[len_at]     ) == 10
					else:
						assert len(lowerCamelCase__     ) == 10







				@pytest.mark.parametrize("""lines, load_json_function"""  ,     [(True, load_json_lines), (False, load_json)]     )
				def 				__a   ( self     :Any  ,     lowerCamelCase__     :Union[str, Any]  ,     lowerCamelCase__     :List[str]  ,     lowerCamelCase__     :Any     ):
					with io.BytesIO() as buffer:
						JsonDatasetWriter(lowerCamelCase__  ,     lowerCamelCase__  ,     lines=lowerCamelCase__  ,     num_proc=2     ).write()
						buffer.seek(0     )
						UpperCamelCase__ :Union[str, Any]						=      load_json_function(lowerCamelCase__     )
					assert isinstance(lowerCamelCase__  ,     lowerCamelCase__     )
					assert isinstance(exported_content[0]  ,     lowerCamelCase__     )
					assert len(lowerCamelCase__     ) == 10







				@pytest.mark.parametrize(
				    """orient, container, keys, len_at"""  ,     [
				        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
				        ("""split""", dict, {"""columns""", """data"""}, """data"""),
				        ("""index""", dict, set("""0123456789"""     ), None),
				        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
				        ("""values""", list, None, None),
				        ("""table""", dict, {"""schema""", """data"""}, """data"""),
				    ]  ,     )
				def 				__a   ( self     :Any  ,     lowerCamelCase__     :Dict  ,     lowerCamelCase__     :str  ,     lowerCamelCase__     :str  ,     lowerCamelCase__     :Dict  ,     lowerCamelCase__     :Dict     ):
					with io.BytesIO() as buffer:
						JsonDatasetWriter(lowerCamelCase__  ,     lowerCamelCase__  ,     lines=lowerCamelCase__  ,     orient=lowerCamelCase__  ,     num_proc=2     ).write()
						buffer.seek(0     )
						UpperCamelCase__ :Optional[Any]						=      load_json(lowerCamelCase__     )
					assert isinstance(lowerCamelCase__  ,     lowerCamelCase__     )
					if keys:
						if container is dict:
							assert exported_content.keys() == keys
						else:
							assert exported_content[0].keys() == keys
					else:
						assert not hasattr(lowerCamelCase__  ,     """keys"""     ) and not hasattr(exported_content[0]  ,     """keys"""     )
					if len_at:
						assert len(exported_content[len_at]     ) == 10
					else:
						assert len(lowerCamelCase__     ) == 10







				def 				__a   ( self     :Tuple  ,     lowerCamelCase__     :List[Any]     ):
					with pytest.raises(lowerCamelCase__     ):
						with io.BytesIO() as buffer:
							JsonDatasetWriter(lowerCamelCase__  ,     lowerCamelCase__  ,     num_proc=0     )







				@pytest.mark.parametrize("""compression, extension"""  ,     [("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")]     )
				def 				__a   ( self     :Union[str, Any]  ,     lowerCamelCase__     :Optional[Any]  ,     lowerCamelCase__     :List[Any]  ,     lowerCamelCase__     :List[str]  ,     lowerCamelCase__     :Optional[int]  ,     lowerCamelCase__     :Tuple     ):
					UpperCamelCase__ :Optional[Any]						=      tmp_path_factory.mktemp("""data"""     ) / f"""test.json.{extension}"""
					UpperCamelCase__ :Optional[int]						=      str(shared_datadir / f"""test_file.json.{extension}"""     )
					JsonDatasetWriter(lowerCamelCase__  ,     lowerCamelCase__  ,     compression=lowerCamelCase__     ).write()

					with fsspec.open(lowerCamelCase__  ,     """rb"""  ,     compression="""infer"""     ) as f:
						UpperCamelCase__ :Union[str, Any]						=      f.read()
					with fsspec.open(lowerCamelCase__  ,     """rb"""  ,     compression="""infer"""     ) as f:
						UpperCamelCase__ :int						=      f.read()
					assert exported_content == original_content


'''simple docstring'''







from __future__ import annotations

import unittest

import numpy as np

from transformers import BlipTextConfig
from transformers.testing_utils import require_tf, slow
from transformers.utils import is_tf_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask


if is_tf_available():
      import tensorflow as tf

      from transformers import TFBlipTextModel
      from transformers.models.blip.modeling_tf_blip import TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST


class        A__   :



    def __init__(       self		,					UpperCamelCase__		,					UpperCamelCase__=12		,					UpperCamelCase__=7		,					UpperCamelCase__=True		,					UpperCamelCase__=True		,					UpperCamelCase__=True		,					UpperCamelCase__=99		,					UpperCamelCase__=32		,					UpperCamelCase__=32		,					UpperCamelCase__=2		,					UpperCamelCase__=4		,					UpperCamelCase__=37		,					UpperCamelCase__=0.1		,					UpperCamelCase__=0.1		,					UpperCamelCase__=512		,					UpperCamelCase__=0.02		,					UpperCamelCase__=0		,					UpperCamelCase__=None		,					)    ->   Union[str, Any]:





           '''simple docstring'''
           A_						       =   parent
           A_						       =   batch_size
           A_						       =   seq_length
           A_						       =   is_training
           A_						       =   use_input_mask
           A_						       =   use_labels
           A_						       =   vocab_size
           A_						       =   hidden_size
           A_						       =   projection_dim
           A_						       =   num_hidden_layers
           A_						       =   num_attention_heads
           A_						       =   intermediate_size
           A_						       =   dropout
           A_						       =   attention_dropout
           A_						       =   max_position_embeddings
           A_						       =   initializer_range
           A_						       =   scope
           A_						       =   bos_token_id



    def        snake_case_      (       self							)    ->   Dict:





           '''simple docstring'''
           A_						       =   ids_tensor([self.batch_size, self.seq_length]		,					self.vocab_size							)

           A_						       =   None
           if self.use_input_mask:
                  A_						       =   random_attention_mask([self.batch_size, self.seq_length]							)

           if input_mask is not None:
                  A_						       =   input_mask.numpy()
                  A_       ,      A_						       =   input_mask.shape
                  A_						       =   np.random.randint(1		,					seq_length - 1		,					size=(batch_size,)							)
                  for batch_idx, start_index in enumerate(UpperCamelCase__							):
                         A_						       =   1
                         A_						       =   0

           A_						       =   self.get_config()

           return config, input_ids, tf.convert_to_tensor(UpperCamelCase__							)



    def        snake_case_      (       self							)    ->   Optional[int]:





           '''simple docstring'''
           return BlipTextConfig(
               vocab_size=self.vocab_size		,					hidden_size=self.hidden_size		,					projection_dim=self.projection_dim		,					num_hidden_layers=self.num_hidden_layers		,					num_attention_heads=self.num_attention_heads		,					intermediate_size=self.intermediate_size		,					dropout=self.dropout		,					attention_dropout=self.attention_dropout		,					max_position_embeddings=self.max_position_embeddings		,					initializer_range=self.initializer_range		,					bos_token_id=self.bos_token_id		,					)



    def        snake_case_      (       self		,					UpperCamelCase__		,					UpperCamelCase__		,					UpperCamelCase__							)    ->   int:





           '''simple docstring'''
           A_						       =   TFBlipTextModel(config=UpperCamelCase__							)
           A_						       =   model(UpperCamelCase__		,					attention_mask=UpperCamelCase__		,					training=UpperCamelCase__							)
           A_						       =   model(UpperCamelCase__		,					training=UpperCamelCase__							)
           self.parent.assertEqual(result.last_hidden_state.shape		,					(self.batch_size, self.seq_length, self.hidden_size)							)
           self.parent.assertEqual(result.pooler_output.shape		,					(self.batch_size, self.hidden_size)							)







    def        snake_case_      (       self							)    ->   List[Any]:





           '''simple docstring'''
           A_						       =   self.prepare_config_and_inputs()
           A_       ,      A_       ,      A_						       =   config_and_inputs
           A_						       =   {"""input_ids""": input_ids, """attention_mask""": input_mask}
           return config, inputs_dict


@require_tf
class        A__   (   _snake_case    ,    unittest.TestCase      ):
    lowercase						       =		(TFBlipTextModel,) if is_tf_available() else ()
    lowercase						       =		False
    lowercase						       =		False
    lowercase						       =		False



    def        snake_case_      (       self							)    ->   Optional[int]:





           '''simple docstring'''
           A_						       =   BlipTextModelTester(self							)
           A_						       =   ConfigTester(self		,					config_class=UpperCamelCase__		,					hidden_size=37							)



    def        snake_case_      (       self							)    ->   Tuple:





           '''simple docstring'''
           self.config_tester.run_common_tests()



    def        snake_case_      (       self							)    ->   str:





           '''simple docstring'''
           A_						       =   self.model_tester.prepare_config_and_inputs()
           self.model_tester.create_and_check_model(*UpperCamelCase__							)



    def        snake_case_      (       self							)    ->   List[Any]:





           '''simple docstring'''
           pass



    def        snake_case_      (       self							)    ->   Union[str, Any]:





           '''simple docstring'''
           pass



    @unittest.skip(reason="""Blip does not use inputs_embeds"""							)
    def        snake_case_      (       self							)    ->   Union[str, Any]:





           '''simple docstring'''
           pass



    @unittest.skip(reason="""BlipTextModel has no base class and is not available in MODEL_MAPPING"""							)
    def        snake_case_      (       self							)    ->   List[Any]:





           '''simple docstring'''
           pass



    @unittest.skip(reason="""BlipTextModel has no base class and is not available in MODEL_MAPPING"""							)
    def        snake_case_      (       self							)    ->   Any:





           '''simple docstring'''
           pass



    @slow
    def        snake_case_      (       self							)    ->   List[Any]:





           '''simple docstring'''
           for model_name in TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  A_						       =   TFBlipTextModel.from_pretrained(UpperCamelCase__							)
                  self.assertIsNotNone(UpperCamelCase__							)







    def        snake_case_      (       self		,					UpperCamelCase__=True							)    ->   str:





           '''simple docstring'''
           super().test_pt_tf_model_equivalence(allow_missing_keys=UpperCamelCase__							)
