




import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def      _SCREAMING_SNAKE_CASE     (				lowercase      :			Tuple      ,				lowercase      :			Dict	):


      '''simple docstring'''



      assert isinstance(_lowerCamelCase      ,				_lowerCamelCase	)
      assert dataset.num_rows == 4
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_1", "col_2", "col_3"]
      for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('keep_in_memory'      ,				[False, True]	)
def      _SCREAMING_SNAKE_CASE     (				lowercase      :			Any      ,				lowercase      :			List[Any]      ,				lowercase      :			str	):


      '''simple docstring'''



      lowerCamelCase_							=       tmp_path / """cache"""
      lowerCamelCase_							=       {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
      with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
            lowerCamelCase_							=       JsonDatasetReader(_lowerCamelCase      ,				cache_dir=_lowerCamelCase      ,				keep_in_memory=_lowerCamelCase	).read()
      _check_json_dataset(_lowerCamelCase      ,				_lowerCamelCase	)

@pytest.mark.parametrize(
    'features'      ,				[
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]      ,				)
def      _SCREAMING_SNAKE_CASE     (				lowercase      :			Dict      ,				lowercase      :			Any      ,				lowercase      :			int	):


      '''simple docstring'''



      lowerCamelCase_							=       tmp_path / """cache"""
      lowerCamelCase_							=       {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
      lowerCamelCase_							=       features.copy() if features else default_expected_features
      lowerCamelCase_							=       (
          Features({feature: Value(_lowerCamelCase	) for feature, dtype in features.items()}	) if features is not None else None
      )
      lowerCamelCase_							=       JsonDatasetReader(_lowerCamelCase      ,				features=_lowerCamelCase      ,				cache_dir=_lowerCamelCase	).read()
      _check_json_dataset(_lowerCamelCase      ,				_lowerCamelCase	)

@pytest.mark.parametrize(
    'features'      ,				[
        None,
        {'col_3': 'float64', 'col_1': 'string', 'col_2': 'int64'},
    ]      ,				)
def      _SCREAMING_SNAKE_CASE     (				lowercase      :			Any      ,				lowercase      :			Optional[int]      ,				lowercase      :			Optional[int]	):


      '''simple docstring'''



      lowerCamelCase_							=       tmp_path / """cache"""
      lowerCamelCase_							=       {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""}
      lowerCamelCase_							=       features.copy() if features else default_expected_features
      lowerCamelCase_							=       (
          Features({feature: Value(_lowerCamelCase	) for feature, dtype in features.items()}	) if features is not None else None
      )
      lowerCamelCase_							=       JsonDatasetReader(_lowerCamelCase      ,				features=_lowerCamelCase      ,				cache_dir=_lowerCamelCase	).read()
      assert isinstance(_lowerCamelCase      ,				_lowerCamelCase	)
      assert dataset.num_rows == 2
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_3", "col_1", "col_2"]
      for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype

def      _SCREAMING_SNAKE_CASE     (				lowercase      :			Dict      ,				lowercase      :			List[str]	):


      '''simple docstring'''



      lowerCamelCase_							=       {"""col_2""": """int64""", """col_3""": """float64""", """col_1""": """string"""}
      lowerCamelCase_							=       features.copy()
      lowerCamelCase_							=       (
          Features({feature: Value(_lowerCamelCase	) for feature, dtype in features.items()}	) if features is not None else None
      )
      lowerCamelCase_							=       tmp_path / """cache"""
      lowerCamelCase_							=       JsonDatasetReader(_lowerCamelCase      ,				features=_lowerCamelCase      ,				cache_dir=_lowerCamelCase	).read()
      assert isinstance(_lowerCamelCase      ,				_lowerCamelCase	)
      assert dataset.num_rows == 2
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_2", "col_3", "col_1"]
      for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('split'      ,				[None, NamedSplit('train'	), 'train', 'test']	)
def      _SCREAMING_SNAKE_CASE     (				lowercase      :			List[Any]      ,				lowercase      :			Dict      ,				lowercase      :			List[Any]	):


      '''simple docstring'''



      lowerCamelCase_							=       tmp_path / """cache"""
      lowerCamelCase_							=       {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
      lowerCamelCase_							=       JsonDatasetReader(_lowerCamelCase      ,				cache_dir=_lowerCamelCase      ,				split=_lowerCamelCase	).read()
      _check_json_dataset(_lowerCamelCase      ,				_lowerCamelCase	)
      assert dataset.split == split if split else "train"

@pytest.mark.parametrize('path_type'      ,				[str, list]	)
def      _SCREAMING_SNAKE_CASE     (				lowercase      :			List[Any]      ,				lowercase      :			List[str]      ,				lowercase      :			List[str]	):


      '''simple docstring'''



      if issubclass(_lowerCamelCase      ,				_lowerCamelCase	):
            lowerCamelCase_							=       jsonl_path
      elif issubclass(_lowerCamelCase      ,				_lowerCamelCase	):
            lowerCamelCase_							=       [jsonl_path]
      lowerCamelCase_							=       tmp_path / """cache"""
      lowerCamelCase_							=       {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
      lowerCamelCase_							=       JsonDatasetReader(_lowerCamelCase      ,				cache_dir=_lowerCamelCase	).read()
      _check_json_dataset(_lowerCamelCase      ,				_lowerCamelCase	)

def      _SCREAMING_SNAKE_CASE     (				lowercase      :			Optional[Any]      ,				lowercase      :			List[str]      ,				lowercase      :			Any=("train",)	):


      '''simple docstring'''



      assert isinstance(_lowerCamelCase      ,				_lowerCamelCase	)
      for split in splits:
            lowerCamelCase_							=       dataset_dict[split]
            assert dataset.num_rows == 4
            assert dataset.num_columns == 3
            assert dataset.column_names == ["col_1", "col_2", "col_3"]
            for feature, expected_dtype in expected_features.items():
                  assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('keep_in_memory'      ,				[False, True]	)
def      _SCREAMING_SNAKE_CASE     (				lowercase      :			Optional[Any]      ,				lowercase      :			int      ,				lowercase      :			Tuple	):


      '''simple docstring'''



      lowerCamelCase_							=       tmp_path / """cache"""
      lowerCamelCase_							=       {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
      with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
            lowerCamelCase_							=       JsonDatasetReader({'train': jsonl_path}      ,				cache_dir=_lowerCamelCase      ,				keep_in_memory=_lowerCamelCase	).read()
      _check_json_datasetdict(_lowerCamelCase      ,				_lowerCamelCase	)

@pytest.mark.parametrize(
    'features'      ,				[
        None,
        {'col_1': 'string', 'col_2': 'int64', 'col_3': 'float64'},
        {'col_1': 'string', 'col_2': 'string', 'col_3': 'string'},
        {'col_1': 'int32', 'col_2': 'int32', 'col_3': 'int32'},
        {'col_1': 'float32', 'col_2': 'float32', 'col_3': 'float32'},
    ]      ,				)
def      _SCREAMING_SNAKE_CASE     (				lowercase      :			int      ,				lowercase      :			List[str]      ,				lowercase      :			List[str]	):


      '''simple docstring'''



      lowerCamelCase_							=       tmp_path / """cache"""
      lowerCamelCase_							=       {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
      lowerCamelCase_							=       features.copy() if features else default_expected_features
      lowerCamelCase_							=       (
          Features({feature: Value(_lowerCamelCase	) for feature, dtype in features.items()}	) if features is not None else None
      )
      lowerCamelCase_							=       JsonDatasetReader({'train': jsonl_path}      ,				features=_lowerCamelCase      ,				cache_dir=_lowerCamelCase	).read()
      _check_json_datasetdict(_lowerCamelCase      ,				_lowerCamelCase	)

@pytest.mark.parametrize('split'      ,				[None, NamedSplit('train'	), 'train', 'test']	)
def      _SCREAMING_SNAKE_CASE     (				lowercase      :			int      ,				lowercase      :			Union[str, Any]      ,				lowercase      :			Optional[Any]	):


      '''simple docstring'''



      if split:
            lowerCamelCase_							=       {split: jsonl_path}
      else:
            lowerCamelCase_							=       """train"""
            lowerCamelCase_							=       {"""train""": jsonl_path, """test""": jsonl_path}
      lowerCamelCase_							=       tmp_path / """cache"""
      lowerCamelCase_							=       {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
      lowerCamelCase_							=       JsonDatasetReader(_lowerCamelCase      ,				cache_dir=_lowerCamelCase	).read()
      _check_json_datasetdict(_lowerCamelCase      ,				_lowerCamelCase      ,				splits=list(path.keys()	)	)
      assert all(dataset[split].split == split for split in path.keys()	)

def      _SCREAMING_SNAKE_CASE     (				lowercase      :			Union[str, Any]	):


      '''simple docstring'''



      return json.load(_lowerCamelCase	)

def      _SCREAMING_SNAKE_CASE     (				lowercase      :			Optional[int]	):


      '''simple docstring'''



      return [json.loads(_lowerCamelCase	) for line in buffer]







class     A:



 '''simple docstring'''


 @pytest.mark.parametrize('lines, load_json_function'		,				[(True, load_json_lines), (False, load_json)]    )
 def 							a__					(							self   :    Optional[Any]		,				A_   :    List[Any]		,				A_   :    List[str]		,				A_   :    Optional[int]    )  ->      Tuple:

       """simple docstring"""






       with io.BytesIO() as buffer:
             JsonDatasetWriter(A_		,				A_		,				lines=A_    ).write()
             buffer.seek(0    )
             lowerCamelCase_							=       load_json_function(A_    )
       assert isinstance(A_		,				A_    )
       assert isinstance(exported_content[0]		,				A_    )
       assert len(A_    ) == 10

 @pytest.mark.parametrize(
     'orient, container, keys, len_at'		,				[
         ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
         ('split', dict, {'columns', 'data'}, 'data'),
         ('index', dict, set('0123456789'    ), None),
         ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
         ('values', list, None, None),
         ('table', dict, {'schema', 'data'}, 'data'),
     ]		,				)
 def 							a__					(							self   :    Optional[Any]		,				A_   :    List[Any]		,				A_   :    Dict		,				A_   :    Union[str, Any]		,				A_   :    Any		,				A_   :    Tuple    )  ->      Tuple:

       """simple docstring"""






       with io.BytesIO() as buffer:
             JsonDatasetWriter(A_		,				A_		,				lines=A_		,				orient=A_    ).write()
             buffer.seek(0    )
             lowerCamelCase_							=       load_json(A_    )
       assert isinstance(A_		,				A_    )
       if keys:
             if container is dict:
                   assert exported_content.keys() == keys
             else:
                   assert exported_content[0].keys() == keys
       else:
             assert not hasattr(A_		,				'keys'    ) and not hasattr(exported_content[0]		,				'keys'    )
       if len_at:
             assert len(exported_content[len_at]    ) == 10
       else:
             assert len(A_    ) == 10

 @pytest.mark.parametrize('lines, load_json_function'		,				[(True, load_json_lines), (False, load_json)]    )
 def 							a__					(							self   :    Union[str, Any]		,				A_   :    List[Any]		,				A_   :    Tuple		,				A_   :    Dict    )  ->      Tuple:

       """simple docstring"""






       with io.BytesIO() as buffer:
             JsonDatasetWriter(A_		,				A_		,				lines=A_		,				num_proc=2    ).write()
             buffer.seek(0    )
             lowerCamelCase_							=       load_json_function(A_    )
       assert isinstance(A_		,				A_    )
       assert isinstance(exported_content[0]		,				A_    )
       assert len(A_    ) == 10

 @pytest.mark.parametrize(
     'orient, container, keys, len_at'		,				[
         ('records', list, {'tokens', 'labels', 'answers', 'id'}, None),
         ('split', dict, {'columns', 'data'}, 'data'),
         ('index', dict, set('0123456789'    ), None),
         ('columns', dict, {'tokens', 'labels', 'answers', 'id'}, 'tokens'),
         ('values', list, None, None),
         ('table', dict, {'schema', 'data'}, 'data'),
     ]		,				)
 def 							a__					(							self   :    Optional[int]		,				A_   :    List[Any]		,				A_   :    List[Any]		,				A_   :    str		,				A_   :    Optional[int]		,				A_   :    Optional[int]    )  ->      Optional[int]:

       """simple docstring"""






       with io.BytesIO() as buffer:
             JsonDatasetWriter(A_		,				A_		,				lines=A_		,				orient=A_		,				num_proc=2    ).write()
             buffer.seek(0    )
             lowerCamelCase_							=       load_json(A_    )
       assert isinstance(A_		,				A_    )
       if keys:
             if container is dict:
                   assert exported_content.keys() == keys
             else:
                   assert exported_content[0].keys() == keys
       else:
             assert not hasattr(A_		,				'keys'    ) and not hasattr(exported_content[0]		,				'keys'    )
       if len_at:
             assert len(exported_content[len_at]    ) == 10
       else:
             assert len(A_    ) == 10

 def 							a__					(							self   :    List[Any]		,				A_   :    List[str]    )  ->      str:

       """simple docstring"""






       with pytest.raises(A_    ):
             with io.BytesIO() as buffer:
                   JsonDatasetWriter(A_		,				A_		,				num_proc=0    )





 @pytest.mark.parametrize('compression, extension'		,				[('gzip', 'gz'), ('bz2', 'bz2'), ('xz', 'xz')]    )
 def 							a__					(							self   :    Union[str, Any]		,				A_   :    Optional[Any]		,				A_   :    Optional[int]		,				A_   :    Optional[Any]		,				A_   :    Optional[Any]		,				A_   :    Union[str, Any]    )  ->      Tuple:

       """simple docstring"""






       lowerCamelCase_							=       tmp_path_factory.mktemp('data'    ) / f"""test.json.{extension}"""
       lowerCamelCase_							=       str(shared_datadir / f"""test_file.json.{extension}"""    )
       JsonDatasetWriter(A_		,				A_		,				compression=A_    ).write()

       with fsspec.open(A_		,				'rb'		,				compression='infer'    ) as f:
             lowerCamelCase_							=       f.read()
       with fsspec.open(A_		,				'rb'		,				compression='infer'    ) as f:
             lowerCamelCase_							=       f.read()
       assert exported_content == original_content










'''simple docstring'''






import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding




class  _A (					__lowercase		):
		lowercase__:   Any				     =   ['''image_processor''', '''tokenizer''']
		lowercase__:   Any				     =   '''CLIPImageProcessor'''
		lowercase__:   Optional[Any]				     =   ('''CLIPTokenizer''', '''CLIPTokenizerFast''')

		def __init__(   self					:   int  ,      __magic_name__					:   Dict=None  ,      __magic_name__					:   Dict=None  ,      **__magic_name__					:   Union[str, Any]			)					->    Any:


					"""simple docstring"""

					__snake_case					:      Optional[Any]							      =     None
					if "feature_extractor" in kwargs:
								warnings.warn(
								    """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
								    """ instead."""  ,      __magic_name__  ,      )
								__snake_case					:      List[Any]							      =     kwargs.pop("""feature_extractor"""			)

					__snake_case					:      List[str]							      =     image_processor if image_processor is not None else feature_extractor
					if image_processor is None:
								raise ValueError("""You need to specify an `image_processor`."""			)
					if tokenizer is None:
								raise ValueError("""You need to specify a `tokenizer`."""			)

					super().__init__(__magic_name__  ,      __magic_name__			)

		def __call__(   self					:   int  ,      __magic_name__					:   List[str]=None  ,      __magic_name__					:   Tuple=None  ,      __magic_name__					:   Any=None  ,      **__magic_name__					:   Union[str, Any]			)					->    Optional[int]:


					"""simple docstring"""

					if text is None and images is None:
								raise ValueError("""You have to specify either text or images. Both cannot be none."""			)

					if text is not None:
								__snake_case					:      int							      =     self.tokenizer(__magic_name__  ,      return_tensors=__magic_name__  ,      **__magic_name__			)

					if images is not None:
								__snake_case					:      str							      =     self.image_processor(__magic_name__  ,      return_tensors=__magic_name__  ,      **__magic_name__			)

					if text is not None and images is not None:
								__snake_case					:      Union[str, Any]							      =     image_features.pixel_values
								return encoding
					elif text is not None:
								return encoding
					else:
								return BatchEncoding(data=dict(**__magic_name__			)  ,      tensor_type=__magic_name__			)

		def 	lowercase__     (   self					:   Optional[int]  ,      *__magic_name__					:   List[Any]  ,      **__magic_name__					:   Any			)					->    Optional[Any]:


					"""simple docstring"""

					return self.tokenizer.batch_decode(*__magic_name__  ,      **__magic_name__			)

		def 	lowercase__     (   self					:   List[str]  ,      *__magic_name__					:   Tuple  ,      **__magic_name__					:   List[Any]			)					->    int:


					"""simple docstring"""

					return self.tokenizer.decode(*__magic_name__  ,      **__magic_name__			)

		@property
		def 	lowercase__     (   self					:   List[Any]			)					->    Optional[Any]:


					"""simple docstring"""

					__snake_case					:      Dict							      =     self.tokenizer.model_input_names
					__snake_case					:      str							      =     self.image_processor.model_input_names
					return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names			)			)

		@property
		def 	lowercase__     (   self					:   int			)					->    List[str]:


					"""simple docstring"""

					warnings.warn(
					    """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""  ,      __magic_name__  ,      )
					return self.image_processor_class

		@property
		def 	lowercase__     (   self					:   Union[str, Any]			)					->    Dict:


					"""simple docstring"""

					warnings.warn(
					    """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""  ,      __magic_name__  ,      )
					return self.image_processor


