
import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases




def 							a		(				snake_case__:   Tuple		,  snake_case__:   Tuple   ):


				'''simple docstring'''
				assert isinstance(snake_case__		,  snake_case__   )
				assert dataset.num_rows == 4
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_1", "col_2", "col_3"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize('''keep_in_memory'''		,  [False, True]   )
def 							a		(				snake_case__:   Optional[int]		,  snake_case__:   Any		,  snake_case__:   Any   ):


				'''simple docstring'''
				lowercase_	    =   tmp_path / '''cache'''
				lowercase_	    =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								lowercase_	    =   JsonDatasetReader(snake_case__		,  cache_dir=snake_case__		,  keep_in_memory=snake_case__   ).read()
				_check_json_dataset(snake_case__		,  snake_case__   )




@pytest.mark.parametrize(
    '''features'''		,  [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]		,  )
def 							a		(				snake_case__:   Optional[Any]		,  snake_case__:   List[str]		,  snake_case__:   Any   ):


				'''simple docstring'''
				lowercase_	    =   tmp_path / '''cache'''
				lowercase_	    =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				lowercase_	    =   features.copy() if features else default_expected_features
				lowercase_	    =   (
				    Features({feature: Value(snake_case__   ) for feature, dtype in features.items()}   ) if features is not None else None
				)
				lowercase_	    =   JsonDatasetReader(snake_case__		,  features=snake_case__		,  cache_dir=snake_case__   ).read()
				_check_json_dataset(snake_case__		,  snake_case__   )




@pytest.mark.parametrize(
    '''features'''		,  [
        None,
        {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''},
    ]		,  )
def 							a		(				snake_case__:   Any		,  snake_case__:   List[str]		,  snake_case__:   Any   ):


				'''simple docstring'''
				lowercase_	    =   tmp_path / '''cache'''
				lowercase_	    =   {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''}
				lowercase_	    =   features.copy() if features else default_expected_features
				lowercase_	    =   (
				    Features({feature: Value(snake_case__   ) for feature, dtype in features.items()}   ) if features is not None else None
				)
				lowercase_	    =   JsonDatasetReader(snake_case__		,  features=snake_case__		,  cache_dir=snake_case__   ).read()
				assert isinstance(snake_case__		,  snake_case__   )
				assert dataset.num_rows == 2
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_3", "col_1", "col_2"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype




def 							a		(				snake_case__:   Optional[int]		,  snake_case__:   List[Any]   ):


				'''simple docstring'''
				# jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
				lowercase_	    =   {'''col_2''': '''int64''', '''col_3''': '''float64''', '''col_1''': '''string'''}
				lowercase_	    =   features.copy()
				lowercase_	    =   (
				    Features({feature: Value(snake_case__   ) for feature, dtype in features.items()}   ) if features is not None else None
				)
				lowercase_	    =   tmp_path / '''cache'''
				lowercase_	    =   JsonDatasetReader(snake_case__		,  features=snake_case__		,  cache_dir=snake_case__   ).read()
				assert isinstance(snake_case__		,  snake_case__   )
				assert dataset.num_rows == 2
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_2", "col_3", "col_1"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize('''split'''		,  [None, NamedSplit('''train'''   ), '''train''', '''test''']   )
def 							a		(				snake_case__:   Tuple		,  snake_case__:   str		,  snake_case__:   Tuple   ):


				'''simple docstring'''
				lowercase_	    =   tmp_path / '''cache'''
				lowercase_	    =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				lowercase_	    =   JsonDatasetReader(snake_case__		,  cache_dir=snake_case__		,  split=snake_case__   ).read()
				_check_json_dataset(snake_case__		,  snake_case__   )
				assert dataset.split == split if split else "train"




@pytest.mark.parametrize('''path_type'''		,  [str, list]   )
def 							a		(				snake_case__:   Dict		,  snake_case__:   str		,  snake_case__:   Optional[Any]   ):


				'''simple docstring'''
				if issubclass(snake_case__		,  snake_case__   ):
								lowercase_	    =   jsonl_path
				elif issubclass(snake_case__		,  snake_case__   ):
								lowercase_	    =   [jsonl_path]
				lowercase_	    =   tmp_path / '''cache'''
				lowercase_	    =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				lowercase_	    =   JsonDatasetReader(snake_case__		,  cache_dir=snake_case__   ).read()
				_check_json_dataset(snake_case__		,  snake_case__   )




def 							a		(				snake_case__:   Optional[Any]		,  snake_case__:   Any		,  snake_case__:   Tuple=("train",)   ):


				'''simple docstring'''
				assert isinstance(snake_case__		,  snake_case__   )
				for split in splits:
								lowercase_	    =   dataset_dict[split]
								assert dataset.num_rows == 4
								assert dataset.num_columns == 3
								assert dataset.column_names == ["col_1", "col_2", "col_3"]
								for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize('''keep_in_memory'''		,  [False, True]   )
def 							a		(				snake_case__:   Optional[int]		,  snake_case__:   int		,  snake_case__:   Tuple   ):


				'''simple docstring'''
				lowercase_	    =   tmp_path / '''cache'''
				lowercase_	    =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								lowercase_	    =   JsonDatasetReader({'''train''': jsonl_path}		,  cache_dir=snake_case__		,  keep_in_memory=snake_case__   ).read()
				_check_json_datasetdict(snake_case__		,  snake_case__   )




@pytest.mark.parametrize(
    '''features'''		,  [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]		,  )
def 							a		(				snake_case__:   Any		,  snake_case__:   int		,  snake_case__:   int   ):


				'''simple docstring'''
				lowercase_	    =   tmp_path / '''cache'''
				lowercase_	    =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				lowercase_	    =   features.copy() if features else default_expected_features
				lowercase_	    =   (
				    Features({feature: Value(snake_case__   ) for feature, dtype in features.items()}   ) if features is not None else None
				)
				lowercase_	    =   JsonDatasetReader({'''train''': jsonl_path}		,  features=snake_case__		,  cache_dir=snake_case__   ).read()
				_check_json_datasetdict(snake_case__		,  snake_case__   )




@pytest.mark.parametrize('''split'''		,  [None, NamedSplit('''train'''   ), '''train''', '''test''']   )
def 							a		(				snake_case__:   List[str]		,  snake_case__:   Union[str, Any]		,  snake_case__:   Any   ):


				'''simple docstring'''
				if split:
								lowercase_	    =   {split: jsonl_path}
				else:
								lowercase_	    =   '''train'''
								lowercase_	    =   {'''train''': jsonl_path, '''test''': jsonl_path}
				lowercase_	    =   tmp_path / '''cache'''
				lowercase_	    =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				lowercase_	    =   JsonDatasetReader(snake_case__		,  cache_dir=snake_case__   ).read()
				_check_json_datasetdict(snake_case__		,  snake_case__		,  splits=list(path.keys()   )   )
				assert all(dataset[split].split == split for split in path.keys()   )




def 							a		(				snake_case__:   List[str]   ):


				'''simple docstring'''
				return json.load(snake_case__   )




def 							a		(				snake_case__:   Tuple   ):


				'''simple docstring'''
				return [json.loads(snake_case__   ) for line in buffer]


class 					lowercase__:



	"""simple docstring"""




	@pytest.mark.parametrize('''lines, load_json_function'''			,				[(True, load_json_lines), (False, load_json)]			)
	def 		_lowercase    (			self    :     List[Any]			,				SCREAMING_SNAKE_CASE_    :     Any			,				SCREAMING_SNAKE_CASE_    :     Optional[int]			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]			)			->	int:
					with io.BytesIO() as buffer:
									JsonDatasetWriter(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			,				lines=SCREAMING_SNAKE_CASE_			).write()
									buffer.seek(0			)
									lowercase_	    =   load_json_function(SCREAMING_SNAKE_CASE_			)
					assert isinstance(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			)
					assert isinstance(exported_content[0]			,				SCREAMING_SNAKE_CASE_			)
					assert len(SCREAMING_SNAKE_CASE_			) == 1_0
	@pytest.mark.parametrize(
	    '''orient, container, keys, len_at'''			,				[
	        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
	        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
	        ('''index''', dict, set('''0123456789'''			), None),
	        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
	        ('''values''', list, None, None),
	        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
	    ]			,				)
	def 		_lowercase    (			self    :     Dict			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]			,				SCREAMING_SNAKE_CASE_    :     List[Any]			,				SCREAMING_SNAKE_CASE_    :     List[str]			,				SCREAMING_SNAKE_CASE_    :     Optional[int]			,				SCREAMING_SNAKE_CASE_    :     List[Any]			)			->	Any:
					with io.BytesIO() as buffer:
									JsonDatasetWriter(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			,				lines=SCREAMING_SNAKE_CASE_			,				orient=SCREAMING_SNAKE_CASE_			).write()
									buffer.seek(0			)
									lowercase_	    =   load_json(SCREAMING_SNAKE_CASE_			)
					assert isinstance(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			)
					if keys:
									if container is dict:
													assert exported_content.keys() == keys
									else:
													assert exported_content[0].keys() == keys
					else:
									assert not hasattr(SCREAMING_SNAKE_CASE_			,				'''keys'''			) and not hasattr(exported_content[0]			,				'''keys'''			)
					if len_at:
									assert len(exported_content[len_at]			) == 1_0
					else:
									assert len(SCREAMING_SNAKE_CASE_			) == 1_0
	@pytest.mark.parametrize('''lines, load_json_function'''			,				[(True, load_json_lines), (False, load_json)]			)
	def 		_lowercase    (			self    :     List[Any]			,				SCREAMING_SNAKE_CASE_    :     str			,				SCREAMING_SNAKE_CASE_    :     int			,				SCREAMING_SNAKE_CASE_    :     Optional[Any]			)			->	Tuple:
					with io.BytesIO() as buffer:
									JsonDatasetWriter(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			,				lines=SCREAMING_SNAKE_CASE_			,				num_proc=2			).write()
									buffer.seek(0			)
									lowercase_	    =   load_json_function(SCREAMING_SNAKE_CASE_			)
					assert isinstance(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			)
					assert isinstance(exported_content[0]			,				SCREAMING_SNAKE_CASE_			)
					assert len(SCREAMING_SNAKE_CASE_			) == 1_0
	@pytest.mark.parametrize(
	    '''orient, container, keys, len_at'''			,				[
	        ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
	        ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
	        ('''index''', dict, set('''0123456789'''			), None),
	        ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
	        ('''values''', list, None, None),
	        ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
	    ]			,				)
	def 		_lowercase    (			self    :     List[str]			,				SCREAMING_SNAKE_CASE_    :     List[str]			,				SCREAMING_SNAKE_CASE_    :     int			,				SCREAMING_SNAKE_CASE_    :     int			,				SCREAMING_SNAKE_CASE_    :     List[str]			,				SCREAMING_SNAKE_CASE_    :     List[Any]			)			->	Optional[int]:
					with io.BytesIO() as buffer:
									JsonDatasetWriter(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			,				lines=SCREAMING_SNAKE_CASE_			,				orient=SCREAMING_SNAKE_CASE_			,				num_proc=2			).write()
									buffer.seek(0			)
									lowercase_	    =   load_json(SCREAMING_SNAKE_CASE_			)
					assert isinstance(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			)
					if keys:
									if container is dict:
													assert exported_content.keys() == keys
									else:
													assert exported_content[0].keys() == keys
					else:
									assert not hasattr(SCREAMING_SNAKE_CASE_			,				'''keys'''			) and not hasattr(exported_content[0]			,				'''keys'''			)
					if len_at:
									assert len(exported_content[len_at]			) == 1_0
					else:
									assert len(SCREAMING_SNAKE_CASE_			) == 1_0
	def 		_lowercase    (			self    :     Any			,				SCREAMING_SNAKE_CASE_    :     Optional[int]			)			->	Optional[int]:
					with pytest.raises(SCREAMING_SNAKE_CASE_			):
									with io.BytesIO() as buffer:
													JsonDatasetWriter(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			,				num_proc=0			)




	@pytest.mark.parametrize('''compression, extension'''			,				[('''gzip''', '''gz'''), ('''bz2''', '''bz2'''), ('''xz''', '''xz''')]			)
	def 		_lowercase    (			self    :     Tuple			,				SCREAMING_SNAKE_CASE_    :     Dict			,				SCREAMING_SNAKE_CASE_    :     Union[str, Any]			,				SCREAMING_SNAKE_CASE_    :     List[Any]			,				SCREAMING_SNAKE_CASE_    :     str			,				SCREAMING_SNAKE_CASE_    :     List[Any]			)			->	Tuple:
					lowercase_	    =   tmp_path_factory.mktemp('''data'''			) / f'''test.json.{extension}'''
					lowercase_	    =   str(shared_datadir / f'''test_file.json.{extension}'''			)
					JsonDatasetWriter(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			,				compression=SCREAMING_SNAKE_CASE_			).write()

					with fsspec.open(SCREAMING_SNAKE_CASE_			,				'''rb'''			,				compression='''infer'''			) as f:
									lowercase_	    =   f.read()
					with fsspec.open(SCREAMING_SNAKE_CASE_			,				'''rb'''			,				compression='''infer'''			) as f:
									lowercase_	    =   f.read()
					assert exported_content == original_content










'''simple docstring'''
import unittest

from datasets import load_dataset

from transformers.pipelines import pipeline
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_torch, slow
@is_pipeline_test
@require_torch
class   __UpperCAmelCase				(     unittest.TestCase				):



					'''simple docstring'''





					@require_torch
					def     __snake_case						(	self     :						str)    ->							Optional[int]:
								A_									= pipeline(
								    task='zero-shot-audio-classification'      ,       model='hf-internal-testing/tiny-clap-htsat-unfused')
								A_									= load_dataset('ashraq/esc50')
								A_									= dataset['train']['audio'][-1]['array']
								A_									= audio_classifier(_lowercase      ,       candidate_labels=['Sound of a dog', 'Sound of vaccum cleaner'])
								self.assertEqual(
								    nested_simplify(_lowercase)      ,       [{'score': 0.5_01, 'label': 'Sound of a dog'}, {'score': 0.4_99, 'label': 'Sound of vaccum cleaner'}]      ,       )



					@unittest.skip('No models are available in TF')
					def     __snake_case						(	self     :						Tuple)    ->							str:
								pass



					@slow
					@require_torch
					def     __snake_case						(	self     :						Optional[Any])    ->							Tuple:
								A_									= pipeline(
								    task='zero-shot-audio-classification'      ,       model='laion/clap-htsat-unfused'      ,       )
								# This is an audio of a dog
								A_									= load_dataset('ashraq/esc50')
								A_									= dataset['train']['audio'][-1]['array']
								A_									= audio_classifier(_lowercase      ,       candidate_labels=['Sound of a dog', 'Sound of vaccum cleaner'])

								self.assertEqual(
								    nested_simplify(_lowercase)      ,       [
								        {'score': 0.9_99, 'label': 'Sound of a dog'},
								        {'score': 0.0_01, 'label': 'Sound of vaccum cleaner'},
								    ]      ,       )

								A_									= audio_classifier([audio] * 5      ,       candidate_labels=['Sound of a dog', 'Sound of vaccum cleaner'])
								self.assertEqual(
								    nested_simplify(_lowercase)      ,       [
								        [
								            {'score': 0.9_99, 'label': 'Sound of a dog'},
								            {'score': 0.0_01, 'label': 'Sound of vaccum cleaner'},
								        ],
								    ]
								    * 5      ,       )
								A_									= audio_classifier(
								    [audio] * 5      ,       candidate_labels=['Sound of a dog', 'Sound of vaccum cleaner']      ,       batch_size=5)
								self.assertEqual(
								    nested_simplify(_lowercase)      ,       [
								        [
								            {'score': 0.9_99, 'label': 'Sound of a dog'},
								            {'score': 0.0_01, 'label': 'Sound of vaccum cleaner'},
								        ],
								    ]
								    * 5      ,       )



					@unittest.skip('No models are available in TF')
					def     __snake_case						(	self     :						List[str])    ->							str:
								pass




