





"""simple docstring"""





import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


def       a__					(					_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		assert isinstance(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    )
		assert dataset.num_rows == 4
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_1", "col_2", "col_3"]
		for feature, expected_dtype in expected_features.items():
				assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("keep_in_memory"  ,	[False, True]    )
def       a__					(					_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	tmp_path / "cache"
		UpperCamelCase   =	{"col_1": "string", "col_2": "int64", "col_3": "float64"}
		with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
				UpperCamelCase   =	JsonDatasetReader(_SCREAMING_SNAKE_CASE  ,	cache_dir=_SCREAMING_SNAKE_CASE  ,	keep_in_memory=_SCREAMING_SNAKE_CASE    ).read()
		_check_json_dataset(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    )


@pytest.mark.parametrize(
    "features"  ,	[
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]  ,	)
def       a__					(					_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	tmp_path / "cache"
		UpperCamelCase   =	{"col_1": "string", "col_2": "int64", "col_3": "float64"}
		UpperCamelCase   =	features.copy() if features else default_expected_features
		UpperCamelCase   =	(
		    Features({feature: Value(_SCREAMING_SNAKE_CASE    ) for feature, dtype in features.items()}    ) if features is not None else None
		)
		UpperCamelCase   =	JsonDatasetReader(_SCREAMING_SNAKE_CASE  ,	features=_SCREAMING_SNAKE_CASE  ,	cache_dir=_SCREAMING_SNAKE_CASE    ).read()
		_check_json_dataset(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    )


@pytest.mark.parametrize(
    "features"  ,	[
        None,
        {"col_3": "float64", "col_1": "string", "col_2": "int64"},
    ]  ,	)
def       a__					(					_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	tmp_path / "cache"
		UpperCamelCase   =	{"col_3": "float64", "col_1": "string", "col_2": "int64"}
		UpperCamelCase   =	features.copy() if features else default_expected_features
		UpperCamelCase   =	(
		    Features({feature: Value(_SCREAMING_SNAKE_CASE    ) for feature, dtype in features.items()}    ) if features is not None else None
		)
		UpperCamelCase   =	JsonDatasetReader(_SCREAMING_SNAKE_CASE  ,	features=_SCREAMING_SNAKE_CASE  ,	cache_dir=_SCREAMING_SNAKE_CASE    ).read()
		assert isinstance(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    )
		assert dataset.num_rows == 2
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_3", "col_1", "col_2"]
		for feature, expected_dtype in expected_features.items():
				assert dataset.features[feature].dtype == expected_dtype


def       a__					(					_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	{"col_2": "int64", "col_3": "float64", "col_1": "string"}
		UpperCamelCase   =	features.copy()
		UpperCamelCase   =	(
		    Features({feature: Value(_SCREAMING_SNAKE_CASE    ) for feature, dtype in features.items()}    ) if features is not None else None
		)
		UpperCamelCase   =	tmp_path / "cache"
		UpperCamelCase   =	JsonDatasetReader(_SCREAMING_SNAKE_CASE  ,	features=_SCREAMING_SNAKE_CASE  ,	cache_dir=_SCREAMING_SNAKE_CASE    ).read()
		assert isinstance(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    )
		assert dataset.num_rows == 2
		assert dataset.num_columns == 3
		assert dataset.column_names == ["col_2", "col_3", "col_1"]
		for feature, expected_dtype in expected_features.items():
				assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("split"  ,	[None, NamedSplit("train"    ), "train", "test"]    )
def       a__					(					_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	tmp_path / "cache"
		UpperCamelCase   =	{"col_1": "string", "col_2": "int64", "col_3": "float64"}
		UpperCamelCase   =	JsonDatasetReader(_SCREAMING_SNAKE_CASE  ,	cache_dir=_SCREAMING_SNAKE_CASE  ,	split=_SCREAMING_SNAKE_CASE    ).read()
		_check_json_dataset(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    )
		assert dataset.split == split if split else "train"


@pytest.mark.parametrize("path_type"  ,	[str, list]    )
def       a__					(					_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		if issubclass(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    ):
				UpperCamelCase   =	jsonl_path
		elif issubclass(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    ):
				UpperCamelCase   =	[jsonl_path]
		UpperCamelCase   =	tmp_path / "cache"
		UpperCamelCase   =	{"col_1": "string", "col_2": "int64", "col_3": "float64"}
		UpperCamelCase   =	JsonDatasetReader(_SCREAMING_SNAKE_CASE  ,	cache_dir=_SCREAMING_SNAKE_CASE    ).read()
		_check_json_dataset(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    )


def       a__					(					_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE=("train",)    ):




		"""simple docstring"""


		assert isinstance(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    )
		for split in splits:
				UpperCamelCase   =	dataset_dict[split]
				assert dataset.num_rows == 4
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_1", "col_2", "col_3"]
				for feature, expected_dtype in expected_features.items():
						assert dataset.features[feature].dtype == expected_dtype


@pytest.mark.parametrize("keep_in_memory"  ,	[False, True]    )
def       a__					(					_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	tmp_path / "cache"
		UpperCamelCase   =	{"col_1": "string", "col_2": "int64", "col_3": "float64"}
		with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
				UpperCamelCase   =	JsonDatasetReader({"train": jsonl_path}  ,	cache_dir=_SCREAMING_SNAKE_CASE  ,	keep_in_memory=_SCREAMING_SNAKE_CASE    ).read()
		_check_json_datasetdict(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    )


@pytest.mark.parametrize(
    "features"  ,	[
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]  ,	)
def       a__					(					_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	tmp_path / "cache"
		UpperCamelCase   =	{"col_1": "string", "col_2": "int64", "col_3": "float64"}
		UpperCamelCase   =	features.copy() if features else default_expected_features
		UpperCamelCase   =	(
		    Features({feature: Value(_SCREAMING_SNAKE_CASE    ) for feature, dtype in features.items()}    ) if features is not None else None
		)
		UpperCamelCase   =	JsonDatasetReader({"train": jsonl_path}  ,	features=_SCREAMING_SNAKE_CASE  ,	cache_dir=_SCREAMING_SNAKE_CASE    ).read()
		_check_json_datasetdict(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    )


@pytest.mark.parametrize("split"  ,	[None, NamedSplit("train"    ), "train", "test"]    )
def       a__					(					_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		if split:
				UpperCamelCase   =	{split: jsonl_path}
		else:
				UpperCamelCase   =	"train"
				UpperCamelCase   =	{"train": jsonl_path, "test": jsonl_path}
		UpperCamelCase   =	tmp_path / "cache"
		UpperCamelCase   =	{"col_1": "string", "col_2": "int64", "col_3": "float64"}
		UpperCamelCase   =	JsonDatasetReader(_SCREAMING_SNAKE_CASE  ,	cache_dir=_SCREAMING_SNAKE_CASE    ).read()
		_check_json_datasetdict(_SCREAMING_SNAKE_CASE  ,	_SCREAMING_SNAKE_CASE  ,	splits=list(path.keys()    )    )
		assert all(dataset[split].split == split for split in path.keys()    )


def       a__					(					_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		return json.load(_SCREAMING_SNAKE_CASE    )


def       a__					(					_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		return [json.loads(_SCREAMING_SNAKE_CASE    ) for line in buffer]






class        _lowerCamelCase      :


						@pytest.mark.parametrize("lines, load_json_function"   ,							[(True, load_json_lines), (False, load_json)]		)
						def     snake_case_   (self   ,							__a   ,							__a   ,							__a		) ->   int:
								with io.BytesIO() as buffer:
										JsonDatasetWriter(__a   ,							__a   ,							lines=__a		).write()
										buffer.seek(0		)
										UpperCamelCase   =	load_json_function(__a		)
								assert isinstance(__a   ,							__a		)
								assert isinstance(exported_content[0]   ,							__a		)
								assert len(__a		) == 10


						@pytest.mark.parametrize(
						    "orient, container, keys, len_at"   ,							[
						        ("records", list, {"tokens", "labels", "answers", "id"}, None),
						        ("split", dict, {"columns", "data"}, "data"),
						        ("index", dict, set("0123456789"		), None),
						        ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
						        ("values", list, None, None),
						        ("table", dict, {"schema", "data"}, "data"),
						    ]   ,							)
						def     snake_case_   (self   ,							__a   ,							__a   ,							__a   ,							__a   ,							__a		) ->   Optional[Any]:
								with io.BytesIO() as buffer:
										JsonDatasetWriter(__a   ,							__a   ,							lines=__a   ,							orient=__a		).write()
										buffer.seek(0		)
										UpperCamelCase   =	load_json(__a		)
								assert isinstance(__a   ,							__a		)
								if keys:
										if container is dict:
												assert exported_content.keys() == keys
										else:
												assert exported_content[0].keys() == keys
								else:
										assert not hasattr(__a   ,							"keys"		) and not hasattr(exported_content[0]   ,							"keys"		)
								if len_at:
										assert len(exported_content[len_at]		) == 10
								else:
										assert len(__a		) == 10


						@pytest.mark.parametrize("lines, load_json_function"   ,							[(True, load_json_lines), (False, load_json)]		)
						def     snake_case_   (self   ,							__a   ,							__a   ,							__a		) ->   Dict:
								with io.BytesIO() as buffer:
										JsonDatasetWriter(__a   ,							__a   ,							lines=__a   ,							num_proc=2		).write()
										buffer.seek(0		)
										UpperCamelCase   =	load_json_function(__a		)
								assert isinstance(__a   ,							__a		)
								assert isinstance(exported_content[0]   ,							__a		)
								assert len(__a		) == 10


						@pytest.mark.parametrize(
						    "orient, container, keys, len_at"   ,							[
						        ("records", list, {"tokens", "labels", "answers", "id"}, None),
						        ("split", dict, {"columns", "data"}, "data"),
						        ("index", dict, set("0123456789"		), None),
						        ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
						        ("values", list, None, None),
						        ("table", dict, {"schema", "data"}, "data"),
						    ]   ,							)
						def     snake_case_   (self   ,							__a   ,							__a   ,							__a   ,							__a   ,							__a		) ->   Optional[int]:
								with io.BytesIO() as buffer:
										JsonDatasetWriter(__a   ,							__a   ,							lines=__a   ,							orient=__a   ,							num_proc=2		).write()
										buffer.seek(0		)
										UpperCamelCase   =	load_json(__a		)
								assert isinstance(__a   ,							__a		)
								if keys:
										if container is dict:
												assert exported_content.keys() == keys
										else:
												assert exported_content[0].keys() == keys
								else:
										assert not hasattr(__a   ,							"keys"		) and not hasattr(exported_content[0]   ,							"keys"		)
								if len_at:
										assert len(exported_content[len_at]		) == 10
								else:
										assert len(__a		) == 10


						def     snake_case_   (self   ,							__a		) ->   Optional[int]:
								with pytest.raises(__a		):
										with io.BytesIO() as buffer:
												JsonDatasetWriter(__a   ,							__a   ,							num_proc=0		)




						@pytest.mark.parametrize("compression, extension"   ,							[("gzip", "gz"), ("bz2", "bz2"), ("xz", "xz")]		)
						def     snake_case_   (self   ,							__a   ,							__a   ,							__a   ,							__a   ,							__a		) ->   List[str]:
								UpperCamelCase   =	tmp_path_factory.mktemp("data"		) / F"test.json.{extension}"
								UpperCamelCase   =	str(shared_datadir / F"test_file.json.{extension}"		)
								JsonDatasetWriter(__a   ,							__a   ,							compression=__a		).write()

								with fsspec.open(__a   ,							"rb"   ,							compression="infer"		) as f:
										UpperCamelCase   =	f.read()
								with fsspec.open(__a   ,							"rb"   ,							compression="infer"		) as f:
										UpperCamelCase   =	f.read()
								assert exported_content == original_content













"""simple docstring"""





lowerCAmelCase__     			=			'''
# Transformers installation
! pip install transformers datasets
# To install from source instead of the last release, comment the command above and uncomment the following one.
# ! pip install git+https://github.com/huggingface/transformers.git
'''

lowerCAmelCase__     			=			[{'''type''': '''code''', '''content''': INSTALL_CONTENT}]
lowerCAmelCase__     			=			{
    '''{processor_class}''': '''FakeProcessorClass''',
    '''{model_class}''': '''FakeModelClass''',
    '''{object_class}''': '''FakeObjectClass''',
}






