



import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases



def    lowercase							( SCREAMING_SNAKE_CASE__							:						Any					,							SCREAMING_SNAKE_CASE__							:						Union[str, Any]					)	->  Dict:
 assert isinstance(__A					,							__A					)
 assert dataset.num_rows == 4
 assert dataset.num_columns == 3
 assert dataset.column_names == ["col_1", "col_2", "col_3"]
 for feature, expected_dtype in expected_features.items():
  assert dataset.features[feature].dtype == expected_dtype



@pytest.mark.parametrize("""keep_in_memory"""					,							[False, True]					)
def    lowercase							( SCREAMING_SNAKE_CASE__							:						Optional[Any]					,							SCREAMING_SNAKE_CASE__							:						int					,							SCREAMING_SNAKE_CASE__							:						int					)	->  List[Any]:
 _snake_case     :	List[Any]											=       tmp_path / "cache"
 _snake_case     :	Optional[Any]											=       {"col_1": "string", "col_2": "int64", "col_3": "float64"}
 with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
  _snake_case     :	Tuple											=       JsonDatasetReader(__A					,							cache_dir=__A					,							keep_in_memory=__A					).read()
 _check_json_dataset(__A					,							__A					)



@pytest.mark.parametrize(
    """features"""					,							[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]					,							)
def    lowercase							( SCREAMING_SNAKE_CASE__							:						Tuple					,							SCREAMING_SNAKE_CASE__							:						int					,							SCREAMING_SNAKE_CASE__							:						List[Any]					)	->  Optional[Any]:
 _snake_case     :	Dict											=       tmp_path / "cache"
 _snake_case     :	Optional[Any]											=       {"col_1": "string", "col_2": "int64", "col_3": "float64"}
 _snake_case     :	List[str]											=       features.copy() if features else default_expected_features
 _snake_case     :	str											=       (
     Features({feature: Value(__A					) for feature, dtype in features.items()}					) if features is not None else None
 )
 _snake_case     :	List[Any]											=       JsonDatasetReader(__A					,							features=__A					,							cache_dir=__A					).read()
 _check_json_dataset(__A					,							__A					)



@pytest.mark.parametrize(
    """features"""					,							[
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ]					,							)
def    lowercase							( SCREAMING_SNAKE_CASE__							:						str					,							SCREAMING_SNAKE_CASE__							:						Dict					,							SCREAMING_SNAKE_CASE__							:						Union[str, Any]					)	->  str:
 _snake_case     :	Any											=       tmp_path / "cache"
 _snake_case     :	Dict											=       {"col_3": "float64", "col_1": "string", "col_2": "int64"}
 _snake_case     :	str											=       features.copy() if features else default_expected_features
 _snake_case     :	Optional[Any]											=       (
     Features({feature: Value(__A					) for feature, dtype in features.items()}					) if features is not None else None
 )
 _snake_case     :	Dict											=       JsonDatasetReader(__A					,							features=__A					,							cache_dir=__A					).read()
 assert isinstance(__A					,							__A					)
 assert dataset.num_rows == 2
 assert dataset.num_columns == 3
 assert dataset.column_names == ["col_3", "col_1", "col_2"]
 for feature, expected_dtype in expected_features.items():
  assert dataset.features[feature].dtype == expected_dtype



def    lowercase							( SCREAMING_SNAKE_CASE__							:						Optional[int]					,							SCREAMING_SNAKE_CASE__							:						Dict					)	->  Any:
 _snake_case     :	Tuple											=       {"col_2": "int64", "col_3": "float64", "col_1": "string"}
 _snake_case     :	Optional[int]											=       features.copy()
 _snake_case     :	str											=       (
     Features({feature: Value(__A					) for feature, dtype in features.items()}					) if features is not None else None
 )
 _snake_case     :	Union[str, Any]											=       tmp_path / "cache"
 _snake_case     :	Union[str, Any]											=       JsonDatasetReader(__A					,							features=__A					,							cache_dir=__A					).read()
 assert isinstance(__A					,							__A					)
 assert dataset.num_rows == 2
 assert dataset.num_columns == 3
 assert dataset.column_names == ["col_2", "col_3", "col_1"]
 for feature, expected_dtype in expected_features.items():
  assert dataset.features[feature].dtype == expected_dtype



@pytest.mark.parametrize("""split"""					,							[None, NamedSplit("""train"""					), """train""", """test"""]					)
def    lowercase							( SCREAMING_SNAKE_CASE__							:						List[Any]					,							SCREAMING_SNAKE_CASE__							:						int					,							SCREAMING_SNAKE_CASE__							:						Dict					)	->  Tuple:
 _snake_case     :	Optional[int]											=       tmp_path / "cache"
 _snake_case     :	int											=       {"col_1": "string", "col_2": "int64", "col_3": "float64"}
 _snake_case     :	Optional[Any]											=       JsonDatasetReader(__A					,							cache_dir=__A					,							split=__A					).read()
 _check_json_dataset(__A					,							__A					)
 assert dataset.split == split if split else "train"



@pytest.mark.parametrize("""path_type"""					,							[str, list]					)
def    lowercase							( SCREAMING_SNAKE_CASE__							:						Optional[Any]					,							SCREAMING_SNAKE_CASE__							:						Optional[Any]					,							SCREAMING_SNAKE_CASE__							:						str					)	->  Any:
 if issubclass(__A					,							__A					):
  _snake_case     :	Optional[Any]											=       jsonl_path
 elif issubclass(__A					,							__A					):
  _snake_case     :	List[Any]											=       [jsonl_path]
 _snake_case     :	str											=       tmp_path / "cache"
 _snake_case     :	str											=       {"col_1": "string", "col_2": "int64", "col_3": "float64"}
 _snake_case     :	Tuple											=       JsonDatasetReader(__A					,							cache_dir=__A					).read()
 _check_json_dataset(__A					,							__A					)



def    lowercase							( SCREAMING_SNAKE_CASE__							:						Dict					,							SCREAMING_SNAKE_CASE__							:						Optional[int]					,							SCREAMING_SNAKE_CASE__							:						List[str]=("train",)					)	->  Optional[int]:
 assert isinstance(__A					,							__A					)
 for split in splits:
  _snake_case     :	Dict											=       dataset_dict[split]
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
   assert dataset.features[feature].dtype == expected_dtype



@pytest.mark.parametrize("""keep_in_memory"""					,							[False, True]					)
def    lowercase							( SCREAMING_SNAKE_CASE__							:						Tuple					,							SCREAMING_SNAKE_CASE__							:						Tuple					,							SCREAMING_SNAKE_CASE__							:						List[Any]					)	->  Tuple:
 _snake_case     :	Optional[int]											=       tmp_path / "cache"
 _snake_case     :	Optional[Any]											=       {"col_1": "string", "col_2": "int64", "col_3": "float64"}
 with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
  _snake_case     :	int											=       JsonDatasetReader({"""train""": jsonl_path}					,							cache_dir=__A					,							keep_in_memory=__A					).read()
 _check_json_datasetdict(__A					,							__A					)



@pytest.mark.parametrize(
    """features"""					,							[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]					,							)
def    lowercase							( SCREAMING_SNAKE_CASE__							:						Dict					,							SCREAMING_SNAKE_CASE__							:						int					,							SCREAMING_SNAKE_CASE__							:						Dict					)	->  Optional[Any]:
 _snake_case     :	int											=       tmp_path / "cache"
 _snake_case     :	List[str]											=       {"col_1": "string", "col_2": "int64", "col_3": "float64"}
 _snake_case     :	List[str]											=       features.copy() if features else default_expected_features
 _snake_case     :	Union[str, Any]											=       (
     Features({feature: Value(__A					) for feature, dtype in features.items()}					) if features is not None else None
 )
 _snake_case     :	List[Any]											=       JsonDatasetReader({"""train""": jsonl_path}					,							features=__A					,							cache_dir=__A					).read()
 _check_json_datasetdict(__A					,							__A					)



@pytest.mark.parametrize("""split"""					,							[None, NamedSplit("""train"""					), """train""", """test"""]					)
def    lowercase							( SCREAMING_SNAKE_CASE__							:						List[Any]					,							SCREAMING_SNAKE_CASE__							:						Any					,							SCREAMING_SNAKE_CASE__							:						Union[str, Any]					)	->  str:
 if split:
  _snake_case     :	List[Any]											=       {split: jsonl_path}
 else:
  _snake_case     :	int											=       "train"
  _snake_case     :	str											=       {"train": jsonl_path, "test": jsonl_path}
 _snake_case     :	List[Any]											=       tmp_path / "cache"
 _snake_case     :	List[Any]											=       {"col_1": "string", "col_2": "int64", "col_3": "float64"}
 _snake_case     :	Any											=       JsonDatasetReader(__A					,							cache_dir=__A					).read()
 _check_json_datasetdict(__A					,							__A					,							splits=list(path.keys()					)					)
 assert all(dataset[split].split == split for split in path.keys()					)



def    lowercase							( SCREAMING_SNAKE_CASE__							:						List[Any]					)	->  List[str]:
 return json.load(__A					)



def    lowercase							( SCREAMING_SNAKE_CASE__							:						Tuple					)	->  Optional[int]:
 return [json.loads(__A					) for line in buffer]







class 		snake_case    :



   '''simple docstring'''






   @pytest.mark.parametrize("""lines, load_json_function"""					,		[(True, load_json_lines), (False, load_json)])
   def   UpperCamelCase_	(       self				:      List[str]					,		lowerCAmelCase				:      str					,		lowerCAmelCase				:      Tuple					,		lowerCAmelCase				:      List[str])					->					List[Any]:

    """simple docstring"""







    with io.BytesIO() as buffer:
     JsonDatasetWriter(__UpperCAmelCase					,		__UpperCAmelCase					,		lines=__UpperCAmelCase).write()
     buffer.seek(0)
     _snake_case     :	Any											=       load_json_function(__UpperCAmelCase)
    assert isinstance(__UpperCAmelCase					,		__UpperCAmelCase)
    assert isinstance(exported_content[0]					,		__UpperCAmelCase)
    assert len(__UpperCAmelCase) == 10

   @pytest.mark.parametrize(
       """orient, container, keys, len_at"""					,		[
           ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
           ("""split""", dict, {"""columns""", """data"""}, """data"""),
           ("""index""", dict, set("""0123456789"""), None),
           ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
           ("""values""", list, None, None),
           ("""table""", dict, {"""schema""", """data"""}, """data"""),
       ]					,		)
   def   UpperCamelCase_	(       self				:      Optional[int]					,		lowerCAmelCase				:      Dict					,		lowerCAmelCase				:      Union[str, Any]					,		lowerCAmelCase				:      Optional[int]					,		lowerCAmelCase				:      List[str]					,		lowerCAmelCase				:      Tuple)					->					Any:

    """simple docstring"""







    with io.BytesIO() as buffer:
     JsonDatasetWriter(__UpperCAmelCase					,		__UpperCAmelCase					,		lines=__UpperCAmelCase					,		orient=__UpperCAmelCase).write()
     buffer.seek(0)
     _snake_case     :	List[str]											=       load_json(__UpperCAmelCase)
    assert isinstance(__UpperCAmelCase					,		__UpperCAmelCase)
    if keys:
     if container is dict:
      assert exported_content.keys() == keys
     else:
      assert exported_content[0].keys() == keys
    else:
     assert not hasattr(__UpperCAmelCase					,		"""keys""") and not hasattr(exported_content[0]					,		"""keys""")
    if len_at:
     assert len(exported_content[len_at]) == 10
    else:
     assert len(__UpperCAmelCase) == 10

   @pytest.mark.parametrize("""lines, load_json_function"""					,		[(True, load_json_lines), (False, load_json)])
   def   UpperCamelCase_	(       self				:      int					,		lowerCAmelCase				:      int					,		lowerCAmelCase				:      Union[str, Any]					,		lowerCAmelCase				:      Optional[int])					->					Optional[int]:

    """simple docstring"""







    with io.BytesIO() as buffer:
     JsonDatasetWriter(__UpperCAmelCase					,		__UpperCAmelCase					,		lines=__UpperCAmelCase					,		num_proc=2).write()
     buffer.seek(0)
     _snake_case     :	int											=       load_json_function(__UpperCAmelCase)
    assert isinstance(__UpperCAmelCase					,		__UpperCAmelCase)
    assert isinstance(exported_content[0]					,		__UpperCAmelCase)
    assert len(__UpperCAmelCase) == 10

   @pytest.mark.parametrize(
       """orient, container, keys, len_at"""					,		[
           ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
           ("""split""", dict, {"""columns""", """data"""}, """data"""),
           ("""index""", dict, set("""0123456789"""), None),
           ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
           ("""values""", list, None, None),
           ("""table""", dict, {"""schema""", """data"""}, """data"""),
       ]					,		)
   def   UpperCamelCase_	(       self				:      List[Any]					,		lowerCAmelCase				:      Tuple					,		lowerCAmelCase				:      List[str]					,		lowerCAmelCase				:      Optional[int]					,		lowerCAmelCase				:      Dict					,		lowerCAmelCase				:      Union[str, Any])					->					List[Any]:

    """simple docstring"""







    with io.BytesIO() as buffer:
     JsonDatasetWriter(__UpperCAmelCase					,		__UpperCAmelCase					,		lines=__UpperCAmelCase					,		orient=__UpperCAmelCase					,		num_proc=2).write()
     buffer.seek(0)
     _snake_case     :	List[str]											=       load_json(__UpperCAmelCase)
    assert isinstance(__UpperCAmelCase					,		__UpperCAmelCase)
    if keys:
     if container is dict:
      assert exported_content.keys() == keys
     else:
      assert exported_content[0].keys() == keys
    else:
     assert not hasattr(__UpperCAmelCase					,		"""keys""") and not hasattr(exported_content[0]					,		"""keys""")
    if len_at:
     assert len(exported_content[len_at]) == 10
    else:
     assert len(__UpperCAmelCase) == 10

   def   UpperCamelCase_	(       self				:      List[Any]					,		lowerCAmelCase				:      int)					->					Any:

    """simple docstring"""







    with pytest.raises(__UpperCAmelCase):
     with io.BytesIO() as buffer:
      JsonDatasetWriter(__UpperCAmelCase					,		__UpperCAmelCase					,		num_proc=0)


   @pytest.mark.parametrize("""compression, extension"""					,		[("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")])
   def   UpperCamelCase_	(       self				:      Optional[Any]					,		lowerCAmelCase				:      List[Any]					,		lowerCAmelCase				:      List[Any]					,		lowerCAmelCase				:      Dict					,		lowerCAmelCase				:      Union[str, Any]					,		lowerCAmelCase				:      List[str])					->					Tuple:

    """simple docstring"""







    _snake_case     :	Union[str, Any]											=       tmp_path_factory.mktemp("""data""") / F'''test.json.{extension}'''
    _snake_case     :	str											=       str(shared_datadir / F'''test_file.json.{extension}''')
    JsonDatasetWriter(__UpperCAmelCase					,		__UpperCAmelCase					,		compression=__UpperCAmelCase).write()

    with fsspec.open(__UpperCAmelCase					,		"""rb"""					,		compression="""infer""") as f:
     _snake_case     :	Optional[Any]											=       f.read()
    with fsspec.open(__UpperCAmelCase					,		"""rb"""					,		compression="""infer""") as f:
     _snake_case     :	int											=       f.read()
    assert exported_content == original_content








from abc import ABC, abstractmethod
from argparse import ArgumentParser




class 							snake_case_       (    __UpperCamelCase      ):

						"""simple docstring"""



						@staticmethod
						@abstractmethod
						def        UpperCAmelCase__					(__UpperCAmelCase:					ArgumentParser   )    ->     Tuple:




											'''simple docstring'''



											raise NotImplementedError()



						@abstractmethod
						def        UpperCAmelCase__					(self:					List[str]   )    ->     List[str]:




											'''simple docstring'''



											raise NotImplementedError()





