



from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder


# @@protoc_insertion_point(imports)

UpperCAmelCase							 =					_symbol_database.Default()


UpperCAmelCase							 =					_descriptor_pool.Default().AddSerializedFile(
    B"""\n\x19sentencepiece_model.proto\x12\rsentencepiece\"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12\"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12\"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18\" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse\"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32\".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL\"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03"""
)

UpperCAmelCase							 =					globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, """sentencepiece_model_pb2""", _globals)
if _descriptor._USE_C_DESCRIPTORS is False:
	UpperCAmelCase							 =					None
	UpperCAmelCase							 =					B"""H\003"""
	# (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
	# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
	# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
	# _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
	# _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
	UpperCAmelCase							 =					45
	UpperCAmelCase							 =					1_581
	UpperCAmelCase							 =					1_517
	UpperCAmelCase							 =					1_570
	UpperCAmelCase							 =					1_584
	UpperCAmelCase							 =					1_793
	UpperCAmelCase							 =					1_795
	UpperCAmelCase							 =					1_916
	UpperCAmelCase							 =					1_864
	UpperCAmelCase							 =					1_905
	UpperCAmelCase							 =					1_919
	UpperCAmelCase							 =					2_429
	UpperCAmelCase							 =					2_208
	UpperCAmelCase							 =					2_418
	UpperCAmelCase							 =					2_323
	UpperCAmelCase							 =					2_407
# @@protoc_insertion_point(module_scope)






import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def 			A_      (	__a     :				List[str] ,		__a     :				Optional[int]		):

				"""simple docstring"""
				assert isinstance(__a ,		__a		)
				assert dataset.num_rows == 4
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_1", "col_2", "col_3"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""keep_in_memory""" ,		[False, True]		)
def 			A_      (	__a     :				Tuple ,		__a     :				Dict ,		__a     :				List[str]		):

				"""simple docstring"""
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								a__				      =		JsonDatasetReader(__a ,		cache_dir=__a ,		keep_in_memory=__a		).read()
				_check_json_dataset(__a ,		__a		)

@pytest.mark.parametrize(
    """features""" ,		[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ] ,		)
def 			A_      (	__a     :				int ,		__a     :				Dict ,		__a     :				List[Any]		):

				"""simple docstring"""
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				a__				      =		features.copy() if features else default_expected_features
				a__				      =		(
				    Features({feature: Value(__a		) for feature, dtype in features.items()}		) if features is not None else None
				)
				a__				      =		JsonDatasetReader(__a ,		features=__a ,		cache_dir=__a		).read()
				_check_json_dataset(__a ,		__a		)

@pytest.mark.parametrize(
    """features""" ,		[
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ] ,		)
def 			A_      (	__a     :				Tuple ,		__a     :				Tuple ,		__a     :				Tuple		):

				"""simple docstring"""
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""}
				a__				      =		features.copy() if features else default_expected_features
				a__				      =		(
				    Features({feature: Value(__a		) for feature, dtype in features.items()}		) if features is not None else None
				)
				a__				      =		JsonDatasetReader(__a ,		features=__a ,		cache_dir=__a		).read()
				assert isinstance(__a ,		__a		)
				assert dataset.num_rows == 2
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_3", "col_1", "col_2"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype

def 			A_      (	__a     :				int ,		__a     :				Optional[Any]		):

				"""simple docstring"""
				# jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
				a__				      =		{"""col_2""": """int64""", """col_3""": """float64""", """col_1""": """string"""}
				a__				      =		features.copy()
				a__				      =		(
				    Features({feature: Value(__a		) for feature, dtype in features.items()}		) if features is not None else None
				)
				a__				      =		tmp_path / """cache"""
				a__				      =		JsonDatasetReader(__a ,		features=__a ,		cache_dir=__a		).read()
				assert isinstance(__a ,		__a		)
				assert dataset.num_rows == 2
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_2", "col_3", "col_1"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""split""" ,		[None, NamedSplit("""train"""		), """train""", """test"""]		)
def 			A_      (	__a     :				List[Any] ,		__a     :				int ,		__a     :				List[str]		):

				"""simple docstring"""
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				a__				      =		JsonDatasetReader(__a ,		cache_dir=__a ,		split=__a		).read()
				_check_json_dataset(__a ,		__a		)
				assert dataset.split == split if split else "train"

@pytest.mark.parametrize("""path_type""" ,		[str, list]		)
def 			A_      (	__a     :				Dict ,		__a     :				Tuple ,		__a     :				List[str]		):

				"""simple docstring"""
				if issubclass(__a ,		__a		):
								a__				      =		jsonl_path
				elif issubclass(__a ,		__a		):
								a__				      =		[jsonl_path]
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				a__				      =		JsonDatasetReader(__a ,		cache_dir=__a		).read()
				_check_json_dataset(__a ,		__a		)

def 			A_      (	__a     :				str ,		__a     :				List[Any] ,		__a     :				Dict=("train",)		):

				"""simple docstring"""
				assert isinstance(__a ,		__a		)
				for split in splits:
								a__				      =		dataset_dict[split]
								assert dataset.num_rows == 4
								assert dataset.num_columns == 3
								assert dataset.column_names == ["col_1", "col_2", "col_3"]
								for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""keep_in_memory""" ,		[False, True]		)
def 			A_      (	__a     :				int ,		__a     :				int ,		__a     :				Dict		):

				"""simple docstring"""
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								a__				      =		JsonDatasetReader({"""train""": jsonl_path} ,		cache_dir=__a ,		keep_in_memory=__a		).read()
				_check_json_datasetdict(__a ,		__a		)

@pytest.mark.parametrize(
    """features""" ,		[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ] ,		)
def 			A_      (	__a     :				Tuple ,		__a     :				Dict ,		__a     :				Optional[Any]		):

				"""simple docstring"""
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				a__				      =		features.copy() if features else default_expected_features
				a__				      =		(
				    Features({feature: Value(__a		) for feature, dtype in features.items()}		) if features is not None else None
				)
				a__				      =		JsonDatasetReader({"""train""": jsonl_path} ,		features=__a ,		cache_dir=__a		).read()
				_check_json_datasetdict(__a ,		__a		)

@pytest.mark.parametrize("""split""" ,		[None, NamedSplit("""train"""		), """train""", """test"""]		)
def 			A_      (	__a     :				Dict ,		__a     :				Any ,		__a     :				Tuple		):

				"""simple docstring"""
				if split:
								a__				      =		{split: jsonl_path}
				else:
								a__				      =		"""train"""
								a__				      =		{"""train""": jsonl_path, """test""": jsonl_path}
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				a__				      =		JsonDatasetReader(__a ,		cache_dir=__a		).read()
				_check_json_datasetdict(__a ,		__a ,		splits=list(path.keys()		)		)
				assert all(dataset[split].split == split for split in path.keys()		)

def 			A_      (	__a     :				Any		):

				"""simple docstring"""
				return json.load(__a		)

def 			A_      (	__a     :				Tuple		):

				"""simple docstring"""
				return [json.loads(__a		) for line in buffer]




class  __snake_case      :







							'''simple docstring'''






							@pytest.mark.parametrize("""lines, load_json_function"""  ,  [(True, load_json_lines), (False, load_json)]		)
							def _a      (							self  ,  a_  ,  a_  ,  a_		):
											with io.BytesIO() as buffer:
															JsonDatasetWriter(a_  ,  a_  ,  lines=a_		).write()
															buffer.seek(0		)
															a__				      =		load_json_function(a_		)
											assert isinstance(a_  ,  a_		)
											assert isinstance(exported_content[0]  ,  a_		)
											assert len(a_		) == 10


							@pytest.mark.parametrize(
							    """orient, container, keys, len_at"""  ,  [
							        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
							        ("""split""", dict, {"""columns""", """data"""}, """data"""),
							        ("""index""", dict, set("""0123456789"""		), None),
							        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
							        ("""values""", list, None, None),
							        ("""table""", dict, {"""schema""", """data"""}, """data"""),
							    ]  ,  )
							def _a      (							self  ,  a_  ,  a_  ,  a_  ,  a_  ,  a_		):
											with io.BytesIO() as buffer:
															JsonDatasetWriter(a_  ,  a_  ,  lines=a_  ,  orient=a_		).write()
															buffer.seek(0		)
															a__				      =		load_json(a_		)
											assert isinstance(a_  ,  a_		)
											if keys:
															if container is dict:
																			assert exported_content.keys() == keys
															else:
																			assert exported_content[0].keys() == keys
											else:
															assert not hasattr(a_  ,  """keys"""		) and not hasattr(exported_content[0]  ,  """keys"""		)
											if len_at:
															assert len(exported_content[len_at]		) == 10
											else:
															assert len(a_		) == 10


							@pytest.mark.parametrize("""lines, load_json_function"""  ,  [(True, load_json_lines), (False, load_json)]		)
							def _a      (							self  ,  a_  ,  a_  ,  a_		):
											with io.BytesIO() as buffer:
															JsonDatasetWriter(a_  ,  a_  ,  lines=a_  ,  num_proc=2		).write()
															buffer.seek(0		)
															a__				      =		load_json_function(a_		)
											assert isinstance(a_  ,  a_		)
											assert isinstance(exported_content[0]  ,  a_		)
											assert len(a_		) == 10


							@pytest.mark.parametrize(
							    """orient, container, keys, len_at"""  ,  [
							        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
							        ("""split""", dict, {"""columns""", """data"""}, """data"""),
							        ("""index""", dict, set("""0123456789"""		), None),
							        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
							        ("""values""", list, None, None),
							        ("""table""", dict, {"""schema""", """data"""}, """data"""),
							    ]  ,  )
							def _a      (							self  ,  a_  ,  a_  ,  a_  ,  a_  ,  a_		):
											with io.BytesIO() as buffer:
															JsonDatasetWriter(a_  ,  a_  ,  lines=a_  ,  orient=a_  ,  num_proc=2		).write()
															buffer.seek(0		)
															a__				      =		load_json(a_		)
											assert isinstance(a_  ,  a_		)
											if keys:
															if container is dict:
																			assert exported_content.keys() == keys
															else:
																			assert exported_content[0].keys() == keys
											else:
															assert not hasattr(a_  ,  """keys"""		) and not hasattr(exported_content[0]  ,  """keys"""		)
											if len_at:
															assert len(exported_content[len_at]		) == 10
											else:
															assert len(a_		) == 10


							def _a      (							self  ,  a_		):
											with pytest.raises(a_		):
															with io.BytesIO() as buffer:
																			JsonDatasetWriter(a_  ,  a_  ,  num_proc=0		)







							@pytest.mark.parametrize("""compression, extension"""  ,  [("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")]		)
							def _a      (							self  ,  a_  ,  a_  ,  a_  ,  a_  ,  a_		):
											a__				      =		tmp_path_factory.mktemp("""data"""		) / F'''test.json.{extension}'''
											a__				      =		str(shared_datadir / F'''test_file.json.{extension}'''		)
											JsonDatasetWriter(a_  ,  a_  ,  compression=a_		).write()

											with fsspec.open(a_  ,  """rb"""  ,  compression="""infer"""		) as f:
															a__				      =		f.read()
											with fsspec.open(a_  ,  """rb"""  ,  compression="""infer"""		) as f:
															a__				      =		f.read()
											assert exported_content == original_content

