




import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
				import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a													=			logging.get_logger(__name__)

a													=			{"vocab_file": "vocab.txt"}

a													=			{
    "vocab_file": {
        "openbmb/cpm-ant-10b": "https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt",
    },
}

a													=			{
    "openbmb/cpm-ant-10b": 1024,
}




def        _SCREAMING_SNAKE_CASE      (		snake_case       )						->				Dict:
						_UpperCAmelCase      =			collections.OrderedDict()
						with open(__SCREAMING_SNAKE_CASE			,     """r"""			,     encoding="""utf-8"""       ) as reader:
												_UpperCAmelCase      =			reader.readlines()
						for index, token in enumerate(__SCREAMING_SNAKE_CASE       ):
												_UpperCAmelCase      =			token.rstrip("""\n"""       )
												_UpperCAmelCase      =			index
						return vocab




class       lowerCamelCase				(		lowercase__     ):






		def __init__(    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE="<unk>"						,		_SCREAMING_SNAKE_CASE=200			):
								_UpperCAmelCase      =			vocab
								_UpperCAmelCase      =			unk_token
								_UpperCAmelCase      =			max_input_chars_per_word






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								_UpperCAmelCase      =			list(_SCREAMING_SNAKE_CASE			)
								if len(_SCREAMING_SNAKE_CASE			) > self.max_input_chars_per_word:
														return [self.unk_token]

								_UpperCAmelCase      =			0
								_UpperCAmelCase      =			[]
								while start < len(_SCREAMING_SNAKE_CASE			):
														_UpperCAmelCase      =			len(_SCREAMING_SNAKE_CASE			)
														_UpperCAmelCase      =			None
														while start < end:
																				_UpperCAmelCase      =			"""""".join(chars[start:end]			)
																				if substr in self.vocab:
																										_UpperCAmelCase      =			substr
																										break
																				end -= 1
														if cur_substr is None:
																				sub_tokens.append(self.unk_token			)
																				start += 1
														else:
																				sub_tokens.append(_SCREAMING_SNAKE_CASE			)
																				_UpperCAmelCase      =			end

								return sub_tokens




class       lowerCamelCase				(		lowercase__     ):
		__a					    =       VOCAB_FILES_NAMES
		__a					    =       PRETRAINED_VOCAB_FILES_MAP
		__a					    =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__a					    =       ['input_ids', 'attention_mask']
		__a					    =       False






		def __init__(    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE="<d>"						,		_SCREAMING_SNAKE_CASE="</d>"						,		_SCREAMING_SNAKE_CASE="<s>"						,		_SCREAMING_SNAKE_CASE="</s>"						,		_SCREAMING_SNAKE_CASE="<pad>"						,		_SCREAMING_SNAKE_CASE="<unk>"						,		_SCREAMING_SNAKE_CASE="</n>"						,		_SCREAMING_SNAKE_CASE="</_>"						,		_SCREAMING_SNAKE_CASE="left"						,		**_SCREAMING_SNAKE_CASE						,		):
								requires_backends(self						,		["""jieba"""]			)
								super().__init__(
								    bod_token=_SCREAMING_SNAKE_CASE						,		eod_token=_SCREAMING_SNAKE_CASE						,		bos_token=_SCREAMING_SNAKE_CASE						,		eos_token=_SCREAMING_SNAKE_CASE						,		pad_token=_SCREAMING_SNAKE_CASE						,		unk_token=_SCREAMING_SNAKE_CASE						,		line_token=_SCREAMING_SNAKE_CASE						,		space_token=_SCREAMING_SNAKE_CASE						,		padding_side=_SCREAMING_SNAKE_CASE						,		**_SCREAMING_SNAKE_CASE						,		)
								_UpperCAmelCase      =			bod_token
								_UpperCAmelCase      =			eod_token
								_UpperCAmelCase      =			load_vocab(_SCREAMING_SNAKE_CASE			)
								_UpperCAmelCase      =			self.encoder[space_token]
								_UpperCAmelCase      =			self.encoder[line_token]

								del self.encoder[space_token]
								del self.encoder[line_token]

								_UpperCAmelCase      =			collections.OrderedDict(sorted(self.encoder.items()						,		key=lambda _SCREAMING_SNAKE_CASE			: x[1]			)			)
								_UpperCAmelCase      =			{v: k for k, v in self.encoder.items()}

								_UpperCAmelCase      =			WordpieceTokenizer(vocab=self.encoder						,		unk_token=self.unk_token			)






		@property
		def 				UpperCAmelCase (    self			):
								return self.encoder[self.bod_token]






		@property
		def 				UpperCAmelCase (    self			):
								return self.encoder[self.eod_token]






		@property
		def 				UpperCAmelCase (    self			):
								return self.encoder["\n"]






		@property
		def 				UpperCAmelCase (    self			):
								return len(self.encoder			)






		def 				UpperCAmelCase (    self			):
								return dict(self.encoder						,		**self.added_tokens_encoder			)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								_UpperCAmelCase      =			[]
								for x in jieba.cut(_SCREAMING_SNAKE_CASE						,		cut_all=_SCREAMING_SNAKE_CASE			):
														output_tokens.extend(self.wordpiece_tokenizer.tokenize(_SCREAMING_SNAKE_CASE			)			)
								return output_tokens






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		**_SCREAMING_SNAKE_CASE			):
								_UpperCAmelCase      =			[i for i in token_ids if i >= 0]
								_UpperCAmelCase      =			[
								    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
								]
								return super()._decode(_SCREAMING_SNAKE_CASE						,		**_SCREAMING_SNAKE_CASE			)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								return token in self.encoder






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								return "".join(_SCREAMING_SNAKE_CASE			)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								return self.encoder.get(_SCREAMING_SNAKE_CASE						,		self.encoder.get(self.unk_token			)			)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								return self.decoder.get(_SCREAMING_SNAKE_CASE						,		self.unk_token			)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE = None			):
								if os.path.isdir(_SCREAMING_SNAKE_CASE			):
														_UpperCAmelCase      =			os.path.join(
														    _SCREAMING_SNAKE_CASE						,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)
								else:
														_UpperCAmelCase      =			(filename_prefix + """-""" if filename_prefix else """""") + save_directory
								_UpperCAmelCase      =			0
								if " " in self.encoder:
														_UpperCAmelCase      =			self.encoder[""" """]
														del self.encoder[" "]
								if "\n" in self.encoder:
														_UpperCAmelCase      =			self.encoder["""\n"""]
														del self.encoder["\n"]
								_UpperCAmelCase      =			collections.OrderedDict(sorted(self.encoder.items()						,		key=lambda _SCREAMING_SNAKE_CASE			: x[1]			)			)
								with open(_SCREAMING_SNAKE_CASE						,		"""w"""						,		encoding="""utf-8"""			) as writer:
														for token, token_index in self.encoder.items():
																				if index != token_index:
																										logger.warning(
																										    F"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
																										    """ Please check that the vocabulary is not corrupted!"""			)
																										_UpperCAmelCase      =			token_index
																				writer.write(token + """\n"""			)
																				index += 1
								return (vocab_file,)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE = None			):
								if token_ids_a is None:
														return [self.bos_token_id] + token_ids_a
								return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE = None						,		_SCREAMING_SNAKE_CASE = False			):

								if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=_SCREAMING_SNAKE_CASE						,		token_ids_a=_SCREAMING_SNAKE_CASE						,		already_has_special_tokens=_SCREAMING_SNAKE_CASE			)

								if token_ids_a is not None:
														return [1] + ([0] * len(_SCREAMING_SNAKE_CASE			)) + [1] + ([0] * len(_SCREAMING_SNAKE_CASE			))
								return [1] + ([0] * len(_SCREAMING_SNAKE_CASE			))





import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases




def        _SCREAMING_SNAKE_CASE      (		snake_case			,     snake_case       )						->				int:
						assert isinstance(snake_case			,     snake_case       )
						assert dataset.num_rows == 4
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_1", "col_2", "col_3"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("""keep_in_memory"""			,     [False, True]       )
def        _SCREAMING_SNAKE_CASE      (		snake_case			,     snake_case			,     snake_case       )						->				Optional[int]:
						_UpperCAmelCase      =			tmp_path / """cache"""
						_UpperCAmelCase      =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												_UpperCAmelCase      =			JsonDatasetReader(snake_case			,     cache_dir=snake_case			,     keep_in_memory=snake_case       ).read()
						_check_json_dataset(snake_case			,     snake_case       )




@pytest.mark.parametrize(
    """features"""			,     [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]			,     )
def        _SCREAMING_SNAKE_CASE      (		snake_case			,     snake_case			,     snake_case       )						->				Optional[Any]:
						_UpperCAmelCase      =			tmp_path / """cache"""
						_UpperCAmelCase      =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						_UpperCAmelCase      =			features.copy() if features else default_expected_features
						_UpperCAmelCase      =			(
						    Features({feature: Value(snake_case       ) for feature, dtype in features.items()}       ) if features is not None else None
						)
						_UpperCAmelCase      =			JsonDatasetReader(snake_case			,     features=snake_case			,     cache_dir=snake_case       ).read()
						_check_json_dataset(snake_case			,     snake_case       )




@pytest.mark.parametrize(
    """features"""			,     [
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ]			,     )
def        _SCREAMING_SNAKE_CASE      (		snake_case			,     snake_case			,     snake_case       )						->				int:
						_UpperCAmelCase      =			tmp_path / """cache"""
						_UpperCAmelCase      =			{"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""}
						_UpperCAmelCase      =			features.copy() if features else default_expected_features
						_UpperCAmelCase      =			(
						    Features({feature: Value(snake_case       ) for feature, dtype in features.items()}       ) if features is not None else None
						)
						_UpperCAmelCase      =			JsonDatasetReader(snake_case			,     features=snake_case			,     cache_dir=snake_case       ).read()
						assert isinstance(snake_case			,     snake_case       )
						assert dataset.num_rows == 2
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_3", "col_1", "col_2"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype




def        _SCREAMING_SNAKE_CASE      (		snake_case			,     snake_case       )						->				Optional[int]:
						# jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
						_UpperCAmelCase      =			{"""col_2""": """int64""", """col_3""": """float64""", """col_1""": """string"""}
						_UpperCAmelCase      =			features.copy()
						_UpperCAmelCase      =			(
						    Features({feature: Value(snake_case       ) for feature, dtype in features.items()}       ) if features is not None else None
						)
						_UpperCAmelCase      =			tmp_path / """cache"""
						_UpperCAmelCase      =			JsonDatasetReader(snake_case			,     features=snake_case			,     cache_dir=snake_case       ).read()
						assert isinstance(snake_case			,     snake_case       )
						assert dataset.num_rows == 2
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_2", "col_3", "col_1"]
						for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("""split"""			,     [None, NamedSplit("""train"""       ), """train""", """test"""]       )
def        _SCREAMING_SNAKE_CASE      (		snake_case			,     snake_case			,     snake_case       )						->				List[Any]:
						_UpperCAmelCase      =			tmp_path / """cache"""
						_UpperCAmelCase      =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						_UpperCAmelCase      =			JsonDatasetReader(snake_case			,     cache_dir=snake_case			,     split=snake_case       ).read()
						_check_json_dataset(snake_case			,     snake_case       )
						assert dataset.split == split if split else "train"




@pytest.mark.parametrize("""path_type"""			,     [str, list]       )
def        _SCREAMING_SNAKE_CASE      (		snake_case			,     snake_case			,     snake_case       )						->				Any:
						if issubclass(snake_case			,     snake_case       ):
												_UpperCAmelCase      =			jsonl_path
						elif issubclass(snake_case			,     snake_case       ):
												_UpperCAmelCase      =			[jsonl_path]
						_UpperCAmelCase      =			tmp_path / """cache"""
						_UpperCAmelCase      =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						_UpperCAmelCase      =			JsonDatasetReader(snake_case			,     cache_dir=snake_case       ).read()
						_check_json_dataset(snake_case			,     snake_case       )




def        _SCREAMING_SNAKE_CASE      (		snake_case			,     snake_case			,     snake_case=("train",)       )						->				str:
						assert isinstance(snake_case			,     snake_case       )
						for split in splits:
												_UpperCAmelCase      =			dataset_dict[split]
												assert dataset.num_rows == 4
												assert dataset.num_columns == 3
												assert dataset.column_names == ["col_1", "col_2", "col_3"]
												for feature, expected_dtype in expected_features.items():
																		assert dataset.features[feature].dtype == expected_dtype




@pytest.mark.parametrize("""keep_in_memory"""			,     [False, True]       )
def        _SCREAMING_SNAKE_CASE      (		snake_case			,     snake_case			,     snake_case       )						->				str:
						_UpperCAmelCase      =			tmp_path / """cache"""
						_UpperCAmelCase      =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
												_UpperCAmelCase      =			JsonDatasetReader({"""train""": jsonl_path}			,     cache_dir=snake_case			,     keep_in_memory=snake_case       ).read()
						_check_json_datasetdict(snake_case			,     snake_case       )




@pytest.mark.parametrize(
    """features"""			,     [
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]			,     )
def        _SCREAMING_SNAKE_CASE      (		snake_case			,     snake_case			,     snake_case       )						->				Tuple:
						_UpperCAmelCase      =			tmp_path / """cache"""
						_UpperCAmelCase      =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						_UpperCAmelCase      =			features.copy() if features else default_expected_features
						_UpperCAmelCase      =			(
						    Features({feature: Value(snake_case       ) for feature, dtype in features.items()}       ) if features is not None else None
						)
						_UpperCAmelCase      =			JsonDatasetReader({"""train""": jsonl_path}			,     features=snake_case			,     cache_dir=snake_case       ).read()
						_check_json_datasetdict(snake_case			,     snake_case       )




@pytest.mark.parametrize("""split"""			,     [None, NamedSplit("""train"""       ), """train""", """test"""]       )
def        _SCREAMING_SNAKE_CASE      (		snake_case			,     snake_case			,     snake_case       )						->				Optional[Any]:
						if split:
												_UpperCAmelCase      =			{split: jsonl_path}
						else:
												_UpperCAmelCase      =			"""train"""
												_UpperCAmelCase      =			{"""train""": jsonl_path, """test""": jsonl_path}
						_UpperCAmelCase      =			tmp_path / """cache"""
						_UpperCAmelCase      =			{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
						_UpperCAmelCase      =			JsonDatasetReader(snake_case			,     cache_dir=snake_case       ).read()
						_check_json_datasetdict(snake_case			,     snake_case			,     splits=list(path.keys()       )       )
						assert all(dataset[split].split == split for split in path.keys()       )




def        _SCREAMING_SNAKE_CASE      (		snake_case       )						->				List[Any]:
						return json.load(snake_case       )




def        _SCREAMING_SNAKE_CASE      (		snake_case       )						->				List[Any]:
						return [json.loads(snake_case       ) for line in buffer]




class       _A				:






		@pytest.mark.parametrize("""lines, load_json_function"""						,		[(True, load_json_lines), (False, load_json)]			)
		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			):
								with io.BytesIO() as buffer:
														JsonDatasetWriter(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		lines=_SCREAMING_SNAKE_CASE			).write()
														buffer.seek(0			)
														_UpperCAmelCase      =			load_json_function(_SCREAMING_SNAKE_CASE			)
								assert isinstance(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)
								assert isinstance(exported_content[0]						,		_SCREAMING_SNAKE_CASE			)
								assert len(_SCREAMING_SNAKE_CASE			) == 10






		@pytest.mark.parametrize(
		    """orient, container, keys, len_at"""						,		[
		        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
		        ("""split""", dict, {"""columns""", """data"""}, """data"""),
		        ("""index""", dict, set("""0123456789"""			), None),
		        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
		        ("""values""", list, None, None),
		        ("""table""", dict, {"""schema""", """data"""}, """data"""),
		    ]						,		)
		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			):
								with io.BytesIO() as buffer:
														JsonDatasetWriter(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		lines=_SCREAMING_SNAKE_CASE						,		orient=_SCREAMING_SNAKE_CASE			).write()
														buffer.seek(0			)
														_UpperCAmelCase      =			load_json(_SCREAMING_SNAKE_CASE			)
								assert isinstance(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)
								if keys:
														if container is dict:
																				assert exported_content.keys() == keys
														else:
																				assert exported_content[0].keys() == keys
								else:
														assert not hasattr(_SCREAMING_SNAKE_CASE						,		"""keys"""			) and not hasattr(exported_content[0]						,		"""keys"""			)
								if len_at:
														assert len(exported_content[len_at]			) == 10
								else:
														assert len(_SCREAMING_SNAKE_CASE			) == 10






		@pytest.mark.parametrize("""lines, load_json_function"""						,		[(True, load_json_lines), (False, load_json)]			)
		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			):
								with io.BytesIO() as buffer:
														JsonDatasetWriter(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		lines=_SCREAMING_SNAKE_CASE						,		num_proc=2			).write()
														buffer.seek(0			)
														_UpperCAmelCase      =			load_json_function(_SCREAMING_SNAKE_CASE			)
								assert isinstance(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)
								assert isinstance(exported_content[0]						,		_SCREAMING_SNAKE_CASE			)
								assert len(_SCREAMING_SNAKE_CASE			) == 10






		@pytest.mark.parametrize(
		    """orient, container, keys, len_at"""						,		[
		        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
		        ("""split""", dict, {"""columns""", """data"""}, """data"""),
		        ("""index""", dict, set("""0123456789"""			), None),
		        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
		        ("""values""", list, None, None),
		        ("""table""", dict, {"""schema""", """data"""}, """data"""),
		    ]						,		)
		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			):
								with io.BytesIO() as buffer:
														JsonDatasetWriter(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		lines=_SCREAMING_SNAKE_CASE						,		orient=_SCREAMING_SNAKE_CASE						,		num_proc=2			).write()
														buffer.seek(0			)
														_UpperCAmelCase      =			load_json(_SCREAMING_SNAKE_CASE			)
								assert isinstance(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			)
								if keys:
														if container is dict:
																				assert exported_content.keys() == keys
														else:
																				assert exported_content[0].keys() == keys
								else:
														assert not hasattr(_SCREAMING_SNAKE_CASE						,		"""keys"""			) and not hasattr(exported_content[0]						,		"""keys"""			)
								if len_at:
														assert len(exported_content[len_at]			) == 10
								else:
														assert len(_SCREAMING_SNAKE_CASE			) == 10






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								with pytest.raises(_SCREAMING_SNAKE_CASE			):
														with io.BytesIO() as buffer:
																				JsonDatasetWriter(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		num_proc=0			)






		@pytest.mark.parametrize("""compression, extension"""						,		[("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")]			)
		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			):
								_UpperCAmelCase      =			tmp_path_factory.mktemp("""data"""			) / F"test.json.{extension}"
								_UpperCAmelCase      =			str(shared_datadir / F"test_file.json.{extension}"			)
								JsonDatasetWriter(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		compression=_SCREAMING_SNAKE_CASE			).write()

								with fsspec.open(_SCREAMING_SNAKE_CASE						,		"""rb"""						,		compression="""infer"""			) as f:
														_UpperCAmelCase      =			f.read()
								with fsspec.open(_SCREAMING_SNAKE_CASE						,		"""rb"""						,		compression="""infer"""			) as f:
														_UpperCAmelCase      =			f.read()
								assert exported_content == original_content