



import collections
import json
import os
import re
from typing import TYPE_CHECKING, List, Optional, Tuple

import numpy as np

from ...tokenization_utils_fast import PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


UpperCAmelCase							 =					logging.get_logger(__name__)

UpperCAmelCase							 =					{"""vocab_file""": """vocab.txt""", """emoji_file""": """emoji.json"""}

UpperCAmelCase							 =					{
    """vocab_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/vocab.txt""",
    },
    """emoji_file""": {
        """abeja/gpt-neox-japanese-2.7b""": """https://huggingface.co/abeja/gpt-neox-japanese-2.7b/resolve/main/emoji.json""",
    },
}

UpperCAmelCase							 =					{
    """abeja/gpt-neox-japanese-2.7b""": 2_048,
}

def 			A_      (	__a     :				Optional[Any] ,		__a     :				Optional[int]		):

				"""simple docstring"""
				with open(__a ,		"""r""" ,		encoding="""utf-8"""		) as f:
								a__				      =		json.loads(f.read()		)

				a__				      =		collections.OrderedDict()
				a__				      =		collections.OrderedDict()
				a__				      =		collections.OrderedDict()
				with open(__a ,		"""r""" ,		encoding="""utf-8"""		) as f:
								a__				      =		f.readlines()
				a__				      =		[[t.rstrip("""\n"""		)] if (t == """,""" or """,""" not in t) else t.rstrip("""\n"""		).split(""","""		) for t in token]
				for idx, b in enumerate(__a		):
								a__				      =		b
								a__				      =		idx
								for wd in b:
												a__				      =		idx

				return vocab, raw_vocab, ids_to_tokens, emoji

class  __snake_case      (   SCREAMING_SNAKE_CASE):







							'''simple docstring'''






							UpperCamelCase__				:      Tuple         =					VOCAB_FILES_NAMES
							UpperCamelCase__				:      List[str]         =					PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase__				:      Dict         =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase__				:      Optional[Any]         =					["""input_ids""", """attention_mask"""]


							def __init__(							self  ,  a_  ,  a_  ,  a_="<|endoftext|>"  ,  a_="<|endoftext|>"  ,  a_="<|startoftext|>"  ,  a_="<|endoftext|>"  ,  a_=False  ,  **a_  ,  ):
											super().__init__(
											    unk_token=a_  ,  pad_token=a_  ,  bos_token=a_  ,  eos_token=a_  ,  do_clean_text=a_  ,  **a_  ,  )
											if not os.path.isfile(a_		):
															raise ValueError(
															    F'''Can\'t find a vocabulary file at path \'{vocab_file}\'. To load the vocabulary from a Google pretrained'''
															    """ model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""		)
											if not os.path.isfile(a_		):
															raise ValueError(
															    F'''Can\'t find a emoji file at path \'{emoji_file}\'. To load the emoji information from a Google'''
															    """ pretrained model use `tokenizer = GPTNeoXJapaneseokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"""		)
											a__				      =		do_clean_text
											a__      ,			a__      ,			a__      ,			a__				      =		load_vocab_and_emoji(a_  ,  a_		)
											a__				      =		SubWordJapaneseTokenizer(
											    vocab=self.vocab  ,  ids_to_tokens=self.ids_to_tokens  ,  emoji=self.emoji		)


							@property
							def _a      (							self		):
											# self.vocab contains support for character fluctuation unique to Japanese, and has a large number of vocab
											return len(self.raw_vocab		)


							def _a      (							self		):
											return dict(self.raw_vocab  ,  **self.added_tokens_encoder		)


							def _a      (							self  ,  a_		):
											return self.subword_tokenizer.tokenize(a_  ,  clean=self.do_clean_text		)


							def _a      (							self  ,  a_		):
											return self.vocab.get(a_  ,  self.vocab.get(self.unk_token		)		)


							def _a      (							self  ,  a_		):
											return self.subword_tokenizer.convert_id_to_token(a_		)


							def _a      (							self  ,  a_		):
											a__				      =		"""""".join(a_		).strip()
											return out_string


							def _a      (							self  ,  a_		):
											a__				      =		[]
											for is_user, text in conversation.iter_texts():
															input_ids.extend(self.encode(a_  ,  add_special_tokens=a_		) + [self.eos_token_id]		)

											if len(a_		) > self.model_max_length:
															a__				      =		input_ids[-self.model_max_length :]
											return input_ids







							def _a      (							self  ,  a_  ,  a_ = None		):
											a__				      =		0
											if os.path.isdir(a_		):
															a__				      =		os.path.join(
															    a_  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
															a__				      =		os.path.join(
															    a_  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""emoji_file"""]		)
											else:
															a__				      =		(
															    (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""vocab_file"""]
															)
															a__				      =		(
															    (filename_prefix + """-""" if filename_prefix else """""") + save_directory + VOCAB_FILES_NAMES["""emoji_file"""]
															)
											with open(a_  ,  """w"""  ,  encoding="""utf-8"""		) as writer:
															for token_index, token in self.ids_to_tokens.items():
																			if index != token_index:
																							logger.warning(
																							    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																							    """ Please check that the vocabulary is not corrupted!"""		)
																							a__				      =		token_index
																			writer.write(""",""".join(a_		) + """\n"""		)
																			index += 1
											with open(a_  ,  """w"""  ,  encoding="""utf-8"""		) as writer:
															json.dump(self.emoji  ,  a_		)
											return vocab_file, emoji_file




class  __snake_case      (   SCREAMING_SNAKE_CASE):







							'''simple docstring'''






							def __init__(							self  ,  a_  ,  a_  ,  a_		):
											a__				      =		vocab  # same as swe
											a__				      =		ids_to_tokens  # same as bpe
											a__				      =		emoji
											a__				      =		np.max([len(a_		) for w in self.vocab.keys()]		)
											a__				      =		re.compile(r"""(https?|ftp)(:\/\/[-_\.!~*\'()a-zA-Z0-9;\/?:\@&=\+$,%#]+)"""		)
											a__				      =		re.compile(r"""[A-Za-z0-9\._+]*@[\-_0-9A-Za-z]+(\.[A-Za-z]+)*"""		)
											a__				      =		re.compile(r"""[\(]{0,1}[0-9]{2,4}[\)\-\(]{0,1}[0-9]{2,4}[\)\-]{0,1}[0-9]{3,4}"""		)
											a__				      =		re.compile(
											    r"""([12]\d{3}[/\-年])*(0?[1-9]|1[0-2])[/\-月]((0?[1-9]|[12][0-9]|3[01])日?)*(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""		)
											a__				      =		re.compile(
											    r"""(明治|大正|昭和|平成|令和|㍾|㍽|㍼|㍻|\u32ff)\d{1,2}年(0?[1-9]|1[0-2])月(0?[1-9]|[12][0-9]|3[01])日(\d{1,2}|:|\d{1,2}時|\d{1,2}分|\(日\)|\(月\)|\(火\)|\(水\)|\(木\)|\(金\)|\(土\)|㈰|㈪|㈫|㈬|㈭|㈮|㈯)*"""		)
											a__				      =		re.compile(
											    r"""((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*億)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*万)*((0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*千)*(0|[1-9]\d*|[1-9]\d{0,2}(,\d{3})+)*(千円|万円|千万円|円|千ドル|万ドル|千万ドル|ドル|千ユーロ|万ユーロ|千万ユーロ|ユーロ)+(\(税込\)|\(税抜\)|\+tax)*"""		)
											a__				      =		"""─━│┃┄┅┆┇┈┉┊┋┌┍┎┏┐┑┒┓└┕┖┗┘┙┚┛├┝┞┟┠┡┢┣┤┥┦┧┨┩┪┫┬┭┮┯┰┱┲┳┴┵┶┷┸┹┺┻┼┽┾┿╀╁╂╃╄╅╆╇╈╉╊╋╌╍╎╏═║╒╓╔╕╖╗╘╙╚╛╜╝╞╟╠╡╢╣╤╥╦╧╨╩╪╫╬╭╮╯╰╱╲╳╴╵╶╷╸╹╺╻╼╽╾╿"""
											a__				      =		"""▀▁▂▃▄▅▆▇█▉▊▋▌▍▎▏▐░▒▓▔▕▖▗▘▙▚▛▜▝▞▟"""
											a__				      =		str.maketrans({k: """<BLOCK>""" for k in keisen + blocks}		)


							def __len__(							self		):
											return len(self.ids_to_tokens		)


							def _a      (							self  ,  a_		):
											a__				      =		self.content_repattera.sub("""<URL>"""  ,  a_		)
											a__				      =		self.content_repattera.sub("""<EMAIL>"""  ,  a_		)
											a__				      =		self.content_repattera.sub("""<TEL>"""  ,  a_		)
											a__				      =		self.content_repattera.sub("""<DATE>"""  ,  a_		)
											a__				      =		self.content_repattera.sub("""<DATE>"""  ,  a_		)
											a__				      =		self.content_repattera.sub("""<PRICE>"""  ,  a_		)
											a__				      =		content.translate(self.content_transa		)
											while "<BLOCK><BLOCK>" in content:
															a__				      =		content.replace("""<BLOCK><BLOCK>"""  ,  """<BLOCK>"""		)
											return content


							def _a      (							self  ,  a_  ,  a_=False		):
											a__				      =		text.replace(""" """  ,  """<SP>"""		)
											a__				      =		text.replace("""　"""  ,  """<SP>"""		)
											a__				      =		text.replace("""\r\n"""  ,  """<BR>"""		)
											a__				      =		text.replace("""\n"""  ,  """<BR>"""		)
											a__				      =		text.replace("""\r"""  ,  """<BR>"""		)
											a__				      =		text.replace("""\t"""  ,  """<TAB>"""		)
											a__				      =		text.replace("""—"""  ,  """ー"""		)
											a__				      =		text.replace("""−"""  ,  """ー"""		)
											for k, v in self.emoji["emoji"].items():
															if k in text:
																			a__				      =		text.replace(a_  ,  a_		)
											if clean:
															a__				      =		self.clean_text(a_		)

											def check_simbol(a_		):
															a__				      =		x.encode()
															if len(a_		) == 1 and len(a_		) == 2:
																			a__				      =		(int(e[0]		) << 8) + int(e[1]		)
																			if (
																			    (c >= 0xc2a1 and c <= 0xc2bf)
																			    or (c >= 0xc780 and c <= 0xc783)
																			    or (c >= 0xcab9 and c <= 0xcbbf)
																			    or (c >= 0xcc80 and c <= 0xcda2)
																			):
																							return True
															return False

											def checkuae(a_		):
															a__				      =		x.encode()
															if len(a_		) == 1 and len(a_		) == 3:
																			a__				      =		(int(e[0]		) << 16) + (int(e[1]		) << 8) + int(e[2]		)
																			if c >= 0xe2_8080 and c <= 0xe2_b07f:
																							return True
															return False

											a__				      =		0
											a__				      =		[]
											while pos < len(a_		):
															a__				      =		min(len(a_		)  ,  pos + self.maxlen + 1		) if text[pos] == """<""" else pos + 3
															a__				      =		[]  # (token_id, token, pos)
															for e in range(a_  ,  a_  ,  -1		):
																			a__				      =		text[pos:e]
																			if wd in self.vocab:
																							if wd[0] == "<" and len(a_		) > 2:
																											a__				      =		[(self.vocab[wd], wd, e)]
																											break
																							else:
																											candidates.append((self.vocab[wd], wd, e)		)
															if len(a_		) > 0:
																			# the smallest token_id is adopted
																			a__      ,			a__      ,			a__				      =		sorted(a_  ,  key=lambda a_		: x[0]		)[0]
																			result.append(a_		)
																			a__				      =		e
															else:
																			a__				      =		pos + 1
																			a__				      =		text[pos:end]
																			if check_simbol(a_		):
																							result.append("""<KIGOU>"""		)
																			elif checkuae(a_		):
																							result.append("""<U2000U2BFF>"""		)
																			else:
																							for i in wd.encode("""utf-8"""		):
																											result.append("""<|byte%d|>""" % i		)
																			a__				      =		end
											return result







							def _a      (							self  ,  a_  ,  a_="\n"		):
											a__				      =		[]
											a__				      =		[]
											a__				      =		self.ids_to_tokens[index][0]
											if word[:6] == "<|byte" and word[-2:] == "|>":
															byte_tokens.append(int(word[6:-2]		)		)
											else:
															if len(a_		) > 0:
																			words.append(bytearray(a_		).decode("""utf-8"""  ,  errors="""replace"""		)		)
																			a__				      =		[]
															if word[:7] == "<|emoji" and word[-2:] == "|>":
																			words.append(self.emoji["""emoji_inv"""][word]		)
															elif word == "<SP>":
																			words.append(""" """		)
															elif word == "<BR>":
																			words.append(a_		)
															elif word == "<TAB>":
																			words.append("""\t"""		)
															elif word == "<BLOCK>":
																			words.append("""▀"""		)
															elif word == "<KIGOU>":
																			words.append("""ǀ"""		)
															elif word == "<U2000U2BFF>":
																			words.append("""‖"""		)
															else:
																			words.append(a_		)
											if len(a_		) > 0:
															words.append(bytearray(a_		).decode("""utf-8"""  ,  errors="""replace"""		)		)
											a__				      =		"""""".join(a_		)
											return text






import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def 			A_      (	__a     :				List[str] ,		__a     :				Optional[int]		):

				"""simple docstring"""
				assert isinstance(__a ,		__a		)
				assert dataset.num_rows == 4
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_1", "col_2", "col_3"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""keep_in_memory""" ,		[False, True]		)
def 			A_      (	__a     :				Tuple ,		__a     :				Dict ,		__a     :				List[str]		):

				"""simple docstring"""
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								a__				      =		JsonDatasetReader(__a ,		cache_dir=__a ,		keep_in_memory=__a		).read()
				_check_json_dataset(__a ,		__a		)

@pytest.mark.parametrize(
    """features""" ,		[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ] ,		)
def 			A_      (	__a     :				int ,		__a     :				Dict ,		__a     :				List[Any]		):

				"""simple docstring"""
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				a__				      =		features.copy() if features else default_expected_features
				a__				      =		(
				    Features({feature: Value(__a		) for feature, dtype in features.items()}		) if features is not None else None
				)
				a__				      =		JsonDatasetReader(__a ,		features=__a ,		cache_dir=__a		).read()
				_check_json_dataset(__a ,		__a		)

@pytest.mark.parametrize(
    """features""" ,		[
        None,
        {"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""},
    ] ,		)
def 			A_      (	__a     :				Tuple ,		__a     :				Tuple ,		__a     :				Tuple		):

				"""simple docstring"""
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_3""": """float64""", """col_1""": """string""", """col_2""": """int64"""}
				a__				      =		features.copy() if features else default_expected_features
				a__				      =		(
				    Features({feature: Value(__a		) for feature, dtype in features.items()}		) if features is not None else None
				)
				a__				      =		JsonDatasetReader(__a ,		features=__a ,		cache_dir=__a		).read()
				assert isinstance(__a ,		__a		)
				assert dataset.num_rows == 2
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_3", "col_1", "col_2"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype

def 			A_      (	__a     :				int ,		__a     :				Optional[Any]		):

				"""simple docstring"""
				# jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
				a__				      =		{"""col_2""": """int64""", """col_3""": """float64""", """col_1""": """string"""}
				a__				      =		features.copy()
				a__				      =		(
				    Features({feature: Value(__a		) for feature, dtype in features.items()}		) if features is not None else None
				)
				a__				      =		tmp_path / """cache"""
				a__				      =		JsonDatasetReader(__a ,		features=__a ,		cache_dir=__a		).read()
				assert isinstance(__a ,		__a		)
				assert dataset.num_rows == 2
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_2", "col_3", "col_1"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""split""" ,		[None, NamedSplit("""train"""		), """train""", """test"""]		)
def 			A_      (	__a     :				List[Any] ,		__a     :				int ,		__a     :				List[str]		):

				"""simple docstring"""
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				a__				      =		JsonDatasetReader(__a ,		cache_dir=__a ,		split=__a		).read()
				_check_json_dataset(__a ,		__a		)
				assert dataset.split == split if split else "train"

@pytest.mark.parametrize("""path_type""" ,		[str, list]		)
def 			A_      (	__a     :				Dict ,		__a     :				Tuple ,		__a     :				List[str]		):

				"""simple docstring"""
				if issubclass(__a ,		__a		):
								a__				      =		jsonl_path
				elif issubclass(__a ,		__a		):
								a__				      =		[jsonl_path]
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				a__				      =		JsonDatasetReader(__a ,		cache_dir=__a		).read()
				_check_json_dataset(__a ,		__a		)

def 			A_      (	__a     :				str ,		__a     :				List[Any] ,		__a     :				Dict=("train",)		):

				"""simple docstring"""
				assert isinstance(__a ,		__a		)
				for split in splits:
								a__				      =		dataset_dict[split]
								assert dataset.num_rows == 4
								assert dataset.num_columns == 3
								assert dataset.column_names == ["col_1", "col_2", "col_3"]
								for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("""keep_in_memory""" ,		[False, True]		)
def 			A_      (	__a     :				int ,		__a     :				int ,		__a     :				Dict		):

				"""simple docstring"""
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								a__				      =		JsonDatasetReader({"""train""": jsonl_path} ,		cache_dir=__a ,		keep_in_memory=__a		).read()
				_check_json_datasetdict(__a ,		__a		)

@pytest.mark.parametrize(
    """features""" ,		[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ] ,		)
def 			A_      (	__a     :				Tuple ,		__a     :				Dict ,		__a     :				Optional[Any]		):

				"""simple docstring"""
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				a__				      =		features.copy() if features else default_expected_features
				a__				      =		(
				    Features({feature: Value(__a		) for feature, dtype in features.items()}		) if features is not None else None
				)
				a__				      =		JsonDatasetReader({"""train""": jsonl_path} ,		features=__a ,		cache_dir=__a		).read()
				_check_json_datasetdict(__a ,		__a		)

@pytest.mark.parametrize("""split""" ,		[None, NamedSplit("""train"""		), """train""", """test"""]		)
def 			A_      (	__a     :				Dict ,		__a     :				Any ,		__a     :				Tuple		):

				"""simple docstring"""
				if split:
								a__				      =		{split: jsonl_path}
				else:
								a__				      =		"""train"""
								a__				      =		{"""train""": jsonl_path, """test""": jsonl_path}
				a__				      =		tmp_path / """cache"""
				a__				      =		{"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
				a__				      =		JsonDatasetReader(__a ,		cache_dir=__a		).read()
				_check_json_datasetdict(__a ,		__a ,		splits=list(path.keys()		)		)
				assert all(dataset[split].split == split for split in path.keys()		)

def 			A_      (	__a     :				Any		):

				"""simple docstring"""
				return json.load(__a		)

def 			A_      (	__a     :				Tuple		):

				"""simple docstring"""
				return [json.loads(__a		) for line in buffer]




class  __snake_case      :







							'''simple docstring'''






							@pytest.mark.parametrize("""lines, load_json_function"""  ,  [(True, load_json_lines), (False, load_json)]		)
							def _a      (							self  ,  a_  ,  a_  ,  a_		):
											with io.BytesIO() as buffer:
															JsonDatasetWriter(a_  ,  a_  ,  lines=a_		).write()
															buffer.seek(0		)
															a__				      =		load_json_function(a_		)
											assert isinstance(a_  ,  a_		)
											assert isinstance(exported_content[0]  ,  a_		)
											assert len(a_		) == 10


							@pytest.mark.parametrize(
							    """orient, container, keys, len_at"""  ,  [
							        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
							        ("""split""", dict, {"""columns""", """data"""}, """data"""),
							        ("""index""", dict, set("""0123456789"""		), None),
							        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
							        ("""values""", list, None, None),
							        ("""table""", dict, {"""schema""", """data"""}, """data"""),
							    ]  ,  )
							def _a      (							self  ,  a_  ,  a_  ,  a_  ,  a_  ,  a_		):
											with io.BytesIO() as buffer:
															JsonDatasetWriter(a_  ,  a_  ,  lines=a_  ,  orient=a_		).write()
															buffer.seek(0		)
															a__				      =		load_json(a_		)
											assert isinstance(a_  ,  a_		)
											if keys:
															if container is dict:
																			assert exported_content.keys() == keys
															else:
																			assert exported_content[0].keys() == keys
											else:
															assert not hasattr(a_  ,  """keys"""		) and not hasattr(exported_content[0]  ,  """keys"""		)
											if len_at:
															assert len(exported_content[len_at]		) == 10
											else:
															assert len(a_		) == 10


							@pytest.mark.parametrize("""lines, load_json_function"""  ,  [(True, load_json_lines), (False, load_json)]		)
							def _a      (							self  ,  a_  ,  a_  ,  a_		):
											with io.BytesIO() as buffer:
															JsonDatasetWriter(a_  ,  a_  ,  lines=a_  ,  num_proc=2		).write()
															buffer.seek(0		)
															a__				      =		load_json_function(a_		)
											assert isinstance(a_  ,  a_		)
											assert isinstance(exported_content[0]  ,  a_		)
											assert len(a_		) == 10


							@pytest.mark.parametrize(
							    """orient, container, keys, len_at"""  ,  [
							        ("""records""", list, {"""tokens""", """labels""", """answers""", """id"""}, None),
							        ("""split""", dict, {"""columns""", """data"""}, """data"""),
							        ("""index""", dict, set("""0123456789"""		), None),
							        ("""columns""", dict, {"""tokens""", """labels""", """answers""", """id"""}, """tokens"""),
							        ("""values""", list, None, None),
							        ("""table""", dict, {"""schema""", """data"""}, """data"""),
							    ]  ,  )
							def _a      (							self  ,  a_  ,  a_  ,  a_  ,  a_  ,  a_		):
											with io.BytesIO() as buffer:
															JsonDatasetWriter(a_  ,  a_  ,  lines=a_  ,  orient=a_  ,  num_proc=2		).write()
															buffer.seek(0		)
															a__				      =		load_json(a_		)
											assert isinstance(a_  ,  a_		)
											if keys:
															if container is dict:
																			assert exported_content.keys() == keys
															else:
																			assert exported_content[0].keys() == keys
											else:
															assert not hasattr(a_  ,  """keys"""		) and not hasattr(exported_content[0]  ,  """keys"""		)
											if len_at:
															assert len(exported_content[len_at]		) == 10
											else:
															assert len(a_		) == 10


							def _a      (							self  ,  a_		):
											with pytest.raises(a_		):
															with io.BytesIO() as buffer:
																			JsonDatasetWriter(a_  ,  a_  ,  num_proc=0		)







							@pytest.mark.parametrize("""compression, extension"""  ,  [("""gzip""", """gz"""), ("""bz2""", """bz2"""), ("""xz""", """xz""")]		)
							def _a      (							self  ,  a_  ,  a_  ,  a_  ,  a_  ,  a_		):
											a__				      =		tmp_path_factory.mktemp("""data"""		) / F'''test.json.{extension}'''
											a__				      =		str(shared_datadir / F'''test_file.json.{extension}'''		)
											JsonDatasetWriter(a_  ,  a_  ,  compression=a_		).write()

											with fsspec.open(a_  ,  """rb"""  ,  compression="""infer"""		) as f:
															a__				      =		f.read()
											with fsspec.open(a_  ,  """rb"""  ,  compression="""infer"""		) as f:
															a__				      =		f.read()
											assert exported_content == original_content

