


import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class 			_a				(     UpperCamelCase__	,      unittest.TestCase     ):
							_lowercase      :     Optional[Any]     =					BertTokenizer
							_lowercase      :     int     =					BertTokenizerFast
							_lowercase      :     Tuple     =					True
							_lowercase      :     Union[str, Any]     =					True
							_lowercase      :     Tuple     =					filter_non_english

							def  lowerCamelCase_					(				self:      List[Any]				)     ->  int:





									"""simple docstring"""







									super().setUp()

									lowercase__				= [
									    '''[UNK]''',
									    '''[CLS]''',
									    '''[SEP]''',
									    '''[PAD]''',
									    '''[MASK]''',
									    '''want''',
									    '''##want''',
									    '''##ed''',
									    '''wa''',
									    '''un''',
									    '''runn''',
									    '''##ing''',
									    ''',''',
									    '''low''',
									    '''lowest''',
									]
									lowercase__				= os.path.join(self.tmpdirname       ,		VOCAB_FILES_NAMES['''vocab_file''']				)
									with open(self.vocab_file       ,		'''w'''       ,		encoding='''utf-8'''				) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]				)				)

							def  lowerCamelCase_					(				self:      Optional[int]       ,		UpperCamelCase_:      Optional[int]				)     ->  Dict:





									"""simple docstring"""







									lowercase__				= '''UNwant\u00E9d,running'''
									lowercase__				= '''unwanted, running'''
									return input_text, output_text

							def  lowerCamelCase_					(				self:      Tuple				)     ->  Optional[int]:





									"""simple docstring"""







									lowercase__				= self.tokenizer_class(self.vocab_file				)

									lowercase__				= tokenizer.tokenize('''UNwant\u00E9d,running'''				)
									self.assertListEqual(UpperCamelCase_       ,		['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']				)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_				)       ,		[9, 6, 7, 12, 10, 11]				)

							def  lowerCamelCase_					(				self:      Tuple				)     ->  List[str]:





									"""simple docstring"""







									if not self.test_rust_tokenizer:
											return

									lowercase__				= self.get_tokenizer()
									lowercase__				= self.get_rust_tokenizer()

									lowercase__				= '''UNwant\u00E9d,running'''

									lowercase__				= tokenizer.tokenize(UpperCamelCase_				)
									lowercase__				= rust_tokenizer.tokenize(UpperCamelCase_				)
									self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

									lowercase__				= tokenizer.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
									lowercase__				= rust_tokenizer.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
									self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

									lowercase__				= self.get_rust_tokenizer()
									lowercase__				= tokenizer.encode(UpperCamelCase_				)
									lowercase__				= rust_tokenizer.encode(UpperCamelCase_				)
									self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

									# With lower casing
									lowercase__				= self.get_tokenizer(do_lower_case=UpperCamelCase_				)
									lowercase__				= self.get_rust_tokenizer(do_lower_case=UpperCamelCase_				)

									lowercase__				= '''UNwant\u00E9d,running'''

									lowercase__				= tokenizer.tokenize(UpperCamelCase_				)
									lowercase__				= rust_tokenizer.tokenize(UpperCamelCase_				)
									self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

									lowercase__				= tokenizer.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
									lowercase__				= rust_tokenizer.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
									self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

									lowercase__				= self.get_rust_tokenizer()
									lowercase__				= tokenizer.encode(UpperCamelCase_				)
									lowercase__				= rust_tokenizer.encode(UpperCamelCase_				)
									self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

							def  lowerCamelCase_					(				self:      Dict				)     ->  List[Any]:





									"""simple docstring"""







									lowercase__				= BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''				)       ,		['''ah''', '''\u535A''', '''\u63A8''', '''zz''']				)

							def  lowerCamelCase_					(				self:      List[Any]				)     ->  Tuple:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''				)       ,		['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,		['''hello''']				)

							def  lowerCamelCase_					(				self:      List[Any]				)     ->  Dict:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_       ,		strip_accents=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,		['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,		['''h\u00E9llo''']				)

							def  lowerCamelCase_					(				self:      Any				)     ->  List[str]:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_       ,		strip_accents=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,		['''hello''']				)

							def  lowerCamelCase_					(				self:      List[Any]				)     ->  Optional[Any]:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,		['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
									self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,		['''hello''']				)

							def  lowerCamelCase_					(				self:      int				)     ->  Union[str, Any]:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''				)       ,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)

							def  lowerCamelCase_					(				self:      str				)     ->  Any:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_       ,		strip_accents=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,		['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)

							def  lowerCamelCase_					(				self:      List[str]				)     ->  Optional[Any]:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_       ,		strip_accents=UpperCamelCase_				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,		['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)

							def  lowerCamelCase_					(				self:      int				)     ->  Tuple:





									"""simple docstring"""







									lowercase__				= BasicTokenizer(do_lower_case=UpperCamelCase_       ,		never_split=['''[UNK]''']				)

									self.assertListEqual(
									    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''				)       ,		['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']				)

							def  lowerCamelCase_					(				self:      Optional[int]				)     ->  Union[str, Any]:





									"""simple docstring"""







									lowercase__				= BasicTokenizer()
									lowercase__				= '''a\n\'ll !!to?\'d of, can\'t.'''
									lowercase__				= ['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
									self.assertListEqual(tokenizer.tokenize(UpperCamelCase_				)       ,		UpperCamelCase_				)

							def  lowerCamelCase_					(				self:      Dict				)     ->  List[Any]:





									"""simple docstring"""







									lowercase__				= ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

									lowercase__				= {}
									for i, token in enumerate(UpperCamelCase_				):
											lowercase__				= i
									lowercase__				= WordpieceTokenizer(vocab=UpperCamelCase_       ,		unk_token='''[UNK]'''				)

									self.assertListEqual(tokenizer.tokenize(''''''				)       ,		[]				)

									self.assertListEqual(tokenizer.tokenize('''unwanted running'''				)       ,		['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']				)

									self.assertListEqual(tokenizer.tokenize('''unwantedX running'''				)       ,		['''[UNK]''', '''runn''', '''##ing''']				)

							def  lowerCamelCase_					(				self:      str				)     ->  int:





									"""simple docstring"""







									self.assertTrue(_is_whitespace(''' '''				)				)
									self.assertTrue(_is_whitespace('''\t'''				)				)
									self.assertTrue(_is_whitespace('''\r'''				)				)
									self.assertTrue(_is_whitespace('''\n'''				)				)
									self.assertTrue(_is_whitespace('''\u00A0'''				)				)

									self.assertFalse(_is_whitespace('''A'''				)				)
									self.assertFalse(_is_whitespace('''-'''				)				)

							def  lowerCamelCase_					(				self:      str				)     ->  Optional[int]:





									"""simple docstring"""







									self.assertTrue(_is_control('''\u0005'''				)				)

									self.assertFalse(_is_control('''A'''				)				)
									self.assertFalse(_is_control(''' '''				)				)
									self.assertFalse(_is_control('''\t'''				)				)
									self.assertFalse(_is_control('''\r'''				)				)

							def  lowerCamelCase_					(				self:      Optional[Any]				)     ->  str:





									"""simple docstring"""







									self.assertTrue(_is_punctuation('''-'''				)				)
									self.assertTrue(_is_punctuation('''$'''				)				)
									self.assertTrue(_is_punctuation('''`'''				)				)
									self.assertTrue(_is_punctuation('''.'''				)				)

									self.assertFalse(_is_punctuation('''A'''				)				)
									self.assertFalse(_is_punctuation(''' '''				)				)

							def  lowerCamelCase_					(				self:      Dict				)     ->  Dict:





									"""simple docstring"""







									lowercase__				= self.get_tokenizer()
									lowercase__				= self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(UpperCamelCase_				) for t in ['''Test''', '''\xad''', '''test''']]       ,		[['''[UNK]'''], [], ['''[UNK]''']]				)

									self.assertListEqual(
									    [rust_tokenizer.tokenize(UpperCamelCase_				) for t in ['''Test''', '''\xad''', '''test''']]       ,		[['''[UNK]'''], [], ['''[UNK]''']]				)

							@slow
							def  lowerCamelCase_					(				self:      Union[str, Any]				)     ->  str:





									"""simple docstring"""







									lowercase__				= self.tokenizer_class.from_pretrained('''bert-base-uncased'''				)

									lowercase__				= tokenizer.encode('''sequence builders'''       ,		add_special_tokens=UpperCamelCase_				)
									lowercase__				= tokenizer.encode('''multi-sequence build'''       ,		add_special_tokens=UpperCamelCase_				)

									lowercase__				= tokenizer.build_inputs_with_special_tokens(UpperCamelCase_				)
									lowercase__				= tokenizer.build_inputs_with_special_tokens(UpperCamelCase_       ,		UpperCamelCase_				)

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]

							def  lowerCamelCase_					(				self:      List[Any]				)     ->  Union[str, Any]:





									"""simple docstring"""







									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'				):
													lowercase__				= self.rust_tokenizer_class.from_pretrained(UpperCamelCase_       ,		**UpperCamelCase_				)

													lowercase__				= f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
													lowercase__				= tokenizer_r.encode_plus(
													    UpperCamelCase_       ,		return_attention_mask=UpperCamelCase_       ,		return_token_type_ids=UpperCamelCase_       ,		return_offsets_mapping=UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_       ,		)

													lowercase__				= tokenizer_r.do_lower_case if hasattr(UpperCamelCase_       ,		'''do_lower_case'''				) else False
													lowercase__				= (
													    [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''A'''),
													        ((1, 2), ''','''),
													        ((3, 5), '''na'''),
													        ((5, 6), '''##ï'''),
													        ((6, 8), '''##ve'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''Allen'''),
													        ((21, 23), '''##NL'''),
													        ((23, 24), '''##P'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													    if not do_lower_case
													    else [
													        ((0, 0), tokenizer_r.cls_token),
													        ((0, 1), '''a'''),
													        ((1, 2), ''','''),
													        ((3, 8), '''naive'''),
													        ((9, 15), tokenizer_r.mask_token),
													        ((16, 21), '''allen'''),
													        ((21, 23), '''##nl'''),
													        ((23, 24), '''##p'''),
													        ((25, 33), '''sentence'''),
													        ((33, 34), '''.'''),
													        ((0, 0), tokenizer_r.sep_token),
													    ]
													)

													self.assertEqual(
													    [e[1] for e in expected_results]       ,		tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']				)				)
													self.assertEqual([e[0] for e in expected_results]       ,		tokens['''offset_mapping''']				)


							def  lowerCamelCase_					(				self:      Any				)     ->  Any:





									"""simple docstring"""







									lowercase__				= ['''的''', '''人''', '''有''']
									lowercase__				= ''''''.join(UpperCamelCase_				)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'				):
													lowercase__				= True
													lowercase__				= self.tokenizer_class.from_pretrained(UpperCamelCase_       ,		**UpperCamelCase_				)
													lowercase__				= self.rust_tokenizer_class.from_pretrained(UpperCamelCase_       ,		**UpperCamelCase_				)

													lowercase__				= tokenizer_p.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
													lowercase__				= tokenizer_r.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)

													lowercase__				= tokenizer_r.convert_ids_to_tokens(UpperCamelCase_				)
													lowercase__				= tokenizer_p.convert_ids_to_tokens(UpperCamelCase_				)

													# it is expected that each Chinese character is not preceded by "##"
													self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)
													self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)

													lowercase__				= False
													lowercase__				= self.rust_tokenizer_class.from_pretrained(UpperCamelCase_       ,		**UpperCamelCase_				)
													lowercase__				= self.tokenizer_class.from_pretrained(UpperCamelCase_       ,		**UpperCamelCase_				)

													lowercase__				= tokenizer_r.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)
													lowercase__				= tokenizer_p.encode(UpperCamelCase_       ,		add_special_tokens=UpperCamelCase_				)

													lowercase__				= tokenizer_r.convert_ids_to_tokens(UpperCamelCase_				)
													lowercase__				= tokenizer_p.convert_ids_to_tokens(UpperCamelCase_				)

													# it is expected that only the first Chinese character is not preceded by "##".
													lowercase__				= [
													    f'##{token}' if idx != 0 else token for idx, token in enumerate(UpperCamelCase_				)
													]
													self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)
													self.assertListEqual(UpperCamelCase_       ,		UpperCamelCase_				)










'''simple docstring'''





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_UpperCamelCase						:						str  =					datasets.utils.logging.get_logger(__name__)


@dataclass
class  snake_case__      (					datasets.BuilderConfig):
					a_        =					None
					a_        =					"utf-8"
					a_        =					None
					a_        =					None
					a_        =					True  # deprecated
					a_        =					None  # deprecated
					a_        =					10 << 20  # 10MB
					a_        =					None






class  snake_case__      (					datasets.ArrowBasedBuilder):
					a_        =					JsonConfig




					def 		A      (  self  :		Union[str, Any]      )    ->							Optional[int]:
								if self.config.block_size is not None:
											logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''      )
											UpperCAmelCase_      :       Union[str, Any]          =       self.config.block_size
								if self.config.use_threads is not True:
											logger.warning(
											    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''      )
								if self.config.newlines_in_values is not None:
											raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''      )
								return datasets.DatasetInfo(features=self.config.features      )




					def 		A      (  self  :		Optional[int]				, _A  :		List[str]      )    ->							int:
								if not self.config.data_files:
											raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"      )
								UpperCAmelCase_      :       Optional[Any]          =       dl_manager.download_and_extract(self.config.data_files      )
								if isinstance(_A				, (str, list, tuple)      ):
											UpperCAmelCase_      :       List[Any]          =       data_files
											if isinstance(_A				, _A      ):
														UpperCAmelCase_      :       List[str]          =       [files]
											UpperCAmelCase_      :       List[str]          =       [dl_manager.iter_files(_A      ) for file in files]
											return [datasets.SplitGenerator(name=datasets.Split.TRAIN				, gen_kwargs={'''files''': files}      )]
								UpperCAmelCase_      :       Tuple          =       []
								for split_name, files in data_files.items():
											if isinstance(_A				, _A      ):
														UpperCAmelCase_      :       Optional[Any]          =       [files]
											UpperCAmelCase_      :       List[Any]          =       [dl_manager.iter_files(_A      ) for file in files]
											splits.append(datasets.SplitGenerator(name=_A				, gen_kwargs={'''files''': files}      )      )
								return splits




					def 		A      (  self  :		List[Any]				, _A  :		pa.Table      )    ->							pa.Table:
								if self.config.features is not None:
											# adding missing columns
											for column_name in set(self.config.features      ) - set(pa_table.column_names      ):
														UpperCAmelCase_      :       Union[str, Any]          =       self.config.features.arrow_schema.field(_A      ).type
														UpperCAmelCase_      :       int          =       pa_table.append_column(_A				, pa.array([None] * len(_A      )				, type=_A      )      )
											# more expensive cast to support nested structures with keys in a different order
											# allows str <-> int/float or str to Audio for example
											UpperCAmelCase_      :       Dict          =       table_cast(_A				, self.config.features.arrow_schema      )
								return pa_table





					def 		A      (  self  :		Optional[Any]				, _A  :		List[Any]      )    ->							Optional[int]:
								for file_idx, file in enumerate(itertools.chain.from_iterable(_A      )      ):
											# If the file is one json object and if we need to look at the list of items in one specific field
											if self.config.field is not None:
														with open(_A				, encoding=self.config.encoding				, errors=self.config.encoding_errors      ) as f:
																	UpperCAmelCase_      :       Optional[int]          =       json.load(_A      )

														# We keep only the field we are interested in
														UpperCAmelCase_      :       Any          =       dataset[self.config.field]

														# We accept two format: a list of dicts or a dict of lists
														if isinstance(_A				, (list, tuple)      ):
																	UpperCAmelCase_      :       Optional[int]          =       set().union(*[row.keys() for row in dataset]      )
																	UpperCAmelCase_      :       str          =       {col: [row.get(_A      ) for row in dataset] for col in keys}
														else:
																	UpperCAmelCase_      :       Tuple          =       dataset
														UpperCAmelCase_      :       List[str]          =       pa.Table.from_pydict(_A      )
														yield file_idx, self._cast_table(_A      )

											# If the file has one json object per line
											else:
														with open(_A				, '''rb'''      ) as f:
																	UpperCAmelCase_      :       Tuple          =       0
																	# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																	# Set a default minimum value of 16kB if the chunk size is really small
																	UpperCAmelCase_      :       Optional[Any]          =       max(self.config.chunksize // 32				, 16 << 10      )
																	UpperCAmelCase_      :       Tuple          =       (
																	    self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
																	)
																	while True:
																				UpperCAmelCase_      :       Optional[Any]          =       f.read(self.config.chunksize      )
																				if not batch:
																							break
																				# Finish current line
																				try:
																							batch += f.readline()
																				except (AttributeError, io.UnsupportedOperation):
																							batch += readline(_A      )
																				# PyArrow only accepts utf-8 encoded bytes
																				if self.config.encoding != "utf-8":
																							UpperCAmelCase_      :       List[Any]          =       batch.decode(self.config.encoding				, errors=_A      ).encode('''utf-8'''      )
																				try:
																							while True:
																										try:
																													UpperCAmelCase_      :       int          =       paj.read_json(
																													    io.BytesIO(_A      )				, read_options=paj.ReadOptions(block_size=_A      )      )
																													break
																										except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																													if (
																													    isinstance(_A				, pa.ArrowInvalid      )
																													    and "straddling" not in str(_A      )
																													    or block_size > len(_A      )
																													):
																																raise
																													else:
																																# Increase the block size in case it was too small.
																																# The block size will be reset for the next file.
																																logger.debug(
																																    F"Batch of {len(_A      )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."      )
																																block_size *= 2
																				except pa.ArrowInvalid as e:
																							try:
																										with open(
																										    _A				, encoding=self.config.encoding				, errors=self.config.encoding_errors      ) as f:
																													UpperCAmelCase_      :       List[Any]          =       json.load(_A      )
																							except json.JSONDecodeError:
																										logger.error(F"Failed to read file '{file}' with error {type(_A      )}: {e}"      )
																										raise e
																							# If possible, parse the file as a list of json objects and exit the loop
																							if isinstance(_A				, _A      ):  # list is the only sequence type supported in JSON
																										try:
																													UpperCAmelCase_      :       Optional[int]          =       set().union(*[row.keys() for row in dataset]      )
																													UpperCAmelCase_      :       Any          =       {col: [row.get(_A      ) for row in dataset] for col in keys}
																													UpperCAmelCase_      :       Optional[int]          =       pa.Table.from_pydict(_A      )
																										except (pa.ArrowInvalid, AttributeError) as e:
																													logger.error(F"Failed to read file '{file}' with error {type(_A      )}: {e}"      )
																													raise ValueError(F"Not able to read records in the JSON file at {file}."      ) from None
																										yield file_idx, self._cast_table(_A      )
																										break
																							else:
																										logger.error(F"Failed to read file '{file}' with error {type(_A      )}: {e}"      )
																										raise ValueError(
																										    F"Not able to read records in the JSON file at {file}. "
																										    F"You should probably indicate the field of the JSON file containing your records. "
																										    F"This JSON file contain the following fields: {str(list(dataset.keys()      )      )}. "
																										    F"Select the correct one and provide it as `field='XXX'` to the dataset loading method. "      ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																				yield (file_idx, batch_idx), self._cast_table(_A      )
																				batch_idx += 1


