






'''simple docstring'''





import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class  snake_case__      (					UpperCamelCase     ,				unittest.TestCase):
					a_        =					RoCBertTokenizer
					a_        =					None
					a_        =					False
					a_        =					True
					a_        =					filter_non_english




					def 		A      (  self  :		List[str]      )    ->							Optional[int]:
								super().setUp()

								UpperCAmelCase_      :       List[str]          =       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
								UpperCAmelCase_      :       List[str]          =       {}
								UpperCAmelCase_      :       Any          =       {}
								for i, value in enumerate(_A      ):
											UpperCAmelCase_      :       Dict          =       i
											UpperCAmelCase_      :       Dict          =       i
								UpperCAmelCase_      :       Union[str, Any]          =       os.path.join(self.tmpdirname				, VOCAB_FILES_NAMES['''vocab_file''']      )
								UpperCAmelCase_      :       Dict          =       os.path.join(self.tmpdirname				, VOCAB_FILES_NAMES['''word_shape_file''']      )
								UpperCAmelCase_      :       Any          =       os.path.join(self.tmpdirname				, VOCAB_FILES_NAMES['''word_pronunciation_file''']      )
								with open(self.vocab_file				, '''w'''				, encoding='''utf-8'''      ) as vocab_writer:
											vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]      )      )
								with open(self.word_shape_file				, '''w'''				, encoding='''utf-8'''      ) as word_shape_writer:
											json.dump(_A				, _A				, ensure_ascii=_A      )
								with open(self.word_pronunciation_file				, '''w'''				, encoding='''utf-8'''      ) as word_pronunciation_writer:
											json.dump(_A				, _A				, ensure_ascii=_A      )




					def 		A      (  self  :		List[Any]      )    ->							List[Any]:
								UpperCAmelCase_      :       Tuple          =       self.tokenizer_class(self.vocab_file				, self.word_shape_file				, self.word_pronunciation_file      )

								UpperCAmelCase_      :       Tuple          =       tokenizer.tokenize('''你好[SEP]你是谁'''      )
								self.assertListEqual(_A				, ['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_A      )				, [5, 6, 2, 5, 7, 8]      )
								self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(_A      )				, [5, 6, 2, 5, 7, 8]      )
								self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(_A      )				, [5, 6, 2, 5, 7, 8]      )




					def 		A      (  self  :		Optional[Any]      )    ->							Tuple:
								UpperCAmelCase_      :       List[Any]          =       RoCBertBasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''      )				, ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']      )




					def 		A      (  self  :		List[Any]      )    ->							Dict:
								UpperCAmelCase_      :       Dict          =       RoCBertBasicTokenizer(do_lower_case=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      )				, ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )				, ['''hello''']      )




					def 		A      (  self  :		str      )    ->							Tuple:
								UpperCAmelCase_      :       str          =       RoCBertBasicTokenizer(do_lower_case=_A				, strip_accents=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )				, ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )				, ['''h\u00E9llo''']      )




					def 		A      (  self  :		Tuple      )    ->							Any:
								UpperCAmelCase_      :       List[str]          =       RoCBertBasicTokenizer(do_lower_case=_A				, strip_accents=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )				, ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )				, ['''hello''']      )




					def 		A      (  self  :		Tuple      )    ->							List[str]:
								UpperCAmelCase_      :       Any          =       RoCBertBasicTokenizer(do_lower_case=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )				, ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']      )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )				, ['''hello''']      )




					def 		A      (  self  :		List[Any]      )    ->							Union[str, Any]:
								UpperCAmelCase_      :       Any          =       RoCBertBasicTokenizer(do_lower_case=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''      )				, ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )




					def 		A      (  self  :		List[str]      )    ->							List[Any]:
								UpperCAmelCase_      :       List[str]          =       RoCBertBasicTokenizer(do_lower_case=_A				, strip_accents=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )				, ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )




					def 		A      (  self  :		Optional[int]      )    ->							int:
								UpperCAmelCase_      :       List[str]          =       RoCBertBasicTokenizer(do_lower_case=_A				, strip_accents=_A      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''      )				, ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']      )




					def 		A      (  self  :		Any      )    ->							Optional[Any]:
								UpperCAmelCase_      :       Union[str, Any]          =       RoCBertBasicTokenizer(do_lower_case=_A				, never_split=['''[UNK]''']      )

								self.assertListEqual(
								    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''      )				, ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']      )




					def 		A      (  self  :		Optional[Any]      )    ->							str:
								UpperCAmelCase_      :       Any          =       ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

								UpperCAmelCase_      :       Any          =       {}
								for i, token in enumerate(_A      ):
											UpperCAmelCase_      :       Optional[Any]          =       i
								UpperCAmelCase_      :       Any          =       RoCBertWordpieceTokenizer(vocab=_A				, unk_token='''[UNK]'''      )

								self.assertListEqual(tokenizer.tokenize(''''''      )				, []      )

								self.assertListEqual(tokenizer.tokenize('''unwanted running'''      )				, ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']      )

								self.assertListEqual(tokenizer.tokenize('''unwantedX running'''      )				, ['''[UNK]''', '''runn''', '''##ing''']      )




					def 		A      (  self  :		int      )    ->							Optional[Any]:
								self.assertTrue(_is_whitespace(''' '''      )      )
								self.assertTrue(_is_whitespace('''\t'''      )      )
								self.assertTrue(_is_whitespace('''\r'''      )      )
								self.assertTrue(_is_whitespace('''\n'''      )      )
								self.assertTrue(_is_whitespace('''\u00A0'''      )      )

								self.assertFalse(_is_whitespace('''A'''      )      )
								self.assertFalse(_is_whitespace('''-'''      )      )




					def 		A      (  self  :		Union[str, Any]      )    ->							Union[str, Any]:
								self.assertTrue(_is_control('''\u0005'''      )      )

								self.assertFalse(_is_control('''A'''      )      )
								self.assertFalse(_is_control(''' '''      )      )
								self.assertFalse(_is_control('''\t'''      )      )
								self.assertFalse(_is_control('''\r'''      )      )




					def 		A      (  self  :		str      )    ->							List[Any]:
								self.assertTrue(_is_punctuation('''-'''      )      )
								self.assertTrue(_is_punctuation('''$'''      )      )
								self.assertTrue(_is_punctuation('''`'''      )      )
								self.assertTrue(_is_punctuation('''.'''      )      )

								self.assertFalse(_is_punctuation('''A'''      )      )
								self.assertFalse(_is_punctuation(''' '''      )      )




					def 		A      (  self  :		Any      )    ->							str:
								UpperCAmelCase_      :       Dict          =       self.get_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(_A      ) for t in ['''Test''', '''\xad''', '''test''']]				, [['''[UNK]'''], [], ['''[UNK]''']]      )

								if self.test_rust_tokenizer:
											UpperCAmelCase_      :       int          =       self.get_rust_tokenizer()
											self.assertListEqual(
											    [rust_tokenizer.tokenize(_A      ) for t in ['''Test''', '''\xad''', '''test''']]				, [['''[UNK]'''], [], ['''[UNK]''']]      )




					def 		A      (  self  :		str      )    ->							str:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
														UpperCAmelCase_      :       Optional[Any]          =       self.rust_tokenizer_class.from_pretrained(_A				, **_A      )

														UpperCAmelCase_      :       str          =       F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
														UpperCAmelCase_      :       Optional[int]          =       tokenizer_r.encode_plus(
														    _A				, return_attention_mask=_A				, return_token_type_ids=_A				, return_offsets_mapping=_A				, add_special_tokens=_A				, )

														UpperCAmelCase_      :       Optional[Any]          =       tokenizer_r.do_lower_case if hasattr(_A				, '''do_lower_case'''      ) else False
														UpperCAmelCase_      :       Union[str, Any]          =       (
														    [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''A'''),
														        ((1, 2), ''','''),
														        ((3, 5), '''na'''),
														        ((5, 6), '''##ï'''),
														        ((6, 8), '''##ve'''),
														        ((9, 15), tokenizer_r.mask_token),
														        ((16, 21), '''Allen'''),
														        ((21, 23), '''##NL'''),
														        ((23, 24), '''##P'''),
														        ((25, 33), '''sentence'''),
														        ((33, 34), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														    if not do_lower_case
														    else [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''a'''),
														        ((1, 2), ''','''),
														        ((3, 8), '''naive'''),
														        ((9, 15), tokenizer_r.mask_token),
														        ((16, 21), '''allen'''),
														        ((21, 23), '''##nl'''),
														        ((23, 24), '''##p'''),
														        ((25, 33), '''sentence'''),
														        ((33, 34), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														)

														self.assertEqual(
														    [e[1] for e in expected_results]				, tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']      )      )
														self.assertEqual([e[0] for e in expected_results]				, tokens['''offset_mapping''']      )




					def 		A      (  self  :		str      )    ->							Any:
								UpperCAmelCase_      :       Dict          =       ['''的''', '''人''', '''有''']
								UpperCAmelCase_      :       List[str]          =       ''''''.join(_A      )
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
											with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
														UpperCAmelCase_      :       Dict          =       True
														UpperCAmelCase_      :       Dict          =       self.tokenizer_class.from_pretrained(_A				, **_A      )
														UpperCAmelCase_      :       Tuple          =       self.rust_tokenizer_class.from_pretrained(_A				, **_A      )

														UpperCAmelCase_      :       List[Any]          =       tokenizer_p.encode(_A				, add_special_tokens=_A      )
														UpperCAmelCase_      :       Any          =       tokenizer_r.encode(_A				, add_special_tokens=_A      )

														UpperCAmelCase_      :       Any          =       tokenizer_r.convert_ids_to_tokens(_A      )
														UpperCAmelCase_      :       Any          =       tokenizer_p.convert_ids_to_tokens(_A      )

														# it is expected that each Chinese character is not preceded by "##"
														self.assertListEqual(_A				, _A      )
														self.assertListEqual(_A				, _A      )

														UpperCAmelCase_      :       int          =       False
														UpperCAmelCase_      :       Any          =       self.rust_tokenizer_class.from_pretrained(_A				, **_A      )
														UpperCAmelCase_      :       int          =       self.tokenizer_class.from_pretrained(_A				, **_A      )

														UpperCAmelCase_      :       Any          =       tokenizer_r.encode(_A				, add_special_tokens=_A      )
														UpperCAmelCase_      :       Union[str, Any]          =       tokenizer_p.encode(_A				, add_special_tokens=_A      )

														UpperCAmelCase_      :       Optional[int]          =       tokenizer_r.convert_ids_to_tokens(_A      )
														UpperCAmelCase_      :       str          =       tokenizer_p.convert_ids_to_tokens(_A      )

														# it is expected that only the first Chinese character is not preceded by "##".
														UpperCAmelCase_      :       Union[str, Any]          =       [
														    F"##{token}" if idx != 0 else token for idx, token in enumerate(_A      )
														]
														self.assertListEqual(_A				, _A      )
														self.assertListEqual(_A				, _A      )




					@slow
					def 		A      (  self  :		Tuple      )    ->							Tuple:
								UpperCAmelCase_      :       Dict          =       self.tokenizer_class(self.vocab_file				, self.word_shape_file				, self.word_pronunciation_file      )

								UpperCAmelCase_      :       int          =       tokenizer.encode('''你好'''				, add_special_tokens=_A      )
								UpperCAmelCase_      :       Dict          =       tokenizer.encode('''你是谁'''				, add_special_tokens=_A      )

								UpperCAmelCase_      :       str          =       tokenizer.build_inputs_with_special_tokens(_A      )
								UpperCAmelCase_      :       str          =       tokenizer.build_inputs_with_special_tokens(_A				, _A      )

								assert encoded_sentence == [1] + text + [2]
								assert encoded_pair == [1] + text + [2] + text_a + [2]





					def 		A      (  self  :		Tuple      )    ->							Dict:
								UpperCAmelCase_      :       List[str]          =       self.get_tokenizers(do_lower_case=_A      )
								for tokenizer in tokenizers:
											with self.subTest(F"{tokenizer.__class__.__name__}"      ):
														UpperCAmelCase_      :       Tuple          =       '''你好，你是谁'''
														UpperCAmelCase_      :       List[Any]          =       tokenizer.tokenize(_A      )
														UpperCAmelCase_      :       int          =       tokenizer.convert_tokens_to_ids(_A      )
														UpperCAmelCase_      :       Union[str, Any]          =       tokenizer.convert_tokens_to_shape_ids(_A      )
														UpperCAmelCase_      :       Union[str, Any]          =       tokenizer.convert_tokens_to_pronunciation_ids(_A      )
														UpperCAmelCase_      :       Union[str, Any]          =       tokenizer.prepare_for_model(
														    _A				, _A				, _A				, add_special_tokens=_A      )

														UpperCAmelCase_      :       List[str]          =       tokenizer.encode_plus(_A				, add_special_tokens=_A      )

														self.assertEqual(_A				, _A      )










'''simple docstring'''





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_UpperCamelCase						:						str  =					datasets.utils.logging.get_logger(__name__)


@dataclass
class  snake_case__      (					datasets.BuilderConfig):
					a_        =					None
					a_        =					"utf-8"
					a_        =					None
					a_        =					None
					a_        =					True  # deprecated
					a_        =					None  # deprecated
					a_        =					10 << 20  # 10MB
					a_        =					None






class  snake_case__      (					datasets.ArrowBasedBuilder):
					a_        =					JsonConfig




					def 		A      (  self  :		Union[str, Any]      )    ->							Optional[int]:
								if self.config.block_size is not None:
											logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''      )
											UpperCAmelCase_      :       Union[str, Any]          =       self.config.block_size
								if self.config.use_threads is not True:
											logger.warning(
											    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''      )
								if self.config.newlines_in_values is not None:
											raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''      )
								return datasets.DatasetInfo(features=self.config.features      )




					def 		A      (  self  :		Optional[int]				, _A  :		List[str]      )    ->							int:
								if not self.config.data_files:
											raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"      )
								UpperCAmelCase_      :       Optional[Any]          =       dl_manager.download_and_extract(self.config.data_files      )
								if isinstance(_A				, (str, list, tuple)      ):
											UpperCAmelCase_      :       List[Any]          =       data_files
											if isinstance(_A				, _A      ):
														UpperCAmelCase_      :       List[str]          =       [files]
											UpperCAmelCase_      :       List[str]          =       [dl_manager.iter_files(_A      ) for file in files]
											return [datasets.SplitGenerator(name=datasets.Split.TRAIN				, gen_kwargs={'''files''': files}      )]
								UpperCAmelCase_      :       Tuple          =       []
								for split_name, files in data_files.items():
											if isinstance(_A				, _A      ):
														UpperCAmelCase_      :       Optional[Any]          =       [files]
											UpperCAmelCase_      :       List[Any]          =       [dl_manager.iter_files(_A      ) for file in files]
											splits.append(datasets.SplitGenerator(name=_A				, gen_kwargs={'''files''': files}      )      )
								return splits




					def 		A      (  self  :		List[Any]				, _A  :		pa.Table      )    ->							pa.Table:
								if self.config.features is not None:
											# adding missing columns
											for column_name in set(self.config.features      ) - set(pa_table.column_names      ):
														UpperCAmelCase_      :       Union[str, Any]          =       self.config.features.arrow_schema.field(_A      ).type
														UpperCAmelCase_      :       int          =       pa_table.append_column(_A				, pa.array([None] * len(_A      )				, type=_A      )      )
											# more expensive cast to support nested structures with keys in a different order
											# allows str <-> int/float or str to Audio for example
											UpperCAmelCase_      :       Dict          =       table_cast(_A				, self.config.features.arrow_schema      )
								return pa_table





					def 		A      (  self  :		Optional[Any]				, _A  :		List[Any]      )    ->							Optional[int]:
								for file_idx, file in enumerate(itertools.chain.from_iterable(_A      )      ):
											# If the file is one json object and if we need to look at the list of items in one specific field
											if self.config.field is not None:
														with open(_A				, encoding=self.config.encoding				, errors=self.config.encoding_errors      ) as f:
																	UpperCAmelCase_      :       Optional[int]          =       json.load(_A      )

														# We keep only the field we are interested in
														UpperCAmelCase_      :       Any          =       dataset[self.config.field]

														# We accept two format: a list of dicts or a dict of lists
														if isinstance(_A				, (list, tuple)      ):
																	UpperCAmelCase_      :       Optional[int]          =       set().union(*[row.keys() for row in dataset]      )
																	UpperCAmelCase_      :       str          =       {col: [row.get(_A      ) for row in dataset] for col in keys}
														else:
																	UpperCAmelCase_      :       Tuple          =       dataset
														UpperCAmelCase_      :       List[str]          =       pa.Table.from_pydict(_A      )
														yield file_idx, self._cast_table(_A      )

											# If the file has one json object per line
											else:
														with open(_A				, '''rb'''      ) as f:
																	UpperCAmelCase_      :       Tuple          =       0
																	# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																	# Set a default minimum value of 16kB if the chunk size is really small
																	UpperCAmelCase_      :       Optional[Any]          =       max(self.config.chunksize // 32				, 16 << 10      )
																	UpperCAmelCase_      :       Tuple          =       (
																	    self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
																	)
																	while True:
																				UpperCAmelCase_      :       Optional[Any]          =       f.read(self.config.chunksize      )
																				if not batch:
																							break
																				# Finish current line
																				try:
																							batch += f.readline()
																				except (AttributeError, io.UnsupportedOperation):
																							batch += readline(_A      )
																				# PyArrow only accepts utf-8 encoded bytes
																				if self.config.encoding != "utf-8":
																							UpperCAmelCase_      :       List[Any]          =       batch.decode(self.config.encoding				, errors=_A      ).encode('''utf-8'''      )
																				try:
																							while True:
																										try:
																													UpperCAmelCase_      :       int          =       paj.read_json(
																													    io.BytesIO(_A      )				, read_options=paj.ReadOptions(block_size=_A      )      )
																													break
																										except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																													if (
																													    isinstance(_A				, pa.ArrowInvalid      )
																													    and "straddling" not in str(_A      )
																													    or block_size > len(_A      )
																													):
																																raise
																													else:
																																# Increase the block size in case it was too small.
																																# The block size will be reset for the next file.
																																logger.debug(
																																    F"Batch of {len(_A      )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."      )
																																block_size *= 2
																				except pa.ArrowInvalid as e:
																							try:
																										with open(
																										    _A				, encoding=self.config.encoding				, errors=self.config.encoding_errors      ) as f:
																													UpperCAmelCase_      :       List[Any]          =       json.load(_A      )
																							except json.JSONDecodeError:
																										logger.error(F"Failed to read file '{file}' with error {type(_A      )}: {e}"      )
																										raise e
																							# If possible, parse the file as a list of json objects and exit the loop
																							if isinstance(_A				, _A      ):  # list is the only sequence type supported in JSON
																										try:
																													UpperCAmelCase_      :       Optional[int]          =       set().union(*[row.keys() for row in dataset]      )
																													UpperCAmelCase_      :       Any          =       {col: [row.get(_A      ) for row in dataset] for col in keys}
																													UpperCAmelCase_      :       Optional[int]          =       pa.Table.from_pydict(_A      )
																										except (pa.ArrowInvalid, AttributeError) as e:
																													logger.error(F"Failed to read file '{file}' with error {type(_A      )}: {e}"      )
																													raise ValueError(F"Not able to read records in the JSON file at {file}."      ) from None
																										yield file_idx, self._cast_table(_A      )
																										break
																							else:
																										logger.error(F"Failed to read file '{file}' with error {type(_A      )}: {e}"      )
																										raise ValueError(
																										    F"Not able to read records in the JSON file at {file}. "
																										    F"You should probably indicate the field of the JSON file containing your records. "
																										    F"This JSON file contain the following fields: {str(list(dataset.keys()      )      )}. "
																										    F"Select the correct one and provide it as `field='XXX'` to the dataset loading method. "      ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																				yield (file_idx, batch_idx), self._cast_table(_A      )
																				batch_idx += 1


