
"""simple docstring"""




import unittest

from transformers import SqueezeBertConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
					    SqueezeBertForMaskedLM,
					    SqueezeBertForMultipleChoice,
					    SqueezeBertForQuestionAnswering,
					    SqueezeBertForSequenceClassification,
					    SqueezeBertForTokenClassification,
					    SqueezeBertModel,
					)



class       lowerCAmelCase_		(a__    ):


				"""simple docstring"""



				def __init__(self  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__=13  ,			SCREAMING_SNAKE_CASE__=7  ,			SCREAMING_SNAKE_CASE__=True  ,			SCREAMING_SNAKE_CASE__=True  ,			SCREAMING_SNAKE_CASE__=False  ,			SCREAMING_SNAKE_CASE__=True  ,			SCREAMING_SNAKE_CASE__=99  ,			SCREAMING_SNAKE_CASE__=32  ,			SCREAMING_SNAKE_CASE__=5  ,			SCREAMING_SNAKE_CASE__=4  ,			SCREAMING_SNAKE_CASE__=64  ,			SCREAMING_SNAKE_CASE__="gelu"  ,			SCREAMING_SNAKE_CASE__=0.1  ,			SCREAMING_SNAKE_CASE__=0.1  ,			SCREAMING_SNAKE_CASE__=5_12  ,			SCREAMING_SNAKE_CASE__=16  ,			SCREAMING_SNAKE_CASE__=2  ,			SCREAMING_SNAKE_CASE__=0.02  ,			SCREAMING_SNAKE_CASE__=3  ,			SCREAMING_SNAKE_CASE__=4  ,			SCREAMING_SNAKE_CASE__=None  ,			SCREAMING_SNAKE_CASE__=2  ,			SCREAMING_SNAKE_CASE__=2  ,			SCREAMING_SNAKE_CASE__=2  ,			SCREAMING_SNAKE_CASE__=2  ,			SCREAMING_SNAKE_CASE__=4  ,			SCREAMING_SNAKE_CASE__=1  ,			)   ->						Tuple:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			Tuple	 =  parent
											SCREAMING_SNAKE_CASE__		:			List[Any]	 =  batch_size
											SCREAMING_SNAKE_CASE__		:			Any	 =  seq_length
											SCREAMING_SNAKE_CASE__		:			int	 =  is_training
											SCREAMING_SNAKE_CASE__		:			List[Any]	 =  use_input_mask
											SCREAMING_SNAKE_CASE__		:			Tuple	 =  use_token_type_ids
											SCREAMING_SNAKE_CASE__		:			Dict	 =  use_labels
											SCREAMING_SNAKE_CASE__		:			str	 =  vocab_size
											SCREAMING_SNAKE_CASE__		:			Optional[Any]	 =  hidden_size
											SCREAMING_SNAKE_CASE__		:			Any	 =  num_hidden_layers
											SCREAMING_SNAKE_CASE__		:			Optional[int]	 =  num_attention_heads
											SCREAMING_SNAKE_CASE__		:			List[Any]	 =  intermediate_size
											SCREAMING_SNAKE_CASE__		:			str	 =  hidden_act
											SCREAMING_SNAKE_CASE__		:			Any	 =  hidden_dropout_prob
											SCREAMING_SNAKE_CASE__		:			str	 =  attention_probs_dropout_prob
											SCREAMING_SNAKE_CASE__		:			List[Any]	 =  max_position_embeddings
											SCREAMING_SNAKE_CASE__		:			Optional[Any]	 =  type_vocab_size
											SCREAMING_SNAKE_CASE__		:			Union[str, Any]	 =  type_sequence_label_size
											SCREAMING_SNAKE_CASE__		:			Tuple	 =  initializer_range
											SCREAMING_SNAKE_CASE__		:			Any	 =  num_labels
											SCREAMING_SNAKE_CASE__		:			int	 =  num_choices
											SCREAMING_SNAKE_CASE__		:			Any	 =  scope
											SCREAMING_SNAKE_CASE__		:			Dict	 =  q_groups
											SCREAMING_SNAKE_CASE__		:			str	 =  k_groups
											SCREAMING_SNAKE_CASE__		:			Optional[int]	 =  v_groups
											SCREAMING_SNAKE_CASE__		:			int	 =  post_attention_groups
											SCREAMING_SNAKE_CASE__		:			Union[str, Any]	 =  intermediate_groups
											SCREAMING_SNAKE_CASE__		:			List[str]	 =  output_groups



				def       __magic_name__       (self       )   ->						int:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			Optional[Any]	 =  ids_tensor([self.batch_size, self.seq_length]  ,			self.vocab_size       )

											SCREAMING_SNAKE_CASE__		:			int	 =  None
											if self.use_input_mask:
																		SCREAMING_SNAKE_CASE__		:			Dict	 =  random_attention_mask([self.batch_size, self.seq_length]       )

											SCREAMING_SNAKE_CASE__		:			List[Any]	 =  None
											SCREAMING_SNAKE_CASE__		:			int	 =  None
											SCREAMING_SNAKE_CASE__		:			Optional[Any]	 =  None
											if self.use_labels:
																		SCREAMING_SNAKE_CASE__		:			Dict	 =  ids_tensor([self.batch_size]  ,			self.type_sequence_label_size       )
																		SCREAMING_SNAKE_CASE__		:			Optional[Any]	 =  ids_tensor([self.batch_size, self.seq_length]  ,			self.num_labels       )
																		SCREAMING_SNAKE_CASE__		:			List[Any]	 =  ids_tensor([self.batch_size]  ,			self.num_choices       )

											SCREAMING_SNAKE_CASE__		:			str	 =  self.get_config()

											return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels



				def       __magic_name__       (self       )   ->						Union[str, Any]:



											"""simple docstring"""




											return SqueezeBertConfig(
											    embedding_size=self.hidden_size  ,			vocab_size=self.vocab_size  ,			hidden_size=self.hidden_size  ,			num_hidden_layers=self.num_hidden_layers  ,			num_attention_heads=self.num_attention_heads  ,			intermediate_size=self.intermediate_size  ,			hidden_act=self.hidden_act  ,			attention_probs_dropout_prob=self.hidden_dropout_prob  ,			attention_dropout=self.attention_probs_dropout_prob  ,			max_position_embeddings=self.max_position_embeddings  ,			initializer_range=self.initializer_range  ,			q_groups=self.q_groups  ,			k_groups=self.k_groups  ,			v_groups=self.v_groups  ,			post_attention_groups=self.post_attention_groups  ,			intermediate_groups=self.intermediate_groups  ,			output_groups=self.output_groups  ,			)



				def       __magic_name__       (self  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__       )   ->						Dict:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			List[str]	 =  SqueezeBertModel(config=_A       )
											model.to(_A       )
											model.eval()
											SCREAMING_SNAKE_CASE__		:			Optional[int]	 =  model(_A  ,			_A       )
											SCREAMING_SNAKE_CASE__		:			Any	 =  model(_A       )
											self.parent.assertEqual(result.last_hidden_state.shape  ,			(self.batch_size, self.seq_length, self.hidden_size)       )



				def       __magic_name__       (self  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__       )   ->						Optional[int]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			Tuple	 =  SqueezeBertForMaskedLM(config=_A       )
											model.to(_A       )
											model.eval()
											SCREAMING_SNAKE_CASE__		:			Dict	 =  model(_A  ,			attention_mask=_A  ,			labels=_A       )
											self.parent.assertEqual(result.logits.shape  ,			(self.batch_size, self.seq_length, self.vocab_size)       )



				def       __magic_name__       (self  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__       )   ->						int:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			Optional[Any]	 =  SqueezeBertForQuestionAnswering(config=_A       )
											model.to(_A       )
											model.eval()
											SCREAMING_SNAKE_CASE__		:			List[str]	 =  model(
											    _A  ,			attention_mask=_A  ,			start_positions=_A  ,			end_positions=_A       )
											self.parent.assertEqual(result.start_logits.shape  ,			(self.batch_size, self.seq_length)       )
											self.parent.assertEqual(result.end_logits.shape  ,			(self.batch_size, self.seq_length)       )



				def       __magic_name__       (self  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__       )   ->						List[str]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			List[Any]	 =  self.num_labels
											SCREAMING_SNAKE_CASE__		:			Optional[int]	 =  SqueezeBertForSequenceClassification(_A       )
											model.to(_A       )
											model.eval()
											SCREAMING_SNAKE_CASE__		:			List[str]	 =  model(_A  ,			attention_mask=_A  ,			labels=_A       )
											self.parent.assertEqual(result.logits.shape  ,			(self.batch_size, self.num_labels)       )



				def       __magic_name__       (self  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__       )   ->						List[Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			Any	 =  self.num_labels
											SCREAMING_SNAKE_CASE__		:			List[Any]	 =  SqueezeBertForTokenClassification(config=_A       )
											model.to(_A       )
											model.eval()

											SCREAMING_SNAKE_CASE__		:			Tuple	 =  model(_A  ,			attention_mask=_A  ,			labels=_A       )
											self.parent.assertEqual(result.logits.shape  ,			(self.batch_size, self.seq_length, self.num_labels)       )



				def       __magic_name__       (self  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__  ,			SCREAMING_SNAKE_CASE__       )   ->						Optional[int]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			Union[str, Any]	 =  self.num_choices
											SCREAMING_SNAKE_CASE__		:			str	 =  SqueezeBertForMultipleChoice(config=_A       )
											model.to(_A       )
											model.eval()
											SCREAMING_SNAKE_CASE__		:			Dict	 =  input_ids.unsqueeze(1       ).expand(-1  ,			self.num_choices  ,			-1       ).contiguous()
											SCREAMING_SNAKE_CASE__		:			Tuple	 =  input_mask.unsqueeze(1       ).expand(-1  ,			self.num_choices  ,			-1       ).contiguous()
											SCREAMING_SNAKE_CASE__		:			str	 =  model(
											    _A  ,			attention_mask=_A  ,			labels=_A  ,			)
											self.parent.assertEqual(result.logits.shape  ,			(self.batch_size, self.num_choices)       )



				def       __magic_name__       (self       )   ->						List[Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			Tuple	 =  self.prepare_config_and_inputs()
											(SCREAMING_SNAKE_CASE__)		:			Any	 =  config_and_inputs
											SCREAMING_SNAKE_CASE__		:			Tuple	 =  {'''input_ids''': input_ids, '''attention_mask''': input_mask}
											return config, inputs_dict






@require_torch
class       lowerCAmelCase_		(a__					,   a__					,   unittest.TestCase    ):


				"""simple docstring"""



				__UpperCamelCase :		Dict   			=  (
				    (
				        SqueezeBertModel,
				        SqueezeBertForMaskedLM,
				        SqueezeBertForMultipleChoice,
				        SqueezeBertForQuestionAnswering,
				        SqueezeBertForSequenceClassification,
				        SqueezeBertForTokenClassification,
				    )
				    if is_torch_available()
				    else None
				)
				__UpperCamelCase :		Tuple   			=  (
				    {
				        '''feature-extraction''': SqueezeBertModel,
				        '''fill-mask''': SqueezeBertForMaskedLM,
				        '''question-answering''': SqueezeBertForQuestionAnswering,
				        '''text-classification''': SqueezeBertForSequenceClassification,
				        '''token-classification''': SqueezeBertForTokenClassification,
				        '''zero-shot''': SqueezeBertForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				__UpperCamelCase :		int   			=  False
				__UpperCamelCase :		str   			=  True
				__UpperCamelCase :		Optional[int]   			=  False



				def       __magic_name__       (self       )   ->						int:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			Any	 =  SqueezeBertModelTester(self       )
											SCREAMING_SNAKE_CASE__		:			str	 =  ConfigTester(self  ,			config_class=_A  ,			dim=37       )



				def       __magic_name__       (self       )   ->						Dict:



											"""simple docstring"""




											self.config_tester.run_common_tests()



				def       __magic_name__       (self       )   ->						Union[str, Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			Union[str, Any]	 =  self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_squeezebert_model(*_A       )



				def       __magic_name__       (self       )   ->						int:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			Optional[int]	 =  self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_squeezebert_for_masked_lm(*_A       )



				def       __magic_name__       (self       )   ->						List[Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			Dict	 =  self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_squeezebert_for_question_answering(*_A       )



				def       __magic_name__       (self       )   ->						Union[str, Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			List[Any]	 =  self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_squeezebert_for_sequence_classification(*_A       )



				def       __magic_name__       (self       )   ->						List[Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			str	 =  self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_squeezebert_for_token_classification(*_A       )



				def       __magic_name__       (self       )   ->						Any:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			Optional[Any]	 =  self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_squeezebert_for_multiple_choice(*_A       )



				@slow
				def       __magic_name__       (self       )   ->						Union[str, Any]:



											"""simple docstring"""




											for model_name in SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																		SCREAMING_SNAKE_CASE__		:			List[Any]	 =  SqueezeBertModel.from_pretrained(_A       )
																		self.assertIsNotNone(_A       )






@require_sentencepiece
@require_tokenizers
@require_torch
class       lowerCAmelCase_		(unittest.TestCase    ):


				"""simple docstring"""



				@slow
				def       __magic_name__       (self       )   ->						Union[str, Any]:



											"""simple docstring"""




											SCREAMING_SNAKE_CASE__		:			int	 =  SqueezeBertForSequenceClassification.from_pretrained("""squeezebert/squeezebert-mnli"""       )

											SCREAMING_SNAKE_CASE__		:			List[Any]	 =  torch.tensor([[1, 2_94_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 13, 15_88, 2]]       )
											SCREAMING_SNAKE_CASE__		:			List[Any]	 =  model(_A       )[0]
											SCREAMING_SNAKE_CASE__		:			Any	 =  torch.Size((1, 3)       )
											self.assertEqual(output.shape  ,			_A       )
											SCREAMING_SNAKE_CASE__		:			int	 =  torch.tensor([[0.6401, -0.0349, -0.6041]]       )
											self.assertTrue(torch.allclose(_A  ,			_A  ,			atol=1E-4       )       )












'''simple docstring'''





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_UpperCamelCase						:						str  =					datasets.utils.logging.get_logger(__name__)


@dataclass
class  snake_case__      (					datasets.BuilderConfig):
					a_        =					None
					a_        =					"utf-8"
					a_        =					None
					a_        =					None
					a_        =					True  # deprecated
					a_        =					None  # deprecated
					a_        =					10 << 20  # 10MB
					a_        =					None






class  snake_case__      (					datasets.ArrowBasedBuilder):
					a_        =					JsonConfig




					def 		A      (  self  :		Union[str, Any]      )    ->							Optional[int]:
								if self.config.block_size is not None:
											logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''      )
											UpperCAmelCase_      :       Union[str, Any]          =       self.config.block_size
								if self.config.use_threads is not True:
											logger.warning(
											    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''      )
								if self.config.newlines_in_values is not None:
											raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''      )
								return datasets.DatasetInfo(features=self.config.features      )




					def 		A      (  self  :		Optional[int]				, _A  :		List[str]      )    ->							int:
								if not self.config.data_files:
											raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"      )
								UpperCAmelCase_      :       Optional[Any]          =       dl_manager.download_and_extract(self.config.data_files      )
								if isinstance(_A				, (str, list, tuple)      ):
											UpperCAmelCase_      :       List[Any]          =       data_files
											if isinstance(_A				, _A      ):
														UpperCAmelCase_      :       List[str]          =       [files]
											UpperCAmelCase_      :       List[str]          =       [dl_manager.iter_files(_A      ) for file in files]
											return [datasets.SplitGenerator(name=datasets.Split.TRAIN				, gen_kwargs={'''files''': files}      )]
								UpperCAmelCase_      :       Tuple          =       []
								for split_name, files in data_files.items():
											if isinstance(_A				, _A      ):
														UpperCAmelCase_      :       Optional[Any]          =       [files]
											UpperCAmelCase_      :       List[Any]          =       [dl_manager.iter_files(_A      ) for file in files]
											splits.append(datasets.SplitGenerator(name=_A				, gen_kwargs={'''files''': files}      )      )
								return splits




					def 		A      (  self  :		List[Any]				, _A  :		pa.Table      )    ->							pa.Table:
								if self.config.features is not None:
											# adding missing columns
											for column_name in set(self.config.features      ) - set(pa_table.column_names      ):
														UpperCAmelCase_      :       Union[str, Any]          =       self.config.features.arrow_schema.field(_A      ).type
														UpperCAmelCase_      :       int          =       pa_table.append_column(_A				, pa.array([None] * len(_A      )				, type=_A      )      )
											# more expensive cast to support nested structures with keys in a different order
											# allows str <-> int/float or str to Audio for example
											UpperCAmelCase_      :       Dict          =       table_cast(_A				, self.config.features.arrow_schema      )
								return pa_table





					def 		A      (  self  :		Optional[Any]				, _A  :		List[Any]      )    ->							Optional[int]:
								for file_idx, file in enumerate(itertools.chain.from_iterable(_A      )      ):
											# If the file is one json object and if we need to look at the list of items in one specific field
											if self.config.field is not None:
														with open(_A				, encoding=self.config.encoding				, errors=self.config.encoding_errors      ) as f:
																	UpperCAmelCase_      :       Optional[int]          =       json.load(_A      )

														# We keep only the field we are interested in
														UpperCAmelCase_      :       Any          =       dataset[self.config.field]

														# We accept two format: a list of dicts or a dict of lists
														if isinstance(_A				, (list, tuple)      ):
																	UpperCAmelCase_      :       Optional[int]          =       set().union(*[row.keys() for row in dataset]      )
																	UpperCAmelCase_      :       str          =       {col: [row.get(_A      ) for row in dataset] for col in keys}
														else:
																	UpperCAmelCase_      :       Tuple          =       dataset
														UpperCAmelCase_      :       List[str]          =       pa.Table.from_pydict(_A      )
														yield file_idx, self._cast_table(_A      )

											# If the file has one json object per line
											else:
														with open(_A				, '''rb'''      ) as f:
																	UpperCAmelCase_      :       Tuple          =       0
																	# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																	# Set a default minimum value of 16kB if the chunk size is really small
																	UpperCAmelCase_      :       Optional[Any]          =       max(self.config.chunksize // 32				, 16 << 10      )
																	UpperCAmelCase_      :       Tuple          =       (
																	    self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
																	)
																	while True:
																				UpperCAmelCase_      :       Optional[Any]          =       f.read(self.config.chunksize      )
																				if not batch:
																							break
																				# Finish current line
																				try:
																							batch += f.readline()
																				except (AttributeError, io.UnsupportedOperation):
																							batch += readline(_A      )
																				# PyArrow only accepts utf-8 encoded bytes
																				if self.config.encoding != "utf-8":
																							UpperCAmelCase_      :       List[Any]          =       batch.decode(self.config.encoding				, errors=_A      ).encode('''utf-8'''      )
																				try:
																							while True:
																										try:
																													UpperCAmelCase_      :       int          =       paj.read_json(
																													    io.BytesIO(_A      )				, read_options=paj.ReadOptions(block_size=_A      )      )
																													break
																										except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																													if (
																													    isinstance(_A				, pa.ArrowInvalid      )
																													    and "straddling" not in str(_A      )
																													    or block_size > len(_A      )
																													):
																																raise
																													else:
																																# Increase the block size in case it was too small.
																																# The block size will be reset for the next file.
																																logger.debug(
																																    F"Batch of {len(_A      )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."      )
																																block_size *= 2
																				except pa.ArrowInvalid as e:
																							try:
																										with open(
																										    _A				, encoding=self.config.encoding				, errors=self.config.encoding_errors      ) as f:
																													UpperCAmelCase_      :       List[Any]          =       json.load(_A      )
																							except json.JSONDecodeError:
																										logger.error(F"Failed to read file '{file}' with error {type(_A      )}: {e}"      )
																										raise e
																							# If possible, parse the file as a list of json objects and exit the loop
																							if isinstance(_A				, _A      ):  # list is the only sequence type supported in JSON
																										try:
																													UpperCAmelCase_      :       Optional[int]          =       set().union(*[row.keys() for row in dataset]      )
																													UpperCAmelCase_      :       Any          =       {col: [row.get(_A      ) for row in dataset] for col in keys}
																													UpperCAmelCase_      :       Optional[int]          =       pa.Table.from_pydict(_A      )
																										except (pa.ArrowInvalid, AttributeError) as e:
																													logger.error(F"Failed to read file '{file}' with error {type(_A      )}: {e}"      )
																													raise ValueError(F"Not able to read records in the JSON file at {file}."      ) from None
																										yield file_idx, self._cast_table(_A      )
																										break
																							else:
																										logger.error(F"Failed to read file '{file}' with error {type(_A      )}: {e}"      )
																										raise ValueError(
																										    F"Not able to read records in the JSON file at {file}. "
																										    F"You should probably indicate the field of the JSON file containing your records. "
																										    F"This JSON file contain the following fields: {str(list(dataset.keys()      )      )}. "
																										    F"Select the correct one and provide it as `field='XXX'` to the dataset loading method. "      ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																				yield (file_idx, batch_idx), self._cast_table(_A      )
																				batch_idx += 1


