
import inspect
import unittest

import numpy as np

from tests.test_modeling_common import floats_tensor
from transformers import DetrConfig, MaskFormerConfig, SwinConfig, is_torch_available, is_vision_available
from transformers.testing_utils import require_torch, require_torch_multi_gpu, require_vision, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import MaskFormerForInstanceSegmentation, MaskFormerModel

							if is_vision_available():
														from transformers import MaskFormerImageProcessor

if is_vision_available():
							from PIL import Image




class   __lowercase						:






	def __init__(self				: Union[str, Any]      ,	snake_case				: str      ,	snake_case				: Optional[Any]=2      ,	snake_case				: str=True      ,	snake_case				: int=False      ,	snake_case				: Optional[Any]=10      ,	snake_case				: Optional[int]=3      ,	snake_case				: str=32 * 4      ,	snake_case				: Optional[Any]=32 * 6      ,	snake_case				: Optional[Any]=4      ,	snake_case				: Optional[Any]=32      ,	) ->			List[Any]:
							_lowercase       :	int           =   parent
							_lowercase       :	List[str]           =   batch_size
							_lowercase       :	Optional[Any]           =   is_training
							_lowercase       :	List[Any]           =   use_auxiliary_loss
							_lowercase       :	Dict           =   num_queries
							_lowercase       :	Union[str, Any]           =   num_channels
							_lowercase       :	Optional[int]           =   min_size
							_lowercase       :	List[Any]           =   max_size
							_lowercase       :	str           =   num_labels
							_lowercase       :	Optional[int]           =   mask_feature_size






	def       _a(self				: Dict    ) ->			Tuple:
							_lowercase       :	Dict           =   floats_tensor([self.batch_size, self.num_channels, self.min_size, self.max_size]    ).to(
							    snake_case    )

							_lowercase       :	Union[str, Any]           =   torch.ones([self.batch_size, self.min_size, self.max_size]      ,	device=snake_case    )

							_lowercase       :	Optional[int]           =   (
							    torch.rand([self.batch_size, self.num_labels, self.min_size, self.max_size]      ,	device=snake_case    ) > 0.5
							).float()
							_lowercase       :	Dict           =   (torch.rand((self.batch_size, self.num_labels)      ,	device=snake_case    ) > 0.5).long()

							_lowercase       :	Optional[int]           =   self.get_config()
							return config, pixel_values, pixel_mask, mask_labels, class_labels






	def       _a(self				: Optional[int]    ) ->			str:
							return MaskFormerConfig.from_backbone_and_decoder_configs(
							    backbone_config=SwinConfig(
							        depths=[1, 1, 1, 1]      ,	)      ,	decoder_config=DetrConfig(
							        decoder_ffn_dim=128      ,	num_queries=self.num_queries      ,	decoder_attention_heads=2      ,	d_model=self.mask_feature_size      ,	)      ,	mask_feature_size=self.mask_feature_size      ,	fpn_feature_size=self.mask_feature_size      ,	num_channels=self.num_channels      ,	num_labels=self.num_labels      ,	)






	def       _a(self				: int    ) ->			Tuple:
							_lowercase      ,    _lowercase      ,    _lowercase      ,    _lowercase      ,    _lowercase       :	Union[str, Any]           =   self.prepare_config_and_inputs()
							_lowercase       :	Dict           =   {"pixel_values": pixel_values, "pixel_mask": pixel_mask}
							return config, inputs_dict






	def       _a(self				: Any      ,	snake_case				: Optional[int]      ,	snake_case				: Union[str, Any]    ) ->			Tuple:
							_lowercase       :	Tuple           =   output.encoder_hidden_states
							_lowercase       :	int           =   output.pixel_decoder_hidden_states
							_lowercase       :	Optional[Any]           =   output.transformer_decoder_hidden_states

							self.parent.assertTrue(len(snake_case    )      ,	len(config.backbone_config.depths    )    )
							self.parent.assertTrue(len(snake_case    )      ,	len(config.backbone_config.depths    )    )
							self.parent.assertTrue(len(snake_case    )      ,	config.decoder_config.decoder_layers    )






	def       _a(self				: Any      ,	snake_case				: Any      ,	snake_case				: Dict      ,	snake_case				: List[Any]      ,	snake_case				: List[str]=False    ) ->			int:
							with torch.no_grad():
													_lowercase       :	Optional[Any]           =   MaskFormerModel(config=snake_case    )
													model.to(snake_case    )
													model.eval()

													_lowercase       :	Optional[Any]           =   model(pixel_values=snake_case      ,	pixel_mask=snake_case    )
													_lowercase       :	List[Any]           =   model(snake_case      ,	output_hidden_states=snake_case    )
							# the correct shape of output.transformer_decoder_hidden_states ensure the correcteness of the
							# encoder and pixel decoder
							self.parent.assertEqual(
							    output.transformer_decoder_last_hidden_state.shape      ,	(self.batch_size, self.num_queries, self.mask_feature_size)      ,	)
							# let's ensure the other two hidden state exists
							self.parent.assertTrue(output.pixel_decoder_last_hidden_state is not None    )
							self.parent.assertTrue(output.encoder_last_hidden_state is not None    )

							if output_hidden_states:
													self.check_output_hidden_state(snake_case      ,	snake_case    )






	def       _a(self				: Optional[Any]      ,	snake_case				: Optional[int]      ,	snake_case				: Union[str, Any]      ,	snake_case				: Optional[Any]      ,	snake_case				: Tuple      ,	snake_case				: Union[str, Any]    ) ->			Any:
							_lowercase       :	List[Any]           =   MaskFormerForInstanceSegmentation(config=snake_case    )
							model.to(snake_case    )
							model.eval()

							def comm_check_on_output(snake_case				: Dict    ):
													# let's still check that all the required stuff is there
													self.parent.assertTrue(result.transformer_decoder_last_hidden_state is not None    )
													self.parent.assertTrue(result.pixel_decoder_last_hidden_state is not None    )
													self.parent.assertTrue(result.encoder_last_hidden_state is not None    )
													# okay, now we need to check the logits shape
													# due to the encoder compression, masks have a //4 spatial size
													self.parent.assertEqual(
													    result.masks_queries_logits.shape      ,	(self.batch_size, self.num_queries, self.min_size // 4, self.max_size // 4)      ,	)
													# + 1 for null class
													self.parent.assertEqual(
													    result.class_queries_logits.shape      ,	(self.batch_size, self.num_queries, self.num_labels + 1)    )

							with torch.no_grad():
													_lowercase       :	Optional[Any]           =   model(pixel_values=snake_case      ,	pixel_mask=snake_case    )
													_lowercase       :	Dict           =   model(snake_case    )

													comm_check_on_output(snake_case    )

													_lowercase       :	Dict           =   model(
													    pixel_values=snake_case      ,	pixel_mask=snake_case      ,	mask_labels=snake_case      ,	class_labels=snake_case    )

							comm_check_on_output(snake_case    )

							self.parent.assertTrue(result.loss is not None    )
							self.parent.assertEqual(result.loss.shape      ,	torch.Size([1]    )    )




@require_torch
class   __lowercase						(						__snake_case   , __snake_case   , unittest.TestCase       ):
	_A       =   (MaskFormerModel, MaskFormerForInstanceSegmentation) if is_torch_available() else ()
	_A       =   (
	    {"feature-extraction": MaskFormerModel, "image-segmentation": MaskFormerForInstanceSegmentation}
	    if is_torch_available()
	    else {}
	)

	_A       =   False
	_A       =   False
	_A       =   False
	_A       =   False






	def       _a(self				: Dict    ) ->			List[Any]:
							_lowercase       :	Optional[Any]           =   MaskFormerModelTester(self    )
							_lowercase       :	Optional[int]           =   ConfigTester(self      ,	config_class=snake_case      ,	has_text_modality=snake_case    )






	def       _a(self				: List[str]    ) ->			Optional[Any]:
							self.config_tester.run_common_tests()






	def       _a(self				: List[str]    ) ->			Union[str, Any]:
							_lowercase      ,    _lowercase       :	Union[str, Any]           =   self.model_tester.prepare_config_and_inputs_for_common()
							self.model_tester.create_and_check_maskformer_model(snake_case      ,	**snake_case      ,	output_hidden_states=snake_case    )






	def       _a(self				: Tuple    ) ->			List[str]:
							_lowercase       :	List[str]           =   self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_maskformer_instance_segmentation_head_model(*snake_case    )






	@unittest.skip(reason="MaskFormer does not use inputs_embeds"    )
	def       _a(self				: Dict    ) ->			int:
							pass






	@unittest.skip(reason="MaskFormer does not have a get_input_embeddings method"    )
	def       _a(self				: Optional[int]    ) ->			Any:
							pass






	@unittest.skip(reason="MaskFormer is not a generative model"    )
	def       _a(self				: Optional[Any]    ) ->			Any:
							pass






	@unittest.skip(reason="MaskFormer does not use token embeddings"    )
	def       _a(self				: Dict    ) ->			int:
							pass






	@require_torch_multi_gpu
	@unittest.skip(
	    reason="MaskFormer has some layers using `add_module` which doesn't work well with `nn.DataParallel`"    )
	def       _a(self				: Optional[Any]    ) ->			Optional[int]:
							pass






	@unittest.skip("Will be fixed soon by reducing the size of the model used for common tests."    )
	def       _a(self				: Optional[int]    ) ->			List[Any]:
							pass






	def       _a(self				: Any    ) ->			Any:
							_lowercase      ,    _lowercase       :	Optional[int]           =   self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
													_lowercase       :	List[str]           =   model_class(snake_case    )
													_lowercase       :	Union[str, Any]           =   inspect.signature(model.forward    )
													# signature.parameters is an OrderedDict => so arg_names order is deterministic
													_lowercase       :	Any           =   [*signature.parameters.keys()]

													_lowercase       :	List[str]           =   ["pixel_values"]
													self.assertListEqual(arg_names[:1]      ,	snake_case    )






	@slow
	def       _a(self				: str    ) ->			List[Any]:
							for model_name in ["facebook/maskformer-swin-small-coco"]:
													_lowercase       :	List[Any]           =   MaskFormerModel.from_pretrained(snake_case    )
													self.assertIsNotNone(snake_case    )






	def       _a(self				: Tuple    ) ->			Dict:
							_lowercase       :	Optional[int]           =   (self.model_tester.min_size,) * 2
							_lowercase       :	List[str]           =   {
							    "pixel_values": torch.randn((2, 3, *size)      ,	device=snake_case    ),
							    "mask_labels": torch.randn((2, 10, *size)      ,	device=snake_case    ),
							    "class_labels": torch.zeros(2      ,	10      ,	device=snake_case    ).long(),
							}

							_lowercase       :	Tuple           =   MaskFormerForInstanceSegmentation(MaskFormerConfig()    ).to(snake_case    )
							_lowercase       :	Any           =   model(**snake_case    )
							self.assertTrue(outputs.loss is not None    )






	def       _a(self				: Optional[Any]    ) ->			str:
							_lowercase      ,    _lowercase       :	Optional[Any]           =   self.model_tester.prepare_config_and_inputs_for_common()
							self.model_tester.create_and_check_maskformer_model(snake_case      ,	**snake_case      ,	output_hidden_states=snake_case    )






	def       _a(self				: Optional[int]    ) ->			Any:
							_lowercase      ,    _lowercase       :	int           =   self.model_tester.prepare_config_and_inputs_for_common()

							for model_class in self.all_model_classes:
													_lowercase       :	List[Any]           =   model_class(snake_case    ).to(snake_case    )
													_lowercase       :	Tuple           =   model(**snake_case      ,	output_attentions=snake_case    )
													self.assertTrue(outputs.attentions is not None    )






	def       _a(self				: Union[str, Any]    ) ->			str:
							if not self.model_tester.is_training:
													return
							# only MaskFormerForInstanceSegmentation has the loss
							_lowercase       :	str           =   self.all_model_classes[1]
							_lowercase      ,    _lowercase      ,    _lowercase      ,    _lowercase      ,    _lowercase       :	Any           =   self.model_tester.prepare_config_and_inputs()

							_lowercase       :	Dict           =   model_class(snake_case    )
							model.to(snake_case    )
							model.train()

							_lowercase       :	List[Any]           =   model(snake_case      ,	mask_labels=snake_case      ,	class_labels=snake_case    ).loss
							loss.backward()






	def       _a(self				: List[str]    ) ->			int:
							# only MaskFormerForInstanceSegmentation has the loss
							_lowercase       :	List[Any]           =   self.all_model_classes[1]
							_lowercase      ,    _lowercase      ,    _lowercase      ,    _lowercase      ,    _lowercase       :	List[Any]           =   self.model_tester.prepare_config_and_inputs()
							_lowercase       :	str           =   True
							_lowercase       :	int           =   True

							_lowercase       :	Optional[int]           =   model_class(snake_case    )
							model.to(snake_case    )
							model.train()

							_lowercase       :	List[Any]           =   model(snake_case      ,	mask_labels=snake_case      ,	class_labels=snake_case    )

							_lowercase       :	Dict           =   outputs.encoder_hidden_states[0]
							encoder_hidden_states.retain_grad()

							_lowercase       :	List[str]           =   outputs.pixel_decoder_hidden_states[0]
							pixel_decoder_hidden_states.retain_grad()
							# we requires_grad=True in inputs_embeds (line 2152), the original implementation don't
							_lowercase       :	Optional[int]           =   outputs.transformer_decoder_hidden_states[0]
							transformer_decoder_hidden_states.retain_grad()

							_lowercase       :	str           =   outputs.attentions[0]
							attentions.retain_grad()

							outputs.loss.backward(retain_graph=snake_case    )

							self.assertIsNotNone(encoder_hidden_states.grad    )
							self.assertIsNotNone(pixel_decoder_hidden_states.grad    )
							self.assertIsNotNone(transformer_decoder_hidden_states.grad    )
							self.assertIsNotNone(attentions.grad    )


UpperCamelCase_						:     Union[str, Any]	       =							1e-4



def 		UpperCamelCase     (       )   ->			Optional[int]:






						'''simple docstring'''

						_lowercase       :	int           =   Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"  )
						return image




@require_vision
@slow
class   __lowercase						(						unittest.TestCase       ):






	@cached_property
	def       _a(self				: Tuple    ) ->			str:
							return (
							    MaskFormerImageProcessor.from_pretrained("facebook/maskformer-swin-small-coco"    )
							    if is_vision_available()
							    else None
							)






	def       _a(self				: Tuple    ) ->			str:
							_lowercase       :	List[str]           =   MaskFormerModel.from_pretrained("facebook/maskformer-swin-small-coco"    ).to(snake_case    )
							_lowercase       :	Optional[Any]           =   self.default_image_processor
							_lowercase       :	List[Any]           =   prepare_img()
							_lowercase       :	List[Any]           =   image_processor(snake_case      ,	return_tensors="pt"    ).to(snake_case    )
							_lowercase       :	Tuple           =   inputs["pixel_values"].shape
							# check size is divisible by 32
							self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0    )
							# check size
							self.assertEqual(snake_case      ,	(1, 3, 800, 1088)    )

							with torch.no_grad():
													_lowercase       :	int           =   model(**snake_case    )

							_lowercase       :	Union[str, Any]           =   torch.tensor(
							    [[-0.04_82, 0.92_28, 0.49_51], [-0.25_47, 0.80_17, 0.85_27], [-0.00_69, 0.33_85, -0.00_89]]    ).to(snake_case    )
							self.assertTrue(
							    torch.allclose(
							        outputs.encoder_last_hidden_state[0, 0, :3, :3]      ,	snake_case      ,	atol=snake_case    )    )

							_lowercase       :	List[Any]           =   torch.tensor(
							    [[-0.84_22, -0.84_34, -0.97_18], [-1.01_44, -0.55_65, -0.41_95], [-1.00_38, -0.44_84, -0.19_61]]    ).to(snake_case    )
							self.assertTrue(
							    torch.allclose(
							        outputs.pixel_decoder_last_hidden_state[0, 0, :3, :3]      ,	snake_case      ,	atol=snake_case    )    )

							_lowercase       :	int           =   torch.tensor(
							    [[0.28_52, -0.01_59, 0.97_35], [0.62_54, 0.18_58, 0.85_29], [-0.06_80, -0.41_16, 1.84_13]]    ).to(snake_case    )
							self.assertTrue(
							    torch.allclose(
							        outputs.transformer_decoder_last_hidden_state[0, :3, :3]      ,	snake_case      ,	atol=snake_case    )    )






	def       _a(self				: Optional[int]    ) ->			Tuple:
							_lowercase       :	int           =   (
							    MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-swin-small-coco"    )
							    .to(snake_case    )
							    .eval()
							)
							_lowercase       :	List[str]           =   self.default_image_processor
							_lowercase       :	List[str]           =   prepare_img()
							_lowercase       :	List[str]           =   image_processor(snake_case      ,	return_tensors="pt"    ).to(snake_case    )
							_lowercase       :	int           =   inputs["pixel_values"].shape
							# check size is divisible by 32
							self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0    )
							# check size
							self.assertEqual(snake_case      ,	(1, 3, 800, 1088)    )

							with torch.no_grad():
													_lowercase       :	Dict           =   model(**snake_case    )
							# masks_queries_logits
							_lowercase       :	Tuple           =   outputs.masks_queries_logits
							self.assertEqual(
							    masks_queries_logits.shape      ,	(1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)      ,	)
							_lowercase       :	List[Any]           =   [
							    [-1.3_73_71_24, -1.7_72_49_37, -1.9_36_42_33],
							    [-1.5_97_72_81, -1.9_86_79_39, -2.1_52_36_95],
							    [-1.5_79_53_98, -1.9_26_98_32, -2.09_39_42],
							]
							_lowercase       :	Any           =   torch.tensor(snake_case    ).to(snake_case    )
							self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]      ,	snake_case      ,	atol=snake_case    )    )
							# class_queries_logits
							_lowercase       :	Union[str, Any]           =   outputs.class_queries_logits
							self.assertEqual(
							    class_queries_logits.shape      ,	(1, model.config.decoder_config.num_queries, model.config.num_labels + 1)    )
							_lowercase       :	Optional[int]           =   torch.tensor(
							    [
							        [1.6_512e00, -5.2_572e00, -3.3_519e00],
							        [3.6_169e-02, -5.9_025e00, -2.9_313e00],
							        [1.0_766e-04, -7.7_630e00, -5.1_263e00],
							    ]    ).to(snake_case    )
							self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]      ,	snake_case      ,	atol=snake_case    )    )






	def       _a(self				: Optional[int]    ) ->			Optional[Any]:
							_lowercase       :	List[Any]           =   (
							    MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-resnet101-coco-stuff"    )
							    .to(snake_case    )
							    .eval()
							)
							_lowercase       :	str           =   self.default_image_processor
							_lowercase       :	str           =   prepare_img()
							_lowercase       :	Any           =   image_processor(snake_case      ,	return_tensors="pt"    ).to(snake_case    )
							_lowercase       :	str           =   inputs["pixel_values"].shape
							# check size is divisible by 32
							self.assertTrue((inputs_shape[-1] % 32) == 0 and (inputs_shape[-2] % 32) == 0    )
							# check size
							self.assertEqual(snake_case      ,	(1, 3, 800, 1088)    )

							with torch.no_grad():
													_lowercase       :	str           =   model(**snake_case    )
							# masks_queries_logits
							_lowercase       :	Union[str, Any]           =   outputs.masks_queries_logits
							self.assertEqual(
							    masks_queries_logits.shape      ,	(1, model.config.decoder_config.num_queries, inputs_shape[-2] // 4, inputs_shape[-1] // 4)      ,	)
							_lowercase       :	Optional[Any]           =   [[-0.90_46, -2.63_66, -4.60_62], [-3.41_79, -5.78_90, -8.80_57], [-4.91_79, -7.65_60, -10.77_11]]
							_lowercase       :	int           =   torch.tensor(snake_case    ).to(snake_case    )
							self.assertTrue(torch.allclose(masks_queries_logits[0, 0, :3, :3]      ,	snake_case      ,	atol=snake_case    )    )
							# class_queries_logits
							_lowercase       :	Dict           =   outputs.class_queries_logits
							self.assertEqual(
							    class_queries_logits.shape      ,	(1, model.config.decoder_config.num_queries, model.config.num_labels + 1)    )
							_lowercase       :	Optional[Any]           =   torch.tensor(
							    [[4.71_88, -3.25_85, -2.88_57], [6.68_71, -2.91_81, -1.24_87], [7.24_49, -2.27_64, -2.18_74]]    ).to(snake_case    )
							self.assertTrue(torch.allclose(outputs.class_queries_logits[0, :3, :3]      ,	snake_case      ,	atol=snake_case    )    )






	def       _a(self				: str    ) ->			List[Any]:
							_lowercase       :	List[str]           =   (
							    MaskFormerForInstanceSegmentation.from_pretrained("facebook/maskformer-swin-small-coco"    )
							    .to(snake_case    )
							    .eval()
							)
							_lowercase       :	Union[str, Any]           =   self.default_image_processor

							_lowercase       :	Optional[int]           =   image_processor(
							    [np.zeros((3, 800, 1333)    ), np.zeros((3, 800, 1333)    )]      ,	segmentation_maps=[np.zeros((384, 384)    ).astype(np.floataa    ), np.zeros((384, 384)    ).astype(np.floataa    )]      ,	return_tensors="pt"      ,	)

							_lowercase       :	Optional[Any]           =   inputs["pixel_values"].to(snake_case    )
							_lowercase       :	Optional[Any]           =   [el.to(snake_case    ) for el in inputs["mask_labels"]]
							_lowercase       :	Any           =   [el.to(snake_case    ) for el in inputs["class_labels"]]

							with torch.no_grad():
													_lowercase       :	str           =   model(**snake_case    )

							self.assertTrue(outputs.loss is not None    )






import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


UpperCamelCase_						:     Union[str, Any]	       =							datasets.utils.logging.get_logger(__name__)




@dataclass
class   __lowercase						(						datasets.BuilderConfig       ):
	_A       =   None
	_A       =   "utf-8"
	_A       =   None
	_A       =   None
	_A       =   True  # deprecated
	_A       =   None  # deprecated
	_A       =   10 << 20  # 10MB
	_A       =   None




class   __lowercase						(						datasets.ArrowBasedBuilder       ):
	_A       =   JsonConfig






	def       _a(self				: List[Any]    ) ->			Dict:
							if self.config.block_size is not None:
													logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"    )
													_lowercase       :	Union[str, Any]           =   self.config.block_size
							if self.config.use_threads is not True:
													logger.warning(
													    "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."    )
							if self.config.newlines_in_values is not None:
													raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported"    )
							return datasets.DatasetInfo(features=self.config.features    )






	def       _a(self				: Any      ,	snake_case				: Union[str, Any]    ) ->			List[str]:
							if not self.config.data_files:
													raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""    )
							_lowercase       :	str           =   dl_manager.download_and_extract(self.config.data_files    )
							if isinstance(snake_case      ,	(str, list, tuple)    ):
													_lowercase       :	int           =   data_files
													if isinstance(snake_case      ,	snake_case    ):
																			_lowercase       :	Union[str, Any]           =   [files]
													_lowercase       :	Union[str, Any]           =   [dl_manager.iter_files(snake_case    ) for file in files]
													return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,	gen_kwargs={"files": files}    )]
							_lowercase       :	Union[str, Any]           =   []
							for split_name, files in data_files.items():
													if isinstance(snake_case      ,	snake_case    ):
																			_lowercase       :	List[Any]           =   [files]
													_lowercase       :	Optional[Any]           =   [dl_manager.iter_files(snake_case    ) for file in files]
													splits.append(datasets.SplitGenerator(name=snake_case      ,	gen_kwargs={"files": files}    )    )
							return splits






	def       _a(self				: List[Any]      ,	snake_case				: pa.Table    ) ->			pa.Table:
							if self.config.features is not None:
													# adding missing columns
													for column_name in set(self.config.features    ) - set(pa_table.column_names    ):
																			_lowercase       :	Any           =   self.config.features.arrow_schema.field(snake_case    ).type
																			_lowercase       :	Optional[Any]           =   pa_table.append_column(snake_case      ,	pa.array([None] * len(snake_case    )      ,	type=snake_case    )    )
													# more expensive cast to support nested structures with keys in a different order
													# allows str <-> int/float or str to Audio for example
													_lowercase       :	str           =   table_cast(snake_case      ,	self.config.features.arrow_schema    )
							return pa_table






	def       _a(self				: str      ,	snake_case				: Tuple    ) ->			Tuple:
							for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case    )    ):
													# If the file is one json object and if we need to look at the list of items in one specific field
													if self.config.field is not None:
																			with open(snake_case      ,	encoding=self.config.encoding      ,	errors=self.config.encoding_errors    ) as f:
																									_lowercase       :	Any           =   json.load(snake_case    )

																			# We keep only the field we are interested in
																			_lowercase       :	List[Any]           =   dataset[self.config.field]

																			# We accept two format: a list of dicts or a dict of lists
																			if isinstance(snake_case      ,	(list, tuple)    ):
																									_lowercase       :	Optional[int]           =   set().union(*[row.keys() for row in dataset]    )
																									_lowercase       :	Union[str, Any]           =   {col: [row.get(snake_case    ) for row in dataset] for col in keys}
																			else:
																									_lowercase       :	List[Any]           =   dataset
																			_lowercase       :	int           =   pa.Table.from_pydict(snake_case    )
																			yield file_idx, self._cast_table(snake_case    )

													# If the file has one json object per line
													else:
																			with open(snake_case      ,	"rb"    ) as f:
																									_lowercase       :	Union[str, Any]           =   0
																									# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																									# Set a default minimum value of 16kB if the chunk size is really small
																									_lowercase       :	Dict           =   max(self.config.chunksize // 32      ,	16 << 10    )
																									_lowercase       :	int           =   (
																									    self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
																									)
																									while True:
																															_lowercase       :	Tuple           =   f.read(self.config.chunksize    )
																															if not batch:
																																					break
																															# Finish current line
																															try:
																																					batch += f.readline()
																															except (AttributeError, io.UnsupportedOperation):
																																					batch += readline(snake_case    )
																															# PyArrow only accepts utf-8 encoded bytes
																															if self.config.encoding != "utf-8":
																																					_lowercase       :	Union[str, Any]           =   batch.decode(self.config.encoding      ,	errors=snake_case    ).encode("utf-8"    )
																															try:
																																					while True:
																																											try:
																																																	_lowercase       :	Optional[Any]           =   paj.read_json(
																																																	    io.BytesIO(snake_case    )      ,	read_options=paj.ReadOptions(block_size=snake_case    )    )
																																																	break
																																											except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																	if (
																																																	    isinstance(snake_case      ,	pa.ArrowInvalid    )
																																																	    and "straddling" not in str(snake_case    )
																																																	    or block_size > len(snake_case    )
																																																	):
																																																							raise
																																																	else:
																																																							# Increase the block size in case it was too small.
																																																							# The block size will be reset for the next file.
																																																							logger.debug(
																																																							    F"""Batch of {len(snake_case    )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""    )
																																																							block_size *= 2
																															except pa.ArrowInvalid as e:
																																					try:
																																											with open(
																																											    snake_case      ,	encoding=self.config.encoding      ,	errors=self.config.encoding_errors    ) as f:
																																																	_lowercase       :	Any           =   json.load(snake_case    )
																																					except json.JSONDecodeError:
																																											logger.error(F"""Failed to read file '{file}' with error {type(snake_case    )}: {e}"""    )
																																											raise e
																																					# If possible, parse the file as a list of json objects and exit the loop
																																					if isinstance(snake_case      ,	snake_case    ):  # list is the only sequence type supported in JSON
																																											try:
																																																	_lowercase       :	int           =   set().union(*[row.keys() for row in dataset]    )
																																																	_lowercase       :	Dict           =   {col: [row.get(snake_case    ) for row in dataset] for col in keys}
																																																	_lowercase       :	str           =   pa.Table.from_pydict(snake_case    )
																																											except (pa.ArrowInvalid, AttributeError) as e:
																																																	logger.error(F"""Failed to read file '{file}' with error {type(snake_case    )}: {e}"""    )
																																																	raise ValueError(F"""Not able to read records in the JSON file at {file}."""    ) from None
																																											yield file_idx, self._cast_table(snake_case    )
																																											break
																																					else:
																																											logger.error(F"""Failed to read file '{file}' with error {type(snake_case    )}: {e}"""    )
																																											raise ValueError(
																																											    F"""Not able to read records in the JSON file at {file}. """
																																											    F"""You should probably indicate the field of the JSON file containing your records. """
																																											    F"""This JSON file contain the following fields: {str(list(dataset.keys()    )    )}. """
																																											    F"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """    ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																															yield (file_idx, batch_idx), self._cast_table(snake_case    )
																															batch_idx += 1




