






import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    EulerAncestralDiscreteScheduler,
    LMSDiscreteScheduler,
    PNDMScheduler,
    StableDiffusionInstructPixaPixPipeline,
    UNetaDConditionModel,
)
from diffusers.image_processor import VaeImageProcessor
from diffusers.utils import floats_tensor, load_image, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    IMAGE_TO_IMAGE_IMAGE_PARAMS,
    TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_PARAMS,
)
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()


class 							__UpperCamelCase     (  _lowerCAmelCase ,      _lowerCAmelCase ,      _lowerCAmelCase ,      unittest.TestCase ):
	__snake_case						:Optional[int] =							StableDiffusionInstructPixaPixPipeline
	__snake_case						:List[Any] =							TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {'height', 'width', 'cross_attention_kwargs'}
	__snake_case						:Any =							TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS
	__snake_case						:int =							IMAGE_TO_IMAGE_IMAGE_PARAMS
	__snake_case						:int =							IMAGE_TO_IMAGE_IMAGE_PARAMS







	def 			_a (	self   :				List[Any]      )		->			Tuple:
		"""simple docstring"""







		torch.manual_seed(0      )
		__lowercase   							=	UNetaDConditionModel(
		    block_out_channels=(32, 64)							,					layers_per_block=2							,					sample_size=32							,					in_channels=8							,					out_channels=4							,					down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")							,					up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")							,					cross_attention_dim=32							,					)
		__lowercase   							=	PNDMScheduler(skip_prk_steps=_lowerCAmelCase      )
		torch.manual_seed(0      )
		__lowercase   							=	AutoencoderKL(
		    block_out_channels=[32, 64]							,					in_channels=3							,					out_channels=3							,					down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]							,					up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]							,					latent_channels=4							,					)
		torch.manual_seed(0      )
		__lowercase   							=	CLIPTextConfig(
		    bos_token_id=0							,					eos_token_id=2							,					hidden_size=32							,					intermediate_size=37							,					layer_norm_eps=1e-05							,					num_attention_heads=4							,					num_hidden_layers=5							,					pad_token_id=1							,					vocab_size=1000							,					)
		__lowercase   							=	CLIPTextModel(_lowerCAmelCase      )
		__lowercase   							=	CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""      )

		__lowercase   							=	{
		    """unet""": unet,
		    """scheduler""": scheduler,
		    """vae""": vae,
		    """text_encoder""": text_encoder,
		    """tokenizer""": tokenizer,
		    """safety_checker""": None,
		    """feature_extractor""": None,
		}
		return components







	def 			_a (	self   :				Optional[int]							,					_lowerCAmelCase   :				str							,					_lowerCAmelCase   :				Optional[Any]=0      )		->			Any:
		"""simple docstring"""







		__lowercase   							=	floats_tensor((1, 3, 32, 32)							,					rng=random.Random(_lowerCAmelCase      )      ).to(_lowerCAmelCase      )
		__lowercase   							=	image.cpu().permute(0							,					2							,					3							,					1      )[0]
		__lowercase   							=	Image.fromarray(np.uinta(_lowerCAmelCase      )      ).convert("""RGB"""      )
		if str(_lowerCAmelCase      ).startswith("""mps"""      ):
			__lowercase   							=	torch.manual_seed(_lowerCAmelCase      )
		else:
			__lowercase   							=	torch.Generator(device=_lowerCAmelCase      ).manual_seed(_lowerCAmelCase      )
		__lowercase   							=	{
		    """prompt""": """A painting of a squirrel eating a burger""",
		    """image""": image,
		    """generator""": generator,
		    """num_inference_steps""": 2,
		    """guidance_scale""": 6.0,
		    """image_guidance_scale""": 1,
		    """output_type""": """numpy""",
		}
		return inputs







	def 			_a (	self   :				str      )		->			Tuple:
		"""simple docstring"""







		__lowercase   							=	"""cpu"""  # ensure determinism for the device-dependent torch.Generator
		__lowercase   							=	self.get_dummy_components()
		__lowercase   							=	StableDiffusionInstructPixaPixPipeline(**_lowerCAmelCase      )
		__lowercase   							=	sd_pipe.to(_lowerCAmelCase      )
		sd_pipe.set_progress_bar_config(disable=_lowerCAmelCase      )

		__lowercase   							=	self.get_dummy_inputs(_lowerCAmelCase      )
		__lowercase   							=	sd_pipe(**_lowerCAmelCase      ).images
		__lowercase   							=	image[0, -3:, -3:, -1]
		assert image.shape == (1, 32, 32, 3)
		__lowercase   							=	np.array([0.7_526, 0.3_750, 0.4_547, 0.6_117, 0.5_866, 0.5_016, 0.4_327, 0.5_642, 0.4_815]      )

		assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1e-3







	def 			_a (	self   :				Any      )		->			Any:
		"""simple docstring"""







		__lowercase   							=	"""cpu"""  # ensure determinism for the device-dependent torch.Generator
		__lowercase   							=	self.get_dummy_components()
		__lowercase   							=	StableDiffusionInstructPixaPixPipeline(**_lowerCAmelCase      )
		__lowercase   							=	sd_pipe.to(_lowerCAmelCase      )
		sd_pipe.set_progress_bar_config(disable=_lowerCAmelCase      )

		__lowercase   							=	self.get_dummy_inputs(_lowerCAmelCase      )
		__lowercase   							=	"""french fries"""
		__lowercase   							=	sd_pipe(**_lowerCAmelCase							,					negative_prompt=_lowerCAmelCase      )
		__lowercase   							=	output.images
		__lowercase   							=	image[0, -3:, -3:, -1]

		assert image.shape == (1, 32, 32, 3)
		__lowercase   							=	np.array([0.7_511, 0.3_642, 0.4_553, 0.6_236, 0.5_797, 0.5_013, 0.4_343, 0.5_611, 0.4_831]      )

		assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1e-3







	def 			_a (	self   :				Optional[Any]      )		->			Dict:
		"""simple docstring"""







		__lowercase   							=	"""cpu"""  # ensure determinism for the device-dependent torch.Generator
		__lowercase   							=	self.get_dummy_components()
		__lowercase   							=	StableDiffusionInstructPixaPixPipeline(**_lowerCAmelCase      )
		__lowercase   							=	sd_pipe.to(_lowerCAmelCase      )
		sd_pipe.set_progress_bar_config(disable=_lowerCAmelCase      )

		__lowercase   							=	self.get_dummy_inputs(_lowerCAmelCase      )
		__lowercase   							=	[inputs["""prompt"""]] * 2

		__lowercase   							=	np.array(inputs["""image"""]      ).astype(np.floataa      ) / 255.0
		__lowercase   							=	torch.from_numpy(_lowerCAmelCase      ).unsqueeze(0      ).to(_lowerCAmelCase      )
		__lowercase   							=	image / 2 + 0.5
		__lowercase   							=	image.permute(0							,					3							,					1							,					2      )
		__lowercase   							=	image.repeat(2							,					1							,					1							,					1      )

		__lowercase   							=	sd_pipe(**_lowerCAmelCase      ).images
		__lowercase   							=	image[-1, -3:, -3:, -1]

		assert image.shape == (2, 32, 32, 3)
		__lowercase   							=	np.array([0.5_812, 0.5_748, 0.5_222, 0.5_908, 0.5_695, 0.7_174, 0.6_804, 0.5_523, 0.5_579]      )

		assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1e-3







	def 			_a (	self   :				int      )		->			List[str]:
		"""simple docstring"""







		__lowercase   							=	"""cpu"""  # ensure determinism for the device-dependent torch.Generator
		__lowercase   							=	self.get_dummy_components()
		__lowercase   							=	EulerAncestralDiscreteScheduler(
		    beta_start=0.00_085							,					beta_end=0.012							,					beta_schedule="""scaled_linear"""      )
		__lowercase   							=	StableDiffusionInstructPixaPixPipeline(**_lowerCAmelCase      )
		__lowercase   							=	sd_pipe.to(_lowerCAmelCase      )
		sd_pipe.set_progress_bar_config(disable=_lowerCAmelCase      )

		__lowercase   							=	self.get_dummy_inputs(_lowerCAmelCase      )
		__lowercase   							=	sd_pipe(**_lowerCAmelCase      ).images
		__lowercase   							=	image[0, -3:, -3:, -1]

		__lowercase   							=	[round(_lowerCAmelCase							,					4      ) for x in image_slice.flatten().tolist()]
		print(""",""".join([str(_lowerCAmelCase      ) for x in slice]      )      )

		assert image.shape == (1, 32, 32, 3)
		__lowercase   							=	np.array([0.7_417, 0.3_842, 0.4_732, 0.5_776, 0.5_891, 0.5_139, 0.4_052, 0.5_673, 0.4_986]      )

		assert np.abs(image_slice.flatten() - expected_slice      ).max() < 1e-3







	def 			_a (	self   :				List[str]      )		->			Dict:
		"""simple docstring"""







		super().test_inference_batch_single_identical(expected_max_diff=3e-3      )







	def 			_a (	self   :				Union[str, Any]      )		->			int:
		"""simple docstring"""







		__lowercase   							=	self.get_dummy_components()
		__lowercase   							=	StableDiffusionInstructPixaPixPipeline(**_lowerCAmelCase      )
		__lowercase   							=	VaeImageProcessor(do_resize=_lowerCAmelCase							,					do_normalize=_lowerCAmelCase      )
		__lowercase   							=	pipe.to(_lowerCAmelCase      )
		pipe.set_progress_bar_config(disable=_lowerCAmelCase      )

		__lowercase   							=	pipe(**self.get_dummy_inputs_by_type(_lowerCAmelCase							,					input_image_type="""pt"""      )      )[0]

		__lowercase   							=	components["""vae"""]
		__lowercase   							=	self.get_dummy_inputs_by_type(_lowerCAmelCase							,					input_image_type="""pt"""      )

		for image_param in self.image_latents_params:
			if image_param in inputs.keys():
				__lowercase   							=	vae.encode(inputs[image_param]      ).latent_dist.mode()

		__lowercase   							=	pipe(**_lowerCAmelCase      )[0]

		__lowercase   							=	np.abs(out - out_latents_inputs      ).max()
		self.assertLess(_lowerCAmelCase							,					1e-4							,					"""passing latents as image input generate different result from passing image"""      )


@slow
@require_torch_gpu
class 							__UpperCamelCase     (  unittest.TestCase ):







	def 			_a (	self   :				Dict      )		->			Optional[int]:
		"""simple docstring"""







		super().tearDown()
		gc.collect()
		torch.cuda.empty_cache()







	def 			_a (	self   :				Dict							,					_lowerCAmelCase   :				str=0      )		->			Optional[int]:
		"""simple docstring"""







		__lowercase   							=	torch.manual_seed(_lowerCAmelCase      )
		__lowercase   							=	load_image(
		    """https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_pix2pix/example.jpg"""      )
		__lowercase   							=	{
		    """prompt""": """turn him into a cyborg""",
		    """image""": image,
		    """generator""": generator,
		    """num_inference_steps""": 3,
		    """guidance_scale""": 7.5,
		    """image_guidance_scale""": 1.0,
		    """output_type""": """numpy""",
		}
		return inputs







	def 			_a (	self   :				Tuple      )		->			Optional[Any]:
		"""simple docstring"""







		__lowercase   							=	StableDiffusionInstructPixaPixPipeline.from_pretrained(
		    """timbrooks/instruct-pix2pix"""							,					safety_checker=_lowerCAmelCase      )
		pipe.to(_lowerCAmelCase      )
		pipe.set_progress_bar_config(disable=_lowerCAmelCase      )
		pipe.enable_attention_slicing()

		__lowercase   							=	self.get_inputs()
		__lowercase   							=	pipe(**_lowerCAmelCase      ).images
		__lowercase   							=	image[0, -3:, -3:, -1].flatten()

		assert image.shape == (1, 512, 512, 3)
		__lowercase   							=	np.array([0.5_902, 0.6_015, 0.6_027, 0.5_983, 0.6_092, 0.6_061, 0.5_765, 0.5_785, 0.5_555]      )

		assert np.abs(expected_slice - image_slice      ).max() < 1e-3







	def 			_a (	self   :				Dict      )		->			Union[str, Any]:
		"""simple docstring"""







		__lowercase   							=	StableDiffusionInstructPixaPixPipeline.from_pretrained(
		    """timbrooks/instruct-pix2pix"""							,					safety_checker=_lowerCAmelCase      )
		__lowercase   							=	LMSDiscreteScheduler.from_config(pipe.scheduler.config      )
		pipe.to(_lowerCAmelCase      )
		pipe.set_progress_bar_config(disable=_lowerCAmelCase      )
		pipe.enable_attention_slicing()

		__lowercase   							=	self.get_inputs()
		__lowercase   							=	pipe(**_lowerCAmelCase      ).images
		__lowercase   							=	image[0, -3:, -3:, -1].flatten()

		assert image.shape == (1, 512, 512, 3)
		__lowercase   							=	np.array([0.6_578, 0.6_817, 0.6_972, 0.6_761, 0.6_856, 0.6_916, 0.6_428, 0.6_516, 0.6_301]      )

		assert np.abs(expected_slice - image_slice      ).max() < 1e-3







	def 			_a (	self   :				Any      )		->			Union[str, Any]:
		"""simple docstring"""







		__lowercase   							=	StableDiffusionInstructPixaPixPipeline.from_pretrained(
		    """timbrooks/instruct-pix2pix"""							,					safety_checker=_lowerCAmelCase      )
		__lowercase   							=	DDIMScheduler.from_config(pipe.scheduler.config      )
		pipe.to(_lowerCAmelCase      )
		pipe.set_progress_bar_config(disable=_lowerCAmelCase      )
		pipe.enable_attention_slicing()

		__lowercase   							=	self.get_inputs()
		__lowercase   							=	pipe(**_lowerCAmelCase      ).images
		__lowercase   							=	image[0, -3:, -3:, -1].flatten()

		assert image.shape == (1, 512, 512, 3)
		__lowercase   							=	np.array([0.3_828, 0.3_834, 0.3_818, 0.3_792, 0.3_865, 0.3_752, 0.3_792, 0.3_847, 0.3_753]      )

		assert np.abs(expected_slice - image_slice      ).max() < 1e-3







	def 			_a (	self   :				Union[str, Any]      )		->			Any:
		"""simple docstring"""







		__lowercase   							=	0

		def callback_fn(_lowerCAmelCase   :				int							,					_lowerCAmelCase   :				int							,					_lowerCAmelCase   :				torch.FloatTensor      ) -> None:
			__lowercase   							=	True
			nonlocal number_of_steps
			number_of_steps += 1
			if step == 1:
				__lowercase   							=	latents.detach().cpu().numpy()
				assert latents.shape == (1, 4, 64, 64)
				__lowercase   							=	latents[0, -3:, -3:, -1]
				__lowercase   							=	np.array([-0.2_463, -0.4_644, -0.9_756, 1.5_176, 1.4_414, 0.7_866, 0.9_897, 0.8_521, 0.7_983]      )

				assert np.abs(latents_slice.flatten() - expected_slice      ).max() < 5e-2
			elif step == 2:
				__lowercase   							=	latents.detach().cpu().numpy()
				assert latents.shape == (1, 4, 64, 64)
				__lowercase   							=	latents[0, -3:, -3:, -1]
				__lowercase   							=	np.array([-0.2_644, -0.4_626, -0.9_653, 1.5_176, 1.4_551, 0.7_686, 0.9_805, 0.8_452, 0.8_115]      )

				assert np.abs(latents_slice.flatten() - expected_slice      ).max() < 5e-2

		__lowercase   							=	False

		__lowercase   							=	StableDiffusionInstructPixaPixPipeline.from_pretrained(
		    """timbrooks/instruct-pix2pix"""							,					safety_checker=_lowerCAmelCase							,					torch_dtype=torch.floataa      )
		__lowercase   							=	pipe.to(_lowerCAmelCase      )
		pipe.set_progress_bar_config(disable=_lowerCAmelCase      )
		pipe.enable_attention_slicing()

		__lowercase   							=	self.get_inputs()
		pipe(**_lowerCAmelCase							,					callback=_lowerCAmelCase							,					callback_steps=1      )
		assert callback_fn.has_been_called
		assert number_of_steps == 3







	def 			_a (	self   :				Optional[int]      )		->			str:
		"""simple docstring"""







		torch.cuda.empty_cache()
		torch.cuda.reset_max_memory_allocated()
		torch.cuda.reset_peak_memory_stats()

		__lowercase   							=	StableDiffusionInstructPixaPixPipeline.from_pretrained(
		    """timbrooks/instruct-pix2pix"""							,					safety_checker=_lowerCAmelCase							,					torch_dtype=torch.floataa      )
		__lowercase   							=	pipe.to(_lowerCAmelCase      )
		pipe.set_progress_bar_config(disable=_lowerCAmelCase      )
		pipe.enable_attention_slicing(1      )
		pipe.enable_sequential_cpu_offload()

		__lowercase   							=	self.get_inputs()
		__lowercase   							=	pipe(**_lowerCAmelCase      )

		__lowercase   							=	torch.cuda.max_memory_allocated()
		# make sure that less than 2.2 GB is allocated
		assert mem_bytes < 2.2 * 10**9







	def 			_a (	self   :				List[Any]      )		->			Optional[int]:
		"""simple docstring"""







		__lowercase   							=	self.get_inputs()
		# resize to resolution that is divisible by 8 but not 16 or 32
		__lowercase   							=	inputs["""image"""].resize((504, 504)      )

		__lowercase   							=	"""timbrooks/instruct-pix2pix"""
		__lowercase   							=	StableDiffusionInstructPixaPixPipeline.from_pretrained(
		    _lowerCAmelCase							,					safety_checker=_lowerCAmelCase							,					)
		pipe.to(_lowerCAmelCase      )
		pipe.set_progress_bar_config(disable=_lowerCAmelCase      )
		pipe.enable_attention_slicing()

		__lowercase   							=	pipe(**_lowerCAmelCase      )
		__lowercase   							=	output.images[0]

		__lowercase   							=	image[255:258, 383:386, -1]

		assert image.shape == (504, 504, 3)
		__lowercase   							=	np.array([0.2_726, 0.2_529, 0.2_664, 0.2_655, 0.2_641, 0.2_642, 0.2_591, 0.2_649, 0.2_590]      )

		assert np.abs(image_slice.flatten() - expected_slice      ).max() < 5e-3









import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


__a:     Tuple   					=					datasets.utils.logging.get_logger(__name__)
@dataclass
class 		SCREAMING_SNAKE_CASE__	(  datasets.BuilderConfig     ):

  '''simple docstring'''







  _lowerCamelCase    =  None
  _lowerCamelCase    =  "utf-8"
  _lowerCamelCase    =  None
  _lowerCamelCase    =  None
  _lowerCamelCase    =  True  # deprecated
  _lowerCamelCase    =  None  # deprecated
  _lowerCamelCase    =  10 << 20  # 10MB
  _lowerCamelCase    =  None



class 		SCREAMING_SNAKE_CASE__	(  datasets.ArrowBasedBuilder     ):

  '''simple docstring'''







  _lowerCamelCase    =  JsonConfig


  def  lowerCamelCase       (      self		:			Any							)						->	Any:


         """simple docstring"""
         if self.config.block_size is not None:
                logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""							)
                _UpperCAmelCase							=    self.config.block_size
         if self.config.use_threads is not True:
                logger.warning(
                    """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."""							)
         if self.config.newlines_in_values is not None:
                raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""							)
         return datasets.DatasetInfo(features=self.config.features							)


  def  lowerCamelCase       (      self		:			List[Any]       ,				lowerCamelCase		:			Optional[int]							)						->	Dict:


         """simple docstring"""
         if not self.config.data_files:
                raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""							)
         _UpperCAmelCase							=    dl_manager.download_and_extract(self.config.data_files							)
         if isinstance(lowerCamelCase       ,				(str, list, tuple)							):
                _UpperCAmelCase							=    data_files
                if isinstance(lowerCamelCase       ,				lowerCamelCase							):
                       _UpperCAmelCase							=    [files]
                _UpperCAmelCase							=    [dl_manager.iter_files(lowerCamelCase							) for file in files]
                return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,				gen_kwargs={"""files""": files}							)]
         _UpperCAmelCase							=    []
         for split_name, files in data_files.items():
                if isinstance(lowerCamelCase       ,				lowerCamelCase							):
                       _UpperCAmelCase							=    [files]
                _UpperCAmelCase							=    [dl_manager.iter_files(lowerCamelCase							) for file in files]
                splits.append(datasets.SplitGenerator(name=lowerCamelCase       ,				gen_kwargs={"""files""": files}							)							)
         return splits


  def  lowerCamelCase       (      self		:			Tuple       ,				lowerCamelCase		:			pa.Table							)						->	pa.Table:


         """simple docstring"""
         if self.config.features is not None:
                # adding missing columns
                for column_name in set(self.config.features							) - set(pa_table.column_names							):
                       _UpperCAmelCase							=    self.config.features.arrow_schema.field(lowerCamelCase							).type
                       _UpperCAmelCase							=    pa_table.append_column(lowerCamelCase       ,				pa.array([None] * len(lowerCamelCase							)       ,				type=lowerCamelCase							)							)
                # more expensive cast to support nested structures with keys in a different order
                # allows str <-> int/float or str to Audio for example
                _UpperCAmelCase							=    table_cast(lowerCamelCase       ,				self.config.features.arrow_schema							)
         return pa_table





  def  lowerCamelCase       (      self		:			Union[str, Any]       ,				lowerCamelCase		:			Optional[int]							)						->	str:


         """simple docstring"""
         for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCamelCase							)							):
                # If the file is one json object and if we need to look at the list of items in one specific field
                if self.config.field is not None:
                       with open(lowerCamelCase       ,				encoding=self.config.encoding       ,				errors=self.config.encoding_errors							) as f:
                              _UpperCAmelCase							=    json.load(lowerCamelCase							)

                       # We keep only the field we are interested in
                       _UpperCAmelCase							=    dataset[self.config.field]

                       # We accept two format: a list of dicts or a dict of lists
                       if isinstance(lowerCamelCase       ,				(list, tuple)							):
                              _UpperCAmelCase							=    set().union(*[row.keys() for row in dataset]							)
                              _UpperCAmelCase							=    {col: [row.get(lowerCamelCase							) for row in dataset] for col in keys}
                       else:
                              _UpperCAmelCase							=    dataset
                       _UpperCAmelCase							=    pa.Table.from_pydict(lowerCamelCase							)
                       yield file_idx, self._cast_table(lowerCamelCase							)

                # If the file has one json object per line
                else:
                       with open(lowerCamelCase       ,				"""rb"""							) as f:
                              _UpperCAmelCase							=    0
                              # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                              # Set a default minimum value of 16kB if the chunk size is really small
                              _UpperCAmelCase							=    max(self.config.chunksize // 32       ,				16 << 10							)
                              _UpperCAmelCase							=    (
                                  self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
                              )
                              while True:
                                     _UpperCAmelCase							=    f.read(self.config.chunksize							)
                                     if not batch:
                                            break
                                     # Finish current line
                                     try:
                                            batch += f.readline()
                                     except (AttributeError, io.UnsupportedOperation):
                                            batch += readline(lowerCamelCase							)
                                     # PyArrow only accepts utf-8 encoded bytes
                                     if self.config.encoding != "utf-8":
                                            _UpperCAmelCase							=    batch.decode(self.config.encoding       ,				errors=lowerCamelCase							).encode("""utf-8"""							)
                                     try:
                                            while True:
                                                   try:
                                                          _UpperCAmelCase							=    paj.read_json(
                                                              io.BytesIO(lowerCamelCase							)       ,				read_options=paj.ReadOptions(block_size=lowerCamelCase							)							)
                                                          break
                                                   except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                                          if (
                                                              isinstance(lowerCamelCase       ,				pa.ArrowInvalid							)
                                                              and "straddling" not in str(lowerCamelCase							)
                                                              or block_size > len(lowerCamelCase							)
                                                          ):
                                                                 raise
                                                          else:
                                                                 # Increase the block size in case it was too small.
                                                                 # The block size will be reset for the next file.
                                                                 logger.debug(
                                                                     f"""Batch of {len(lowerCamelCase							)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""							)
                                                                 block_size *= 2
                                     except pa.ArrowInvalid as e:
                                            try:
                                                   with open(
                                                       lowerCamelCase       ,				encoding=self.config.encoding       ,				errors=self.config.encoding_errors							) as f:
                                                          _UpperCAmelCase							=    json.load(lowerCamelCase							)
                                            except json.JSONDecodeError:
                                                   logger.error(f"""Failed to read file '{file}' with error {type(lowerCamelCase							)}: {e}"""							)
                                                   raise e
                                            # If possible, parse the file as a list of json objects and exit the loop
                                            if isinstance(lowerCamelCase       ,				lowerCamelCase							):  # list is the only sequence type supported in JSON
                                                   try:
                                                          _UpperCAmelCase							=    set().union(*[row.keys() for row in dataset]							)
                                                          _UpperCAmelCase							=    {col: [row.get(lowerCamelCase							) for row in dataset] for col in keys}
                                                          _UpperCAmelCase							=    pa.Table.from_pydict(lowerCamelCase							)
                                                   except (pa.ArrowInvalid, AttributeError) as e:
                                                          logger.error(f"""Failed to read file '{file}' with error {type(lowerCamelCase							)}: {e}"""							)
                                                          raise ValueError(f"""Not able to read records in the JSON file at {file}."""							) from None
                                                   yield file_idx, self._cast_table(lowerCamelCase							)
                                                   break
                                            else:
                                                   logger.error(f"""Failed to read file '{file}' with error {type(lowerCamelCase							)}: {e}"""							)
                                                   raise ValueError(
                                                       f"""Not able to read records in the JSON file at {file}. """
                                                       f"""You should probably indicate the field of the JSON file containing your records. """
                                                       f"""This JSON file contain the following fields: {str(list(dataset.keys()							)							)}. """
                                                       f"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """							) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                     yield (file_idx, batch_idx), self._cast_table(lowerCamelCase							)
                                     batch_idx += 1