

import gc
import random
import unittest

import numpy as np
import torch
from PIL import Image
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    EulerAncestralDiscreteScheduler,
    LMSDiscreteScheduler,
    PNDMScheduler,
    StableDiffusionInstructPixaPixPipeline,
    UNetaDConditionModel,
)
from diffusers.image_processor import VaeImageProcessor
from diffusers.utils import floats_tensor, load_image, slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import (
    IMAGE_TO_IMAGE_IMAGE_PARAMS,
    TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_VARIATION_PARAMS,
)
from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()







class 					__lowercase  (						_SCREAMING_SNAKE_CASE		,       _SCREAMING_SNAKE_CASE		,       _SCREAMING_SNAKE_CASE		,       unittest.TestCase				):





							"""simple docstring"""

							UpperCAmelCase_						:		Optional[int]								=	StableDiffusionInstructPixaPixPipeline
							UpperCAmelCase_						:		str								=	TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {'''height''', '''width''', '''cross_attention_kwargs'''}
							UpperCAmelCase_						:		List[Any]								=	TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS
							UpperCAmelCase_						:		Optional[int]								=	IMAGE_TO_IMAGE_IMAGE_PARAMS
							UpperCAmelCase_						:		Any								=	IMAGE_TO_IMAGE_IMAGE_PARAMS




							def       snake_case			(					self						)		->       List[Any]:
														torch.manual_seed(0						)
														A      :					Tuple      		=					UNetaDConditionModel(
														    block_out_channels=(32, 64)    ,						layers_per_block=2    ,						sample_size=32    ,						in_channels=8    ,						out_channels=4    ,						down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''')    ,						up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''')    ,						cross_attention_dim=32    ,						)
														A      :					Any      		=					PNDMScheduler(skip_prk_steps=__UpperCAmelCase						)
														torch.manual_seed(0						)
														A      :					Dict      		=					AutoencoderKL(
														    block_out_channels=[32, 64]    ,						in_channels=3    ,						out_channels=3    ,						down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D''']    ,						up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D''']    ,						latent_channels=4    ,						)
														torch.manual_seed(0						)
														A      :					str      		=					CLIPTextConfig(
														    bos_token_id=0    ,						eos_token_id=2    ,						hidden_size=32    ,						intermediate_size=37    ,						layer_norm_eps=1E-05    ,						num_attention_heads=4    ,						num_hidden_layers=5    ,						pad_token_id=1    ,						vocab_size=10_00    ,						)
														A      :					Dict      		=					CLIPTextModel(__UpperCAmelCase						)
														A      :					Dict      		=					CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''						)

														A      :					int      		=					{
														    '''unet''': unet,
														    '''scheduler''': scheduler,
														    '''vae''': vae,
														    '''text_encoder''': text_encoder,
														    '''tokenizer''': tokenizer,
														    '''safety_checker''': None,
														    '''feature_extractor''': None,
														}
														return components




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase=0						)		->       Dict:
														A      :					Any      		=					floats_tensor((1, 3, 32, 32)    ,						rng=random.Random(__UpperCAmelCase						)						).to(__UpperCAmelCase						)
														A      :					Dict      		=					image.cpu().permute(0    ,						2    ,						3    ,						1						)[0]
														A      :					Optional[Any]      		=					Image.fromarray(np.uinta(__UpperCAmelCase						)						).convert('''RGB'''						)
														if str(__UpperCAmelCase						).startswith('''mps'''						):
																					A      :					str      		=					torch.manual_seed(__UpperCAmelCase						)
														else:
																					A      :					Optional[Any]      		=					torch.Generator(device=__UpperCAmelCase						).manual_seed(__UpperCAmelCase						)
														A      :					List[Any]      		=					{
														    '''prompt''': '''A painting of a squirrel eating a burger''',
														    '''image''': image,
														    '''generator''': generator,
														    '''num_inference_steps''': 2,
														    '''guidance_scale''': 6.0,
														    '''image_guidance_scale''': 1,
														    '''output_type''': '''numpy''',
														}
														return inputs




							def       snake_case			(					self						)		->       Union[str, Any]:
														A      :					Optional[Any]      		=					'''cpu'''  # ensure determinism for the device-dependent torch.Generator
														A      :					Dict      		=					self.get_dummy_components()
														A      :					str      		=					StableDiffusionInstructPixaPixPipeline(**__UpperCAmelCase						)
														A      :					Any      		=					sd_pipe.to(__UpperCAmelCase						)
														sd_pipe.set_progress_bar_config(disable=__UpperCAmelCase						)

														A      :					Optional[Any]      		=					self.get_dummy_inputs(__UpperCAmelCase						)
														A      :					int      		=					sd_pipe(**__UpperCAmelCase						).images
														A      :					List[str]      		=					image[0, -3:, -3:, -1]
														assert image.shape == (1, 32, 32, 3)
														A      :					Optional[Any]      		=					np.array([0.7_5_2_6, 0.3_7_5_0, 0.4_5_4_7, 0.6_1_1_7, 0.5_8_6_6, 0.5_0_1_6, 0.4_3_2_7, 0.5_6_4_2, 0.4_8_1_5]						)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1E-3




							def       snake_case			(					self						)		->       int:
														A      :					Any      		=					'''cpu'''  # ensure determinism for the device-dependent torch.Generator
														A      :					Optional[Any]      		=					self.get_dummy_components()
														A      :					Any      		=					StableDiffusionInstructPixaPixPipeline(**__UpperCAmelCase						)
														A      :					str      		=					sd_pipe.to(__UpperCAmelCase						)
														sd_pipe.set_progress_bar_config(disable=__UpperCAmelCase						)

														A      :					str      		=					self.get_dummy_inputs(__UpperCAmelCase						)
														A      :					Optional[int]      		=					'''french fries'''
														A      :					Any      		=					sd_pipe(**__UpperCAmelCase    ,						negative_prompt=__UpperCAmelCase						)
														A      :					Optional[int]      		=					output.images
														A      :					str      		=					image[0, -3:, -3:, -1]

														assert image.shape == (1, 32, 32, 3)
														A      :					Any      		=					np.array([0.7_5_1_1, 0.3_6_4_2, 0.4_5_5_3, 0.6_2_3_6, 0.5_7_9_7, 0.5_0_1_3, 0.4_3_4_3, 0.5_6_1_1, 0.4_8_3_1]						)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1E-3




							def       snake_case			(					self						)		->       Tuple:
														A      :					List[Any]      		=					'''cpu'''  # ensure determinism for the device-dependent torch.Generator
														A      :					List[Any]      		=					self.get_dummy_components()
														A      :					Optional[int]      		=					StableDiffusionInstructPixaPixPipeline(**__UpperCAmelCase						)
														A      :					Tuple      		=					sd_pipe.to(__UpperCAmelCase						)
														sd_pipe.set_progress_bar_config(disable=__UpperCAmelCase						)

														A      :					str      		=					self.get_dummy_inputs(__UpperCAmelCase						)
														A      :					int      		=					[inputs['''prompt''']] * 2

														A      :					List[str]      		=					np.array(inputs['''image''']						).astype(np.floataa						) / 2_5_5.0
														A      :					Any      		=					torch.from_numpy(__UpperCAmelCase						).unsqueeze(0						).to(__UpperCAmelCase						)
														A      :					Union[str, Any]      		=					image / 2 + 0.5
														A      :					str      		=					image.permute(0    ,						3    ,						1    ,						2						)
														A      :					List[str]      		=					image.repeat(2    ,						1    ,						1    ,						1						)

														A      :					Union[str, Any]      		=					sd_pipe(**__UpperCAmelCase						).images
														A      :					int      		=					image[-1, -3:, -3:, -1]

														assert image.shape == (2, 32, 32, 3)
														A      :					Any      		=					np.array([0.5_8_1_2, 0.5_7_4_8, 0.5_2_2_2, 0.5_9_0_8, 0.5_6_9_5, 0.7_1_7_4, 0.6_8_0_4, 0.5_5_2_3, 0.5_5_7_9]						)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1E-3




							def       snake_case			(					self						)		->       Dict:
														A      :					int      		=					'''cpu'''  # ensure determinism for the device-dependent torch.Generator
														A      :					List[Any]      		=					self.get_dummy_components()
														A      :					Union[str, Any]      		=					EulerAncestralDiscreteScheduler(
														    beta_start=0.0_0_0_8_5    ,						beta_end=0.0_1_2    ,						beta_schedule='''scaled_linear'''						)
														A      :					int      		=					StableDiffusionInstructPixaPixPipeline(**__UpperCAmelCase						)
														A      :					Tuple      		=					sd_pipe.to(__UpperCAmelCase						)
														sd_pipe.set_progress_bar_config(disable=__UpperCAmelCase						)

														A      :					Tuple      		=					self.get_dummy_inputs(__UpperCAmelCase						)
														A      :					Optional[Any]      		=					sd_pipe(**__UpperCAmelCase						).images
														A      :					Dict      		=					image[0, -3:, -3:, -1]

														A      :					Optional[int]      		=					[round(__UpperCAmelCase    ,						4						) for x in image_slice.flatten().tolist()]
														print(''','''.join([str(__UpperCAmelCase						) for x in slice]						)						)

														assert image.shape == (1, 32, 32, 3)
														A      :					Dict      		=					np.array([0.7_4_1_7, 0.3_8_4_2, 0.4_7_3_2, 0.5_7_7_6, 0.5_8_9_1, 0.5_1_3_9, 0.4_0_5_2, 0.5_6_7_3, 0.4_9_8_6]						)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 1E-3




							def       snake_case			(					self						)		->       Union[str, Any]:
														super().test_inference_batch_single_identical(expected_max_diff=3E-3						)






							def       snake_case			(					self						)		->       Union[str, Any]:
														A      :					int      		=					self.get_dummy_components()
														A      :					int      		=					StableDiffusionInstructPixaPixPipeline(**__UpperCAmelCase						)
														A      :					Any      		=					VaeImageProcessor(do_resize=__UpperCAmelCase    ,						do_normalize=__UpperCAmelCase						)
														A      :					Optional[Any]      		=					pipe.to(__UpperCAmelCase						)
														pipe.set_progress_bar_config(disable=__UpperCAmelCase						)

														A      :					Union[str, Any]      		=					pipe(**self.get_dummy_inputs_by_type(__UpperCAmelCase    ,						input_image_type='''pt'''						)						)[0]

														A      :					Optional[Any]      		=					components['''vae''']
														A      :					Dict      		=					self.get_dummy_inputs_by_type(__UpperCAmelCase    ,						input_image_type='''pt'''						)

														for image_param in self.image_latents_params:
																					if image_param in inputs.keys():
																												A      :					Optional[Any]      		=					vae.encode(inputs[image_param]						).latent_dist.mode()

														A      :					List[str]      		=					pipe(**__UpperCAmelCase						)[0]

														A      :					Any      		=					np.abs(out - out_latents_inputs						).max()
														self.assertLess(__UpperCAmelCase    ,						1E-4    ,						'''passing latents as image input generate different result from passing image'''						)







@slow
@require_torch_gpu
class 					__lowercase  (						unittest.TestCase				):





							"""simple docstring"""

							def       snake_case			(					self						)		->       str:
														super().tearDown()
														gc.collect()
														torch.cuda.empty_cache()




							def       snake_case			(					self    ,						__UpperCAmelCase=0						)		->       Tuple:
														A      :					List[str]      		=					torch.manual_seed(__UpperCAmelCase						)
														A      :					List[str]      		=					load_image(
														    '''https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_pix2pix/example.jpg'''						)
														A      :					Dict      		=					{
														    '''prompt''': '''turn him into a cyborg''',
														    '''image''': image,
														    '''generator''': generator,
														    '''num_inference_steps''': 3,
														    '''guidance_scale''': 7.5,
														    '''image_guidance_scale''': 1.0,
														    '''output_type''': '''numpy''',
														}
														return inputs




							def       snake_case			(					self						)		->       Optional[int]:
														A      :					Union[str, Any]      		=					StableDiffusionInstructPixaPixPipeline.from_pretrained(
														    '''timbrooks/instruct-pix2pix'''    ,						safety_checker=__UpperCAmelCase						)
														pipe.to(__UpperCAmelCase						)
														pipe.set_progress_bar_config(disable=__UpperCAmelCase						)
														pipe.enable_attention_slicing()

														A      :					Optional[int]      		=					self.get_inputs()
														A      :					Any      		=					pipe(**__UpperCAmelCase						).images
														A      :					Union[str, Any]      		=					image[0, -3:, -3:, -1].flatten()

														assert image.shape == (1, 5_12, 5_12, 3)
														A      :					Dict      		=					np.array([0.5_9_0_2, 0.6_0_1_5, 0.6_0_2_7, 0.5_9_8_3, 0.6_0_9_2, 0.6_0_6_1, 0.5_7_6_5, 0.5_7_8_5, 0.5_5_5_5]						)

														assert np.abs(expected_slice - image_slice						).max() < 1E-3




							def       snake_case			(					self						)		->       Dict:
														A      :					str      		=					StableDiffusionInstructPixaPixPipeline.from_pretrained(
														    '''timbrooks/instruct-pix2pix'''    ,						safety_checker=__UpperCAmelCase						)
														A      :					List[str]      		=					LMSDiscreteScheduler.from_config(pipe.scheduler.config						)
														pipe.to(__UpperCAmelCase						)
														pipe.set_progress_bar_config(disable=__UpperCAmelCase						)
														pipe.enable_attention_slicing()

														A      :					Union[str, Any]      		=					self.get_inputs()
														A      :					Union[str, Any]      		=					pipe(**__UpperCAmelCase						).images
														A      :					str      		=					image[0, -3:, -3:, -1].flatten()

														assert image.shape == (1, 5_12, 5_12, 3)
														A      :					Dict      		=					np.array([0.6_5_7_8, 0.6_8_1_7, 0.6_9_7_2, 0.6_7_6_1, 0.6_8_5_6, 0.6_9_1_6, 0.6_4_2_8, 0.6_5_1_6, 0.6_3_0_1]						)

														assert np.abs(expected_slice - image_slice						).max() < 1E-3




							def       snake_case			(					self						)		->       List[Any]:
														A      :					List[Any]      		=					StableDiffusionInstructPixaPixPipeline.from_pretrained(
														    '''timbrooks/instruct-pix2pix'''    ,						safety_checker=__UpperCAmelCase						)
														A      :					List[str]      		=					DDIMScheduler.from_config(pipe.scheduler.config						)
														pipe.to(__UpperCAmelCase						)
														pipe.set_progress_bar_config(disable=__UpperCAmelCase						)
														pipe.enable_attention_slicing()

														A      :					str      		=					self.get_inputs()
														A      :					List[Any]      		=					pipe(**__UpperCAmelCase						).images
														A      :					str      		=					image[0, -3:, -3:, -1].flatten()

														assert image.shape == (1, 5_12, 5_12, 3)
														A      :					Any      		=					np.array([0.3_8_2_8, 0.3_8_3_4, 0.3_8_1_8, 0.3_7_9_2, 0.3_8_6_5, 0.3_7_5_2, 0.3_7_9_2, 0.3_8_4_7, 0.3_7_5_3]						)

														assert np.abs(expected_slice - image_slice						).max() < 1E-3




							def       snake_case			(					self						)		->       Optional[Any]:
														A      :					Optional[Any]      		=					0

														def callback_fn(__UpperCAmelCase    ,						__UpperCAmelCase    ,						__UpperCAmelCase						) -> None:
																					A      :					List[Any]      		=					True
																					nonlocal number_of_steps
																					number_of_steps += 1
																					if step == 1:
																												A      :					Optional[Any]      		=					latents.detach().cpu().numpy()
																												assert latents.shape == (1, 4, 64, 64)
																												A      :					Union[str, Any]      		=					latents[0, -3:, -3:, -1]
																												A      :					List[str]      		=					np.array([-0.2_4_6_3, -0.4_6_4_4, -0.9_7_5_6, 1.5_1_7_6, 1.4_4_1_4, 0.7_8_6_6, 0.9_8_9_7, 0.8_5_2_1, 0.7_9_8_3]						)

																												assert np.abs(latents_slice.flatten() - expected_slice						).max() < 5E-2
																					elif step == 2:
																												A      :					int      		=					latents.detach().cpu().numpy()
																												assert latents.shape == (1, 4, 64, 64)
																												A      :					Optional[int]      		=					latents[0, -3:, -3:, -1]
																												A      :					str      		=					np.array([-0.2_6_4_4, -0.4_6_2_6, -0.9_6_5_3, 1.5_1_7_6, 1.4_5_5_1, 0.7_6_8_6, 0.9_8_0_5, 0.8_4_5_2, 0.8_1_1_5]						)

																												assert np.abs(latents_slice.flatten() - expected_slice						).max() < 5E-2

														A      :					Optional[Any]      		=					False

														A      :					str      		=					StableDiffusionInstructPixaPixPipeline.from_pretrained(
														    '''timbrooks/instruct-pix2pix'''    ,						safety_checker=__UpperCAmelCase    ,						torch_dtype=torch.floataa						)
														A      :					List[str]      		=					pipe.to(__UpperCAmelCase						)
														pipe.set_progress_bar_config(disable=__UpperCAmelCase						)
														pipe.enable_attention_slicing()

														A      :					List[Any]      		=					self.get_inputs()
														pipe(**__UpperCAmelCase    ,						callback=__UpperCAmelCase    ,						callback_steps=1						)
														assert callback_fn.has_been_called
														assert number_of_steps == 3




							def       snake_case			(					self						)		->       Tuple:
														torch.cuda.empty_cache()
														torch.cuda.reset_max_memory_allocated()
														torch.cuda.reset_peak_memory_stats()

														A      :					Optional[Any]      		=					StableDiffusionInstructPixaPixPipeline.from_pretrained(
														    '''timbrooks/instruct-pix2pix'''    ,						safety_checker=__UpperCAmelCase    ,						torch_dtype=torch.floataa						)
														A      :					Dict      		=					pipe.to(__UpperCAmelCase						)
														pipe.set_progress_bar_config(disable=__UpperCAmelCase						)
														pipe.enable_attention_slicing(1						)
														pipe.enable_sequential_cpu_offload()

														A      :					Optional[Any]      		=					self.get_inputs()
														A      :					str      		=					pipe(**__UpperCAmelCase						)

														A      :					Dict      		=					torch.cuda.max_memory_allocated()
														# make sure that less than 2.2 GB is allocated
														assert mem_bytes < 2.2 * 10**9






							def       snake_case			(					self						)		->       int:
														A      :					Union[str, Any]      		=					self.get_inputs()
														# resize to resolution that is divisible by 8 but not 16 or 32
														A      :					Union[str, Any]      		=					inputs['''image'''].resize((5_04, 5_04)						)

														A      :					str      		=					'''timbrooks/instruct-pix2pix'''
														A      :					Tuple      		=					StableDiffusionInstructPixaPixPipeline.from_pretrained(
														    __UpperCAmelCase    ,						safety_checker=__UpperCAmelCase    ,						)
														pipe.to(__UpperCAmelCase						)
														pipe.set_progress_bar_config(disable=__UpperCAmelCase						)
														pipe.enable_attention_slicing()

														A      :					Tuple      		=					pipe(**__UpperCAmelCase						)
														A      :					List[str]      		=					output.images[0]

														A      :					List[Any]      		=					image[2_55:2_58, 3_83:3_86, -1]

														assert image.shape == (5_04, 5_04, 3)
														A      :					Any      		=					np.array([0.2_7_2_6, 0.2_5_2_9, 0.2_6_6_4, 0.2_6_5_5, 0.2_6_4_1, 0.2_6_4_2, 0.2_5_9_1, 0.2_6_4_9, 0.2_5_9_0]						)

														assert np.abs(image_slice.flatten() - expected_slice						).max() < 5E-3









import os
import posixpath
import uuid
from dataclasses import dataclass
from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, Union

import numpy as np
import pyarrow as pa

import datasets
from datasets.arrow_writer import ArrowWriter, ParquetWriter
from datasets.config import MAX_SHARD_SIZE
from datasets.filesystems import (
    is_remote_filesystem,
    rename,
)
from datasets.iterable_dataset import _BaseExamplesIterable
from datasets.utils.py_utils import convert_file_size_to_int


lowercase			:				Dict   =     datasets.utils.logging.get_logger(__name__)

if TYPE_CHECKING:
		import pyspark







@dataclass
class 					__lowercase  (						datasets.BuilderConfig				):





							"""simple docstring"""

							UpperCAmelCase_						:		Optional[datasets.Features]								=	None






def        snake_case__		(    lowerCamelCase_     ,				lowerCamelCase_     ,				):
							import pyspark

							def generate_fn():
														A      :					Dict      		=					df.select('''*'''     ,				pyspark.sql.functions.spark_partition_id().alias('''part_id'''		)		)
														for partition_id in partition_order:
																					A      :					Optional[Any]      		=					df_with_partition_id.select('''*'''		).where(F'part_id = {partition_id}'		).drop('''part_id'''		)
																					A      :					Dict      		=					partition_df.collect()
																					A      :					List[str]      		=					0
																					for row in rows:
																												yield F'{partition_id}_{row_id}', row.asDict()
																												row_id += 1

							return generate_fn







class 					__lowercase  (						_BaseExamplesIterable				):





							"""simple docstring"""

							def __init__(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase=None    ,						)		->       List[str]:
														A      :					List[Any]      		=					df
														A      :					List[str]      		=					partition_order or range(self.df.rdd.getNumPartitions()						)
														A      :					Optional[int]      		=					_generate_iterable_examples(self.df    ,						self.partition_order						)




							def __iter__(					self						)		->       str:
														yield from self.generate_examples_fn()




							def       snake_case			(					self    ,						__UpperCAmelCase						)		->       "SparkExamplesIterable":
														A      :					Tuple      		=					list(range(self.df.rdd.getNumPartitions()						)						)
														generator.shuffle(__UpperCAmelCase						)
														return SparkExamplesIterable(self.df    ,						partition_order=__UpperCAmelCase						)




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase						)		->       "SparkExamplesIterable":
														A      :					Tuple      		=					self.split_shard_indices_by_worker(__UpperCAmelCase    ,						__UpperCAmelCase						)
														return SparkExamplesIterable(self.df    ,						partition_order=__UpperCAmelCase						)






							@property
							def       snake_case			(					self						)		->       int:
														return len(self.partition_order						)







class 					__lowercase  (						datasets.DatasetBuilder				):





							"""simple docstring"""

							UpperCAmelCase_						:		List[str]								=	SparkConfig




							def __init__(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase = None    ,						__UpperCAmelCase = None    ,						**__UpperCAmelCase    ,						)		->       Optional[Any]:
														import pyspark

														A      :					Tuple      		=					pyspark.sql.SparkSession.builder.getOrCreate()
														A      :					List[Any]      		=					df
														A      :					List[Any]      		=					working_dir

														super().__init__(
														    cache_dir=__UpperCAmelCase    ,						config_name=str(self.df.semanticHash()						)    ,						**__UpperCAmelCase    ,						)




							def       snake_case			(					self						)		->       Optional[Any]:
														# Returns the path of the created file.
														def create_cache_and_write_probe(__UpperCAmelCase						):
																					# makedirs with exist_ok will recursively create the directory. It will not throw an error if directories
																					# already exist.
																					os.makedirs(self._cache_dir    ,						exist_ok=__UpperCAmelCase						)
																					A      :					Optional[Any]      		=					os.path.join(self._cache_dir    ,						'''fs_test''' + uuid.uuida().hex						)
																					# Opening the file in append mode will create a new file unless it already exists, in which case it will not
																					# change the file contents.
																					open(__UpperCAmelCase    ,						'''a'''						)
																					return [probe_file]

														if self._spark.conf.get('''spark.master'''    ,						''''''						).startswith('''local'''						):
																					return

														# If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS
														# accessible to the driver.
														# TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error.
														if self._cache_dir:
																					A      :					Optional[int]      		=					(
																					    self._spark.sparkContext.parallelize(range(1						)    ,						1						).mapPartitions(__UpperCAmelCase						).collect()
																					)
																					if os.path.isfile(probe[0]						):
																												return

														raise ValueError(
														    '''When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir'''						)




							def       snake_case			(					self						)		->       List[Any]:
														return datasets.DatasetInfo(features=self.config.features						)




							def       snake_case			(					self    ,						__UpperCAmelCase						)		->       str:
														return [datasets.SplitGenerator(name=datasets.Split.TRAIN						)]




							def       snake_case			(					self    ,						__UpperCAmelCase						)		->       List[Any]:
														import pyspark

														def get_arrow_batch_size(__UpperCAmelCase						):
																					for batch in it:
																												yield pa.RecordBatch.from_pydict({'''batch_bytes''': [batch.nbytes]}						)

														A      :					Any      		=					self.df.count()
														A      :					Tuple      		=					df_num_rows if df_num_rows <= 1_00 else 1_00
														# Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample.
														A      :					Dict      		=					(
														    self.df.limit(__UpperCAmelCase						)
														    .repartition(1						)
														    .mapInArrow(__UpperCAmelCase    ,						'''batch_bytes: long'''						)
														    .agg(pyspark.sql.functions.sum('''batch_bytes'''						).alias('''sample_bytes'''						)						)
														    .collect()[0]
														    .sample_bytes
														    / sample_num_rows
														)
														A      :					str      		=					approx_bytes_per_row * df_num_rows
														if approx_total_size > max_shard_size:
																					# Make sure there is at least one row per partition.
																					A      :					Union[str, Any]      		=					min(__UpperCAmelCase    ,						int(approx_total_size / max_shard_size						)						)
																					A      :					List[Any]      		=					self.df.repartition(__UpperCAmelCase						)




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase    ,						__UpperCAmelCase    ,						)		->       Iterable[Tuple[int, bool, Union[int, tuple]]]:
														import pyspark

														A      :					Dict      		=					ParquetWriter if file_format == '''parquet''' else ArrowWriter
														A      :					Optional[Any]      		=					os.path.join(self._working_dir    ,						os.path.basename(__UpperCAmelCase						)						) if self._working_dir else fpath
														A      :					int      		=					file_format == '''parquet'''

														# Define these so that we don't reference self in write_arrow, which will result in a pickling error due to
														# pickling the SparkContext.
														A      :					Any      		=					self.config.features
														A      :					Any      		=					self._writer_batch_size
														A      :					int      		=					self._fs.storage_options

														def write_arrow(__UpperCAmelCase						):
																					# Within the same SparkContext, no two task attempts will share the same attempt ID.
																					A      :					List[Any]      		=					pyspark.TaskContext().taskAttemptId()
																					A      :					Union[str, Any]      		=					next(__UpperCAmelCase    ,						__UpperCAmelCase						)
																					if first_batch is None:
																												# Some partitions might not receive any data.
																												return pa.RecordBatch.from_arrays(
																												    [[task_id], [0], [0]]    ,						names=['''task_id''', '''num_examples''', '''num_bytes''']    ,						)
																					A      :					Optional[int]      		=					0
																					A      :					Optional[int]      		=					writer_class(
																					    features=__UpperCAmelCase    ,						path=working_fpath.replace('''SSSSS'''    ,						f'{shard_id:05d}'						).replace('''TTTTT'''    ,						f'{task_id:05d}'						)    ,						writer_batch_size=__UpperCAmelCase    ,						storage_options=__UpperCAmelCase    ,						embed_local_files=__UpperCAmelCase    ,						)
																					A      :					Tuple      		=					pa.Table.from_batches([first_batch]						)
																					writer.write_table(__UpperCAmelCase						)
																					for batch in it:
																												if max_shard_size is not None and writer._num_bytes >= max_shard_size:
																																			A		,		A      :					str      		=					writer.finalize()
																																			writer.close()
																																			yield pa.RecordBatch.from_arrays(
																																			    [[task_id], [num_examples], [num_bytes]]    ,						names=['''task_id''', '''num_examples''', '''num_bytes''']    ,						)
																																			shard_id += 1
																																			A      :					List[str]      		=					writer_class(
																																			    features=writer._features    ,						path=working_fpath.replace('''SSSSS'''    ,						f'{shard_id:05d}'						).replace('''TTTTT'''    ,						f'{task_id:05d}'						)    ,						writer_batch_size=__UpperCAmelCase    ,						storage_options=__UpperCAmelCase    ,						embed_local_files=__UpperCAmelCase    ,						)
																												A      :					str      		=					pa.Table.from_batches([batch]						)
																												writer.write_table(__UpperCAmelCase						)

																					if writer._num_bytes > 0:
																												A		,		A      :					Tuple      		=					writer.finalize()
																												writer.close()
																												yield pa.RecordBatch.from_arrays(
																												    [[task_id], [num_examples], [num_bytes]]    ,						names=['''task_id''', '''num_examples''', '''num_bytes''']    ,						)

																					if working_fpath != fpath:
																												for file in os.listdir(os.path.dirname(__UpperCAmelCase						)						):
																																			A      :					Union[str, Any]      		=					os.path.join(os.path.dirname(__UpperCAmelCase						)    ,						os.path.basename(__UpperCAmelCase						)						)
																																			shutil.move(__UpperCAmelCase    ,						__UpperCAmelCase						)

														A      :					Union[str, Any]      		=					(
														    self.df.mapInArrow(__UpperCAmelCase    ,						'''task_id: long, num_examples: long, num_bytes: long'''						)
														    .groupBy('''task_id'''						)
														    .agg(
														        pyspark.sql.functions.sum('''num_examples'''						).alias('''total_num_examples'''						)    ,						pyspark.sql.functions.sum('''num_bytes'''						).alias('''total_num_bytes'''						)    ,						pyspark.sql.functions.count('''num_bytes'''						).alias('''num_shards'''						)    ,						pyspark.sql.functions.collect_list('''num_examples'''						).alias('''shard_lengths'''						)    ,						)
														    .collect()
														)
														for row in stats:
																					yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths)




							def       snake_case			(					self    ,						__UpperCAmelCase    ,						__UpperCAmelCase = "arrow"    ,						__UpperCAmelCase = None    ,						__UpperCAmelCase = None    ,						**__UpperCAmelCase    ,						)		->       List[str]:
														self._validate_cache_dir()

														A      :					int      		=					convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE						)
														self._repartition_df_if_needed(__UpperCAmelCase						)
														A      :					Any      		=					not is_remote_filesystem(self._fs						)
														A      :					Union[str, Any]      		=					os.path.join if is_local else posixpath.join

														A      :					Any      		=					'''-TTTTT-SSSSS-of-NNNNN'''
														A      :					Any      		=					f'{self.name}-{split_generator.name}{SUFFIX}.{file_format}'
														A      :					List[Any]      		=					path_join(self._output_dir    ,						__UpperCAmelCase						)

														A      :					Union[str, Any]      		=					0
														A      :					Any      		=					0
														A      :					Tuple      		=					0
														A      :					Union[str, Any]      		=					[]
														A      :					List[str]      		=					[]

														for task_id, content in self._prepare_split_single(__UpperCAmelCase    ,						__UpperCAmelCase    ,						__UpperCAmelCase						):
																					(
																					    (
																					    A
																					)		,		(
																					    A
																					)		,		(
																					    A
																					)		,		(
																					    A
																					)		,		
																					)      :					List[Any]      		=					content
																					if num_bytes > 0:
																												total_num_examples += num_examples
																												total_num_bytes += num_bytes
																												total_shards += num_shards
																												task_id_and_num_shards.append((task_id, num_shards)						)
																												all_shard_lengths.extend(__UpperCAmelCase						)

														A      :					List[str]      		=					total_num_examples
														A      :					List[str]      		=					total_num_bytes

														# should rename everything at the end
														logger.debug(f'Renaming {total_shards} shards.'						)
														if total_shards > 1:
																					A      :					Tuple      		=					all_shard_lengths

																					# Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a
																					# pickling error due to pickling the SparkContext.
																					A      :					Union[str, Any]      		=					self._fs

																					# use the -SSSSS-of-NNNNN pattern
																					def _rename_shard(
																					    __UpperCAmelCase    ,						__UpperCAmelCase    ,						__UpperCAmelCase    ,						):
																												rename(
																												    __UpperCAmelCase    ,						fpath.replace('''SSSSS'''    ,						f'{shard_id:05d}'						).replace('''TTTTT'''    ,						f'{task_id:05d}'						)    ,						fpath.replace('''TTTTT-SSSSS'''    ,						f'{global_shard_id:05d}'						).replace('''NNNNN'''    ,						f'{total_shards:05d}'						)    ,						)

																					A      :					Tuple      		=					[]
																					A      :					Optional[int]      		=					0
																					for i in range(len(__UpperCAmelCase						)						):
																												A		,		A      :					Tuple      		=					task_id_and_num_shards[i]
																												for shard_id in range(__UpperCAmelCase						):
																																			args.append([task_id, shard_id, global_shard_id]						)
																																			global_shard_id += 1
																					self._spark.sparkContext.parallelize(__UpperCAmelCase    ,						len(__UpperCAmelCase						)						).map(lambda __UpperCAmelCase						: _rename_shard(*__UpperCAmelCase						)						).collect()
														else:
																					# don't use any pattern
																					A      :					int      		=					0
																					A      :					Tuple      		=					task_id_and_num_shards[0][0]
																					self._rename(
																					    fpath.replace('''SSSSS'''    ,						f'{shard_id:05d}'						).replace('''TTTTT'''    ,						f'{task_id:05d}'						)    ,						fpath.replace(__UpperCAmelCase    ,						''''''						)    ,						)






							def       snake_case			(					self    ,						__UpperCAmelCase    ,						)		->       SparkExamplesIterable:
														return SparkExamplesIterable(self.df						)






