
"""simple docstring"""


import math
from typing import Dict, Iterable, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    is_torch_available,
    is_torch_tensor,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_torch_available():
						import torch

if is_vision_available():
						import PIL


A_    							=					logging.get_logger(__name__)






def  lowercase						(	lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__						):
			def constraint_to_multiple_of(lowerCAmelCase__	,lowerCAmelCase__	,lowerCAmelCase__=0	,lowerCAmelCase__=None						):
						lowerCamelCase_					=  round(val / multiple						) * multiple

						if max_val is not None and x > max_val:
									lowerCamelCase_					=  math.floor(val / multiple						) * multiple

						if x < min_val:
									lowerCamelCase_					=  math.ceil(val / multiple						) * multiple

						return x

			lowerCamelCase_					=  (output_size, output_size) if isinstance(lowerCAmelCase__	,lowerCAmelCase__						) else output_size

			lowerCamelCase_		, lowerCamelCase_					=  get_image_size(lowerCAmelCase__						)
			lowerCamelCase_		, lowerCamelCase_					=  output_size

			# determine new height and width
			lowerCamelCase_					=  output_height / input_height
			lowerCamelCase_					=  output_width / input_width

			if keep_aspect_ratio:
						# scale as little as possible
						if abs(1 - scale_width						) < abs(1 - scale_height						):
									# fit width
									lowerCamelCase_					=  scale_width
						else:
									# fit height
									lowerCamelCase_					=  scale_height

			lowerCamelCase_					=  constraint_to_multiple_of(scale_height * input_height	,multiple=lowerCAmelCase__						)
			lowerCamelCase_					=  constraint_to_multiple_of(scale_width * input_width	,multiple=lowerCAmelCase__						)

			return (new_height, new_width)




class  __lowerCamelCase  ( lowerCAmelCase			):
				a__:      int          =    ['pixel_values']




				def __init__(				self							,						UpperCAmelCase = True							,						UpperCAmelCase = None							,						UpperCAmelCase = PILImageResampling.BILINEAR							,						UpperCAmelCase = False							,						UpperCAmelCase = 1							,						UpperCAmelCase = True							,						UpperCAmelCase = 1 / 255							,						UpperCAmelCase = True							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						**UpperCAmelCase							,						):
							super().__init__(**UpperCAmelCase   )
							lowerCamelCase_					=  size if size is not None else {'''height''': 384, '''width''': 384}
							lowerCamelCase_					=  get_size_dict(UpperCAmelCase   )
							lowerCamelCase_					=  do_resize
							lowerCamelCase_					=  size
							lowerCamelCase_					=  keep_aspect_ratio
							lowerCamelCase_					=  ensure_multiple_of
							lowerCamelCase_					=  resample
							lowerCamelCase_					=  do_rescale
							lowerCamelCase_					=  rescale_factor
							lowerCamelCase_					=  do_normalize
							lowerCamelCase_					=  image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
							lowerCamelCase_					=  image_std if image_std is not None else IMAGENET_STANDARD_STD




				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase							,						UpperCAmelCase = False							,						UpperCAmelCase = 1							,						UpperCAmelCase = PILImageResampling.BICUBIC							,						UpperCAmelCase = None							,						**UpperCAmelCase							,						):
							lowerCamelCase_					=  get_size_dict(UpperCAmelCase   )
							if "height" not in size or "width" not in size:
										raise ValueError(f"The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}"   )
							lowerCamelCase_					=  get_resize_output_image_size(
							    UpperCAmelCase							,						output_size=(size['''height'''], size['''width'''])							,						keep_aspect_ratio=UpperCAmelCase							,						multiple=UpperCAmelCase							,						)
							return resize(UpperCAmelCase							,						size=UpperCAmelCase							,						resample=UpperCAmelCase							,						data_format=UpperCAmelCase							,						**UpperCAmelCase   )




				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase							,						UpperCAmelCase = None							,						**UpperCAmelCase							,						):
							return rescale(UpperCAmelCase							,						scale=UpperCAmelCase							,						data_format=UpperCAmelCase							,						**UpperCAmelCase   )




				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase							,						UpperCAmelCase							,						UpperCAmelCase = None							,						**UpperCAmelCase							,						):
							return normalize(UpperCAmelCase							,						mean=UpperCAmelCase							,						std=UpperCAmelCase							,						data_format=UpperCAmelCase							,						**UpperCAmelCase   )




				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						UpperCAmelCase = ChannelDimension.FIRST							,						**UpperCAmelCase							,						):
							lowerCamelCase_					=  do_resize if do_resize is not None else self.do_resize
							lowerCamelCase_					=  size if size is not None else self.size
							lowerCamelCase_					=  get_size_dict(UpperCAmelCase   )
							lowerCamelCase_					=  keep_aspect_ratio if keep_aspect_ratio is not None else self.keep_aspect_ratio
							lowerCamelCase_					=  ensure_multiple_of if ensure_multiple_of is not None else self.ensure_multiple_of
							lowerCamelCase_					=  resample if resample is not None else self.resample
							lowerCamelCase_					=  do_rescale if do_rescale is not None else self.do_rescale
							lowerCamelCase_					=  rescale_factor if rescale_factor is not None else self.rescale_factor
							lowerCamelCase_					=  do_normalize if do_normalize is not None else self.do_normalize
							lowerCamelCase_					=  image_mean if image_mean is not None else self.image_mean
							lowerCamelCase_					=  image_std if image_std is not None else self.image_std

							lowerCamelCase_					=  make_list_of_images(UpperCAmelCase   )

							if not valid_images(UpperCAmelCase   ):
										raise ValueError(
										    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
										    '''torch.Tensor, tf.Tensor or jax.ndarray.'''   )

							if do_resize and size is None or resample is None:
										raise ValueError('''Size and resample must be specified if do_resize is True.'''   )

							if do_rescale and rescale_factor is None:
										raise ValueError('''Rescale factor must be specified if do_rescale is True.'''   )

							if do_normalize and (image_mean is None or image_std is None):
										raise ValueError('''Image mean and std must be specified if do_normalize is True.'''   )

							# All transformations expect numpy arrays.
							lowerCamelCase_					=  [to_numpy_array(UpperCAmelCase   ) for image in images]

							if do_resize:
										lowerCamelCase_					=  [self.resize(image=UpperCAmelCase							,						size=UpperCAmelCase							,						resample=UpperCAmelCase   ) for image in images]

							if do_rescale:
										lowerCamelCase_					=  [self.rescale(image=UpperCAmelCase							,						scale=UpperCAmelCase   ) for image in images]

							if do_normalize:
										lowerCamelCase_					=  [self.normalize(image=UpperCAmelCase							,						mean=UpperCAmelCase							,						std=UpperCAmelCase   ) for image in images]

							lowerCamelCase_					=  [to_channel_dimension_format(UpperCAmelCase							,						UpperCAmelCase   ) for image in images]

							lowerCamelCase_					=  {'''pixel_values''': images}
							return BatchFeature(data=UpperCAmelCase							,						tensor_type=UpperCAmelCase   )





				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase = None   ):
							lowerCamelCase_					=  outputs.logits

							# Resize logits and compute semantic segmentation maps
							if target_sizes is not None:
										if len(UpperCAmelCase   ) != len(UpperCAmelCase   ):
													raise ValueError(
													    '''Make sure that you pass in as many target sizes as the batch dimension of the logits'''   )

										if is_torch_tensor(UpperCAmelCase   ):
													lowerCamelCase_					=  target_sizes.numpy()

										lowerCamelCase_					=  []

										for idx in range(len(UpperCAmelCase   )   ):
													lowerCamelCase_					=  torch.nn.functional.interpolate(
													    logits[idx].unsqueeze(dim=0   )							,						size=target_sizes[idx]							,						mode='''bilinear'''							,						align_corners=UpperCAmelCase   )
													lowerCamelCase_					=  resized_logits[0].argmax(dim=0   )
													semantic_segmentation.append(UpperCAmelCase   )
							else:
										lowerCamelCase_					=  logits.argmax(dim=1   )
										lowerCamelCase_					=  [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0]   )]

							return semantic_segmentation











from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)






def 	UpperCamelCase	(							snake_case__  : List[Any]						,snake_case__  : List[str]	):




     '''simple docstring'''






     __snake_case      :int 							=    []
     for part_id in partition_order:
          __snake_case      :int 							=    df.where(f'''SPARK_PARTITION_ID() = {part_id}'''	).collect()
          for row_idx, row in enumerate(snake_case__	):
               expected_row_ids_and_row_dicts.append((f'''{part_id}_{row_idx}''', row.asDict())	)
     return expected_row_ids_and_row_dicts






@require_not_windows
@require_dill_gt_0_3_2
def 	UpperCamelCase	(							):




     '''simple docstring'''






     __snake_case      :List[str] 							=    pyspark.sql.SparkSession.builder.master("""local[*]"""	).appName("""pyspark"""	).getOrCreate()
     __snake_case      :Any 							=    spark.range(100	).repartition(1	)
     __snake_case      :int 							=    Spark(snake_case__	)
     # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
     # that each partition can hold 2 rows.
     spark_builder._repartition_df_if_needed(max_shard_size=16	)
     # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
     assert spark_builder.df.rdd.getNumPartitions() == 50






@require_not_windows
@require_dill_gt_0_3_2
def 	UpperCamelCase	(							):




     '''simple docstring'''






     __snake_case      :Tuple 							=    pyspark.sql.SparkSession.builder.master("""local[*]"""	).appName("""pyspark"""	).getOrCreate()
     __snake_case      :int 							=    spark.range(10	).repartition(2	)
     __snake_case      :str 							=    [1, 0]
     __snake_case      :List[Any] 							=    _generate_iterable_examples(snake_case__						,snake_case__	)  # Reverse the partitions.
     __snake_case      :Union[str, Any] 							=    _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case__						,snake_case__	)

     for i, (row_id, row_dict) in enumerate(generate_fn()	):
          __snake_case							,			__snake_case      :Union[str, Any] 							=    expected_row_ids_and_row_dicts[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def 	UpperCamelCase	(							):




     '''simple docstring'''






     __snake_case      :Union[str, Any] 							=    pyspark.sql.SparkSession.builder.master("""local[*]"""	).appName("""pyspark"""	).getOrCreate()
     __snake_case      :Tuple 							=    spark.range(10	).repartition(1	)
     __snake_case      :Dict 							=    SparkExamplesIterable(snake_case__	)
     assert it.n_shards == 1
     for i, (row_id, row_dict) in enumerate(snake_case__	):
          assert row_id == f'''0_{i}'''
          assert row_dict == {"id": i}






@require_not_windows
@require_dill_gt_0_3_2
def 	UpperCamelCase	(							):




     '''simple docstring'''






     __snake_case      :List[str] 							=    pyspark.sql.SparkSession.builder.master("""local[*]"""	).appName("""pyspark"""	).getOrCreate()
     __snake_case      :Union[str, Any] 							=    spark.range(30	).repartition(3	)
     # Mock the generator so that shuffle reverses the partition indices.
     with patch("""numpy.random.Generator"""	) as generator_mock:
          __snake_case      :Dict 							=    lambda snake_case__	: x.reverse()
          __snake_case      :int 							=    _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case__						,[2, 1, 0]	)

          __snake_case      :Dict 							=    SparkExamplesIterable(snake_case__	).shuffle_data_sources(snake_case__	)
          assert shuffled_it.n_shards == 3
          for i, (row_id, row_dict) in enumerate(snake_case__	):
               __snake_case							,			__snake_case      :List[str] 							=    expected_row_ids_and_row_dicts[i]
               assert row_id == expected_row_id
               assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def 	UpperCamelCase	(							):




     '''simple docstring'''






     __snake_case      :Union[str, Any] 							=    pyspark.sql.SparkSession.builder.master("""local[*]"""	).appName("""pyspark"""	).getOrCreate()
     __snake_case      :Tuple 							=    spark.range(20	).repartition(4	)

     # Partitions 0 and 2
     __snake_case      :List[Any] 							=    SparkExamplesIterable(snake_case__	).shard_data_sources(worker_id=0						,num_workers=2	)
     assert shard_it_a.n_shards == 2
     __snake_case      :Dict 							=    _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case__						,[0, 2]	)
     for i, (row_id, row_dict) in enumerate(snake_case__	):
          __snake_case							,			__snake_case      :Tuple 							=    expected_row_ids_and_row_dicts_a[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict

     # Partitions 1 and 3
     __snake_case      :str 							=    SparkExamplesIterable(snake_case__	).shard_data_sources(worker_id=1						,num_workers=2	)
     assert shard_it_a.n_shards == 2
     __snake_case      :Optional[int] 							=    _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case__						,[1, 3]	)
     for i, (row_id, row_dict) in enumerate(snake_case__	):
          __snake_case							,			__snake_case      :Dict 							=    expected_row_ids_and_row_dicts_a[i]
          assert row_id == expected_row_id
          assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def 	UpperCamelCase	(							):




     '''simple docstring'''






     __snake_case      :Union[str, Any] 							=    pyspark.sql.SparkSession.builder.master("""local[*]"""	).appName("""pyspark"""	).getOrCreate()
     __snake_case      :Tuple 							=    spark.range(100	).repartition(1	)
     __snake_case      :Dict 							=    Spark(snake_case__	)
     # Choose a small max_shard_size for maximum partitioning.
     spark_builder._repartition_df_if_needed(max_shard_size=1	)
     # The new number of partitions should not be greater than the number of rows.
     assert spark_builder.df.rdd.getNumPartitions() == 100


