



'''simple docstring'''





import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam






class __A     (    datasets.BeamBasedBuilder		):



 '''simple docstring'''







 def    a__    (self			)       ->	Optional[int]:


      """simple docstring"""



      return datasets.DatasetInfo(
          features=datasets.Features({'''content''': datasets.Value('''string'''			)}			)					,  supervised_keys=A					,  )



 def    a__    (self					,  A					,  A			)       ->	Optional[Any]:


      """simple docstring"""



      return [datasets.SplitGenerator(name=datasets.Split.TRAIN					,  gen_kwargs={'''examples''': get_test_dummy_examples()}			)]



 def    a__    (self					,  A					,  A			)       ->	str:


      """simple docstring"""



      import apache_beam as beam

      return pipeline | "Load Examples" >> beam.Create(A			)






class __A     (    datasets.BeamBasedBuilder		):



 '''simple docstring'''







 def    a__    (self			)       ->	int:


      """simple docstring"""



      return datasets.DatasetInfo(
          features=datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''			)}			)}			)					,  supervised_keys=A					,  )



 def    a__    (self					,  A					,  A			)       ->	str:


      """simple docstring"""



      return [
          datasets.SplitGenerator(name=datasets.Split.TRAIN					,  gen_kwargs={'''examples''': get_test_nested_examples()}			)
      ]



 def    a__    (self					,  A					,  A			)       ->	int:


      """simple docstring"""



      import apache_beam as beam

      return pipeline | "Load Examples" >> beam.Create(A			)





def lowerCAmelCase    ():



     """simple docstring"""



     return [(i, {"content": content}) for i, content in enumerate(['''foo''', '''bar''', '''foobar'''])]




def lowerCAmelCase    ():



     """simple docstring"""



     return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['''foo''', '''bar''', '''foobar'''])]






class __A     (    A		):



 '''simple docstring'''







 @require_beam
 def    a__    (self			)       ->	List[Any]:


      """simple docstring"""



      _a           =    len(get_test_dummy_examples()			)
      with tempfile.TemporaryDirectory() as tmp_cache_dir:
           _a           =    DummyBeamDataset(cache_dir=A					,  beam_runner='''DirectRunner'''			)
           builder.download_and_prepare()
           self.assertTrue(
               os.path.exists(
                   os.path.join(A					,  builder.name					,  '''default'''					,  '''0.0.0'''					,  f'''{builder.name}-train.arrow'''			)			)			)
           self.assertDictEqual(builder.info.features					,  datasets.Features({'''content''': datasets.Value('''string'''			)}			)			)
           _a           =    builder.as_dataset()
           self.assertEqual(dset['''train'''].num_rows					,  A			)
           self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples					,  A			)
           self.assertDictEqual(dset['''train'''][0]					,  get_test_dummy_examples()[0][1]			)
           self.assertDictEqual(
               dset['''train'''][expected_num_examples - 1]					,  get_test_dummy_examples()[expected_num_examples - 1][1]			)
           self.assertTrue(
               os.path.exists(os.path.join(A					,  builder.name					,  '''default'''					,  '''0.0.0'''					,  '''dataset_info.json'''			)			)			)
           del dset



 @require_beam
 def    a__    (self			)       ->	Optional[Any]:


      """simple docstring"""



      import apache_beam as beam

      _a           =    beam.io.parquetio.WriteToParquet

      _a           =    len(get_test_dummy_examples()			)
      with tempfile.TemporaryDirectory() as tmp_cache_dir:
           _a           =    DummyBeamDataset(cache_dir=A					,  beam_runner='''DirectRunner'''			)
           with patch('''apache_beam.io.parquetio.WriteToParquet'''			) as write_parquet_mock:
                _a           =    partial(A					,  num_shards=2			)
                builder.download_and_prepare()
           self.assertTrue(
               os.path.exists(
                   os.path.join(
                       A					,  builder.name					,  '''default'''					,  '''0.0.0'''					,  f'''{builder.name}-train-00000-of-00002.arrow'''			)			)			)
           self.assertTrue(
               os.path.exists(
                   os.path.join(
                       A					,  builder.name					,  '''default'''					,  '''0.0.0'''					,  f'''{builder.name}-train-00000-of-00002.arrow'''			)			)			)
           self.assertDictEqual(builder.info.features					,  datasets.Features({'''content''': datasets.Value('''string'''			)}			)			)
           _a           =    builder.as_dataset()
           self.assertEqual(dset['''train'''].num_rows					,  A			)
           self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples					,  A			)
           # Order is not preserved when sharding, so we just check that all the elements are there
           self.assertListEqual(sorted(dset['''train''']['''content''']			)					,  sorted(['''foo''', '''bar''', '''foobar''']			)			)
           self.assertTrue(
               os.path.exists(os.path.join(A					,  builder.name					,  '''default'''					,  '''0.0.0'''					,  '''dataset_info.json'''			)			)			)
           del dset



 @require_beam
 def    a__    (self			)       ->	Optional[Any]:


      """simple docstring"""



      with tempfile.TemporaryDirectory() as tmp_cache_dir:
           _a           =    DummyBeamDataset(cache_dir=A			)
           self.assertRaises(datasets.builder.MissingBeamOptions					,  builder.download_and_prepare			)



 @require_beam
 def    a__    (self			)       ->	str:


      """simple docstring"""



      _a           =    len(get_test_nested_examples()			)
      with tempfile.TemporaryDirectory() as tmp_cache_dir:
           _a           =    NestedBeamDataset(cache_dir=A					,  beam_runner='''DirectRunner'''			)
           builder.download_and_prepare()
           self.assertTrue(
               os.path.exists(
                   os.path.join(A					,  builder.name					,  '''default'''					,  '''0.0.0'''					,  f'''{builder.name}-train.arrow'''			)			)			)
           self.assertDictEqual(
               builder.info.features					,  datasets.Features({'''a''': datasets.Sequence({'''b''': datasets.Value('''string'''			)}			)}			)			)
           _a           =    builder.as_dataset()
           self.assertEqual(dset['''train'''].num_rows					,  A			)
           self.assertEqual(dset['''train'''].info.splits['''train'''].num_examples					,  A			)
           self.assertDictEqual(dset['''train'''][0]					,  get_test_nested_examples()[0][1]			)
           self.assertDictEqual(
               dset['''train'''][expected_num_examples - 1]					,  get_test_nested_examples()[expected_num_examples - 1][1]			)
           self.assertTrue(
               os.path.exists(os.path.join(A					,  builder.name					,  '''default'''					,  '''0.0.0'''					,  '''dataset_info.json'''			)			)			)
           del dset






'''simple docstring'''





from typing import Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature
from ...image_transforms import get_image_size, pad, rescale, to_channel_dimension_format
from ...image_utils import ChannelDimension, ImageInput, make_list_of_images, to_numpy_array, valid_images
from ...utils import TensorType, logging


UpperCamelCase_    						=       logging.get_logger(__name__)





class 		__SCREAMING_SNAKE_CASE							(			lowercase__      ):
				lowerCamelCase_		       =    ['pixel_values']







				def __init__(       self     :		List[Any]		,					UpperCAmelCase__     :		bool = True		,					UpperCAmelCase__     :		Union[int, float] = 1 / 255		,					UpperCAmelCase__     :		bool = True		,					UpperCAmelCase__     :		int = 8		,					**UpperCAmelCase__     :		str		,					):
										'''simple docstring'''
										super().__init__(**UpperCAmelCase__   )

										lowercase :							Union[str, Any]    		=do_rescale
										lowercase :							List[Any]    		=rescale_factor
										lowercase :							Tuple    		=do_pad
										lowercase :							List[str]    		=pad_size







				def  lowerCamelCase_	(       self     :		List[str]		,					UpperCAmelCase__     :		np.ndarray		,					UpperCAmelCase__     :		float		,					UpperCAmelCase__     :		Optional[Union[str, ChannelDimension]] = None		,					**UpperCAmelCase__     :		Optional[int]   ):
										'''simple docstring'''
										return rescale(UpperCAmelCase__		,					scale=UpperCAmelCase__		,					data_format=UpperCAmelCase__		,					**UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		Tuple		,					UpperCAmelCase__     :		np.ndarray		,					UpperCAmelCase__     :		int		,					UpperCAmelCase__     :		Optional[Union[str, ChannelDimension]] = None   ):
										'''simple docstring'''
										lowercase		,	lowercase :							Union[str, Any]    		=get_image_size(UpperCAmelCase__   )
										lowercase :							Tuple    		=(old_height // size + 1) * size - old_height
										lowercase :							Tuple    		=(old_width // size + 1) * size - old_width

										return pad(UpperCAmelCase__		,					((0, pad_height), (0, pad_width))		,					mode='''symmetric'''		,					data_format=UpperCAmelCase__   )







				def  lowerCamelCase_	(       self     :		List[str]		,					UpperCAmelCase__     :		ImageInput		,					UpperCAmelCase__     :		Optional[bool] = None		,					UpperCAmelCase__     :		Optional[float] = None		,					UpperCAmelCase__     :		Optional[bool] = None		,					UpperCAmelCase__     :		Optional[int] = None		,					UpperCAmelCase__     :		Optional[Union[str, TensorType]] = None		,					UpperCAmelCase__     :		Union[str, ChannelDimension] = ChannelDimension.FIRST		,					**UpperCAmelCase__     :		Optional[Any]		,					):
										'''simple docstring'''
										lowercase :							int    		=do_rescale if do_rescale is not None else self.do_rescale
										lowercase :							Tuple    		=rescale_factor if rescale_factor is not None else self.rescale_factor
										lowercase :							int    		=do_pad if do_pad is not None else self.do_pad
										lowercase :							List[Any]    		=pad_size if pad_size is not None else self.pad_size

										lowercase :							Any    		=make_list_of_images(UpperCAmelCase__   )

										if not valid_images(UpperCAmelCase__   ):
																raise ValueError(
																    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
																    '''torch.Tensor, tf.Tensor or jax.ndarray.'''   )

										if do_rescale and rescale_factor is None:
																raise ValueError('''Rescale factor must be specified if do_rescale is True.'''   )

										# All transformations expect numpy arrays.
										lowercase :							Dict    		=[to_numpy_array(UpperCAmelCase__   ) for image in images]

										if do_rescale:
																lowercase :							Tuple    		=[self.rescale(image=UpperCAmelCase__		,					scale=UpperCAmelCase__   ) for image in images]

										if do_pad:
																lowercase :							Union[str, Any]    		=[self.pad(UpperCAmelCase__		,					size=UpperCAmelCase__   ) for image in images]

										lowercase :							Dict    		=[to_channel_dimension_format(UpperCAmelCase__		,					UpperCAmelCase__   ) for image in images]

										lowercase :							Any    		={'''pixel_values''': images}
										return BatchFeature(data=UpperCAmelCase__		,					tensor_type=UpperCAmelCase__   )





