
import shutil
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import (
    is_pt_tf_cross_test,
    require_tf,
    require_torch,
    require_torchvision,
    require_vision,
)
from transformers.utils import is_tf_available, is_torch_available, is_vision_available


if is_vision_available():
  from PIL import Image

  from transformers import AutoProcessor, SamImageProcessor, SamProcessor

if is_torch_available():
  import torch

if is_tf_available():
  import tensorflow as tf

@require_vision
@require_torchvision
class __snake_case						( unittest.TestCase	):
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	List[str]			      = tempfile.mkdtemp()
           lowercase				:	Union[str, Any]			      = SamImageProcessor()
           lowercase				:	Tuple			      = SamProcessor(snake_case )
           processor.save_pretrained(self.tmpdirname )
    def  _SCREAMING_SNAKE_CASE			(    self						,**snake_case ):



           '''simple docstring'''







           return AutoProcessor.from_pretrained(self.tmpdirname						,**snake_case ).image_processor
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           shutil.rmtree(self.tmpdirname )
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	int			      = [np.random.randint(255						,size=(3, 30, 400)						,dtype=np.uinta )]

           lowercase				:	Optional[Any]			      = [Image.fromarray(np.moveaxis(snake_case						,0						,-1 ) ) for x in image_inputs]

           return image_inputs
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	str			      = SamProcessor(image_processor=self.get_image_processor() )
           processor.save_pretrained(self.tmpdirname )

           lowercase				:	Any			      = self.get_image_processor(do_normalize=snake_case						,padding_value=1.0 )

           lowercase				:	Tuple			      = SamProcessor.from_pretrained(self.tmpdirname						,do_normalize=snake_case						,padding_value=1.0 )

           self.assertEqual(processor.image_processor.to_json_string()						,image_processor_add_kwargs.to_json_string() )
           self.assertIsInstance(processor.image_processor						,snake_case )
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	Any			      = self.get_image_processor()

           lowercase				:	Tuple			      = SamProcessor(image_processor=snake_case )

           lowercase				:	List[Any]			      = self.prepare_image_inputs()

           lowercase				:	Optional[int]			      = image_processor(snake_case						,return_tensors="""np""" )
           lowercase				:	Optional[Any]			      = processor(images=snake_case						,return_tensors="""np""" )

           input_feat_extract.pop("""original_sizes""" )  # pop original_sizes as it is popped in the processor
           input_feat_extract.pop("""reshaped_input_sizes""" )  # pop original_sizes as it is popped in the processor

           for key in input_feat_extract.keys():
                  self.assertAlmostEqual(input_feat_extract[key].sum()						,input_processor[key].sum()						,delta=1e-2 )
    @require_torch
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	Any			      = self.get_image_processor()

           lowercase				:	int			      = SamProcessor(image_processor=snake_case )
           lowercase				:	Optional[Any]			      = [torch.ones((1, 3, 5, 5) )]

           lowercase				:	Any			      = [[1764, 2646]]

           lowercase				:	int			      = [[683, 1024]]
           lowercase				:	Any			      = processor.post_process_masks(snake_case						,snake_case						,snake_case )
           self.assertEqual(masks[0].shape						,(1, 3, 1764, 2646) )

           lowercase				:	str			      = processor.post_process_masks(
               snake_case						,torch.tensor(snake_case )						,torch.tensor(snake_case ) )
           self.assertEqual(masks[0].shape						,(1, 3, 1764, 2646) )

           # should also work with np
           lowercase				:	Union[str, Any]			      = [np.ones((1, 3, 5, 5) )]
           lowercase				:	Any			      = processor.post_process_masks(snake_case						,np.array(snake_case )						,np.array(snake_case ) )

           self.assertEqual(masks[0].shape						,(1, 3, 1764, 2646) )

           lowercase				:	List[str]			      = [[1, 0], [0, 1]]
           with self.assertRaises(snake_case ):
                  lowercase				:	int			      = processor.post_process_masks(snake_case						,np.array(snake_case )						,np.array(snake_case ) )






@require_vision
@require_tf
class __snake_case						( unittest.TestCase	):
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	List[Any]			      = tempfile.mkdtemp()
           lowercase				:	Dict			      = SamImageProcessor()
           lowercase				:	int			      = SamProcessor(snake_case )
           processor.save_pretrained(self.tmpdirname )
    def  _SCREAMING_SNAKE_CASE			(    self						,**snake_case ):



           '''simple docstring'''







           return AutoProcessor.from_pretrained(self.tmpdirname						,**snake_case ).image_processor
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           shutil.rmtree(self.tmpdirname )
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	Optional[Any]			      = [np.random.randint(255						,size=(3, 30, 400)						,dtype=np.uinta )]

           lowercase				:	Optional[Any]			      = [Image.fromarray(np.moveaxis(snake_case						,0						,-1 ) ) for x in image_inputs]

           return image_inputs
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	Dict			      = SamProcessor(image_processor=self.get_image_processor() )
           processor.save_pretrained(self.tmpdirname )

           lowercase				:	Dict			      = self.get_image_processor(do_normalize=snake_case						,padding_value=1.0 )

           lowercase				:	List[Any]			      = SamProcessor.from_pretrained(self.tmpdirname						,do_normalize=snake_case						,padding_value=1.0 )

           self.assertEqual(processor.image_processor.to_json_string()						,image_processor_add_kwargs.to_json_string() )
           self.assertIsInstance(processor.image_processor						,snake_case )
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	List[Any]			      = self.get_image_processor()

           lowercase				:	Any			      = SamProcessor(image_processor=snake_case )

           lowercase				:	Any			      = self.prepare_image_inputs()

           lowercase				:	List[str]			      = image_processor(snake_case						,return_tensors="""np""" )
           lowercase				:	Optional[Any]			      = processor(images=snake_case						,return_tensors="""np""" )

           input_feat_extract.pop("""original_sizes""" )  # pop original_sizes as it is popped in the processor
           input_feat_extract.pop("""reshaped_input_sizes""" )  # pop reshaped_input_sizes as it is popped in the processor

           for key in input_feat_extract.keys():
                  self.assertAlmostEqual(input_feat_extract[key].sum()						,input_processor[key].sum()						,delta=1e-2 )
    @require_tf
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	Tuple			      = self.get_image_processor()

           lowercase				:	Optional[int]			      = SamProcessor(image_processor=snake_case )
           lowercase				:	Dict			      = [tf.ones((1, 3, 5, 5) )]

           lowercase				:	List[Any]			      = [[1764, 2646]]

           lowercase				:	Any			      = [[683, 1024]]
           lowercase				:	Any			      = processor.post_process_masks(snake_case						,snake_case						,snake_case						,return_tensors="""tf""" )
           self.assertEqual(masks[0].shape						,(1, 3, 1764, 2646) )

           lowercase				:	str			      = processor.post_process_masks(
               snake_case						,tf.convert_to_tensor(snake_case )						,tf.convert_to_tensor(snake_case )						,return_tensors="""tf"""						,)
           self.assertEqual(masks[0].shape						,(1, 3, 1764, 2646) )

           # should also work with np
           lowercase				:	Optional[int]			      = [np.ones((1, 3, 5, 5) )]
           lowercase				:	Union[str, Any]			      = processor.post_process_masks(
               snake_case						,np.array(snake_case )						,np.array(snake_case )						,return_tensors="""tf""" )

           self.assertEqual(masks[0].shape						,(1, 3, 1764, 2646) )

           lowercase				:	int			      = [[1, 0], [0, 1]]
           with self.assertRaises(tf.errors.InvalidArgumentError ):
                  lowercase				:	Union[str, Any]			      = processor.post_process_masks(
                      snake_case						,np.array(snake_case )						,np.array(snake_case )						,return_tensors="""tf""" )






@require_vision
@require_torchvision
class __snake_case						( unittest.TestCase	):
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	Dict			      = tempfile.mkdtemp()
           lowercase				:	Dict			      = SamImageProcessor()
           lowercase				:	Union[str, Any]			      = SamProcessor(snake_case )
           processor.save_pretrained(self.tmpdirname )
    def  _SCREAMING_SNAKE_CASE			(    self						,**snake_case ):



           '''simple docstring'''







           return AutoProcessor.from_pretrained(self.tmpdirname						,**snake_case ).image_processor
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           shutil.rmtree(self.tmpdirname )
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	Dict			      = [np.random.randint(255						,size=(3, 30, 400)						,dtype=np.uinta )]

           lowercase				:	List[str]			      = [Image.fromarray(np.moveaxis(snake_case						,0						,-1 ) ) for x in image_inputs]

           return image_inputs
    @is_pt_tf_cross_test
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	Optional[int]			      = self.get_image_processor()

           lowercase				:	int			      = SamProcessor(image_processor=snake_case )
           lowercase				:	Optional[Any]			      = np.random.randint(0						,2						,size=(1, 3, 5, 5) ).astype(np.floataa )
           lowercase				:	Optional[int]			      = [tf.convert_to_tensor(snake_case )]
           lowercase				:	Optional[int]			      = [torch.tensor(snake_case )]

           lowercase				:	Optional[Any]			      = [[1764, 2646]]

           lowercase				:	List[Any]			      = [[683, 1024]]
           lowercase				:	Any			      = processor.post_process_masks(
               snake_case						,snake_case						,snake_case						,return_tensors="""tf""" )
           lowercase				:	Optional[int]			      = processor.post_process_masks(
               snake_case						,snake_case						,snake_case						,return_tensors="""pt""" )

           self.assertTrue(np.all(tf_masks[0].numpy() == pt_masks[0].numpy() ) )
    @is_pt_tf_cross_test
    def  _SCREAMING_SNAKE_CASE			(    self ):



           '''simple docstring'''







           lowercase				:	Tuple			      = self.get_image_processor()

           lowercase				:	List[Any]			      = SamProcessor(image_processor=snake_case )

           lowercase				:	Dict			      = self.prepare_image_inputs()

           lowercase				:	List[str]			      = image_processor(snake_case						,return_tensors="""pt""" )["""pixel_values"""].numpy()
           lowercase				:	int			      = processor(images=snake_case						,return_tensors="""pt""" )["""pixel_values"""].numpy()

           lowercase				:	List[Any]			      = image_processor(snake_case						,return_tensors="""tf""" )["""pixel_values"""].numpy()
           lowercase				:	str			      = processor(images=snake_case						,return_tensors="""tf""" )["""pixel_values"""].numpy()

           self.assertTrue(np.allclose(snake_case						,snake_case ) )
           self.assertTrue(np.allclose(snake_case						,snake_case ) )
           self.assertTrue(np.allclose(snake_case						,snake_case ) )




"""simple docstring"""



from typing import Optional

import pyspark

from .. import Features, NamedSplit
from ..download import DownloadMode
from ..packaged_modules.spark.spark import Spark
from .abc import AbstractDatasetReader







class    __a						( __snake_case							):







				def __init__( self      ,	UpperCAmelCase      ,	UpperCAmelCase = None      ,	UpperCAmelCase = None      ,	UpperCAmelCase = True      ,	UpperCAmelCase = None      ,	UpperCAmelCase = False      ,	UpperCAmelCase = None      ,	UpperCAmelCase = True      ,	UpperCAmelCase = "arrow"      ,	**UpperCAmelCase      ,	):

					'''simple docstring'''






					super().__init__(
					    split=UpperCAmelCase      ,	features=UpperCAmelCase      ,	cache_dir=UpperCAmelCase      ,	keep_in_memory=UpperCAmelCase      ,	streaming=UpperCAmelCase      ,	**UpperCAmelCase      ,	)
					lowerCAmelCase_      							=			load_from_cache_file
					lowerCAmelCase_      							=			file_format
					lowerCAmelCase_      							=			Spark(
					    df=UpperCAmelCase      ,	features=UpperCAmelCase      ,	cache_dir=UpperCAmelCase      ,	working_dir=UpperCAmelCase      ,	**UpperCAmelCase      ,	)







				def      lowerCamelCase_     ( self   ):

					'''simple docstring'''






					if self.streaming:
						return self.builder.as_streaming_dataset(split=self.split   )
					lowerCAmelCase_      							=			None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD
					self.builder.download_and_prepare(
					    download_mode=UpperCAmelCase      ,	file_format=self._file_format      ,	)
					return self.builder.as_dataset(split=self.split   )