from typing import List, Union

from ..utils import (
    add_end_docstrings,
    is_tf_available,
    is_torch_available,
    is_vision_available,
    logging,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_vision_available():
      from PIL import Image

      from ..image_utils import load_image

if is_tf_available():
      from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_VISION_2_SEQ_MAPPING

if is_torch_available():
      import torch

      from ..models.auto.modeling_auto import MODEL_FOR_VISION_2_SEQ_MAPPING

lowercase_  : Any								=					logging.get_logger(__name__)






@add_end_docstrings(UpperCamelCase_     )
class 		_lowerCamelCase		( UpperCamelCase_     ):
    def __init__(	self			,			*lowerCAmelCase			,			**lowerCAmelCase )							->       Optional[Any]:
     super().__init__(*lowerCAmelCase			,			**lowerCAmelCase )
     requires_backends(self			,			'''vision''' )
     self.check_model_type(
         TF_MODEL_FOR_VISION_2_SEQ_MAPPING if self.framework == '''tf''' else MODEL_FOR_VISION_2_SEQ_MAPPING )
    def     UpperCamelCase_					(	self			,			lowerCAmelCase=None			,			lowerCAmelCase=None			,			lowerCAmelCase=None )							->       Optional[int]:
     SCREAMING_SNAKE_CASE__: List[Any]=   {}
     SCREAMING_SNAKE_CASE__: Tuple=   {}

     if prompt is not None:
      SCREAMING_SNAKE_CASE__: Union[str, Any]=   prompt

     if generate_kwargs is not None:
      SCREAMING_SNAKE_CASE__: List[Any]=   generate_kwargs
     if max_new_tokens is not None:
      if "generate_kwargs" not in forward_kwargs:
       SCREAMING_SNAKE_CASE__: Any=   {}
      if "max_new_tokens" in forward_kwargs["generate_kwargs"]:
       raise ValueError(
           '''\'max_new_tokens\' is defined twice, once in \'generate_kwargs\' and once as a direct parameter,'''
           ''' please use only one''' )
      SCREAMING_SNAKE_CASE__: str=   max_new_tokens
     return preprocess_params, forward_kwargs, {}
    def __call__(	self			,			lowerCAmelCase			,			**lowerCAmelCase )							->       Optional[Any]:
     return super().__call__(lowerCAmelCase			,			**lowerCAmelCase )
    def     UpperCamelCase_					(	self			,			lowerCAmelCase			,			lowerCAmelCase=None )							->       int:
     SCREAMING_SNAKE_CASE__: List[str]=   load_image(lowerCAmelCase )

     if prompt is not None:
      if not isinstance(lowerCAmelCase			,			lowerCAmelCase ):
       raise ValueError(
           f'Received an invalid text input, got - {type(lowerCAmelCase )} - but expected a single string. '
           '''Note also that one single text can be provided for conditional image to text generation.''' )

      SCREAMING_SNAKE_CASE__: List[Any]=   self.model.config.model_type

      if model_type == "git":
       SCREAMING_SNAKE_CASE__: Tuple=   self.image_processor(images=lowerCAmelCase			,			return_tensors=self.framework )
       SCREAMING_SNAKE_CASE__: int=   self.tokenizer(text=lowerCAmelCase			,			add_special_tokens=lowerCAmelCase ).input_ids
       SCREAMING_SNAKE_CASE__: int=   [self.tokenizer.cls_token_id] + input_ids
       SCREAMING_SNAKE_CASE__: List[Any]=   torch.tensor(lowerCAmelCase ).unsqueeze(0 )
       model_inputs.update({'''input_ids''': input_ids} )

      elif model_type == "pix2struct":
       SCREAMING_SNAKE_CASE__: List[Any]=   self.image_processor(images=lowerCAmelCase			,			header_text=lowerCAmelCase			,			return_tensors=self.framework )

      elif model_type != "vision-encoder-decoder":
       # vision-encoder-decoder does not support conditional generation
       SCREAMING_SNAKE_CASE__: Any=   self.image_processor(images=lowerCAmelCase			,			return_tensors=self.framework )
       SCREAMING_SNAKE_CASE__: Tuple=   self.tokenizer(lowerCAmelCase			,			return_tensors=self.framework )
       model_inputs.update(lowerCAmelCase )

      else:
       raise ValueError(f'Model type {model_type} does not support conditional text generation' )

     else:
      SCREAMING_SNAKE_CASE__: Tuple=   self.image_processor(images=lowerCAmelCase			,			return_tensors=self.framework )

     if self.model.config.model_type == "git" and prompt is None:
      SCREAMING_SNAKE_CASE__: Dict=   None

     return model_inputs
    def     UpperCamelCase_					(	self			,			lowerCAmelCase			,			lowerCAmelCase=None )							->       List[Any]:
     # Git model sets `model_inputs["input_ids"] = None` in `preprocess` (when `prompt=None`). In batch model, the
     # pipeline will group them into a list of `None`, which fail `_forward`. Avoid this by checking it first.
     if (
         "input_ids" in model_inputs
         and isinstance(model_inputs['''input_ids''']			,			lowerCAmelCase )
         and all(x is None for x in model_inputs['''input_ids'''] )
     ):
      SCREAMING_SNAKE_CASE__: Dict=   None

     if generate_kwargs is None:
      SCREAMING_SNAKE_CASE__: int=   {}
     # FIXME: We need to pop here due to a difference in how `generation.py` and `generation.tf_utils.py`
     #  parse inputs. In the Tensorflow version, `generate` raises an error if we don't use `input_ids` whereas
     #  the PyTorch version matches it with `self.model.main_input_name` or `self.model.encoder.main_input_name`
     #  in the `_prepare_model_inputs` method.
     SCREAMING_SNAKE_CASE__: Tuple=   model_inputs.pop(self.model.main_input_name )
     SCREAMING_SNAKE_CASE__: Tuple=   self.model.generate(lowerCAmelCase			,			**lowerCAmelCase			,			**lowerCAmelCase )
     return model_outputs



    def     UpperCamelCase_					(	self			,			lowerCAmelCase )							->       int:
     SCREAMING_SNAKE_CASE__: List[str]=   []
     for output_ids in model_outputs:
      SCREAMING_SNAKE_CASE__: str=   {
          '''generated_text''': self.tokenizer.decode(
              lowerCAmelCase			,			skip_special_tokens=lowerCAmelCase			,			)
      }
      records.append(lowerCAmelCase )
     return records








import shutil
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import (
    is_pt_tf_cross_test,
    require_tf,
    require_torch,
    require_torchvision,
    require_vision,
)
from transformers.utils import is_tf_available, is_torch_available, is_vision_available


if is_vision_available():
       from PIL import Image

       from transformers import AutoProcessor, SamImageProcessor, SamProcessor

if is_torch_available():
       import torch

if is_tf_available():
       import tensorflow as tf




@require_vision
@require_torchvision
class        SCREAMING_SNAKE_CASE__					(		unittest.TestCase ):



 def 							UpperCamelCase     (				self					):
        A__		   =		tempfile.mkdtemp()
        A__		   =		SamImageProcessor()
        A__		   =		SamProcessor(__lowerCamelCase					)
        processor.save_pretrained(self.tmpdirname					)



 def 							UpperCamelCase     (				self,**__lowerCamelCase					):
        return AutoProcessor.from_pretrained(self.tmpdirname,**__lowerCamelCase					).image_processor



 def 							UpperCamelCase     (				self					):
        shutil.rmtree(self.tmpdirname					)



 def 							UpperCamelCase     (				self					):
        A__		   =		[np.random.randint(255,size=(3, 30, 400),dtype=np.uinta					)]

        A__		   =		[Image.fromarray(np.moveaxis(__lowerCamelCase,0,-1					)					) for x in image_inputs]

        return image_inputs



 def 							UpperCamelCase     (				self					):
        A__		   =		SamProcessor(image_processor=self.get_image_processor()					)
        processor.save_pretrained(self.tmpdirname					)

        A__		   =		self.get_image_processor(do_normalize=__lowerCamelCase,padding_value=1.0					)

        A__		   =		SamProcessor.from_pretrained(self.tmpdirname,do_normalize=__lowerCamelCase,padding_value=1.0					)

        self.assertEqual(processor.image_processor.to_json_string(),image_processor_add_kwargs.to_json_string()					)
        self.assertIsInstance(processor.image_processor,__lowerCamelCase					)



 def 							UpperCamelCase     (				self					):
        A__		   =		self.get_image_processor()

        A__		   =		SamProcessor(image_processor=__lowerCamelCase					)

        A__		   =		self.prepare_image_inputs()

        A__		   =		image_processor(__lowerCamelCase,return_tensors='''np'''					)
        A__		   =		processor(images=__lowerCamelCase,return_tensors='''np'''					)

        input_feat_extract.pop('''original_sizes'''					)  # pop original_sizes as it is popped in the processor
        input_feat_extract.pop('''reshaped_input_sizes'''					)  # pop original_sizes as it is popped in the processor

        for key in input_feat_extract.keys():
               self.assertAlmostEqual(input_feat_extract[key].sum(),input_processor[key].sum(),delta=1E-2					)




 @require_torch
 def 							UpperCamelCase     (				self					):
        A__		   =		self.get_image_processor()

        A__		   =		SamProcessor(image_processor=__lowerCamelCase					)
        A__		   =		[torch.ones((1, 3, 5, 5)					)]

        A__		   =		[[1764, 2646]]

        A__		   =		[[683, 1024]]
        A__		   =		processor.post_process_masks(__lowerCamelCase,__lowerCamelCase,__lowerCamelCase					)
        self.assertEqual(masks[0].shape,(1, 3, 1764, 2646)					)

        A__		   =		processor.post_process_masks(
            __lowerCamelCase,torch.tensor(__lowerCamelCase					),torch.tensor(__lowerCamelCase					)					)
        self.assertEqual(masks[0].shape,(1, 3, 1764, 2646)					)

        # should also work with np
        A__		   =		[np.ones((1, 3, 5, 5)					)]
        A__		   =		processor.post_process_masks(__lowerCamelCase,np.array(__lowerCamelCase					),np.array(__lowerCamelCase					)					)

        self.assertEqual(masks[0].shape,(1, 3, 1764, 2646)					)

        A__		   =		[[1, 0], [0, 1]]
        with self.assertRaises(__lowerCamelCase					):
               A__		   =		processor.post_process_masks(__lowerCamelCase,np.array(__lowerCamelCase					),np.array(__lowerCamelCase					)					)




@require_vision
@require_tf
class        SCREAMING_SNAKE_CASE__					(		unittest.TestCase ):



 def 							UpperCamelCase     (				self					):
        A__		   =		tempfile.mkdtemp()
        A__		   =		SamImageProcessor()
        A__		   =		SamProcessor(__lowerCamelCase					)
        processor.save_pretrained(self.tmpdirname					)



 def 							UpperCamelCase     (				self,**__lowerCamelCase					):
        return AutoProcessor.from_pretrained(self.tmpdirname,**__lowerCamelCase					).image_processor



 def 							UpperCamelCase     (				self					):
        shutil.rmtree(self.tmpdirname					)



 def 							UpperCamelCase     (				self					):
        A__		   =		[np.random.randint(255,size=(3, 30, 400),dtype=np.uinta					)]

        A__		   =		[Image.fromarray(np.moveaxis(__lowerCamelCase,0,-1					)					) for x in image_inputs]

        return image_inputs



 def 							UpperCamelCase     (				self					):
        A__		   =		SamProcessor(image_processor=self.get_image_processor()					)
        processor.save_pretrained(self.tmpdirname					)

        A__		   =		self.get_image_processor(do_normalize=__lowerCamelCase,padding_value=1.0					)

        A__		   =		SamProcessor.from_pretrained(self.tmpdirname,do_normalize=__lowerCamelCase,padding_value=1.0					)

        self.assertEqual(processor.image_processor.to_json_string(),image_processor_add_kwargs.to_json_string()					)
        self.assertIsInstance(processor.image_processor,__lowerCamelCase					)



 def 							UpperCamelCase     (				self					):
        A__		   =		self.get_image_processor()

        A__		   =		SamProcessor(image_processor=__lowerCamelCase					)

        A__		   =		self.prepare_image_inputs()

        A__		   =		image_processor(__lowerCamelCase,return_tensors='''np'''					)
        A__		   =		processor(images=__lowerCamelCase,return_tensors='''np'''					)

        input_feat_extract.pop('''original_sizes'''					)  # pop original_sizes as it is popped in the processor
        input_feat_extract.pop('''reshaped_input_sizes'''					)  # pop reshaped_input_sizes as it is popped in the processor

        for key in input_feat_extract.keys():
               self.assertAlmostEqual(input_feat_extract[key].sum(),input_processor[key].sum(),delta=1E-2					)




 @require_tf
 def 							UpperCamelCase     (				self					):
        A__		   =		self.get_image_processor()

        A__		   =		SamProcessor(image_processor=__lowerCamelCase					)
        A__		   =		[tf.ones((1, 3, 5, 5)					)]

        A__		   =		[[1764, 2646]]

        A__		   =		[[683, 1024]]
        A__		   =		processor.post_process_masks(__lowerCamelCase,__lowerCamelCase,__lowerCamelCase,return_tensors='''tf'''					)
        self.assertEqual(masks[0].shape,(1, 3, 1764, 2646)					)

        A__		   =		processor.post_process_masks(
            __lowerCamelCase,tf.convert_to_tensor(__lowerCamelCase					),tf.convert_to_tensor(__lowerCamelCase					),return_tensors='''tf''',)
        self.assertEqual(masks[0].shape,(1, 3, 1764, 2646)					)

        # should also work with np
        A__		   =		[np.ones((1, 3, 5, 5)					)]
        A__		   =		processor.post_process_masks(
            __lowerCamelCase,np.array(__lowerCamelCase					),np.array(__lowerCamelCase					),return_tensors='''tf'''					)

        self.assertEqual(masks[0].shape,(1, 3, 1764, 2646)					)

        A__		   =		[[1, 0], [0, 1]]
        with self.assertRaises(tf.errors.InvalidArgumentError					):
               A__		   =		processor.post_process_masks(
                   __lowerCamelCase,np.array(__lowerCamelCase					),np.array(__lowerCamelCase					),return_tensors='''tf'''					)




@require_vision
@require_torchvision
class        SCREAMING_SNAKE_CASE__					(		unittest.TestCase ):



 def 							UpperCamelCase     (				self					):
        A__		   =		tempfile.mkdtemp()
        A__		   =		SamImageProcessor()
        A__		   =		SamProcessor(__lowerCamelCase					)
        processor.save_pretrained(self.tmpdirname					)



 def 							UpperCamelCase     (				self,**__lowerCamelCase					):
        return AutoProcessor.from_pretrained(self.tmpdirname,**__lowerCamelCase					).image_processor



 def 							UpperCamelCase     (				self					):
        shutil.rmtree(self.tmpdirname					)



 def 							UpperCamelCase     (				self					):
        A__		   =		[np.random.randint(255,size=(3, 30, 400),dtype=np.uinta					)]

        A__		   =		[Image.fromarray(np.moveaxis(__lowerCamelCase,0,-1					)					) for x in image_inputs]

        return image_inputs



 @is_pt_tf_cross_test
 def 							UpperCamelCase     (				self					):
        A__		   =		self.get_image_processor()

        A__		   =		SamProcessor(image_processor=__lowerCamelCase					)
        A__		   =		np.random.randint(0,2,size=(1, 3, 5, 5)					).astype(np.floataa					)
        A__		   =		[tf.convert_to_tensor(__lowerCamelCase					)]
        A__		   =		[torch.tensor(__lowerCamelCase					)]

        A__		   =		[[1764, 2646]]

        A__		   =		[[683, 1024]]
        A__		   =		processor.post_process_masks(
            __lowerCamelCase,__lowerCamelCase,__lowerCamelCase,return_tensors='''tf'''					)
        A__		   =		processor.post_process_masks(
            __lowerCamelCase,__lowerCamelCase,__lowerCamelCase,return_tensors='''pt'''					)

        self.assertTrue(np.all(tf_masks[0].numpy() == pt_masks[0].numpy()					)					)




 @is_pt_tf_cross_test
 def 							UpperCamelCase     (				self					):
        A__		   =		self.get_image_processor()

        A__		   =		SamProcessor(image_processor=__lowerCamelCase					)

        A__		   =		self.prepare_image_inputs()

        A__		   =		image_processor(__lowerCamelCase,return_tensors='''pt'''					)['''pixel_values'''].numpy()
        A__		   =		processor(images=__lowerCamelCase,return_tensors='''pt'''					)['''pixel_values'''].numpy()

        A__		   =		image_processor(__lowerCamelCase,return_tensors='''tf'''					)['''pixel_values'''].numpy()
        A__		   =		processor(images=__lowerCamelCase,return_tensors='''tf'''					)['''pixel_values'''].numpy()

        self.assertTrue(np.allclose(__lowerCamelCase,__lowerCamelCase					)					)
        self.assertTrue(np.allclose(__lowerCamelCase,__lowerCamelCase					)					)
        self.assertTrue(np.allclose(__lowerCamelCase,__lowerCamelCase					)					)





