


'''simple docstring'''

from __future__ import annotations

import os
import tempfile
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import is_tensorflow_text_available, is_tf_available
from transformers.testing_utils import require_tensorflow_text, require_tf, slow

from ..test_modeling_tf_common import floats_tensor
from .test_framework_agnostic import GenerationIntegrationTestsMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import (
		    AutoTokenizer,
		    TFAutoModelForCausalLM,
		    TFAutoModelForSeqaSeqLM,
		    TFAutoModelForSpeechSeqaSeq,
		    TFAutoModelForVisionaSeq,
		    TFBartForConditionalGeneration,
		    TFLogitsProcessorList,
		    TFMinLengthLogitsProcessor,
		    tf_top_k_top_p_filtering,
		)

if is_tensorflow_text_available():
		import tensorflow_text as text

@require_tf
class 		snake_case				(    unittest.TestCase						):

				"""simple docstring"""




				def    lowercase__							(	self )    ->      List[Any]:







					"""simple docstring"""






					snake_case__			:			Optional[int]             =		tf.convert_to_tensor(
					    [
					        [
					            8.2_220_991,  # 3rd highest value; idx. 0
					            -0.5_620_044,
					            5.23_229_752,
					            4.0_386_393,
					            -6.8_798_378,
					            -0.54_785_802,
					            -3.2_012_153,
					            2.92_777_176,
					            1.88_171_953,
					            7.35_341_276,  # 5th highest value; idx. 9
					            8.43_207_833,  # 2nd highest value; idx. 10
					            -9.85_711_836,
					            -5.96_209_236,
					            -1.13_039_161,
					            -7.1_115_294,
					            -0.8_369_633,
					            -5.3_186_408,
					            7.06_427_407,
					            0.81_369_344,
					            -0.82_023_817,
					            -5.9_179_796,
					            0.58_813_443,
					            -6.99_778_438,
					            4.71_551_189,
					            -0.18_771_637,
					            7.44_020_759,  # 4th highest value; idx. 25
					            9.38_450_987,  # 1st highest value; idx. 26
					            2.12_662_941,
					            -9.32_562_038,
					            2.35_652_522,
					        ],  # cummulative prob of 5 highest values <= 0.6
					        [
					            0.58_425_518,
					            4.53_139_238,
					            -5.57_510_464,
					            -6.28_030_699,
					            -7.19_529_503,
					            -4.02_122_551,
					            1.39_337_037,
					            -6.06_707_057,
					            1.59_480_517,
					            -9.643_119,
					            0.03_907_799,
					            0.67_231_762,
					            -8.88_206_726,
					            6.27_115_922,  # 4th highest value; idx. 13
					            2.28_520_723,
					            4.82_767_506,
					            4.30_421_368,
					            8.8_275_313,  # 2nd highest value; idx. 17
					            5.44_029_958,  # 5th highest value; idx. 18
					            -4.4_735_794,
					            7.38_579_536,  # 3rd highest value; idx. 20
					            -2.91_051_663,
					            2.61_946_077,
					            -2.5_674_762,
					            -9.48_959_302,
					            -4.02_922_645,
					            -1.35_416_918,
					            9.67_702_323,  # 1st highest value; idx. 27
					            -5.89_478_553,
					            1.85_370_467,
					        ],  # cummulative prob of 5 highest values <= 0.6
					    ]       ,    dtype=tf.floataa       ,    )

					snake_case__			:			List[str]             =		tf.convert_to_tensor(
					    [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]]       ,    dtype=tf.intaa       ,    )  # expected non filtered idx as noted above

					snake_case__			:			Optional[int]             =		tf.convert_to_tensor(
					    [8.222_099, 7.3_534_126, 8.432_078, 7.4_402_075, 9.38_451, 6.271_159, 8.827_531, 5.4_402_995, 7.3_857_956, 9.677_023]       ,    dtype=tf.floataa       ,    )  # expected non filtered values as noted above

					snake_case__			:			int             =		tf_top_k_top_p_filtering(__a       ,    top_k=10       ,    top_p=0.6       ,    min_tokens_to_keep=4 )

					snake_case__			:			int             =		output[output != -float('''inf''' )]
					snake_case__			:			int             =		tf.cast(
					    tf.where(tf.not_equal(__a       ,    tf.constant(-float('''inf''' )       ,    dtype=tf.floataa ) ) )       ,    dtype=tf.intaa       ,    )

					tf.debugging.assert_near(__a       ,    __a       ,    rtol=1E-1_2 )
					tf.debugging.assert_equal(__a       ,    __a )



@require_tf
class 		snake_case				(    unittest.TestCase				,    a__						):

				"""simple docstring"""




				if is_tf_available():
								_lowerCAmelCase           =    {
								    """AutoModelForCausalLM""": TFAutoModelForCausalLM,
								    """AutoModelForSpeechSeq2Seq""": TFAutoModelForSpeechSeqaSeq,
								    """AutoModelForSeq2SeqLM""": TFAutoModelForSeqaSeqLM,
								    """AutoModelForVision2Seq""": TFAutoModelForVisionaSeq,
								    """LogitsProcessorList""": TFLogitsProcessorList,
								    """MinLengthLogitsProcessor""": TFMinLengthLogitsProcessor,
								    """create_tensor_fn""": tf.convert_to_tensor,
								    """floats_tensor""": floats_tensor,
								    """return_tensors""": """tf""",
								}





				@slow
				def    lowercase__							(	self )    ->      Tuple:







					"""simple docstring"""






					snake_case__			:			Dict             =		TFAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2''' )
					snake_case__			:			Any             =		2
					snake_case__			:			int             =		2

					class 		snake_case				(    tf.Module						):

									"""simple docstring"""




									def __init__(	self       ,    lowerCamelCase )    ->      Any:







										"""simple docstring"""






										super(__a       ,    self ).__init__()
										snake_case__			:			List[Any]             =		model





									@tf.function(
									    input_signature=(
									        tf.TensorSpec((None, input_length)       ,    tf.intaa       ,    name='''input_ids''' ),
									        tf.TensorSpec((None, input_length)       ,    tf.intaa       ,    name='''attention_mask''' ),
									    )       ,    jit_compile=__a       ,    )
									def    lowercase__							(	self       ,    lowerCamelCase       ,    lowerCamelCase )    ->      Optional[Any]:







										"""simple docstring"""






										snake_case__			:			Union[str, Any]             =		self.model.generate(
										    input_ids=__a       ,    attention_mask=__a       ,    max_new_tokens=__a       ,    return_dict_in_generate=__a       ,    )
										return {"sequences": outputs["sequences"]}



					snake_case__			:			List[str]             =		[[2, 0], [102, 103]]
					snake_case__			:			List[str]             =		[[1, 0], [1, 1]]
					snake_case__			:			Union[str, Any]             =		DummyModel(model=__a )
					with tempfile.TemporaryDirectory() as tmp_dir:
						tf.saved_model.save(__a       ,    __a       ,    signatures={'''serving_default''': dummy_model.serving} )
						snake_case__			:			Any             =		tf.saved_model.load(__a ).signatures["""serving_default"""]
						for batch_size in range(1       ,    len(__a ) + 1 ):
							snake_case__			:			Dict             =		{
							    """input_ids""": tf.constant(dummy_input_ids[:batch_size] ),
							    """attention_mask""": tf.constant(dummy_attention_masks[:batch_size] ),
							}
							snake_case__			:			Union[str, Any]             =		serving_func(**__a )["""sequences"""]
							snake_case__			:			Tuple             =		test_model.generate(**__a       ,    max_new_tokens=__a )
							tf.debugging.assert_equal(__a       ,    __a )





				@slow
				def    lowercase__							(	self )    ->      Optional[Any]:







					"""simple docstring"""






					snake_case__			:			Optional[int]             =		TFAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2''' )
					snake_case__			:			int             =		1
					snake_case__			:			Union[str, Any]             =		2

					class 		snake_case				(    tf.Module						):

									"""simple docstring"""




									def __init__(	self       ,    lowerCamelCase )    ->      Union[str, Any]:







										"""simple docstring"""






										super(__a       ,    self ).__init__()
										snake_case__			:			Optional[Any]             =		model





									@tf.function(
									    input_signature=(
									        tf.TensorSpec((batch_size, None)       ,    tf.intaa       ,    name='''input_ids''' ),
									        tf.TensorSpec((batch_size, None)       ,    tf.intaa       ,    name='''attention_mask''' ),
									    )       ,    jit_compile=__a       ,    )
									def    lowercase__							(	self       ,    lowerCamelCase       ,    lowerCamelCase )    ->      str:







										"""simple docstring"""






										snake_case__			:			Optional[Any]             =		self.model.generate(
										    input_ids=__a       ,    attention_mask=__a       ,    max_new_tokens=__a       ,    return_dict_in_generate=__a       ,    )
										return {"sequences": outputs["sequences"]}



					snake_case__			:			Tuple             =		[[2], [102, 103]]
					snake_case__			:			int             =		[[1], [1, 1]]
					snake_case__			:			int             =		DummyModel(model=__a )
					with tempfile.TemporaryDirectory() as tmp_dir:
						tf.saved_model.save(__a       ,    __a       ,    signatures={'''serving_default''': dummy_model.serving} )
						snake_case__			:			List[str]             =		tf.saved_model.load(__a ).signatures["""serving_default"""]
						for input_row in range(len(__a ) ):
							snake_case__			:			Union[str, Any]             =		{
							    """input_ids""": tf.constant([dummy_input_ids[input_row]] ),
							    """attention_mask""": tf.constant([dummy_attention_masks[input_row]] ),
							}
							snake_case__			:			List[str]             =		serving_func(**__a )["""sequences"""]
							snake_case__			:			Optional[int]             =		test_model.generate(**__a       ,    max_new_tokens=__a )
							tf.debugging.assert_equal(__a       ,    __a )





				@slow
				@require_tensorflow_text
				def    lowercase__							(	self )    ->      Union[str, Any]:







					"""simple docstring"""






					with tempfile.TemporaryDirectory() as tmp_dir:
						# file needed to load the TF tokenizer
						hf_hub_download(repo_id='''google/flan-t5-small'''       ,    filename='''spiece.model'''       ,    local_dir=__a )

						class 		snake_case				(    tf.keras.layers.Layer						):

										"""simple docstring"""




										def __init__(	self )    ->      int:







											"""simple docstring"""






											super().__init__()
											snake_case__			:			Dict             =		text.SentencepieceTokenizer(
											    model=tf.io.gfile.GFile(os.path.join(__a       ,    '''spiece.model''' )       ,    '''rb''' ).read() )
											snake_case__			:			List[str]             =		TFAutoModelForSeqaSeqLM.from_pretrained('''hf-internal-testing/tiny-random-t5''' )





										def    lowercase__							(	self       ,    lowerCamelCase       ,    *lowerCamelCase       ,    **lowerCamelCase )    ->      Union[str, Any]:







											"""simple docstring"""






											snake_case__			:			List[str]             =		self.tokenizer.tokenize(__a )
											snake_case__			:			str             =		text.pad_model_inputs(
											    __a       ,    max_seq_length=64       ,    pad_value=self.model.config.pad_token_id )
											snake_case__			:			Tuple             =		self.model.generate(input_ids=__a       ,    attention_mask=__a )
											return self.tokenizer.detokenize(__a )



						snake_case__			:			Optional[Any]             =		CompleteSentenceTransformer()
						snake_case__			:			List[str]             =		tf.keras.layers.Input(shape=(1,)       ,    dtype=tf.string       ,    name='''inputs''' )
						snake_case__			:			Any             =		complete_model(__a )
						snake_case__			:			Optional[int]             =		tf.keras.Model(__a       ,    __a )
						keras_model.save(__a )





				def    lowercase__							(	self )    ->      Optional[Any]:







					"""simple docstring"""






					snake_case__			:			int             =		{
					    """do_sample""": True,
					    """num_beams""": 1,
					    """top_p""": 0.7,
					    """top_k""": 10,
					    """temperature""": 0.7,
					}
					snake_case__			:			Tuple             =		14

					snake_case__			:			Union[str, Any]             =		AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2''' )
					snake_case__			:			Union[str, Any]             =		"""Hello, my dog is cute and"""
					snake_case__			:			Optional[int]             =		tokenizer(__a       ,    return_tensors='''tf''' )
					snake_case__			:			Tuple             =		TFAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2''' )

					snake_case__			:			int             =		638
					# forces the generation to happen on CPU, to avoid GPU-related quirks
					with tf.device(''':/CPU:0''' ):
						tf.random.set_seed(0 )
						snake_case__			:			int             =		model.generate(**__a       ,    eos_token_id=__a       ,    **__a )
					self.assertTrue(expectation == len(generated_tokens[0] ) )

					snake_case__			:			Optional[int]             =		[638, 198]
					with tf.device(''':/CPU:0''' ):
						tf.random.set_seed(0 )
						snake_case__			:			Optional[int]             =		model.generate(**__a       ,    eos_token_id=__a       ,    **__a )
					self.assertTrue(expectation == len(generated_tokens[0] ) )





				def    lowercase__							(	self )    ->      Any:







					"""simple docstring"""






					snake_case__			:			List[str]             =		AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bart''' )
					snake_case__			:			List[str]             =		"""Hugging Face is a technology company based in New York and Paris."""
					snake_case__			:			Dict             =		bart_tokenizer(__a       ,    return_tensors='''tf''' ).input_ids
					snake_case__			:			List[str]             =		TFBartForConditionalGeneration.from_pretrained('''hf-internal-testing/tiny-random-bart''' )
					snake_case__			:			Dict             =		bart_model.generate(__a ).numpy()

					class 		snake_case				(    a__						):

									"""simple docstring"""




									def    lowercase__							(	self       ,    lowerCamelCase       ,    lowerCamelCase=None       ,    **lowerCamelCase )    ->      List[str]:







										"""simple docstring"""






										return super().call(__a       ,    **__a )



					snake_case__			:			Union[str, Any]             =		FakeBart.from_pretrained('''hf-internal-testing/tiny-random-bart''' )
					snake_case__			:			Optional[Any]             =		bart_model.generate(__a       ,    foo='''bar''' ).numpy()
					self.assertTrue(np.array_equal(__a       ,    __a ) )

					class 		snake_case				(    bart_model.model.encoder.__class__						):

									"""simple docstring"""




									def    lowercase__							(	self       ,    lowerCamelCase       ,    **lowerCamelCase )    ->      Tuple:







										"""simple docstring"""






										return super().call(__a       ,    **__a )



					snake_case__			:			Any             =		FakeEncoder(bart_model.config       ,    bart_model.model.shared )
					snake_case__			:			Optional[int]             =		fake_encoder

					# Normal generation still works (the output will be different because the encoder weights are different)
					snake_case__			:			Any             =		bart_model.generate(__a ).numpy()
					with self.assertRaises(__a ):
						# FakeEncoder.call() accepts **kwargs -> no filtering -> value error due to unexpected input "foo"
						bart_model.generate(__a       ,    foo='''bar''' )











'''simple docstring'''







from typing import Dict, Iterable, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format, to_pil_image
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_pytesseract_available, is_vision_available, logging, requires_backends


if is_vision_available():
      import PIL

# soft dependency
if is_pytesseract_available():
      import pytesseract

lowercase      :   Optional[Any]      =       logging.get_logger(__name__)



def        lowerCamelCase__				(      __lowercase     ,   __lowercase     ,   __lowercase				):
     return [
         int(1_000 * (box[0] / width)				),
         int(1_000 * (box[1] / height)				),
         int(1_000 * (box[2] / width)				),
         int(1_000 * (box[3] / height)				),
     ]



def        lowerCamelCase__				(      __lowercase     ,   __lowercase     ,   __lowercase				):
     snake_case			:      List[Any]        =			to_pil_image(__lowercase				)
     snake_case ,		snake_case			:      Union[str, Any]        =			pil_image.size
     snake_case			:      Optional[Any]        =			pytesseract.image_to_data(__lowercase     ,   lang=__lowercase     ,   output_type="""dict"""     ,   config=__lowercase				)
     snake_case ,		snake_case ,		snake_case ,		snake_case ,		snake_case			:      Dict        =			data["""text"""], data["""left"""], data["""top"""], data["""width"""], data["""height"""]

     # filter empty words and corresponding coordinates
     snake_case			:      Tuple        =			[idx for idx, word in enumerate(__lowercase				) if not word.strip()]
     snake_case			:      Tuple        =			[word for idx, word in enumerate(__lowercase				) if idx not in irrelevant_indices]
     snake_case			:      str        =			[coord for idx, coord in enumerate(__lowercase				) if idx not in irrelevant_indices]
     snake_case			:      Tuple        =			[coord for idx, coord in enumerate(__lowercase				) if idx not in irrelevant_indices]
     snake_case			:      Optional[Any]        =			[coord for idx, coord in enumerate(__lowercase				) if idx not in irrelevant_indices]
     snake_case			:      Optional[Any]        =			[coord for idx, coord in enumerate(__lowercase				) if idx not in irrelevant_indices]

     # turn coordinates into (left, top, left+width, top+height) format
     snake_case			:      Dict        =			[]
     for x, y, w, h in zip(__lowercase     ,   __lowercase     ,   __lowercase     ,   __lowercase				):
          snake_case			:      Tuple        =			[x, y, x + w, y + h]
          actual_boxes.append(__lowercase				)

     # finally, normalize the bounding boxes
     snake_case			:      str        =			[]
     for box in actual_boxes:
          normalized_boxes.append(normalize_box(__lowercase     ,   __lowercase     ,   __lowercase				)				)

     assert len(__lowercase				) == len(__lowercase				), "Not as many words as there are bounding boxes"

     return words, normalized_boxes







class 					_a (a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Dict          =					["""pixel_values"""]







     def __init__(				self							,__a = True							,__a = None							,__a = PILImageResampling.BILINEAR							,__a = True							,__a = 1 / 255							,__a = True							,__a = None							,__a = None							,__a = True							,__a = None							,__a = ""							,**__a							,)      ->				None:
          super().__init__(**__a							)
          snake_case			:      int        =			size if size is not None else {"""height""": 224, """width""": 224}
          snake_case			:      Union[str, Any]        =			get_size_dict(__a							)

          snake_case			:      List[str]        =			do_resize
          snake_case			:      Dict        =			size
          snake_case			:      Dict        =			resample
          snake_case			:      Union[str, Any]        =			do_rescale
          snake_case			:      Union[str, Any]        =			rescale_value
          snake_case			:      List[str]        =			do_normalize
          snake_case			:      int        =			image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
          snake_case			:      int        =			image_std if image_std is not None else IMAGENET_STANDARD_STD
          snake_case			:      List[Any]        =			apply_ocr
          snake_case			:      Any        =			ocr_lang
          snake_case			:      Optional[int]        =			tesseract_config







     def 			snake_case_      (				self							,__a							,__a							,__a = PILImageResampling.BILINEAR							,__a = None							,**__a							,)      ->				np.ndarray:
          snake_case			:      List[str]        =			get_size_dict(__a							)
          if "height" not in size or "width" not in size:
               raise ValueError(F'''The size dictionary must contain the keys \'height\' and \'width\'. Got {size.keys()}'''							)
          snake_case			:      Tuple        =			(size["""height"""], size["""width"""])
          return resize(__a							,size=__a							,resample=__a							,data_format=__a							,**__a							)







     def 			snake_case_      (				self							,__a							,__a							,__a = None							,**__a							,)      ->				np.ndarray:
          return rescale(__a							,scale=__a							,data_format=__a							,**__a							)







     def 			snake_case_      (				self							,__a							,__a							,__a							,__a = None							,**__a							,)      ->				np.ndarray:
          return normalize(__a							,mean=__a							,std=__a							,data_format=__a							,**__a							)







     def 			snake_case_      (				self							,__a							,__a = None							,__a = None							,__a=None							,__a = None							,__a = None							,__a = None							,__a = None							,__a = None							,__a = None							,__a = None							,__a = None							,__a = None							,__a = ChannelDimension.FIRST							,**__a							,)      ->				PIL.Image.Image:
          snake_case			:      int        =			do_resize if do_resize is not None else self.do_resize
          snake_case			:      List[Any]        =			size if size is not None else self.size
          snake_case			:      Any        =			get_size_dict(__a							)
          snake_case			:      Any        =			resample if resample is not None else self.resample
          snake_case			:      str        =			do_rescale if do_rescale is not None else self.do_rescale
          snake_case			:      List[str]        =			rescale_factor if rescale_factor is not None else self.rescale_factor
          snake_case			:      Dict        =			do_normalize if do_normalize is not None else self.do_normalize
          snake_case			:      Optional[int]        =			image_mean if image_mean is not None else self.image_mean
          snake_case			:      Tuple        =			image_std if image_std is not None else self.image_std
          snake_case			:      List[str]        =			apply_ocr if apply_ocr is not None else self.apply_ocr
          snake_case			:      str        =			ocr_lang if ocr_lang is not None else self.ocr_lang
          snake_case			:      Dict        =			tesseract_config if tesseract_config is not None else self.tesseract_config

          snake_case			:      Any        =			make_list_of_images(__a							)

          if not valid_images(__a							):
               raise ValueError(
                   """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
                   """torch.Tensor, tf.Tensor or jax.ndarray."""							)

          if do_resize and size is None:
               raise ValueError("""Size must be specified if do_resize is True."""							)

          if do_rescale and rescale_factor is None:
               raise ValueError("""Rescale factor must be specified if do_rescale is True."""							)

          if do_normalize and (image_mean is None or image_std is None):
               raise ValueError("""If do_normalize is True, image_mean and image_std must be specified."""							)

          # All transformations expect numpy arrays.
          snake_case			:      Union[str, Any]        =			[to_numpy_array(__a							) for image in images]

          # Tesseract OCR to get words + normalized bounding boxes
          if apply_ocr:
               requires_backends(self							,"""pytesseract"""							)
               snake_case			:      int        =			[]
               snake_case			:      Optional[Any]        =			[]
               for image in images:
                    snake_case ,		snake_case			:      Tuple        =			apply_tesseract(__a							,__a							,__a							)
                    words_batch.append(__a							)
                    boxes_batch.append(__a							)

          if do_resize:
               snake_case			:      Any        =			[self.resize(image=__a							,size=__a							,resample=__a							) for image in images]

          if do_rescale:
               snake_case			:      Tuple        =			[self.rescale(image=__a							,scale=__a							) for image in images]

          if do_normalize:
               snake_case			:      Optional[Any]        =			[self.normalize(image=__a							,mean=__a							,std=__a							) for image in images]

          snake_case			:      Dict        =			[to_channel_dimension_format(__a							,__a							) for image in images]

          snake_case			:      List[Any]        =			BatchFeature(data={"""pixel_values""": images}							,tensor_type=__a							)

          if apply_ocr:
               snake_case			:      Optional[Any]        =			words_batch
               snake_case			:      Optional[int]        =			boxes_batch
          return data



