


from __future__ import annotations

import os
import tempfile
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import is_tensorflow_text_available, is_tf_available
from transformers.testing_utils import require_tensorflow_text, require_tf, slow

from ..test_modeling_tf_common import floats_tensor
from .test_framework_agnostic import GenerationIntegrationTestsMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import (
     AutoTokenizer,
     TFAutoModelForCausalLM,
     TFAutoModelForSeqaSeqLM,
     TFAutoModelForSpeechSeqaSeq,
     TFAutoModelForVisionaSeq,
     TFBartForConditionalGeneration,
     TFLogitsProcessorList,
     TFMinLengthLogitsProcessor,
     tf_top_k_top_p_filtering,
 )

if is_tensorflow_text_available():
 import tensorflow_text as text






@require_tf
class _lowerCamelCase(		unittest.TestCase  ):
       def      UpperCamelCase	(   self)			->  Optional[Any]:




           """simple docstring"""

           _lowercase			:				str 						=						tf.convert_to_tensor(
               [
                   [
                       8.2_2_2_0_9_9_1,  # 3rd highest value; idx. 0
                       -0.5_6_2_0_0_4_4,
                       5.2_3_2_2_9_7_5_2,
                       4.0_3_8_6_3_9_3,
                       -6.8_7_9_8_3_7_8,
                       -0.5_4_7_8_5_8_0_2,
                       -3.2_0_1_2_1_5_3,
                       2.9_2_7_7_7_1_7_6,
                       1.8_8_1_7_1_9_5_3,
                       7.3_5_3_4_1_2_7_6,  # 5th highest value; idx. 9
                       8.4_3_2_0_7_8_3_3,  # 2nd highest value; idx. 10
                       -9.8_5_7_1_1_8_3_6,
                       -5.9_6_2_0_9_2_3_6,
                       -1.1_3_0_3_9_1_6_1,
                       -7.1_1_1_5_2_9_4,
                       -0.8_3_6_9_6_3_3,
                       -5.3_1_8_6_4_0_8,
                       7.0_6_4_2_7_4_0_7,
                       0.8_1_3_6_9_3_4_4,
                       -0.8_2_0_2_3_8_1_7,
                       -5.9_1_7_9_7_9_6,
                       0.5_8_8_1_3_4_4_3,
                       -6.9_9_7_7_8_4_3_8,
                       4.7_1_5_5_1_1_8_9,
                       -0.1_8_7_7_1_6_3_7,
                       7.4_4_0_2_0_7_5_9,  # 4th highest value; idx. 25
                       9.3_8_4_5_0_9_8_7,  # 1st highest value; idx. 26
                       2.1_2_6_6_2_9_4_1,
                       -9.3_2_5_6_2_0_3_8,
                       2.3_5_6_5_2_5_2_2,
                   ],  # cummulative prob of 5 highest values <= 0.6
                   [
                       0.5_8_4_2_5_5_1_8,
                       4.5_3_1_3_9_2_3_8,
                       -5.5_7_5_1_0_4_6_4,
                       -6.2_8_0_3_0_6_9_9,
                       -7.1_9_5_2_9_5_0_3,
                       -4.0_2_1_2_2_5_5_1,
                       1.3_9_3_3_7_0_3_7,
                       -6.0_6_7_0_7_0_5_7,
                       1.5_9_4_8_0_5_1_7,
                       -9.6_4_3_1_1_9,
                       0.0_3_9_0_7_7_9_9,
                       0.6_7_2_3_1_7_6_2,
                       -8.8_8_2_0_6_7_2_6,
                       6.2_7_1_1_5_9_2_2,  # 4th highest value; idx. 13
                       2.2_8_5_2_0_7_2_3,
                       4.8_2_7_6_7_5_0_6,
                       4.3_0_4_2_1_3_6_8,
                       8.8_2_7_5_3_1_3,  # 2nd highest value; idx. 17
                       5.4_4_0_2_9_9_5_8,  # 5th highest value; idx. 18
                       -4.4_7_3_5_7_9_4,
                       7.3_8_5_7_9_5_3_6,  # 3rd highest value; idx. 20
                       -2.9_1_0_5_1_6_6_3,
                       2.6_1_9_4_6_0_7_7,
                       -2.5_6_7_4_7_6_2,
                       -9.4_8_9_5_9_3_0_2,
                       -4.0_2_9_2_2_6_4_5,
                       -1.3_5_4_1_6_9_1_8,
                       9.6_7_7_0_2_3_2_3,  # 1st highest value; idx. 27
                       -5.8_9_4_7_8_5_5_3,
                       1.8_5_3_7_0_4_6_7,
                   ],  # cummulative prob of 5 highest values <= 0.6
               ], dtype=tf.floataa, )

           _lowercase			:				List[Any] 						=						tf.convert_to_tensor(
               [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]], dtype=tf.intaa, )  # expected non filtered idx as noted above

           _lowercase			:				Optional[Any] 						=						tf.convert_to_tensor(
               [8.2_2_2_0_9_9, 7.3_5_3_4_1_2_6, 8.4_3_2_0_7_8, 7.4_4_0_2_0_7_5, 9.3_8_4_5_1, 6.2_7_1_1_5_9, 8.8_2_7_5_3_1, 5.4_4_0_2_9_9_5, 7.3_8_5_7_9_5_6, 9.6_7_7_0_2_3], dtype=tf.floataa, )  # expected non filtered values as noted above

           _lowercase			:				Tuple 						=						tf_top_k_top_p_filtering(lowerCamelCase, top_k=10, top_p=0.6, min_tokens_to_keep=4)

           _lowercase			:				Optional[Any] 						=						output[output != -float('inf')]
           _lowercase			:				Tuple 						=						tf.cast(
               tf.where(tf.not_equal(lowerCamelCase, tf.constant(-float('inf'), dtype=tf.floataa))), dtype=tf.intaa, )

           tf.debugging.assert_near(lowerCamelCase, lowerCamelCase, rtol=1E-12)
           tf.debugging.assert_equal(lowerCamelCase, lowerCamelCase)






@require_tf
class _lowerCamelCase(		unittest.TestCase,					_a  ):
       # setting framework_dependent_parameters needs to be gated, just like its contents' imports
       if is_tf_available():
              lowercase_					:      Optional[int]       					=     {
                  """AutoModelForCausalLM""": TFAutoModelForCausalLM,
                  """AutoModelForSpeechSeq2Seq""": TFAutoModelForSpeechSeqaSeq,
                  """AutoModelForSeq2SeqLM""": TFAutoModelForSeqaSeqLM,
                  """AutoModelForVision2Seq""": TFAutoModelForVisionaSeq,
                  """LogitsProcessorList""": TFLogitsProcessorList,
                  """MinLengthLogitsProcessor""": TFMinLengthLogitsProcessor,
                  """create_tensor_fn""": tf.convert_to_tensor,
                  """floats_tensor""": floats_tensor,
                  """return_tensors""": """tf""",
              }
       @slow
       def      UpperCamelCase	(   self)			->  int:




           """simple docstring"""

           _lowercase			:				Optional[int] 						=						TFAutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2')
           _lowercase			:				Any 						=						2
           _lowercase			:				Any 						=						2






           class _lowerCamelCase(		tf.Module  ):
                  def __init__(   self, lowerCamelCase)			->  Dict:




                      """simple docstring"""

                      super(lowerCamelCase, self).__init__()
                      _lowercase			:				Optional[int] 						=						model




                  @tf.function(
                      input_signature=(
                          tf.TensorSpec((None, input_length), tf.intaa, name='input_ids'),
                          tf.TensorSpec((None, input_length), tf.intaa, name='attention_mask'),
                      ), jit_compile=lowerCamelCase, )
                  def      UpperCamelCase	(   self, lowerCamelCase, lowerCamelCase)			->  Any:




                      """simple docstring"""

                      _lowercase			:				Any 						=						self.model.generate(
                          input_ids=lowerCamelCase, attention_mask=lowerCamelCase, max_new_tokens=lowerCamelCase, return_dict_in_generate=lowerCamelCase, )
                      return {"sequences": outputs["sequences"]}


           _lowercase			:				Optional[Any] 						=						[[2, 0], [1_02, 1_03]]
           _lowercase			:				Any 						=						[[1, 0], [1, 1]]
           _lowercase			:				Any 						=						DummyModel(model=lowerCamelCase)
           with tempfile.TemporaryDirectory() as tmp_dir:
               tf.saved_model.save(lowerCamelCase, lowerCamelCase, signatures={'serving_default': dummy_model.serving})
               _lowercase			:				Optional[Any] 						=						tf.saved_model.load(lowerCamelCase).signatures['serving_default']
               for batch_size in range(1, len(lowerCamelCase) + 1):
                   _lowercase			:				List[Any] 						=						{
                       'input_ids': tf.constant(dummy_input_ids[:batch_size]),
                       'attention_mask': tf.constant(dummy_attention_masks[:batch_size]),
                   }
                   _lowercase			:				Optional[Any] 						=						serving_func(**lowerCamelCase)['sequences']
                   _lowercase			:				Optional[int] 						=						test_model.generate(**lowerCamelCase, max_new_tokens=lowerCamelCase)
                   tf.debugging.assert_equal(lowerCamelCase, lowerCamelCase)
       @slow
       def      UpperCamelCase	(   self)			->  Dict:




           """simple docstring"""

           _lowercase			:				Optional[int] 						=						TFAutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2')
           _lowercase			:				List[Any] 						=						1
           _lowercase			:				List[str] 						=						2






           class _lowerCamelCase(		tf.Module  ):
                  def __init__(   self, lowerCamelCase)			->  List[str]:




                      """simple docstring"""

                      super(lowerCamelCase, self).__init__()
                      _lowercase			:				str 						=						model




                  @tf.function(
                      input_signature=(
                          tf.TensorSpec((batch_size, None), tf.intaa, name='input_ids'),
                          tf.TensorSpec((batch_size, None), tf.intaa, name='attention_mask'),
                      ), jit_compile=lowerCamelCase, )
                  def      UpperCamelCase	(   self, lowerCamelCase, lowerCamelCase)			->  Any:




                      """simple docstring"""

                      _lowercase			:				Union[str, Any] 						=						self.model.generate(
                          input_ids=lowerCamelCase, attention_mask=lowerCamelCase, max_new_tokens=lowerCamelCase, return_dict_in_generate=lowerCamelCase, )
                      return {"sequences": outputs["sequences"]}


           _lowercase			:				List[str] 						=						[[2], [1_02, 1_03]]
           _lowercase			:				str 						=						[[1], [1, 1]]
           _lowercase			:				int 						=						DummyModel(model=lowerCamelCase)
           with tempfile.TemporaryDirectory() as tmp_dir:
               tf.saved_model.save(lowerCamelCase, lowerCamelCase, signatures={'serving_default': dummy_model.serving})
               _lowercase			:				Optional[int] 						=						tf.saved_model.load(lowerCamelCase).signatures['serving_default']
               for input_row in range(len(lowerCamelCase)):
                   _lowercase			:				Dict 						=						{
                       'input_ids': tf.constant([dummy_input_ids[input_row]]),
                       'attention_mask': tf.constant([dummy_attention_masks[input_row]]),
                   }
                   _lowercase			:				str 						=						serving_func(**lowerCamelCase)['sequences']
                   _lowercase			:				Optional[Any] 						=						test_model.generate(**lowerCamelCase, max_new_tokens=lowerCamelCase)
                   tf.debugging.assert_equal(lowerCamelCase, lowerCamelCase)
       @slow
       @require_tensorflow_text
       def      UpperCamelCase	(   self)			->  Any:




           """simple docstring"""

           with tempfile.TemporaryDirectory() as tmp_dir:
               # file needed to load the TF tokenizer
               hf_hub_download(repo_id='google/flan-t5-small', filename='spiece.model', local_dir=lowerCamelCase)






               class _lowerCamelCase(		tf.keras.layers.Layer  ):
                      def __init__(   self)			->  Optional[Any]:




                          """simple docstring"""

                          super().__init__()
                          _lowercase			:				Union[str, Any] 						=						text.SentencepieceTokenizer(
                              model=tf.io.gfile.GFile(os.path.join(lowerCamelCase, 'spiece.model'), 'rb').read())
                          _lowercase			:				Tuple 						=						TFAutoModelForSeqaSeqLM.from_pretrained('hf-internal-testing/tiny-random-t5')




                      def      UpperCamelCase	(   self, lowerCamelCase, *lowerCamelCase, **lowerCamelCase)			->  Optional[Any]:




                          """simple docstring"""

                          _lowercase			:				int 						=						self.tokenizer.tokenize(lowerCamelCase)
                          _lowercase						,							_lowercase			:				str 						=						text.pad_model_inputs(
                              lowerCamelCase, max_seq_length=64, pad_value=self.model.config.pad_token_id)
                          _lowercase			:				int 						=						self.model.generate(input_ids=lowerCamelCase, attention_mask=lowerCamelCase)
                          return self.tokenizer.detokenize(lowerCamelCase)


               _lowercase			:				int 						=						CompleteSentenceTransformer()
               _lowercase			:				Tuple 						=						tf.keras.layers.Input(shape=(1,), dtype=tf.string, name='inputs')
               _lowercase			:				Dict 						=						complete_model(lowerCamelCase)
               _lowercase			:				Tuple 						=						tf.keras.Model(lowerCamelCase, lowerCamelCase)
               keras_model.save(lowerCamelCase)
       def      UpperCamelCase	(   self)			->  str:




           """simple docstring"""

           _lowercase			:				Optional[int] 						=						{
               'do_sample': True,
               'num_beams': 1,
               'top_p': 0.7,
               'top_k': 10,
               'temperature': 0.7,
           }
           _lowercase			:				Union[str, Any] 						=						14

           _lowercase			:				Tuple 						=						AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2')
           _lowercase			:				Optional[int] 						=						'Hello, my dog is cute and'
           _lowercase			:				List[Any] 						=						tokenizer(lowerCamelCase, return_tensors='tf')
           _lowercase			:				int 						=						TFAutoModelForCausalLM.from_pretrained('hf-internal-testing/tiny-random-gpt2')

           _lowercase			:				Optional[int] 						=						6_38
           # forces the generation to happen on CPU, to avoid GPU-related quirks
           with tf.device(':/CPU:0'):
               tf.random.set_seed(0)
               _lowercase			:				Any 						=						model.generate(**lowerCamelCase, eos_token_id=lowerCamelCase, **lowerCamelCase)
           self.assertTrue(expectation == len(generated_tokens[0]))

           _lowercase			:				Any 						=						[6_38, 1_98]
           with tf.device(':/CPU:0'):
               tf.random.set_seed(0)
               _lowercase			:				List[Any] 						=						model.generate(**lowerCamelCase, eos_token_id=lowerCamelCase, **lowerCamelCase)
           self.assertTrue(expectation == len(generated_tokens[0]))




       def      UpperCamelCase	(   self)			->  int:




           """simple docstring"""

           _lowercase			:				Dict 						=						AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bart')
           _lowercase			:				Union[str, Any] 						=						'Hugging Face is a technology company based in New York and Paris.'
           _lowercase			:				Tuple 						=						bart_tokenizer(lowerCamelCase, return_tensors='tf').input_ids
           _lowercase			:				Optional[int] 						=						TFBartForConditionalGeneration.from_pretrained('hf-internal-testing/tiny-random-bart')
           _lowercase			:				List[Any] 						=						bart_model.generate(lowerCamelCase).numpy()






           class _lowerCamelCase(		_a  ):
                  def      UpperCamelCase	(   self, lowerCamelCase, lowerCamelCase=None, **lowerCamelCase)			->  Union[str, Any]:




                      """simple docstring"""

                      return super().call(lowerCamelCase, **lowerCamelCase)


           _lowercase			:				Union[str, Any] 						=						FakeBart.from_pretrained('hf-internal-testing/tiny-random-bart')
           _lowercase			:				Tuple 						=						bart_model.generate(lowerCamelCase, foo='bar').numpy()
           self.assertTrue(np.array_equal(lowerCamelCase, lowerCamelCase))






           class _lowerCamelCase(		bart_model.model.encoder.__class__  ):
                  def      UpperCamelCase	(   self, lowerCamelCase, **lowerCamelCase)			->  Optional[Any]:




                      """simple docstring"""

                      return super().call(lowerCamelCase, **lowerCamelCase)


           _lowercase			:				Any 						=						FakeEncoder(bart_model.config, bart_model.model.shared)
           _lowercase			:				List[str] 						=						fake_encoder

           # Normal generation still works (the output will be different because the encoder weights are different)
           _lowercase			:				List[Any] 						=						bart_model.generate(lowerCamelCase).numpy()
           with self.assertRaises(lowerCamelCase):
               # FakeEncoder.call() accepts **kwargs -> no filtering -> value error due to unexpected input "foo"
               bart_model.generate(lowerCamelCase, foo='bar')







import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType


class 				_a						(       SCREAMING_SNAKE_CASE			):





							'''simple docstring'''

							A				:int =       ["image_processor", "tokenizer"]
							A				:Any =       "LayoutLMv3ImageProcessor"
							A				:str =       ("LayoutLMv3Tokenizer", "LayoutLMv3TokenizerFast")



							def __init__(							self ,					__UpperCAmelCase=None ,					__UpperCAmelCase=None ,					**__UpperCAmelCase				):







											"""simple docstring"""



											a__	:							str      						= None
											if "feature_extractor" in kwargs:
															warnings.warn(
															    "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
															    " instead." ,					__UpperCAmelCase ,					)
															a__	:							List[str]      						= kwargs.pop("feature_extractor"				)

											a__	:							Optional[int]      						= image_processor if image_processor is not None else feature_extractor
											if image_processor is None:
															raise ValueError("You need to specify an `image_processor`."				)
											if tokenizer is None:
															raise ValueError("You need to specify a `tokenizer`."				)

											super().__init__(__UpperCAmelCase ,					__UpperCAmelCase				)



							def __call__(							self ,					__UpperCAmelCase ,					__UpperCAmelCase = None ,					__UpperCAmelCase = None ,					__UpperCAmelCase = None ,					__UpperCAmelCase = None ,					__UpperCAmelCase = True ,					__UpperCAmelCase = False ,					__UpperCAmelCase = None ,					__UpperCAmelCase = None ,					__UpperCAmelCase = 0 ,					__UpperCAmelCase = None ,					__UpperCAmelCase = None ,					__UpperCAmelCase = None ,					__UpperCAmelCase = False ,					__UpperCAmelCase = False ,					__UpperCAmelCase = False ,					__UpperCAmelCase = False ,					__UpperCAmelCase = True ,					__UpperCAmelCase = None ,					**__UpperCAmelCase ,					):







											"""simple docstring"""



											if self.image_processor.apply_ocr and (boxes is not None):
															raise ValueError(
															    "You cannot provide bounding boxes if you initialized the image processor with apply_ocr set to True."				)

											if self.image_processor.apply_ocr and (word_labels is not None):
															raise ValueError(
															    "You cannot provide word labels if you initialized the image processor with apply_ocr set to True."				)

											# first, apply the image processor
											a__	:							List[str]      						= self.image_processor(images=__UpperCAmelCase ,					return_tensors=__UpperCAmelCase				)

											# second, apply the tokenizer
											if text is not None and self.image_processor.apply_ocr and text_pair is None:
															if isinstance(__UpperCAmelCase ,					__UpperCAmelCase				):
																			a__	:							Union[str, Any]      						= [text]  # add batch dimension (as the image processor always adds a batch dimension)
															a__	:							Optional[Any]      						= features["words"]

											a__	:							Union[str, Any]      						= self.tokenizer(
											    text=text if text is not None else features["words"] ,					text_pair=text_pair if text_pair is not None else None ,					boxes=boxes if boxes is not None else features["boxes"] ,					word_labels=__UpperCAmelCase ,					add_special_tokens=__UpperCAmelCase ,					padding=__UpperCAmelCase ,					truncation=__UpperCAmelCase ,					max_length=__UpperCAmelCase ,					stride=__UpperCAmelCase ,					pad_to_multiple_of=__UpperCAmelCase ,					return_token_type_ids=__UpperCAmelCase ,					return_attention_mask=__UpperCAmelCase ,					return_overflowing_tokens=__UpperCAmelCase ,					return_special_tokens_mask=__UpperCAmelCase ,					return_offsets_mapping=__UpperCAmelCase ,					return_length=__UpperCAmelCase ,					verbose=__UpperCAmelCase ,					return_tensors=__UpperCAmelCase ,					**__UpperCAmelCase ,					)

											# add pixel values
											a__	:							Dict      						= features.pop("pixel_values"				)
											if return_overflowing_tokens is True:
															a__	:							str      						= self.get_overflowing_images(__UpperCAmelCase ,					encoded_inputs["overflow_to_sample_mapping"]				)
											a__	:							Tuple      						= images

											return encoded_inputs



							def 		_A	(							self ,					__UpperCAmelCase ,					__UpperCAmelCase				):







											"""simple docstring"""



											a__	:							Any      						= []
											for sample_idx in overflow_to_sample_mapping:
															images_with_overflow.append(images[sample_idx]				)

											if len(__UpperCAmelCase				) != len(__UpperCAmelCase				):
															raise ValueError(
															    "Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got"
															    f' {len(__UpperCAmelCase				)} and {len(__UpperCAmelCase				)}'				)

											return images_with_overflow



							def 		_A	(							self ,					*__UpperCAmelCase ,					**__UpperCAmelCase				):







											"""simple docstring"""



											return self.tokenizer.batch_decode(*__UpperCAmelCase ,					**__UpperCAmelCase				)



							def 		_A	(							self ,					*__UpperCAmelCase ,					**__UpperCAmelCase				):







											"""simple docstring"""



											return self.tokenizer.decode(*__UpperCAmelCase ,					**__UpperCAmelCase				)



							@property
							def 		_A	(							self				):







											"""simple docstring"""



											return ["input_ids", "bbox", "attention_mask", "pixel_values"]



							@property
							def 		_A	(							self				):







											"""simple docstring"""



											warnings.warn(
											    "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead." ,					__UpperCAmelCase ,					)
											return self.image_processor_class




							@property
							def 		_A	(							self				):







											"""simple docstring"""



											warnings.warn(
											    "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead." ,					__UpperCAmelCase ,					)
											return self.image_processor


