





'''simple docstring'''






from __future__ import annotations

import os
import tempfile
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import is_tensorflow_text_available, is_tf_available
from transformers.testing_utils import require_tensorflow_text, require_tf, slow

from ..test_modeling_tf_common import floats_tensor
from .test_framework_agnostic import GenerationIntegrationTestsMixin


if is_tf_available():
	import tensorflow as tf

	from transformers import (
	    AutoTokenizer,
	    TFAutoModelForCausalLM,
	    TFAutoModelForSeqaSeqLM,
	    TFAutoModelForSpeechSeqaSeq,
	    TFAutoModelForVisionaSeq,
	    TFBartForConditionalGeneration,
	    TFLogitsProcessorList,
	    TFMinLengthLogitsProcessor,
	    tf_top_k_top_p_filtering,
	)

if is_tensorflow_text_available():
	import tensorflow_text as text




@require_tf
class  _A (					unittest.TestCase		):

		def 	lowercase__     (   self					:   Optional[Any]			)					->    List[str]:


					"""simple docstring"""

					__snake_case					:      Tuple							      =     tf.convert_to_tensor(
					    [
					        [
					            8.2220991,  # 3rd highest value; idx. 0
					            -0.5620044,
					            5.23229752,
					            4.0386393,
					            -6.8798378,
					            -0.54785802,
					            -3.2012153,
					            2.92777176,
					            1.88171953,
					            7.35341276,  # 5th highest value; idx. 9
					            8.43207833,  # 2nd highest value; idx. 10
					            -9.85711836,
					            -5.96209236,
					            -1.13039161,
					            -7.1115294,
					            -0.8369633,
					            -5.3186408,
					            7.06427407,
					            0.81369344,
					            -0.82023817,
					            -5.9179796,
					            0.58813443,
					            -6.99778438,
					            4.71551189,
					            -0.18771637,
					            7.44020759,  # 4th highest value; idx. 25
					            9.38450987,  # 1st highest value; idx. 26
					            2.12662941,
					            -9.32562038,
					            2.35652522,
					        ],  # cummulative prob of 5 highest values <= 0.6
					        [
					            0.58425518,
					            4.53139238,
					            -5.57510464,
					            -6.28030699,
					            -7.19529503,
					            -4.02122551,
					            1.39337037,
					            -6.06707057,
					            1.59480517,
					            -9.643119,
					            0.03907799,
					            0.67231762,
					            -8.88206726,
					            6.27115922,  # 4th highest value; idx. 13
					            2.28520723,
					            4.82767506,
					            4.30421368,
					            8.8275313,  # 2nd highest value; idx. 17
					            5.44029958,  # 5th highest value; idx. 18
					            -4.4735794,
					            7.38579536,  # 3rd highest value; idx. 20
					            -2.91051663,
					            2.61946077,
					            -2.5674762,
					            -9.48959302,
					            -4.02922645,
					            -1.35416918,
					            9.67702323,  # 1st highest value; idx. 27
					            -5.89478553,
					            1.85370467,
					        ],  # cummulative prob of 5 highest values <= 0.6
					    ]  ,      dtype=tf.floataa  ,      )

					__snake_case					:      Union[str, Any]							      =     tf.convert_to_tensor(
					    [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]]  ,      dtype=tf.intaa  ,      )  # expected non filtered idx as noted above

					__snake_case					:      Any							      =     tf.convert_to_tensor(
					    [8.222099, 7.3534126, 8.432078, 7.4402075, 9.38451, 6.271159, 8.827531, 5.4402995, 7.3857956, 9.677023]  ,      dtype=tf.floataa  ,      )  # expected non filtered values as noted above

					__snake_case					:      Dict							      =     tf_top_k_top_p_filtering(__magic_name__  ,      top_k=10  ,      top_p=0.6  ,      min_tokens_to_keep=4			)

					__snake_case					:      List[Any]							      =     output[output != -float("""inf"""			)]
					__snake_case					:      List[str]							      =     tf.cast(
					    tf.where(tf.not_equal(__magic_name__  ,      tf.constant(-float("""inf"""			)  ,      dtype=tf.floataa			)			)			)  ,      dtype=tf.intaa  ,      )

					tf.debugging.assert_near(__magic_name__  ,      __magic_name__  ,      rtol=1E-12			)
					tf.debugging.assert_equal(__magic_name__  ,      __magic_name__			)





@require_tf
class  _A (					unittest.TestCase						,					__lowercase		):
		# setting framework_dependent_parameters needs to be gated, just like its contents' imports
		if is_tf_available():
				lowercase__:   Optional[Any]				     =   {
				    '''AutoModelForCausalLM''': TFAutoModelForCausalLM,
				    '''AutoModelForSpeechSeq2Seq''': TFAutoModelForSpeechSeqaSeq,
				    '''AutoModelForSeq2SeqLM''': TFAutoModelForSeqaSeqLM,
				    '''AutoModelForVision2Seq''': TFAutoModelForVisionaSeq,
				    '''LogitsProcessorList''': TFLogitsProcessorList,
				    '''MinLengthLogitsProcessor''': TFMinLengthLogitsProcessor,
				    '''create_tensor_fn''': tf.convert_to_tensor,
				    '''floats_tensor''': floats_tensor,
				    '''return_tensors''': '''tf''',
				}

		@slow
		def 	lowercase__     (   self					:   Optional[Any]			)					->    List[Any]:


					"""simple docstring"""

					__snake_case					:      Dict							      =     TFAutoModelForCausalLM.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""			)
					__snake_case					:      Union[str, Any]							      =     2
					__snake_case					:      str							      =     2




					class  _A (					tf.Module		):

							def __init__(   self					:   str  ,      __magic_name__					:   List[str]			)					->    Tuple:


										"""simple docstring"""

										super(__magic_name__  ,      self			).__init__()
										__snake_case					:      Optional[int]							      =     model

							@tf.function(
							    input_signature=(
							        tf.TensorSpec((None, input_length)  ,      tf.intaa  ,      name="""input_ids"""			),
							        tf.TensorSpec((None, input_length)  ,      tf.intaa  ,      name="""attention_mask"""			),
							    )  ,      jit_compile=__magic_name__  ,      )
							def 	lowercase__     (   self					:   Optional[Any]  ,      __magic_name__					:   Optional[Any]  ,      __magic_name__					:   Optional[int]			)					->    Any:


										"""simple docstring"""

										__snake_case					:      Optional[Any]							      =     self.model.generate(
										    input_ids=__magic_name__  ,      attention_mask=__magic_name__  ,      max_new_tokens=__magic_name__  ,      return_dict_in_generate=__magic_name__  ,      )
										return {"sequences": outputs["sequences"]}





					__snake_case					:      int							      =     [[2, 0], [1_02, 1_03]]
					__snake_case					:      Dict							      =     [[1, 0], [1, 1]]
					__snake_case					:      Tuple							      =     DummyModel(model=__magic_name__			)
					with tempfile.TemporaryDirectory() as tmp_dir:
								tf.saved_model.save(__magic_name__  ,      __magic_name__  ,      signatures={"""serving_default""": dummy_model.serving}			)
								__snake_case					:      Any							      =     tf.saved_model.load(__magic_name__			).signatures["""serving_default"""]
								for batch_size in range(1  ,      len(__magic_name__			) + 1			):
											__snake_case					:      Dict							      =     {
											    """input_ids""": tf.constant(dummy_input_ids[:batch_size]			),
											    """attention_mask""": tf.constant(dummy_attention_masks[:batch_size]			),
											}
											__snake_case					:      Any							      =     serving_func(**__magic_name__			)["""sequences"""]
											__snake_case					:      str							      =     test_model.generate(**__magic_name__  ,      max_new_tokens=__magic_name__			)
											tf.debugging.assert_equal(__magic_name__  ,      __magic_name__			)

		@slow
		def 	lowercase__     (   self					:   Optional[int]			)					->    Optional[int]:


					"""simple docstring"""

					__snake_case					:      Optional[int]							      =     TFAutoModelForCausalLM.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""			)
					__snake_case					:      str							      =     1
					__snake_case					:      Any							      =     2




					class  _A (					tf.Module		):

							def __init__(   self					:   str  ,      __magic_name__					:   List[str]			)					->    List[Any]:


										"""simple docstring"""

										super(__magic_name__  ,      self			).__init__()
										__snake_case					:      Tuple							      =     model

							@tf.function(
							    input_signature=(
							        tf.TensorSpec((batch_size, None)  ,      tf.intaa  ,      name="""input_ids"""			),
							        tf.TensorSpec((batch_size, None)  ,      tf.intaa  ,      name="""attention_mask"""			),
							    )  ,      jit_compile=__magic_name__  ,      )
							def 	lowercase__     (   self					:   List[Any]  ,      __magic_name__					:   str  ,      __magic_name__					:   int			)					->    str:


										"""simple docstring"""

										__snake_case					:      Optional[Any]							      =     self.model.generate(
										    input_ids=__magic_name__  ,      attention_mask=__magic_name__  ,      max_new_tokens=__magic_name__  ,      return_dict_in_generate=__magic_name__  ,      )
										return {"sequences": outputs["sequences"]}





					__snake_case					:      List[str]							      =     [[2], [1_02, 1_03]]
					__snake_case					:      Optional[Any]							      =     [[1], [1, 1]]
					__snake_case					:      int							      =     DummyModel(model=__magic_name__			)
					with tempfile.TemporaryDirectory() as tmp_dir:
								tf.saved_model.save(__magic_name__  ,      __magic_name__  ,      signatures={"""serving_default""": dummy_model.serving}			)
								__snake_case					:      List[str]							      =     tf.saved_model.load(__magic_name__			).signatures["""serving_default"""]
								for input_row in range(len(__magic_name__			)			):
											__snake_case					:      int							      =     {
											    """input_ids""": tf.constant([dummy_input_ids[input_row]]			),
											    """attention_mask""": tf.constant([dummy_attention_masks[input_row]]			),
											}
											__snake_case					:      Optional[int]							      =     serving_func(**__magic_name__			)["""sequences"""]
											__snake_case					:      List[str]							      =     test_model.generate(**__magic_name__  ,      max_new_tokens=__magic_name__			)
											tf.debugging.assert_equal(__magic_name__  ,      __magic_name__			)

		@slow
		@require_tensorflow_text
		def 	lowercase__     (   self					:   List[str]			)					->    Dict:


					"""simple docstring"""

					with tempfile.TemporaryDirectory() as tmp_dir:
								# file needed to load the TF tokenizer
								hf_hub_download(repo_id="""google/flan-t5-small"""  ,      filename="""spiece.model"""  ,      local_dir=__magic_name__			)




								class  _A (					tf.keras.layers.Layer		):

										def __init__(   self					:   Optional[int]			)					->    int:


													"""simple docstring"""

													super().__init__()
													__snake_case					:      Tuple							      =     text.SentencepieceTokenizer(
													    model=tf.io.gfile.GFile(os.path.join(__magic_name__  ,      """spiece.model"""			)  ,      """rb"""			).read()			)
													__snake_case					:      List[str]							      =     TFAutoModelForSeqaSeqLM.from_pretrained("""hf-internal-testing/tiny-random-t5"""			)

										def 	lowercase__     (   self					:   Optional[int]  ,      __magic_name__					:   Tuple  ,      *__magic_name__					:   List[Any]  ,      **__magic_name__					:   Any			)					->    Optional[int]:


													"""simple docstring"""

													__snake_case					:      Dict							      =     self.tokenizer.tokenize(__magic_name__			)
													__snake_case      ,     __snake_case					:      List[Any]							      =     text.pad_model_inputs(
													    __magic_name__  ,      max_seq_length=64  ,      pad_value=self.model.config.pad_token_id			)
													__snake_case					:      Optional[Any]							      =     self.model.generate(input_ids=__magic_name__  ,      attention_mask=__magic_name__			)
													return self.tokenizer.detokenize(__magic_name__			)





								__snake_case					:      Tuple							      =     CompleteSentenceTransformer()
								__snake_case					:      Optional[Any]							      =     tf.keras.layers.Input(shape=(1,)  ,      dtype=tf.string  ,      name="""inputs"""			)
								__snake_case					:      Any							      =     complete_model(__magic_name__			)
								__snake_case					:      Any							      =     tf.keras.Model(__magic_name__  ,      __magic_name__			)
								keras_model.save(__magic_name__			)

		def 	lowercase__     (   self					:   str			)					->    Dict:


					"""simple docstring"""

					__snake_case					:      Any							      =     {
					    """do_sample""": True,
					    """num_beams""": 1,
					    """top_p""": 0.7,
					    """top_k""": 10,
					    """temperature""": 0.7,
					}
					__snake_case					:      Union[str, Any]							      =     14

					__snake_case					:      Dict							      =     AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""			)
					__snake_case					:      List[str]							      =     """Hello, my dog is cute and"""
					__snake_case					:      Dict							      =     tokenizer(__magic_name__  ,      return_tensors="""tf"""			)
					__snake_case					:      Union[str, Any]							      =     TFAutoModelForCausalLM.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""			)

					__snake_case					:      Union[str, Any]							      =     6_38
					# forces the generation to happen on CPU, to avoid GPU-related quirks
					with tf.device(""":/CPU:0"""			):
								tf.random.set_seed(0			)
								__snake_case					:      Any							      =     model.generate(**__magic_name__  ,      eos_token_id=__magic_name__  ,      **__magic_name__			)
					self.assertTrue(expectation == len(generated_tokens[0]			)			)

					__snake_case					:      Dict							      =     [6_38, 1_98]
					with tf.device(""":/CPU:0"""			):
								tf.random.set_seed(0			)
								__snake_case					:      Tuple							      =     model.generate(**__magic_name__  ,      eos_token_id=__magic_name__  ,      **__magic_name__			)
					self.assertTrue(expectation == len(generated_tokens[0]			)			)

		def 	lowercase__     (   self					:   Any			)					->    List[str]:


					"""simple docstring"""

					__snake_case					:      Tuple							      =     AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bart"""			)
					__snake_case					:      Union[str, Any]							      =     """Hugging Face is a technology company based in New York and Paris."""
					__snake_case					:      Tuple							      =     bart_tokenizer(__magic_name__  ,      return_tensors="""tf"""			).input_ids
					__snake_case					:      Tuple							      =     TFBartForConditionalGeneration.from_pretrained("""hf-internal-testing/tiny-random-bart"""			)
					__snake_case					:      Any							      =     bart_model.generate(__magic_name__			).numpy()




					class  _A (					__lowercase		):

							def 	lowercase__     (   self					:   Union[str, Any]  ,      __magic_name__					:   int  ,      __magic_name__					:   Dict=None  ,      **__magic_name__					:   Dict			)					->    str:


										"""simple docstring"""

										return super().call(__magic_name__  ,      **__magic_name__			)





					__snake_case					:      Dict							      =     FakeBart.from_pretrained("""hf-internal-testing/tiny-random-bart"""			)
					__snake_case					:      Any							      =     bart_model.generate(__magic_name__  ,      foo="""bar"""			).numpy()
					self.assertTrue(np.array_equal(__magic_name__  ,      __magic_name__			)			)




					class  _A (					bart_model.model.encoder.__class__		):

							def 	lowercase__     (   self					:   Union[str, Any]  ,      __magic_name__					:   Tuple  ,      **__magic_name__					:   Any			)					->    int:


										"""simple docstring"""

										return super().call(__magic_name__  ,      **__magic_name__			)





					__snake_case					:      int							      =     FakeEncoder(bart_model.config  ,      bart_model.model.shared			)
					__snake_case					:      Optional[Any]							      =     fake_encoder

					# Normal generation still works (the output will be different because the encoder weights are different)
					__snake_case					:      Dict							      =     bart_model.generate(__magic_name__			).numpy()
					with self.assertRaises(__magic_name__			):
								# FakeEncoder.call() accepts **kwargs -> no filtering -> value error due to unexpected input "foo"
								bart_model.generate(__magic_name__  ,      foo="""bar"""			)







'''simple docstring'''





# Algorithm for the pigeonhole sorting



def 			a_	(				_lowerCAmelCase			) -> List[str]:
  __lowerCamelCase			:			int       			=					min(_lowerCAmelCase			)  # min() finds the minimum value
  __lowerCamelCase			:			List[Any]       			=					max(_lowerCAmelCase			)  # max() finds the maximum value

  __lowerCamelCase			:			List[Any]       			=					max_val - min_val + 1  # size is difference of max and min values plus one

  # list of pigeonholes of size equal to the variable size
  __lowerCamelCase			:			List[Any]       			=					[0] * size

  # Populate the pigeonholes.
  for x in a:
    assert isinstance(_lowerCAmelCase			,_lowerCAmelCase			), "integers only please"
    holes[x - min_val] += 1

  # Putting the elements back into the array in an order.
  __lowerCamelCase			:			Tuple       			=					0
  for count in range(_lowerCAmelCase			):
    while holes[count] > 0:
      holes[count] -= 1
      __lowerCamelCase			:			List[Any]       			=					count + min_val
      i += 1

def 			a_	(				) -> str:
  __lowerCamelCase			:			Optional[Any]       			=					[8, 3, 2, 7, 4, 6, 8]
  pigeonhole_sort(_lowerCAmelCase			)
  print('Sorted order is:'			,' '.join(_lowerCAmelCase			)			)


if __name__ == "__main__":
   main()






