





"""simple docstring"""





from __future__ import annotations

import os
import tempfile
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import is_tensorflow_text_available, is_tf_available
from transformers.testing_utils import require_tensorflow_text, require_tf, slow

from ..test_modeling_tf_common import floats_tensor
from .test_framework_agnostic import GenerationIntegrationTestsMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import (
      AutoTokenizer,
      TFAutoModelForCausalLM,
      TFAutoModelForSeqaSeqLM,
      TFAutoModelForSpeechSeqaSeq,
      TFAutoModelForVisionaSeq,
      TFBartForConditionalGeneration,
      TFLogitsProcessorList,
      TFMinLengthLogitsProcessor,
      tf_top_k_top_p_filtering,
  )

if is_tensorflow_text_available():
  import tensorflow_text as text




@require_tf
class 							snake_case							(			unittest.TestCase ):

      '''simple docstring'''



      def       A_					(							self       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						tf.convert_to_tensor(
            [
                [
                    8.2_2_2_0_9_9_1,  # 3rd highest value; idx. 0
                    -0.5_6_2_0_0_4_4,
                    5.2_3_2_2_9_7_5_2,
                    4.0_3_8_6_3_9_3,
                    -6.8_7_9_8_3_7_8,
                    -0.5_4_7_8_5_8_0_2,
                    -3.2_0_1_2_1_5_3,
                    2.9_2_7_7_7_1_7_6,
                    1.8_8_1_7_1_9_5_3,
                    7.3_5_3_4_1_2_7_6,  # 5th highest value; idx. 9
                    8.4_3_2_0_7_8_3_3,  # 2nd highest value; idx. 10
                    -9.8_5_7_1_1_8_3_6,
                    -5.9_6_2_0_9_2_3_6,
                    -1.1_3_0_3_9_1_6_1,
                    -7.1_1_1_5_2_9_4,
                    -0.8_3_6_9_6_3_3,
                    -5.3_1_8_6_4_0_8,
                    7.0_6_4_2_7_4_0_7,
                    0.8_1_3_6_9_3_4_4,
                    -0.8_2_0_2_3_8_1_7,
                    -5.9_1_7_9_7_9_6,
                    0.5_8_8_1_3_4_4_3,
                    -6.9_9_7_7_8_4_3_8,
                    4.7_1_5_5_1_1_8_9,
                    -0.1_8_7_7_1_6_3_7,
                    7.4_4_0_2_0_7_5_9,  # 4th highest value; idx. 25
                    9.3_8_4_5_0_9_8_7,  # 1st highest value; idx. 26
                    2.1_2_6_6_2_9_4_1,
                    -9.3_2_5_6_2_0_3_8,
                    2.3_5_6_5_2_5_2_2,
                ],  # cummulative prob of 5 highest values <= 0.6
                [
                    0.5_8_4_2_5_5_1_8,
                    4.5_3_1_3_9_2_3_8,
                    -5.5_7_5_1_0_4_6_4,
                    -6.2_8_0_3_0_6_9_9,
                    -7.1_9_5_2_9_5_0_3,
                    -4.0_2_1_2_2_5_5_1,
                    1.3_9_3_3_7_0_3_7,
                    -6.0_6_7_0_7_0_5_7,
                    1.5_9_4_8_0_5_1_7,
                    -9.6_4_3_1_1_9,
                    0.0_3_9_0_7_7_9_9,
                    0.6_7_2_3_1_7_6_2,
                    -8.8_8_2_0_6_7_2_6,
                    6.2_7_1_1_5_9_2_2,  # 4th highest value; idx. 13
                    2.2_8_5_2_0_7_2_3,
                    4.8_2_7_6_7_5_0_6,
                    4.3_0_4_2_1_3_6_8,
                    8.8_2_7_5_3_1_3,  # 2nd highest value; idx. 17
                    5.4_4_0_2_9_9_5_8,  # 5th highest value; idx. 18
                    -4.4_7_3_5_7_9_4,
                    7.3_8_5_7_9_5_3_6,  # 3rd highest value; idx. 20
                    -2.9_1_0_5_1_6_6_3,
                    2.6_1_9_4_6_0_7_7,
                    -2.5_6_7_4_7_6_2,
                    -9.4_8_9_5_9_3_0_2,
                    -4.0_2_9_2_2_6_4_5,
                    -1.3_5_4_1_6_9_1_8,
                    9.6_7_7_0_2_3_2_3,  # 1st highest value; idx. 27
                    -5.8_9_4_7_8_5_5_3,
                    1.8_5_3_7_0_4_6_7,
                ],  # cummulative prob of 5 highest values <= 0.6
            ]						,				dtype=tf.floataa						,				)

        __UpperCAmelCase    :						str       =						tf.convert_to_tensor(
            [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]]						,				dtype=tf.intaa						,				)  # expected non filtered idx as noted above

        __UpperCAmelCase    :						Optional[int]       =						tf.convert_to_tensor(
            [8.2_2_2_0_9_9, 7.3_5_3_4_1_2_6, 8.4_3_2_0_7_8, 7.4_4_0_2_0_7_5, 9.3_8_4_5_1, 6.2_7_1_1_5_9, 8.8_2_7_5_3_1, 5.4_4_0_2_9_9_5, 7.3_8_5_7_9_5_6, 9.6_7_7_0_2_3]						,				dtype=tf.floataa						,				)  # expected non filtered values as noted above

        __UpperCAmelCase    :						str       =						tf_top_k_top_p_filtering(a_						,				top_k=10						,				top_p=0.6						,				min_tokens_to_keep=4 )

        __UpperCAmelCase    :						Union[str, Any]       =						output[output != -float('''inf''' )]
        __UpperCAmelCase    :						List[Any]       =						tf.cast(
            tf.where(tf.not_equal(a_						,				tf.constant(-float('''inf''' )						,				dtype=tf.floataa ) ) )						,				dtype=tf.intaa						,				)

        tf.debugging.assert_near(a_						,				a_						,				rtol=1e-12 )
        tf.debugging.assert_equal(a_						,				a_ )




@require_tf
class 							snake_case							(			unittest.TestCase      ,  __lowerCamelCase ):

      '''simple docstring'''



      if is_tf_available():
            _A     :       Tuple         =       {
                '''AutoModelForCausalLM''': TFAutoModelForCausalLM,
                '''AutoModelForSpeechSeq2Seq''': TFAutoModelForSpeechSeqaSeq,
                '''AutoModelForSeq2SeqLM''': TFAutoModelForSeqaSeqLM,
                '''AutoModelForVision2Seq''': TFAutoModelForVisionaSeq,
                '''LogitsProcessorList''': TFLogitsProcessorList,
                '''MinLengthLogitsProcessor''': TFMinLengthLogitsProcessor,
                '''create_tensor_fn''': tf.convert_to_tensor,
                '''floats_tensor''': floats_tensor,
                '''return_tensors''': '''tf''',
            }

      @slow
      def       A_					(							self       :      int ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						TFAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2''' )
        __UpperCAmelCase    :						Dict       =						2
        __UpperCAmelCase    :						int       =						2




        class 							snake_case							(			tf.Module ):

              '''simple docstring'''



              def __init__(							self       :      Union[str, Any]						,				__lowercase       :      Union[str, Any] ):


                '''simple docstring'''




                super(a_						,				self ).__init__()
                __UpperCAmelCase    :						Tuple       =						model







              @tf.function(
                  input_signature=(
                      tf.TensorSpec((None, input_length)						,				tf.intaa						,				name='''input_ids''' ),
                      tf.TensorSpec((None, input_length)						,				tf.intaa						,				name='''attention_mask''' ),
                  )						,				jit_compile=a_						,				)
              def       A_					(							self       :      Dict						,				__lowercase       :      str						,				__lowercase       :      Dict ):


                '''simple docstring'''




                __UpperCAmelCase    :						int       =						self.model.generate(
                    input_ids=a_						,				attention_mask=a_						,				max_new_tokens=a_						,				return_dict_in_generate=a_						,				)
                return {"sequences": outputs["sequences"]}


        __UpperCAmelCase    :						int       =						[[2, 0], [102, 103]]
        __UpperCAmelCase    :						Optional[Any]       =						[[1, 0], [1, 1]]
        __UpperCAmelCase    :						Dict       =						DummyModel(model=a_ )
        with tempfile.TemporaryDirectory() as tmp_dir:
          tf.saved_model.save(a_						,				a_						,				signatures={'''serving_default''': dummy_model.serving} )
          __UpperCAmelCase    :						List[str]       =						tf.saved_model.load(a_ ).signatures["serving_default"]
          for batch_size in range(1						,				len(a_ ) + 1 ):
            __UpperCAmelCase    :						Tuple       =						{
                "input_ids": tf.constant(dummy_input_ids[:batch_size] ),
                "attention_mask": tf.constant(dummy_attention_masks[:batch_size] ),
            }
            __UpperCAmelCase    :						Dict       =						serving_func(**a_ )["sequences"]
            __UpperCAmelCase    :						str       =						test_model.generate(**a_						,				max_new_tokens=a_ )
            tf.debugging.assert_equal(a_						,				a_ )

      @slow
      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						TFAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2''' )
        __UpperCAmelCase    :						int       =						1
        __UpperCAmelCase    :						Tuple       =						2




        class 							snake_case							(			tf.Module ):

              '''simple docstring'''



              def __init__(							self       :      str						,				__lowercase       :      Optional[int] ):


                '''simple docstring'''




                super(a_						,				self ).__init__()
                __UpperCAmelCase    :						Tuple       =						model







              @tf.function(
                  input_signature=(
                      tf.TensorSpec((batch_size, None)						,				tf.intaa						,				name='''input_ids''' ),
                      tf.TensorSpec((batch_size, None)						,				tf.intaa						,				name='''attention_mask''' ),
                  )						,				jit_compile=a_						,				)
              def       A_					(							self       :      List[str]						,				__lowercase       :      Any						,				__lowercase       :      Union[str, Any] ):


                '''simple docstring'''




                __UpperCAmelCase    :						Dict       =						self.model.generate(
                    input_ids=a_						,				attention_mask=a_						,				max_new_tokens=a_						,				return_dict_in_generate=a_						,				)
                return {"sequences": outputs["sequences"]}


        __UpperCAmelCase    :						Optional[int]       =						[[2], [102, 103]]
        __UpperCAmelCase    :						Union[str, Any]       =						[[1], [1, 1]]
        __UpperCAmelCase    :						List[Any]       =						DummyModel(model=a_ )
        with tempfile.TemporaryDirectory() as tmp_dir:
          tf.saved_model.save(a_						,				a_						,				signatures={'''serving_default''': dummy_model.serving} )
          __UpperCAmelCase    :						Optional[Any]       =						tf.saved_model.load(a_ ).signatures["serving_default"]
          for input_row in range(len(a_ ) ):
            __UpperCAmelCase    :						List[str]       =						{
                "input_ids": tf.constant([dummy_input_ids[input_row]] ),
                "attention_mask": tf.constant([dummy_attention_masks[input_row]] ),
            }
            __UpperCAmelCase    :						Optional[Any]       =						serving_func(**a_ )["sequences"]
            __UpperCAmelCase    :						Optional[Any]       =						test_model.generate(**a_						,				max_new_tokens=a_ )
            tf.debugging.assert_equal(a_						,				a_ )

      @slow
      @require_tensorflow_text
      def       A_					(							self       :      Any ):


        '''simple docstring'''




        with tempfile.TemporaryDirectory() as tmp_dir:
          # file needed to load the TF tokenizer
          hf_hub_download(repo_id='''google/flan-t5-small'''						,				filename='''spiece.model'''						,				local_dir=a_ )




          class 							snake_case							(			tf.keras.layers.Layer ):

                '''simple docstring'''



                def __init__(							self       :      Optional[Any] ):


                  '''simple docstring'''




                  super().__init__()
                  __UpperCAmelCase    :						int       =						text.SentencepieceTokenizer(
                      model=tf.io.gfile.GFile(os.path.join(a_						,				'''spiece.model''' )						,				'''rb''' ).read() )
                  __UpperCAmelCase    :						Tuple       =						TFAutoModelForSeqaSeqLM.from_pretrained('''hf-internal-testing/tiny-random-t5''' )







                def       A_					(							self       :      Tuple						,				__lowercase       :      Any						,				*__lowercase       :      Optional[int]						,				**__lowercase       :      Tuple ):


                  '''simple docstring'''




                  __UpperCAmelCase    :						int       =						self.tokenizer.tokenize(a_ )
                  __UpperCAmelCase    :						List[Any]       =						text.pad_model_inputs(
                      a_						,				max_seq_length=64						,				pad_value=self.model.config.pad_token_id )
                  __UpperCAmelCase    :						Tuple       =						self.model.generate(input_ids=a_						,				attention_mask=a_ )
                  return self.tokenizer.detokenize(a_ )


          __UpperCAmelCase    :						Dict       =						CompleteSentenceTransformer()
          __UpperCAmelCase    :						List[Any]       =						tf.keras.layers.Input(shape=(1,)						,				dtype=tf.string						,				name='''inputs''' )
          __UpperCAmelCase    :						str       =						complete_model(a_ )
          __UpperCAmelCase    :						Any       =						tf.keras.Model(a_						,				a_ )
          keras_model.save(a_ )

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						{
            "do_sample": True,
            "num_beams": 1,
            "top_p": 0.7,
            "top_k": 10,
            "temperature": 0.7,
        }
        __UpperCAmelCase    :						Tuple       =						14

        __UpperCAmelCase    :						int       =						AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2''' )
        __UpperCAmelCase    :						Dict       =						"Hello, my dog is cute and"
        __UpperCAmelCase    :						Dict       =						tokenizer(a_						,				return_tensors='''tf''' )
        __UpperCAmelCase    :						Tuple       =						TFAutoModelForCausalLM.from_pretrained('''hf-internal-testing/tiny-random-gpt2''' )

        __UpperCAmelCase    :						int       =						638
        # forces the generation to happen on CPU, to avoid GPU-related quirks
        with tf.device(''':/CPU:0''' ):
          tf.random.set_seed(0 )
          __UpperCAmelCase    :						List[Any]       =						model.generate(**a_						,				eos_token_id=a_						,				**a_ )
        self.assertTrue(expectation == len(generated_tokens[0] ) )

        __UpperCAmelCase    :						Any       =						[638, 198]
        with tf.device(''':/CPU:0''' ):
          tf.random.set_seed(0 )
          __UpperCAmelCase    :						Optional[Any]       =						model.generate(**a_						,				eos_token_id=a_						,				**a_ )
        self.assertTrue(expectation == len(generated_tokens[0] ) )







      def       A_					(							self       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bart''' )
        __UpperCAmelCase    :						Tuple       =						"Hugging Face is a technology company based in New York and Paris."
        __UpperCAmelCase    :						Optional[Any]       =						bart_tokenizer(a_						,				return_tensors='''tf''' ).input_ids
        __UpperCAmelCase    :						str       =						TFBartForConditionalGeneration.from_pretrained('''hf-internal-testing/tiny-random-bart''' )
        __UpperCAmelCase    :						Any       =						bart_model.generate(a_ ).numpy()




        class 							snake_case							(			__lowerCamelCase ):

              '''simple docstring'''



              def       A_					(							self       :      int						,				__lowercase       :      Union[str, Any]						,				__lowercase       :      str=None						,				**__lowercase       :      Union[str, Any] ):


                '''simple docstring'''




                return super().call(a_						,				**a_ )


        __UpperCAmelCase    :						Union[str, Any]       =						FakeBart.from_pretrained('''hf-internal-testing/tiny-random-bart''' )
        __UpperCAmelCase    :						int       =						bart_model.generate(a_						,				foo='''bar''' ).numpy()
        self.assertTrue(np.array_equal(a_						,				a_ ) )




        class 							snake_case							(			bart_model.model.encoder.__class__ ):

              '''simple docstring'''



              def       A_					(							self       :      str						,				__lowercase       :      Tuple						,				**__lowercase       :      List[Any] ):


                '''simple docstring'''




                return super().call(a_						,				**a_ )


        __UpperCAmelCase    :						List[Any]       =						FakeEncoder(bart_model.config						,				bart_model.model.shared )
        __UpperCAmelCase    :						int       =						fake_encoder

        # Normal generation still works (the output will be different because the encoder weights are different)
        __UpperCAmelCase    :						Optional[Any]       =						bart_model.generate(a_ ).numpy()
        with self.assertRaises(a_ ):
          # FakeEncoder.call() accepts **kwargs -> no filtering -> value error due to unexpected input "foo"
          bart_model.generate(a_						,				foo='''bar''' )






"""simple docstring"""





def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  )						->Tuple:





  """simple docstring"""


  return (pointa[0] - pointa[0]) ** 2 + (pointa[1] - pointa[1]) ** 2


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_=0  )						->Dict:





  """simple docstring"""


  return sorted(UpperCAmelCase_  ,			key=lambda UpperCAmelCase_  : x[column]  )


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_=float('''inf'''  )  )						->str:





  """simple docstring"""


  for i in range(points_counts - 1  ):
    for j in range(i + 1  ,			UpperCAmelCase_  ):
      __UpperCAmelCase    :						List[Any]       =						euclidean_distance_sqr(points[i]  ,			points[j]  )
      if current_dis < min_dis:
        __UpperCAmelCase    :						Tuple       =						current_dis
  return min_dis


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_=float('''inf'''  )  )						->str:





  """simple docstring"""


  for i in range(min(6  ,			points_counts - 1  )  ,			UpperCAmelCase_  ):
    for j in range(max(0  ,			i - 6  )  ,			UpperCAmelCase_  ):
      __UpperCAmelCase    :						Dict       =						euclidean_distance_sqr(points[i]  ,			points[j]  )
      if current_dis < min_dis:
        __UpperCAmelCase    :						Tuple       =						current_dis
  return min_dis


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  )						->Any:





  """simple docstring"""


  if points_counts <= 3:
    return dis_between_closest_pair(UpperCAmelCase_  ,			UpperCAmelCase_  )

  # recursion
  __UpperCAmelCase    :						Any       =						points_counts // 2
  __UpperCAmelCase    :						Any       =						closest_pair_of_points_sqr(
      UpperCAmelCase_  ,			points_sorted_on_y[:mid]  ,			UpperCAmelCase_  )
  __UpperCAmelCase    :						Tuple       =						closest_pair_of_points_sqr(
      UpperCAmelCase_  ,			points_sorted_on_y[mid:]  ,			points_counts - mid  )
  __UpperCAmelCase    :						List[Any]       =						min(UpperCAmelCase_  ,			UpperCAmelCase_  )
  __UpperCAmelCase    :						int       =						[]
  for point in points_sorted_on_x:
    if abs(point[0] - points_sorted_on_x[mid][0]  ) < closest_pair_dis:
      cross_strip.append(UpperCAmelCase_  )

  __UpperCAmelCase    :						Union[str, Any]       =						dis_between_closest_in_strip(
      UpperCAmelCase_  ,			len(UpperCAmelCase_  )  ,			UpperCAmelCase_  )
  return min(UpperCAmelCase_  ,			UpperCAmelCase_  )


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  )						->List[Any]:





  """simple docstring"""


  __UpperCAmelCase    :						str       =						column_based_sort(UpperCAmelCase_  ,			column=0  )
  __UpperCAmelCase    :						Any       =						column_based_sort(UpperCAmelCase_  ,			column=1  )
  return (
      closest_pair_of_points_sqr(
          UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  )
  ) ** 0.5


if __name__ == "__main__":
  lowercase__				:Optional[Any]				   =       [(2, 3), (1_2, 3_0), (4_0, 5_0), (5, 1), (1_2, 1_0), (3, 4)]
  print('Distance:', closest_pair_of_points(points, len(points)))