




'''simple docstring'''







from __future__ import annotations

import os
import tempfile
import unittest

import numpy as np
from huggingface_hub import hf_hub_download

from transformers import is_tensorflow_text_available, is_tf_available
from transformers.testing_utils import require_tensorflow_text, require_tf, slow

from ..test_modeling_tf_common import floats_tensor
from .test_framework_agnostic import GenerationIntegrationTestsMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import (
          AutoTokenizer,
          TFAutoModelForCausalLM,
          TFAutoModelForSeqaSeqLM,
          TFAutoModelForSpeechSeqaSeq,
          TFAutoModelForVisionaSeq,
          TFBartForConditionalGeneration,
          TFLogitsProcessorList,
          TFMinLengthLogitsProcessor,
          tf_top_k_top_p_filtering,
      )

if is_tensorflow_text_available():
      import tensorflow_text as text







@require_tf
class 					_a (unittest.TestCase	):





     '''simple docstring'''



     def 			snake_case_      (				self							)      ->				List[Any]:
          snake_case			:      Optional[int]        =			tf.convert_to_tensor(
              [
                  [
                      8.2_220_991,  # 3rd highest value; idx. 0
                      -0.5_620_044,
                      5.23_229_752,
                      4.0_386_393,
                      -6.8_798_378,
                      -0.54_785_802,
                      -3.2_012_153,
                      2.92_777_176,
                      1.88_171_953,
                      7.35_341_276,  # 5th highest value; idx. 9
                      8.43_207_833,  # 2nd highest value; idx. 10
                      -9.85_711_836,
                      -5.96_209_236,
                      -1.13_039_161,
                      -7.1_115_294,
                      -0.8_369_633,
                      -5.3_186_408,
                      7.06_427_407,
                      0.81_369_344,
                      -0.82_023_817,
                      -5.9_179_796,
                      0.58_813_443,
                      -6.99_778_438,
                      4.71_551_189,
                      -0.18_771_637,
                      7.44_020_759,  # 4th highest value; idx. 25
                      9.38_450_987,  # 1st highest value; idx. 26
                      2.12_662_941,
                      -9.32_562_038,
                      2.35_652_522,
                  ],  # cummulative prob of 5 highest values <= 0.6
                  [
                      0.58_425_518,
                      4.53_139_238,
                      -5.57_510_464,
                      -6.28_030_699,
                      -7.19_529_503,
                      -4.02_122_551,
                      1.39_337_037,
                      -6.06_707_057,
                      1.59_480_517,
                      -9.643_119,
                      0.03_907_799,
                      0.67_231_762,
                      -8.88_206_726,
                      6.27_115_922,  # 4th highest value; idx. 13
                      2.28_520_723,
                      4.82_767_506,
                      4.30_421_368,
                      8.8_275_313,  # 2nd highest value; idx. 17
                      5.44_029_958,  # 5th highest value; idx. 18
                      -4.4_735_794,
                      7.38_579_536,  # 3rd highest value; idx. 20
                      -2.91_051_663,
                      2.61_946_077,
                      -2.5_674_762,
                      -9.48_959_302,
                      -4.02_922_645,
                      -1.35_416_918,
                      9.67_702_323,  # 1st highest value; idx. 27
                      -5.89_478_553,
                      1.85_370_467,
                  ],  # cummulative prob of 5 highest values <= 0.6
              ]							,dtype=tf.floataa							,)

          snake_case			:      List[str]        =			tf.convert_to_tensor(
              [[0, 0], [0, 9], [0, 10], [0, 25], [0, 26], [1, 13], [1, 17], [1, 18], [1, 20], [1, 27]]							,dtype=tf.intaa							,)  # expected non filtered idx as noted above

          snake_case			:      Optional[int]        =			tf.convert_to_tensor(
              [8.222_099, 7.3_534_126, 8.432_078, 7.4_402_075, 9.38_451, 6.271_159, 8.827_531, 5.4_402_995, 7.3_857_956, 9.677_023]							,dtype=tf.floataa							,)  # expected non filtered values as noted above

          snake_case			:      int        =			tf_top_k_top_p_filtering(__a							,top_k=10							,top_p=0.6							,min_tokens_to_keep=4							)

          snake_case			:      int        =			output[output != -float("""inf"""							)]
          snake_case			:      int        =			tf.cast(
              tf.where(tf.not_equal(__a							,tf.constant(-float("""inf"""							)							,dtype=tf.floataa							)							)							)							,dtype=tf.intaa							,)

          tf.debugging.assert_near(__a							,__a							,rtol=1E-12							)
          tf.debugging.assert_equal(__a							,__a							)







@require_tf
class 					_a (unittest.TestCase,		a__	):





     '''simple docstring'''



     if is_tf_available():
          lowerCAmelCase_  :     int          =					{
              """AutoModelForCausalLM""": TFAutoModelForCausalLM,
              """AutoModelForSpeechSeq2Seq""": TFAutoModelForSpeechSeqaSeq,
              """AutoModelForSeq2SeqLM""": TFAutoModelForSeqaSeqLM,
              """AutoModelForVision2Seq""": TFAutoModelForVisionaSeq,
              """LogitsProcessorList""": TFLogitsProcessorList,
              """MinLengthLogitsProcessor""": TFMinLengthLogitsProcessor,
              """create_tensor_fn""": tf.convert_to_tensor,
              """floats_tensor""": floats_tensor,
              """return_tensors""": """tf""",
          }







     @slow
     def 			snake_case_      (				self							)      ->				Tuple:
          # TF-only test: tf.saved_model export
          snake_case			:      Dict        =			TFAutoModelForCausalLM.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""							)
          snake_case			:      Any        =			2
          snake_case			:      int        =			2







          class 					_a (tf.Module	):





               '''simple docstring'''



               def __init__(				self							,__a							)      ->				Any:
                    super(__a							,self							).__init__()
                    snake_case			:      List[Any]        =			model







               @tf.function(
                   input_signature=(
                       tf.TensorSpec((None, input_length)							,tf.intaa							,name="""input_ids"""							),
                       tf.TensorSpec((None, input_length)							,tf.intaa							,name="""attention_mask"""							),
                   )							,jit_compile=__a							,)
               def 			snake_case_      (				self							,__a							,__a							)      ->				Optional[Any]:
                    snake_case			:      Union[str, Any]        =			self.model.generate(
                        input_ids=__a							,attention_mask=__a							,max_new_tokens=__a							,return_dict_in_generate=__a							,)
                    return {"sequences": outputs["sequences"]}
          snake_case			:      List[str]        =			[[2, 0], [102, 103]]
          snake_case			:      List[str]        =			[[1, 0], [1, 1]]
          snake_case			:      Union[str, Any]        =			DummyModel(model=__a							)
          with tempfile.TemporaryDirectory() as tmp_dir:
               tf.saved_model.save(__a							,__a							,signatures={"""serving_default""": dummy_model.serving}							)
               snake_case			:      Any        =			tf.saved_model.load(__a							).signatures["""serving_default"""]
               for batch_size in range(1							,len(__a							) + 1							):
                    snake_case			:      Dict        =			{
                        """input_ids""": tf.constant(dummy_input_ids[:batch_size]							),
                        """attention_mask""": tf.constant(dummy_attention_masks[:batch_size]							),
                    }
                    snake_case			:      Union[str, Any]        =			serving_func(**__a							)["""sequences"""]
                    snake_case			:      Tuple        =			test_model.generate(**__a							,max_new_tokens=__a							)
                    tf.debugging.assert_equal(__a							,__a							)







     @slow
     def 			snake_case_      (				self							)      ->				Optional[Any]:
          # TF-only test: tf.saved_model export
          snake_case			:      Optional[int]        =			TFAutoModelForCausalLM.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""							)
          snake_case			:      int        =			1
          snake_case			:      Union[str, Any]        =			2







          class 					_a (tf.Module	):





               '''simple docstring'''



               def __init__(				self							,__a							)      ->				Union[str, Any]:
                    super(__a							,self							).__init__()
                    snake_case			:      Optional[Any]        =			model







               @tf.function(
                   input_signature=(
                       tf.TensorSpec((batch_size, None)							,tf.intaa							,name="""input_ids"""							),
                       tf.TensorSpec((batch_size, None)							,tf.intaa							,name="""attention_mask"""							),
                   )							,jit_compile=__a							,)
               def 			snake_case_      (				self							,__a							,__a							)      ->				str:
                    snake_case			:      Optional[Any]        =			self.model.generate(
                        input_ids=__a							,attention_mask=__a							,max_new_tokens=__a							,return_dict_in_generate=__a							,)
                    return {"sequences": outputs["sequences"]}
          snake_case			:      Tuple        =			[[2], [102, 103]]
          snake_case			:      int        =			[[1], [1, 1]]
          snake_case			:      int        =			DummyModel(model=__a							)
          with tempfile.TemporaryDirectory() as tmp_dir:
               tf.saved_model.save(__a							,__a							,signatures={"""serving_default""": dummy_model.serving}							)
               snake_case			:      List[str]        =			tf.saved_model.load(__a							).signatures["""serving_default"""]
               for input_row in range(len(__a							)							):
                    snake_case			:      Union[str, Any]        =			{
                        """input_ids""": tf.constant([dummy_input_ids[input_row]]							),
                        """attention_mask""": tf.constant([dummy_attention_masks[input_row]]							),
                    }
                    snake_case			:      List[str]        =			serving_func(**__a							)["""sequences"""]
                    snake_case			:      Optional[int]        =			test_model.generate(**__a							,max_new_tokens=__a							)
                    tf.debugging.assert_equal(__a							,__a							)







     @slow
     @require_tensorflow_text
     def 			snake_case_      (				self							)      ->				Union[str, Any]:
          # TF-only test: tf.saved_model export
          with tempfile.TemporaryDirectory() as tmp_dir:
               # file needed to load the TF tokenizer
               hf_hub_download(repo_id="""google/flan-t5-small"""							,filename="""spiece.model"""							,local_dir=__a							)







               class 					_a (tf.keras.layers.Layer	):





                    '''simple docstring'''



                    def __init__(				self							)      ->				int:
                         super().__init__()
                         snake_case			:      Dict        =			text.SentencepieceTokenizer(
                             model=tf.io.gfile.GFile(os.path.join(__a							,"""spiece.model"""							)							,"""rb"""							).read()							)
                         snake_case			:      List[str]        =			TFAutoModelForSeqaSeqLM.from_pretrained("""hf-internal-testing/tiny-random-t5"""							)







                    def 			snake_case_      (				self							,__a							,*__a							,**__a							)      ->				Union[str, Any]:
                         snake_case			:      List[str]        =			self.tokenizer.tokenize(__a							)
                         snake_case ,		snake_case			:      str        =			text.pad_model_inputs(
                             __a							,max_seq_length=64							,pad_value=self.model.config.pad_token_id							)
                         snake_case			:      Tuple        =			self.model.generate(input_ids=__a							,attention_mask=__a							)
                         return self.tokenizer.detokenize(__a							)
               snake_case			:      Optional[Any]        =			CompleteSentenceTransformer()
               snake_case			:      List[str]        =			tf.keras.layers.Input(shape=(1,)							,dtype=tf.string							,name="""inputs"""							)
               snake_case			:      Any        =			complete_model(__a							)
               snake_case			:      Optional[int]        =			tf.keras.Model(__a							,__a							)
               keras_model.save(__a							)







     def 			snake_case_      (				self							)      ->				Optional[Any]:
          # Has PT equivalent: this test relies on random sampling
          snake_case			:      int        =			{
              """do_sample""": True,
              """num_beams""": 1,
              """top_p""": 0.7,
              """top_k""": 10,
              """temperature""": 0.7,
          }
          snake_case			:      Tuple        =			14

          snake_case			:      Union[str, Any]        =			AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""							)
          snake_case			:      Union[str, Any]        =			"""Hello, my dog is cute and"""
          snake_case			:      Optional[int]        =			tokenizer(__a							,return_tensors="""tf"""							)
          snake_case			:      Tuple        =			TFAutoModelForCausalLM.from_pretrained("""hf-internal-testing/tiny-random-gpt2"""							)

          snake_case			:      int        =			638
          # forces the generation to happen on CPU, to avoid GPU-related quirks
          with tf.device(""":/CPU:0"""							):
               tf.random.set_seed(0							)
               snake_case			:      int        =			model.generate(**__a							,eos_token_id=__a							,**__a							)
          self.assertTrue(expectation == len(generated_tokens[0]							)							)

          snake_case			:      Optional[int]        =			[638, 198]
          with tf.device(""":/CPU:0"""							):
               tf.random.set_seed(0							)
               snake_case			:      Optional[int]        =			model.generate(**__a							,eos_token_id=__a							,**__a							)
          self.assertTrue(expectation == len(generated_tokens[0]							)							)







     def 			snake_case_      (				self							)      ->				Any:
          # Has PT equivalent: ample use of framework-specific code
          snake_case			:      List[str]        =			AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bart"""							)
          snake_case			:      List[str]        =			"""Hugging Face is a technology company based in New York and Paris."""
          snake_case			:      Dict        =			bart_tokenizer(__a							,return_tensors="""tf"""							).input_ids
          snake_case			:      List[str]        =			TFBartForConditionalGeneration.from_pretrained("""hf-internal-testing/tiny-random-bart"""							)
          snake_case			:      Dict        =			bart_model.generate(__a							).numpy()







          class 					_a (a__	):





               '''simple docstring'''



               def 			snake_case_      (				self							,__a							,__a=None							,**__a							)      ->				List[str]:
                    return super().call(__a							,**__a							)
          snake_case			:      Union[str, Any]        =			FakeBart.from_pretrained("""hf-internal-testing/tiny-random-bart"""							)
          snake_case			:      Optional[Any]        =			bart_model.generate(__a							,foo="""bar"""							).numpy()
          self.assertTrue(np.array_equal(__a							,__a							)							)







          class 					_a (bart_model.model.encoder.__class__	):





               '''simple docstring'''



               def 			snake_case_      (				self							,__a							,**__a							)      ->				Tuple:
                    return super().call(__a							,**__a							)
          snake_case			:      Any        =			FakeEncoder(bart_model.config							,bart_model.model.shared							)
          snake_case			:      Optional[int]        =			fake_encoder

          # Normal generation still works (the output will be different because the encoder weights are different)
          snake_case			:      Any        =			bart_model.generate(__a							).numpy()
          with self.assertRaises(__a							):
               # FakeEncoder.call() accepts **kwargs -> no filtering -> value error due to unexpected input "foo"
               bart_model.generate(__a							,foo="""bar"""							)









'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


lowercase      :   Tuple      =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase      :   Dict      =       {
    """vocab_file""": {
        """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """unc-nlp/lxmert-base-uncased""": (
            """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase      :   List[Any]      =       {
    """unc-nlp/lxmert-base-uncased""": 512,
}

lowercase      :   Tuple      =       {
    """unc-nlp/lxmert-base-uncased""": {"""do_lower_case""": True},
}







class 					_a (a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Optional[int]          =					VOCAB_FILES_NAMES
     lowerCAmelCase_  :     Optional[int]          =					PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase_  :     Any          =					PRETRAINED_INIT_CONFIGURATION
     lowerCAmelCase_  :     Optional[int]          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase_  :     Optional[int]          =					LxmertTokenizer







     def __init__(				self							,__a=None							,__a=None							,__a=True							,__a="[UNK]"							,__a="[SEP]"							,__a="[PAD]"							,__a="[CLS]"							,__a="[MASK]"							,__a=True							,__a=None							,**__a							,)      ->				str:
          super().__init__(
              __a							,tokenizer_file=__a							,do_lower_case=__a							,unk_token=__a							,sep_token=__a							,pad_token=__a							,cls_token=__a							,mask_token=__a							,tokenize_chinese_chars=__a							,strip_accents=__a							,**__a							,)

          snake_case			:      List[Any]        =			json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
          if (
              normalizer_state.get("""lowercase"""							,__a							) != do_lower_case
              or normalizer_state.get("""strip_accents"""							,__a							) != strip_accents
              or normalizer_state.get("""handle_chinese_chars"""							,__a							) != tokenize_chinese_chars
          ):
               snake_case			:      Union[str, Any]        =			getattr(__a							,normalizer_state.pop("""type"""							)							)
               snake_case			:      Optional[int]        =			do_lower_case
               snake_case			:      Optional[int]        =			strip_accents
               snake_case			:      List[Any]        =			tokenize_chinese_chars
               snake_case			:      Union[str, Any]        =			normalizer_class(**__a							)

          snake_case			:      str        =			do_lower_case







     def 			snake_case_      (				self							,__a							,__a=None							)      ->				Optional[int]:
          snake_case			:      Tuple        =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				List[int]:
          snake_case			:      int        =			[self.sep_token_id]
          snake_case			:      Any        =			[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep							) * [0]
          return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				Tuple[str]:
          snake_case			:      Optional[int]        =			self._tokenizer.model.save(__a							,name=__a							)
          return tuple(__a							)



