from __future__ import annotations

import random
import unittest

from transformers import TransfoXLConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import (
				    TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFTransfoXLForSequenceClassification,
				    TFTransfoXLLMHeadModel,
				    TFTransfoXLModel,
				)





class     lowerCAmelCase__     :


		'''simple docstring'''






		def __init__(       self     :     List[Any]     ,  snake_case__     :     Any     ,  )      ->	List[Any]:
				_lowerCamelCase				   =  parent
				_lowerCamelCase				   =  1_3
				_lowerCamelCase				   =  7
				_lowerCamelCase				   =  3_0
				_lowerCamelCase				   =  self.seq_length + self.mem_len
				_lowerCamelCase				   =  1_5
				_lowerCamelCase				   =  True
				_lowerCamelCase				   =  True
				_lowerCamelCase				   =  9_9
				_lowerCamelCase				   =  [1_0, 5_0, 8_0]
				_lowerCamelCase				   =  3_2
				_lowerCamelCase				   =  3_2
				_lowerCamelCase				   =  4
				_lowerCamelCase				   =  8
				_lowerCamelCase				   =  1_2_8
				_lowerCamelCase				   =  2
				_lowerCamelCase				   =  2
				_lowerCamelCase				   =  None
				_lowerCamelCase				   =  1
				_lowerCamelCase				   =  0
				_lowerCamelCase				   =  3
				_lowerCamelCase				   =  self.vocab_size - 1
				_lowerCamelCase				   =  0.01
		def   _snake_case       (       self     :     int		)      ->	Dict:
				_lowerCamelCase				   =  ids_tensor([self.batch_size, self.seq_length]     ,  self.vocab_size		)
				_lowerCamelCase				   =  ids_tensor([self.batch_size, self.seq_length]     ,  self.vocab_size		)

				_lowerCamelCase				   =  None
				if self.use_labels:
						_lowerCamelCase				   =  ids_tensor([self.batch_size, self.seq_length]     ,  self.vocab_size		)

				_lowerCamelCase				   =  TransfoXLConfig(
				    vocab_size=self.vocab_size     ,  mem_len=self.mem_len     ,  clamp_len=self.clamp_len     ,  cutoffs=self.cutoffs     ,  d_model=self.hidden_size     ,  d_embed=self.d_embed     ,  n_head=self.num_attention_heads     ,  d_head=self.d_head     ,  d_inner=self.d_inner     ,  div_val=self.div_val     ,  n_layer=self.num_hidden_layers     ,  eos_token_id=self.eos_token_id     ,  pad_token_id=self.vocab_size - 1     ,  init_range=self.init_range     ,  num_labels=self.num_labels     ,  )

				return (config, input_ids_a, input_ids_a, lm_labels)
		def   _snake_case       (       self     :     Any		)      ->	int:
				random.seed(self.seed		)
				tf.random.set_seed(self.seed		)
		def   _snake_case       (       self     :     str     ,  snake_case__     :     Union[str, Any]     ,  snake_case__     :     Any     ,  snake_case__     :     Dict     ,  snake_case__     :     Optional[int]		)      ->	Optional[Any]:
				_lowerCamelCase				   =  TFTransfoXLModel(snake_case__		)

				_lowerCamelCase   ,							_lowerCamelCase				   =  model(snake_case__		).to_tuple()

				_lowerCamelCase				   =  {'input_ids': input_ids_a, 'mems': mems_a}

				_lowerCamelCase   ,							_lowerCamelCase				   =  model(snake_case__		).to_tuple()

				self.parent.assertEqual(hidden_states_a.shape     ,  (self.batch_size, self.seq_length, self.hidden_size)		)
				self.parent.assertEqual(hidden_states_a.shape     ,  (self.batch_size, self.seq_length, self.hidden_size)		)
				self.parent.assertListEqual(
				    [mem.shape for mem in mems_a]     ,  [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers     ,  )
				self.parent.assertListEqual(
				    [mem.shape for mem in mems_a]     ,  [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers     ,  )
		def   _snake_case       (       self     :     List[str]     ,  snake_case__     :     Tuple     ,  snake_case__     :     Optional[Any]     ,  snake_case__     :     str     ,  snake_case__     :     Optional[int]		)      ->	Tuple:
				_lowerCamelCase				   =  TFTransfoXLLMHeadModel(snake_case__		)

				_lowerCamelCase   ,							_lowerCamelCase				   =  model(snake_case__		).to_tuple()

				_lowerCamelCase				   =  {'input_ids': input_ids_a, 'labels': lm_labels}
				_lowerCamelCase   ,							_lowerCamelCase				   =  model(snake_case__		).to_tuple()

				_lowerCamelCase   ,							_lowerCamelCase				   =  model([input_ids_a, mems_a]		).to_tuple()

				_lowerCamelCase				   =  {'input_ids': input_ids_a, 'mems': mems_a, 'labels': lm_labels}

				_lowerCamelCase   ,							_lowerCamelCase				   =  model(snake_case__		).to_tuple()

				self.parent.assertEqual(lm_logits_a.shape     ,  (self.batch_size, self.seq_length, self.vocab_size)		)
				self.parent.assertListEqual(
				    [mem.shape for mem in mems_a]     ,  [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers     ,  )

				self.parent.assertEqual(lm_logits_a.shape     ,  (self.batch_size, self.seq_length, self.vocab_size)		)
				self.parent.assertListEqual(
				    [mem.shape for mem in mems_a]     ,  [(self.mem_len, self.batch_size, self.hidden_size)] * self.num_hidden_layers     ,  )
		def   _snake_case       (       self     :     int     ,  snake_case__     :     Union[str, Any]     ,  snake_case__     :     str     ,  snake_case__     :     Any     ,  snake_case__     :     Union[str, Any]		)      ->	Optional[Any]:
				_lowerCamelCase				   =  TFTransfoXLForSequenceClassification(snake_case__		)
				_lowerCamelCase				   =  model(snake_case__		)
				self.parent.assertEqual(result.logits.shape     ,  (self.batch_size, self.num_labels)		)



		def   _snake_case       (       self     :     Tuple		)      ->	str:
				_lowerCamelCase				   =  self.prepare_config_and_inputs()
				((_lowerCamelCase)   ,							(_lowerCamelCase)   ,							(_lowerCamelCase)   ,							(_lowerCamelCase))				   =  config_and_inputs
				_lowerCamelCase				   =  {'input_ids': input_ids_a}
				return config, inputs_dict





@require_tf
class     lowerCAmelCase__     (				SCREAMING_SNAKE_CASE_     ,SCREAMING_SNAKE_CASE_     ,unittest.TestCase			):


		'''simple docstring'''






		lowerCAmelCase_     =   (
		    (TFTransfoXLModel, TFTransfoXLLMHeadModel, TFTransfoXLForSequenceClassification) if is_tf_available() else ()
		)
		lowerCAmelCase_     =   () if is_tf_available() else ()
		lowerCAmelCase_     =   (
		    {
		        'feature-extraction': TFTransfoXLModel,
		        'text-classification': TFTransfoXLForSequenceClassification,
		        'text-generation': TFTransfoXLLMHeadModel,
		        'zero-shot': TFTransfoXLForSequenceClassification,
		    }
		    if is_tf_available()
		    else {}
		)
		# TODO: add this test when TFTransfoXLLMHead has a linear output layer implemented
		lowerCAmelCase_     =   False
		lowerCAmelCase_     =   False
		lowerCAmelCase_     =   False
		lowerCAmelCase_     =   False
		def   _snake_case       (       self     :     Optional[Any]     ,  snake_case__     :     Union[str, Any]     ,  snake_case__     :     Union[str, Any]     ,  snake_case__     :     Tuple     ,  snake_case__     :     Dict     ,  snake_case__     :     str		)      ->	List[Any]:
				if pipeline_test_casse_name == "TextGenerationPipelineTests":
						# Get `ValueError: AttributeError: 'NoneType' object has no attribute 'new_ones'` or `AssertionError`.
						# `TransfoXLConfig` was never used in pipeline tests: cannot create a simple
						# tokenizer.
						return True

				return False
		def   _snake_case       (       self     :     Union[str, Any]		)      ->	List[str]:
				_lowerCamelCase				   =  TFTransfoXLModelTester(self		)
				_lowerCamelCase				   =  ConfigTester(self     ,  config_class=snake_case__     ,  d_embed=3_7		)
		def   _snake_case       (       self     :     List[str]		)      ->	Tuple:
				self.config_tester.run_common_tests()
		def   _snake_case       (       self     :     str		)      ->	Any:
				self.model_tester.set_seed()
				_lowerCamelCase				   =  self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_transfo_xl_model(*snake_case__		)
		def   _snake_case       (       self     :     List[Any]		)      ->	Tuple:
				self.model_tester.set_seed()
				_lowerCamelCase				   =  self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_transfo_xl_lm_head(*snake_case__		)
		def   _snake_case       (       self     :     Any		)      ->	Union[str, Any]:
				_lowerCamelCase				   =  self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_transfo_xl_for_sequence_classification(*snake_case__		)
		def   _snake_case       (       self     :     Optional[int]		)      ->	Tuple:
				_lowerCamelCase   ,							_lowerCamelCase				   =  self.model_tester.prepare_config_and_inputs_for_common()
				_lowerCamelCase				   =  [TFTransfoXLForSequenceClassification]

				for model_class in self.all_model_classes:
						_lowerCamelCase				   =  model_class(snake_case__		)
						assert isinstance(model.get_input_embeddings()     ,  tf.keras.layers.Layer		)
						if model_class in list_other_models_with_output_ebd:
								_lowerCamelCase				   =  model.get_output_embeddings()
								assert isinstance(snake_case__     ,  tf.keras.layers.Layer		)
								_lowerCamelCase				   =  model.get_bias()
								assert name is None
						else:
								_lowerCamelCase				   =  model.get_output_embeddings()
								assert x is None
								_lowerCamelCase				   =  model.get_bias()
								assert name is None
		def   _snake_case       (       self     :     int		)      ->	Dict:
				# TODO JP: Make TransfoXL XLA compliant
				pass
		@slow
		def   _snake_case       (       self     :     Any		)      ->	Union[str, Any]:
				for model_name in TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
						_lowerCamelCase				   =  TFTransfoXLModel.from_pretrained(snake_case__		)
						self.assertIsNotNone(snake_case__		)



		@unittest.skip(reason='This model doesn\'t play well with fit() due to not returning a single loss.'		)
		def   _snake_case       (       self     :     str		)      ->	List[Any]:
				pass





@require_tf
class     lowerCAmelCase__     (				unittest.TestCase			):


		'''simple docstring'''






		@unittest.skip('Skip test until #12651 is resolved.'		)
		@slow
		def   _snake_case       (       self     :     List[str]		)      ->	Optional[Any]:
				_lowerCamelCase				   =  TFTransfoXLLMHeadModel.from_pretrained('transfo-xl-wt103'		)
				# fmt: off
				_lowerCamelCase				   =  tf.convert_to_tensor([[3_3,1_2_9_7,2,1,1_0_0_9,4,1_1_0_9,1_1_7_3_9,4_7_6_2,3_5_8,5,2_5,2_4_5,2_2,1_7_0_6,1_7,2_0_0_9_8,5,3_2_1_5,2_1,3_7,1_1_1_0,3,1_3,1_0_4_1,4,2_4,6_0_3,4_9_0,2,7_1_4_7_7,2_0_0_9_8,1_0_4_4_4_7,2,2_0_9_6_1,1,2_6_0_4,4,1,3_2_9,3,6_2_2_4,8_3_1,1_6_0_0_2,2,8,6_0_3,7_8_9_6_7,2_9_5_4_6,2_3,8_0_3,2_0,2_5,4_1_6,5,8,2_3_2,4,2_7_7,6,1_8_5_5,4_6_0_1,3,2_9_5_4_6,5_4,8,3_6_0_9,5,5_7_2_1_1,4_9,4,1,2_7_7,1_8,8,1_7_5_5,1_5_6_9_1,3,3_4_1,2_5,4_1_6,6_9_3,4_2_5_7_3,7_1,1_7,4_0_1,9_4,3_1,1_7_9_1_9,2,2_9_5_4_6,7_8_7_3,1_8,1,4_3_5,2_3,1_1_0_1_1,7_5_5,5,5_1_6_7,3,7_9_8_3,9_8,8_4,2,2_9_5_4_6,3_2_6_7,8,3_6_0_9,4,1,4_8_6_5,1_0_7_5,2,6_0_8_7,7_1,6,3_4_6,8,5_8_5_4,3,2_9_5_4_6,8_2_4,1_4_0_0,1_8_6_8,2,1_9,1_6_0,2,3_1_1,8,5_4_9_6,2,2_0_9_2_0,1_7,2_5,1_5_0_9_7,3,2_4,2_4,0]]     ,  dtype=tf.intaa		)  # noqa: E231
				# fmt: on
				#  In 1991 , the remains of Russian Tsar Nicholas II and his family
				#  ( except for Alexei and Maria ) are discovered .
				#  The voice of Nicholas's young son , Tsarevich Alexei Nikolaevich , narrates the
				#  remainder of the story . 1883 Western Siberia ,
				#  a young Grigori Rasputin is asked by his father and a group of men to perform magic .
				#  Rasputin has a vision and denounces one of the men as a horse thief . Although his
				#  father initially slaps him for making such an accusation , Rasputin watches as the
				#  man is chased outside and beaten . Twenty years later , Rasputin sees a vision of
				#  the Virgin Mary , prompting him to become a priest . Rasputin quickly becomes famous ,
				#  with people , even a bishop , begging for his blessing . <eod> </s> <eos>

				# fmt: off
				_lowerCamelCase				   =  [3_3,1_2_9_7,2,1,1_0_0_9,4,1_1_0_9,1_1_7_3_9,4_7_6_2,3_5_8,5,2_5,2_4_5,2_2,1_7_0_6,1_7,2_0_0_9_8,5,3_2_1_5,2_1,3_7,1_1_1_0,3,1_3,1_0_4_1,4,2_4,6_0_3,4_9_0,2,7_1_4_7_7,2_0_0_9_8,1_0_4_4_4_7,2,2_0_9_6_1,1,2_6_0_4,4,1,3_2_9,3,6_2_2_4,8_3_1,1_6_0_0_2,2,8,6_0_3,7_8_9_6_7,2_9_5_4_6,2_3,8_0_3,2_0,2_5,4_1_6,5,8,2_3_2,4,2_7_7,6,1_8_5_5,4_6_0_1,3,2_9_5_4_6,5_4,8,3_6_0_9,5,5_7_2_1_1,4_9,4,1,2_7_7,1_8,8,1_7_5_5,1_5_6_9_1,3,3_4_1,2_5,4_1_6,6_9_3,4_2_5_7_3,7_1,1_7,4_0_1,9_4,3_1,1_7_9_1_9,2,2_9_5_4_6,7_8_7_3,1_8,1,4_3_5,2_3,1_1_0_1_1,7_5_5,5,5_1_6_7,3,7_9_8_3,9_8,8_4,2,2_9_5_4_6,3_2_6_7,8,3_6_0_9,4,1,4_8_6_5,1_0_7_5,2,6_0_8_7,7_1,6,3_4_6,8,5_8_5_4,3,2_9_5_4_6,8_2_4,1_4_0_0,1_8_6_8,2,1_9,1_6_0,2,3_1_1,8,5_4_9_6,2,2_0_9_2_0,1_7,2_5,1_5_0_9_7,3,2_4,2_4,0,3_3,1,1_8_5_7,2,1,1_0_0_9,4,1_1_0_9,1_1_7_3_9,4_7_6_2,3_5_8,5,2_5,2_4_5,2_8,1_1_1_0,3,1_3,1_0_4_1,4,2_4,6_0_3,4_9_0,2,7_1_4_7_7,2_0_0_9_8,1_0_4_4_4_7,2,2_0_9_6_1,1,2_6_0_4,4,1,3_2_9,3,0]  # noqa: E231
				# fmt: on
				#  In 1991, the remains of Russian Tsar Nicholas II and his family (
				#  except for Alexei and Maria ) are discovered. The voice of young son,
				#  Tsarevich Alexei Nikolaevich, narrates the remainder of the story.
				#  1883 Western Siberia, a young Grigori Rasputin is asked by his father
				#  and a group of men to perform magic. Rasputin has a vision and
				#  denounces one of the men as a horse thief. Although his father initially
				#  slaps him for making such an accusation, Rasputin watches as the man
				#  is chased outside and beaten. Twenty years later, Rasputin sees a vision
				#  of the Virgin Mary, prompting him to become a priest.
				#  Rasputin quickly becomes famous, with people, even a bishop, begging for
				#  his blessing. <unk> <unk> <eos> In the 1990s, the remains of Russian Tsar
				# Nicholas II and his family were discovered. The voice of <unk> young son,
				# Tsarevich Alexei Nikolaevich, narrates the remainder of the story.<eos>

				_lowerCamelCase				   =  model.generate(snake_case__     ,  max_length=2_0_0     ,  do_sample=snake_case__		)
				self.assertListEqual(output_ids[0].numpy().tolist()     ,  snake_case__		)
from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel


def 	lowerCamelCase					(	UpperCamelCase     :				Any			,       UpperCamelCase     :				str			,       UpperCamelCase     :				Dict=None			,       UpperCamelCase     :				str=None							)   ->    List[str]:
		if attention_mask is None:
				_lowerCamelCase				   =  tf.cast(tf.math.not_equal(UpperCamelCase			,       config.pad_token_id							)			,       tf.inta							)
		return {"input_ids": input_ids, "attention_mask": attention_mask}





@require_tf
class     lowerCAmelCase__     :


		'''simple docstring'''






		lowerCAmelCase_     =   OPTConfig
		lowerCAmelCase_     =   {}
		lowerCAmelCase_     =   'gelu'
		def __init__(       self     :     Dict     ,  snake_case__     :     List[Any]     ,  snake_case__     :     Any=1_3     ,  snake_case__     :     List[str]=7     ,  snake_case__     :     Optional[Any]=True     ,  snake_case__     :     int=False     ,  snake_case__     :     Any=9_9     ,  snake_case__     :     Optional[int]=1_6     ,  snake_case__     :     str=2     ,  snake_case__     :     List[str]=4     ,  snake_case__     :     Union[str, Any]=4     ,  snake_case__     :     Union[str, Any]="gelu"     ,  snake_case__     :     Optional[Any]=0.1     ,  snake_case__     :     Union[str, Any]=0.1     ,  snake_case__     :     Any=2_0     ,  snake_case__     :     Any=2     ,  snake_case__     :     Optional[int]=1     ,  snake_case__     :     Optional[Any]=0     ,  snake_case__     :     int=1_6     ,  snake_case__     :     List[Any]=1_6     ,  )      ->	List[Any]:
				_lowerCamelCase				   =  parent
				_lowerCamelCase				   =  batch_size
				_lowerCamelCase				   =  seq_length
				_lowerCamelCase				   =  is_training
				_lowerCamelCase				   =  use_labels
				_lowerCamelCase				   =  vocab_size
				_lowerCamelCase				   =  hidden_size
				_lowerCamelCase				   =  num_hidden_layers
				_lowerCamelCase				   =  num_attention_heads
				_lowerCamelCase				   =  intermediate_size
				_lowerCamelCase				   =  hidden_act
				_lowerCamelCase				   =  hidden_dropout_prob
				_lowerCamelCase				   =  attention_probs_dropout_prob
				_lowerCamelCase				   =  max_position_embeddings
				_lowerCamelCase				   =  eos_token_id
				_lowerCamelCase				   =  pad_token_id
				_lowerCamelCase				   =  bos_token_id
				_lowerCamelCase				   =  embed_dim
				_lowerCamelCase				   =  word_embed_proj_dim
				_lowerCamelCase				   =  False
		def   _snake_case       (       self     :     List[Any]		)      ->	Optional[int]:
				_lowerCamelCase				   =  ids_tensor([self.batch_size, self.seq_length - 1]     ,  self.vocab_size		)
				_lowerCamelCase				   =  tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size		)     ,  1		)
				_lowerCamelCase				   =  tf.concat([input_ids, eos_tensor]     ,  axis=1		)

				_lowerCamelCase				   =  self.config_cls(
				    vocab_size=self.vocab_size     ,  hidden_size=self.hidden_size     ,  num_hidden_layers=self.num_hidden_layers     ,  num_attention_heads=self.num_attention_heads     ,  ffn_dim=self.intermediate_size     ,  dropout=self.hidden_dropout_prob     ,  attention_dropout=self.attention_probs_dropout_prob     ,  max_position_embeddings=self.max_position_embeddings     ,  eos_token_id=self.eos_token_id     ,  bos_token_id=self.bos_token_id     ,  pad_token_id=self.pad_token_id     ,  embed_dim=self.embed_dim     ,  word_embed_proj_dim=self.word_embed_proj_dim     ,  is_encoder_decoder=snake_case__     ,  **self.config_updates     ,  )
				_lowerCamelCase				   =  prepare_opt_inputs_dict(snake_case__     ,  snake_case__		)
				return config, inputs_dict



		def   _snake_case       (       self     :     Dict     ,  snake_case__     :     Union[str, Any]     ,  snake_case__     :     List[Any]		)      ->	Optional[int]:
				_lowerCamelCase				   =  TFOPTModel(config=snake_case__		)
				_lowerCamelCase				   =  inputs_dict['input_ids']

				_lowerCamelCase				   =  input_ids[:1, :]
				_lowerCamelCase				   =  inputs_dict['attention_mask'][:1, :]
				_lowerCamelCase				   =  1

				# first forward pass
				_lowerCamelCase				   =  model(snake_case__     ,  attention_mask=snake_case__     ,  use_cache=snake_case__		)

				_lowerCamelCase   ,							_lowerCamelCase				   =  outputs.to_tuple()

				# create hypothetical next token and extent to next_input_ids
				_lowerCamelCase				   =  ids_tensor((self.batch_size, 3)     ,  config.vocab_size		)
				_lowerCamelCase				   =  tf.cast(ids_tensor((self.batch_size, 3)     ,  2		)     ,  tf.inta		)

				# append to next input_ids and
				_lowerCamelCase				   =  tf.concat([input_ids, next_tokens]     ,  axis=-1		)
				_lowerCamelCase				   =  tf.concat([attention_mask, next_attn_mask]     ,  axis=-1		)

				_lowerCamelCase				   =  model(snake_case__     ,  attention_mask=snake_case__		)[0]
				_lowerCamelCase				   =  model(snake_case__     ,  attention_mask=snake_case__     ,  past_key_values=snake_case__		)[0]

				self.parent.assertEqual(next_tokens.shape[1]     ,  output_from_past.shape[1]		)

				# select random slice
				_lowerCamelCase				   =  int(ids_tensor((1,)     ,  output_from_past.shape[-1]		)		)
				_lowerCamelCase				   =  output_from_no_past[:, -3:, random_slice_idx]
				_lowerCamelCase				   =  output_from_past[:, :, random_slice_idx]

				# test that outputs are equal for slice
				tf.debugging.assert_near(snake_case__     ,  snake_case__     ,  rtol=1e-3		)





@require_tf
class     lowerCAmelCase__     (				SCREAMING_SNAKE_CASE_     ,SCREAMING_SNAKE_CASE_     ,unittest.TestCase			):


		'''simple docstring'''






		lowerCAmelCase_     =   (TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
		lowerCAmelCase_     =   (TFOPTForCausalLM,) if is_tf_available() else ()
		lowerCAmelCase_     =   (
		    {'feature-extraction': TFOPTModel, 'text-generation': TFOPTForCausalLM} if is_tf_available() else {}
		)
		lowerCAmelCase_     =   False
		lowerCAmelCase_     =   False
		lowerCAmelCase_     =   False
		lowerCAmelCase_     =   10
		def   _snake_case       (       self     :     List[Any]		)      ->	int:
				_lowerCamelCase				   =  TFOPTModelTester(self		)
				_lowerCamelCase				   =  ConfigTester(self     ,  config_class=snake_case__		)
		def   _snake_case       (       self     :     List[str]		)      ->	Union[str, Any]:
				self.config_tester.run_common_tests()
		def   _snake_case       (       self     :     Dict		)      ->	Union[str, Any]:
				_lowerCamelCase				   =  self.model_tester.prepare_config_and_inputs_for_common()
				self.model_tester.check_decoder_model_past_large_inputs(*snake_case__		)



		def   _snake_case       (       self     :     Optional[int]		)      ->	Optional[Any]:
				_lowerCamelCase   ,							_lowerCamelCase				   =  self.model_tester.prepare_config_and_inputs_for_common()

				def _get_word_embedding_weight(snake_case__     :     List[Any]     ,  snake_case__     :     Dict		):
						if hasattr(snake_case__     ,  'weight'		):
								return embedding_layer.weight
						else:
								# Here we build the word embeddings weights if not exists.
								# And then we retry to get the attribute once built.
								model.build()
								if hasattr(snake_case__     ,  'weight'		):
										return embedding_layer.weight
								else:
										return None

				for model_class in self.all_model_classes:
						for size in [config.vocab_size - 1_0, config.vocab_size + 1_0]:
								# build the embeddings
								_lowerCamelCase				   =  model_class(config=snake_case__		)
								_lowerCamelCase				   =  _get_word_embedding_weight(snake_case__     ,  model.get_input_embeddings()		)
								_lowerCamelCase				   =  _get_word_embedding_weight(snake_case__     ,  model.get_output_embeddings()		)

								# reshape the embeddings
								model.resize_token_embeddings(snake_case__		)
								_lowerCamelCase				   =  _get_word_embedding_weight(snake_case__     ,  model.get_input_embeddings()		)
								_lowerCamelCase				   =  _get_word_embedding_weight(snake_case__     ,  model.get_output_embeddings()		)

								# check that the resized embeddings size matches the desired size.
								_lowerCamelCase				   =  size if size is not None else config.vocab_size

								self.assertEqual(new_input_embeddings.shape[0]     ,  snake_case__		)

								# check that weights remain the same after resizing
								_lowerCamelCase				   =  True
								for pa, pa in zip(old_input_embeddings.value()     ,  new_input_embeddings.value()		):
										if tf.math.reduce_sum(tf.math.abs(pa - pa		)		) > 0:
												_lowerCamelCase				   =  False
								self.assertTrue(snake_case__		)

								if old_output_embeddings is not None and new_output_embeddings is not None:
										self.assertEqual(new_output_embeddings.shape[0]     ,  snake_case__		)

										_lowerCamelCase				   =  True
										for pa, pa in zip(old_output_embeddings.value()     ,  new_output_embeddings.value()		):
												if tf.math.reduce_sum(tf.math.abs(pa - pa		)		) > 0:
														_lowerCamelCase				   =  False
										self.assertTrue(snake_case__		)



def 	lowerCamelCase					(	UpperCamelCase     :				str							)   ->    List[str]:
		return tf.constant(UpperCamelCase			,       dtype=tf.intaa							)





@require_tf
class     lowerCAmelCase__     (				unittest.TestCase			):


		'''simple docstring'''






		lowerCAmelCase_     =   99
		def   _snake_case       (       self     :     Any		)      ->	int:
				_lowerCamelCase				   =  tf.ones((4, 1)     ,  dtype=tf.intaa		) * 2
				_lowerCamelCase				   =  tf.concat([ids_tensor((4, 6)     ,  self.vocab_size - 3		) + 3, eos_column_vector]     ,  axis=1		)
				_lowerCamelCase				   =  input_ids.shape[0]
				_lowerCamelCase				   =  OPTConfig(
				    vocab_size=self.vocab_size     ,  hidden_size=2_4     ,  num_hidden_layers=2     ,  num_attention_heads=2     ,  ffn_dim=3_2     ,  max_position_embeddings=4_8     ,  eos_token_id=2     ,  pad_token_id=1     ,  bos_token_id=0     ,  )
				return config, input_ids, batch_size





@require_sentencepiece
@require_tf
class     lowerCAmelCase__     (				unittest.TestCase			):


		'''simple docstring'''






		@slow
		def   _snake_case       (       self     :     List[Any]		)      ->	str:
				_lowerCamelCase				   =  TFOPTModel.from_pretrained('facebook/opt-350m'		)
				_lowerCamelCase				   =  _long_tensor([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]		)
				_lowerCamelCase				   =  tf.not_equal(snake_case__     ,  model.config.pad_token_id		)
				with tf.GradientTape():
						_lowerCamelCase				   =  model(input_ids=snake_case__     ,  attention_mask=snake_case__		).last_hidden_state
				_lowerCamelCase				   =  (1, 1_1, 5_1_2)
				self.assertEqual(output.shape     ,  snake_case__		)
				_lowerCamelCase				   =  tf.constant(
				    [[-0.2873, -1.9218, -0.3033], [-1.2710, -0.1338, -0.1902], [0.4095, 0.1214, -1.3121]]		)
				self.assertTrue(np.allclose(output[:, :3, :3]     ,  snake_case__     ,  atol=4e-3		)		)

				_lowerCamelCase				   =  tf.function(snake_case__     ,  jit_compile=snake_case__		)
				_lowerCamelCase				   =  xla_generate(snake_case__     ,  snake_case__		)[0]
				self.assertTrue(np.allclose(output[:, :3, :3]     ,  snake_case__     ,  atol=4e-2		)		)





@require_tf
@slow
class     lowerCAmelCase__     (				unittest.TestCase			):


		'''simple docstring'''






		def   _snake_case       (       self     :     Tuple		)      ->	List[str]:
				super().setUp()
				_lowerCamelCase				   =  'facebook/opt-350m'



		def   _snake_case       (       self     :     Union[str, Any]		)      ->	Dict:
				_lowerCamelCase				   =  TFOPTForCausalLM.from_pretrained(self.path_model		)
				_lowerCamelCase				   =  GPTaTokenizer.from_pretrained(self.path_model		)

				_lowerCamelCase				   =  [
				    'Today is a beautiful day and I want to',
				    'In the city of',
				    'Paris is the capital of France and',
				    'Computers and mobile phones have taken',
				]
				# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
				_lowerCamelCase				   =  tokenizer(snake_case__     ,  return_tensors='tf'     ,  padding=snake_case__     ,  add_special_tokens=snake_case__		)
				_lowerCamelCase				   =  tf.math.reduce_mean(model(inputs.input_ids     ,  attention_mask=inputs.attention_mask		)[0]     ,  axis=-1		)
				_lowerCamelCase				   =  tf.constant(
				    [
				        [1.3851, -13.8923, -10.5229, -10.7533, -0.2309, -10.2384, -0.5365, -9.0947, -5.1670],
				        [-4.7073, -10.6276, -3.9415, -21.5242, -0.2822, -0.2822, -0.2822, -0.2822, -0.2822],
				        [0.6247, -3.4229, -8.9179, -1.4297, -14.1650, 1.4146, -9.0218, -0.2703, -0.2703],
				        [6.4783, -1.9913, -10.7926, -2.3336, 1.5092, -0.9974, -6.8213, 1.3477, 1.3477],
				    ]		)
				self.assertTrue(np.allclose(snake_case__     ,  snake_case__     ,  atol=1e-4		)		)

				_lowerCamelCase				   =  tf.function(snake_case__     ,  jit_compile=snake_case__		)
				_lowerCamelCase				   =  tf.math.reduce_mean(xla_generate(inputs.input_ids     ,  attention_mask=inputs.attention_mask		)[0]     ,  axis=-1		)
				self.assertTrue(np.allclose(snake_case__     ,  snake_case__     ,  atol=1e-4		)		)





@require_tf
@slow
class     lowerCAmelCase__     (				unittest.TestCase			):


		'''simple docstring'''






		@property
		def   _snake_case       (       self     :     str		)      ->	List[str]:
				return [
				    "Today is a beautiful day and I want",
				    "In the city of",
				    "Paris is the capital of France and",
				    "Computers and mobile phones have taken",
				]
		def   _snake_case       (       self     :     Dict		)      ->	Optional[int]:
				_lowerCamelCase				   =  'facebook/opt-125m'

				_lowerCamelCase				   =  [
				    'Today is a beautiful day and I want to',
				    'In the city of New York, the city',
				    'Paris is the capital of France and the capital',
				    'Computers and mobile phones have taken over the',
				]

				_lowerCamelCase				   =  []
				_lowerCamelCase				   =  GPTaTokenizer.from_pretrained(snake_case__		)
				_lowerCamelCase				   =  TFOPTForCausalLM.from_pretrained(snake_case__		)

				for prompt in self.prompts:
						_lowerCamelCase				   =  tokenizer(snake_case__     ,  return_tensors='tf'		).input_ids

						_lowerCamelCase				   =  model.generate(snake_case__     ,  max_length=1_0		)

						_lowerCamelCase				   =  tokenizer.batch_decode(snake_case__     ,  skip_special_tokens=snake_case__		)
						predicted_outputs += generated_string

				self.assertListEqual(snake_case__     ,  snake_case__		)
		def   _snake_case       (       self     :     int		)      ->	Dict:
				_lowerCamelCase				   =  'facebook/opt-350m'

				_lowerCamelCase				   =  GPTaTokenizer.from_pretrained(snake_case__		)
				_lowerCamelCase				   =  TFOPTForCausalLM.from_pretrained(snake_case__		)

				_lowerCamelCase				   =  'left'

				# use different length sentences to test batching
				_lowerCamelCase				   =  [
				    'Hello, my dog is a little',
				    'Today, I',
				]

				_lowerCamelCase				   =  tokenizer(snake_case__     ,  return_tensors='tf'     ,  padding=snake_case__		)
				_lowerCamelCase				   =  inputs['input_ids']

				_lowerCamelCase				   =  model.generate(input_ids=snake_case__     ,  attention_mask=inputs['attention_mask']		)

				_lowerCamelCase				   =  tokenizer(sentences[0]     ,  return_tensors='tf'		).input_ids
				_lowerCamelCase				   =  model.generate(input_ids=snake_case__		)

				_lowerCamelCase				   =  inputs_non_padded.shape[-1] - tf.math.reduce_sum(
				    tf.cast(inputs['attention_mask'][-1]     ,  tf.intaa		)		)
				_lowerCamelCase				   =  tokenizer(sentences[1]     ,  return_tensors='tf'		).input_ids
				_lowerCamelCase				   =  model.generate(input_ids=snake_case__     ,  max_length=model.config.max_length - num_paddings		)

				_lowerCamelCase				   =  tokenizer.batch_decode(snake_case__     ,  skip_special_tokens=snake_case__		)
				_lowerCamelCase				   =  tokenizer.decode(output_non_padded[0]     ,  skip_special_tokens=snake_case__		)
				_lowerCamelCase				   =  tokenizer.decode(output_padded[0]     ,  skip_special_tokens=snake_case__		)

				_lowerCamelCase				   =  [
				    'Hello, my dog is a little bit of a dork.\nI\'m a little bit',
				    'Today, I was in the middle of a conversation with a friend about the',
				]
				self.assertListEqual(snake_case__     ,  snake_case__		)
				self.assertListEqual(snake_case__     ,  [non_padded_sentence, padded_sentence]		)



		def   _snake_case       (       self     :     List[Any]		)      ->	Optional[Any]:
				_lowerCamelCase				   =  'facebook/opt-350m'

				_lowerCamelCase				   =  [
				    'Today is a beautiful day and I want to',
				    'In the city of San Francisco, the city',
				    'Paris is the capital of France and the capital',
				    'Computers and mobile phones have taken over the',
				]

				_lowerCamelCase				   =  []
				_lowerCamelCase				   =  GPTaTokenizer.from_pretrained(snake_case__		)
				_lowerCamelCase				   =  TFOPTForCausalLM.from_pretrained(snake_case__		)

				for prompt in self.prompts:
						_lowerCamelCase				   =  tokenizer(snake_case__     ,  return_tensors='tf'		).input_ids

						_lowerCamelCase				   =  model.generate(snake_case__     ,  max_length=1_0		)

						_lowerCamelCase				   =  tokenizer.batch_decode(snake_case__     ,  skip_special_tokens=snake_case__		)
						predicted_outputs += generated_string

				self.assertListEqual(snake_case__     ,  snake_case__		)