


'''simple docstring'''



from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel







def     _UpperCamelCase      (		UpperCamelCase__ ,      UpperCamelCase__ ,      UpperCamelCase__=None ,      UpperCamelCase__=None					):
			if attention_mask is None:
						UpperCAmelCase__  :					Tuple        =     tf.cast(tf.math.not_equal(_lowercase ,      config.pad_token_id					) ,      tf.inta					)
			return {"input_ids": input_ids, "attention_mask": attention_mask}


@require_tf
class 		_snake_case					:
					lowerCAmelCase      :List[str]   =					OPTConfig
					lowerCAmelCase      :int   =					{}
					lowerCAmelCase      :List[str]   =					'''gelu'''




					def __init__(      self							,							_lowerCamelCase							,							_lowerCamelCase=13							,							_lowerCamelCase=7							,							_lowerCamelCase=True							,							_lowerCamelCase=False							,							_lowerCamelCase=99							,							_lowerCamelCase=16							,							_lowerCamelCase=2							,							_lowerCamelCase=4							,							_lowerCamelCase=4							,							_lowerCamelCase="gelu"							,							_lowerCamelCase=0.1							,							_lowerCamelCase=0.1							,							_lowerCamelCase=20							,							_lowerCamelCase=2							,							_lowerCamelCase=1							,							_lowerCamelCase=0							,							_lowerCamelCase=16							,							_lowerCamelCase=16							,							):
								UpperCAmelCase__  :					Any        =     parent
								UpperCAmelCase__  :					List[Any]        =     batch_size
								UpperCAmelCase__  :					Optional[int]        =     seq_length
								UpperCAmelCase__  :					Tuple        =     is_training
								UpperCAmelCase__  :					Any        =     use_labels
								UpperCAmelCase__  :					Optional[Any]        =     vocab_size
								UpperCAmelCase__  :					Tuple        =     hidden_size
								UpperCAmelCase__  :					str        =     num_hidden_layers
								UpperCAmelCase__  :					int        =     num_attention_heads
								UpperCAmelCase__  :					Union[str, Any]        =     intermediate_size
								UpperCAmelCase__  :					Tuple        =     hidden_act
								UpperCAmelCase__  :					int        =     hidden_dropout_prob
								UpperCAmelCase__  :					Optional[int]        =     attention_probs_dropout_prob
								UpperCAmelCase__  :					Dict        =     max_position_embeddings
								UpperCAmelCase__  :					Union[str, Any]        =     eos_token_id
								UpperCAmelCase__  :					List[str]        =     pad_token_id
								UpperCAmelCase__  :					Optional[int]        =     bos_token_id
								UpperCAmelCase__  :					int        =     embed_dim
								UpperCAmelCase__  :					str        =     word_embed_proj_dim
								UpperCAmelCase__  :					str        =     False




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Optional[Any]        =     ids_tensor([self.batch_size, self.seq_length - 1]							,							self.vocab_size)
								UpperCAmelCase__  :					Optional[int]        =     tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size)							,							1)
								UpperCAmelCase__  :					List[str]        =     tf.concat([input_ids, eos_tensor]							,							axis=1)

								UpperCAmelCase__  :					Dict        =     self.config_cls(
								    vocab_size=self.vocab_size							,							hidden_size=self.hidden_size							,							num_hidden_layers=self.num_hidden_layers							,							num_attention_heads=self.num_attention_heads							,							ffn_dim=self.intermediate_size							,							dropout=self.hidden_dropout_prob							,							attention_dropout=self.attention_probs_dropout_prob							,							max_position_embeddings=self.max_position_embeddings							,							eos_token_id=self.eos_token_id							,							bos_token_id=self.bos_token_id							,							pad_token_id=self.pad_token_id							,							embed_dim=self.embed_dim							,							word_embed_proj_dim=self.word_embed_proj_dim							,							is_encoder_decoder=UpperCamelCase__							,							**self.config_updates							,							)
								UpperCAmelCase__  :					Tuple        =     prepare_opt_inputs_dict(UpperCamelCase__							,							UpperCamelCase__)
								return config, inputs_dict




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase):
								UpperCAmelCase__  :					Optional[Any]        =     TFOPTModel(config=UpperCamelCase__)
								UpperCAmelCase__  :					Dict        =     inputs_dict['''input_ids''']

								UpperCAmelCase__  :					Tuple        =     input_ids[:1, :]
								UpperCAmelCase__  :					int        =     inputs_dict['''attention_mask'''][:1, :]
								UpperCAmelCase__  :					Optional[int]        =     1

								# first forward pass
								UpperCAmelCase__  :					int        =     model(UpperCamelCase__							,							attention_mask=UpperCamelCase__							,							use_cache=UpperCamelCase__)

								UpperCAmelCase__  :					str        =     outputs.to_tuple()

								# create hypothetical next token and extent to next_input_ids
								UpperCAmelCase__  :					Optional[int]        =     ids_tensor((self.batch_size, 3)							,							config.vocab_size)
								UpperCAmelCase__  :					List[Any]        =     tf.cast(ids_tensor((self.batch_size, 3)							,							2)							,							tf.inta)

								# append to next input_ids and
								UpperCAmelCase__  :					int        =     tf.concat([input_ids, next_tokens]							,							axis=-1)
								UpperCAmelCase__  :					Optional[int]        =     tf.concat([attention_mask, next_attn_mask]							,							axis=-1)

								UpperCAmelCase__  :					Tuple        =     model(UpperCamelCase__							,							attention_mask=UpperCamelCase__)[0]
								UpperCAmelCase__  :					Any        =     model(UpperCamelCase__							,							attention_mask=UpperCamelCase__							,							past_key_values=UpperCamelCase__)[0]

								self.parent.assertEqual(next_tokens.shape[1]							,							output_from_past.shape[1])

								# select random slice
								UpperCAmelCase__  :					List[Any]        =     int(ids_tensor((1,)							,							output_from_past.shape[-1]))
								UpperCAmelCase__  :					List[str]        =     output_from_no_past[:, -3:, random_slice_idx]
								UpperCAmelCase__  :					List[str]        =     output_from_past[:, :, random_slice_idx]

								# test that outputs are equal for slice
								tf.debugging.assert_near(UpperCamelCase__							,							UpperCamelCase__							,							rtol=1e-3)


@require_tf
class 		_snake_case					(     UpperCamelCase_       ,   UpperCamelCase_       ,   unittest.TestCase	):
					lowerCAmelCase      :Tuple   =					(TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
					lowerCAmelCase      :List[Any]   =					(TFOPTForCausalLM,) if is_tf_available() else ()
					lowerCAmelCase      :Dict   =					(
					    {'''feature-extraction''': TFOPTModel, '''text-generation''': TFOPTForCausalLM} if is_tf_available() else {}
					)
					lowerCAmelCase      :Dict   =					False
					lowerCAmelCase      :Optional[int]   =					False
					lowerCAmelCase      :Optional[int]   =					False
					lowerCAmelCase      :Optional[int]   =					10




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Optional[int]        =     TFOPTModelTester(self)
								UpperCAmelCase__  :					Dict        =     ConfigTester(self							,							config_class=UpperCamelCase__)




					def 				snake_case__						(      self):
								self.config_tester.run_common_tests()




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					int        =     self.model_tester.prepare_config_and_inputs_for_common()
								self.model_tester.check_decoder_model_past_large_inputs(*UpperCamelCase__)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Tuple        =     self.model_tester.prepare_config_and_inputs_for_common()

								def _get_word_embedding_weight(_lowerCamelCase							,							_lowerCamelCase):
											if hasattr(UpperCamelCase__							,							"""weight"""):
														return embedding_layer.weight
											else:
														# Here we build the word embeddings weights if not exists.
														# And then we retry to get the attribute once built.
														model.build()
														if hasattr(UpperCamelCase__							,							"""weight"""):
																	return embedding_layer.weight
														else:
																	return None

								for model_class in self.all_model_classes:
											for size in [config.vocab_size - 10, config.vocab_size + 10]:
														# build the embeddings
														UpperCAmelCase__  :					Tuple        =     model_class(config=UpperCamelCase__)
														UpperCAmelCase__  :					int        =     _get_word_embedding_weight(UpperCamelCase__							,							model.get_input_embeddings())
														UpperCAmelCase__  :					List[Any]        =     _get_word_embedding_weight(UpperCamelCase__							,							model.get_output_embeddings())

														# reshape the embeddings
														model.resize_token_embeddings(UpperCamelCase__)
														UpperCAmelCase__  :					List[str]        =     _get_word_embedding_weight(UpperCamelCase__							,							model.get_input_embeddings())
														UpperCAmelCase__  :					str        =     _get_word_embedding_weight(UpperCamelCase__							,							model.get_output_embeddings())

														# check that the resized embeddings size matches the desired size.
														UpperCAmelCase__  :					str        =     size if size is not None else config.vocab_size

														self.assertEqual(new_input_embeddings.shape[0]							,							UpperCamelCase__)

														# check that weights remain the same after resizing
														UpperCAmelCase__  :					Tuple        =     True
														for pa, pa in zip(old_input_embeddings.value()							,							new_input_embeddings.value()):
																	if tf.math.reduce_sum(tf.math.abs(pa - pa)) > 0:
																				UpperCAmelCase__  :					Any        =     False
														self.assertTrue(UpperCamelCase__)

														if old_output_embeddings is not None and new_output_embeddings is not None:
																	self.assertEqual(new_output_embeddings.shape[0]							,							UpperCamelCase__)

																	UpperCAmelCase__  :					Union[str, Any]        =     True
																	for pa, pa in zip(old_output_embeddings.value()							,							new_output_embeddings.value()):
																				if tf.math.reduce_sum(tf.math.abs(pa - pa)) > 0:
																							UpperCAmelCase__  :					Tuple        =     False
																	self.assertTrue(UpperCamelCase__)







def     _UpperCamelCase      (		UpperCamelCase__					):
			return tf.constant(_lowercase ,      dtype=tf.intaa					)


@require_tf
class 		_snake_case					(     unittest.TestCase	):
					lowerCAmelCase      :Union[str, Any]   =					99




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Tuple        =     tf.ones((4, 1)							,							dtype=tf.intaa) * 2
								UpperCAmelCase__  :					Union[str, Any]        =     tf.concat([ids_tensor((4, 6)							,							self.vocab_size - 3) + 3, eos_column_vector]							,							axis=1)
								UpperCAmelCase__  :					Dict        =     input_ids.shape[0]
								UpperCAmelCase__  :					Union[str, Any]        =     OPTConfig(
								    vocab_size=self.vocab_size							,							hidden_size=24							,							num_hidden_layers=2							,							num_attention_heads=2							,							ffn_dim=32							,							max_position_embeddings=48							,							eos_token_id=2							,							pad_token_id=1							,							bos_token_id=0							,							)
								return config, input_ids, batch_size


@require_sentencepiece
@require_tf
class 		_snake_case					(     unittest.TestCase	):




					@slow
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[Any]        =     TFOPTModel.from_pretrained("""facebook/opt-350m""")
								UpperCAmelCase__  :					Optional[int]        =     _long_tensor([[0, 3_1414, 232, 328, 740, 1140, 1_2695, 69, 4_6078, 1588, 2]])
								UpperCAmelCase__  :					Union[str, Any]        =     tf.not_equal(UpperCamelCase__							,							model.config.pad_token_id)
								with tf.GradientTape():
											UpperCAmelCase__  :					str        =     model(input_ids=UpperCamelCase__							,							attention_mask=UpperCamelCase__).last_hidden_state
								UpperCAmelCase__  :					Tuple        =     (1, 11, 512)
								self.assertEqual(output.shape							,							UpperCamelCase__)
								UpperCAmelCase__  :					str        =     tf.constant(
								    [[-0.2873, -1.9218, -0.3033], [-1.2710, -0.1338, -0.1902], [0.4095, 0.1214, -1.3121]])
								self.assertTrue(np.allclose(output[:, :3, :3]							,							UpperCamelCase__							,							atol=4e-3))

								UpperCAmelCase__  :					Dict        =     tf.function(UpperCamelCase__							,							jit_compile=UpperCamelCase__)
								UpperCAmelCase__  :					int        =     xla_generate(UpperCamelCase__							,							UpperCamelCase__)[0]
								self.assertTrue(np.allclose(output[:, :3, :3]							,							UpperCamelCase__							,							atol=4e-2))


@require_tf
@slow
class 		_snake_case					(     unittest.TestCase	):




					def 				snake_case__						(      self):
								super().setUp()
								UpperCAmelCase__  :					Dict        =     '''facebook/opt-350m'''




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Dict        =     TFOPTForCausalLM.from_pretrained(self.path_model)
								UpperCAmelCase__  :					Optional[int]        =     GPTaTokenizer.from_pretrained(self.path_model)

								UpperCAmelCase__  :					Optional[int]        =     [
								    '''Today is a beautiful day and I want to''',
								    '''In the city of''',
								    '''Paris is the capital of France and''',
								    '''Computers and mobile phones have taken''',
								]
								# verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
								UpperCAmelCase__  :					int        =     tokenizer(UpperCamelCase__							,							return_tensors="""tf"""							,							padding=UpperCamelCase__							,							add_special_tokens=UpperCamelCase__)
								UpperCAmelCase__  :					List[str]        =     tf.math.reduce_mean(model(inputs.input_ids							,							attention_mask=inputs.attention_mask)[0]							,							axis=-1)
								UpperCAmelCase__  :					str        =     tf.constant(
								    [
								        [1.3851, -13.8923, -10.5229, -10.7533, -0.2309, -10.2384, -0.5365, -9.0947, -5.1670],
								        [-4.7073, -10.6276, -3.9415, -21.5242, -0.2822, -0.2822, -0.2822, -0.2822, -0.2822],
								        [0.6247, -3.4229, -8.9179, -1.4297, -14.1650, 1.4146, -9.0218, -0.2703, -0.2703],
								        [6.4783, -1.9913, -10.7926, -2.3336, 1.5092, -0.9974, -6.8213, 1.3477, 1.3477],
								    ])
								self.assertTrue(np.allclose(UpperCamelCase__							,							UpperCamelCase__							,							atol=1e-4))

								UpperCAmelCase__  :					Dict        =     tf.function(UpperCamelCase__							,							jit_compile=UpperCamelCase__)
								UpperCAmelCase__  :					Dict        =     tf.math.reduce_mean(xla_generate(inputs.input_ids							,							attention_mask=inputs.attention_mask)[0]							,							axis=-1)
								self.assertTrue(np.allclose(UpperCamelCase__							,							UpperCamelCase__							,							atol=1e-4))


@require_tf
@slow
class 		_snake_case					(     unittest.TestCase	):




					@property
					def 				snake_case__						(      self):
								return [
								    "Today is a beautiful day and I want",
								    "In the city of",
								    "Paris is the capital of France and",
								    "Computers and mobile phones have taken",
								]




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					str        =     '''facebook/opt-125m'''

								UpperCAmelCase__  :					Optional[Any]        =     [
								    '''Today is a beautiful day and I want to''',
								    '''In the city of New York, the city''',
								    '''Paris is the capital of France and the capital''',
								    '''Computers and mobile phones have taken over the''',
								]

								UpperCAmelCase__  :					Optional[int]        =     []
								UpperCAmelCase__  :					Dict        =     GPTaTokenizer.from_pretrained(UpperCamelCase__)
								UpperCAmelCase__  :					Dict        =     TFOPTForCausalLM.from_pretrained(UpperCamelCase__)

								for prompt in self.prompts:
											UpperCAmelCase__  :					List[Any]        =     tokenizer(UpperCamelCase__							,							return_tensors="""tf""").input_ids

											UpperCAmelCase__  :					Dict        =     model.generate(UpperCamelCase__							,							max_length=10)

											UpperCAmelCase__  :					int        =     tokenizer.batch_decode(UpperCamelCase__							,							skip_special_tokens=UpperCamelCase__)
											predicted_outputs += generated_string

								self.assertListEqual(UpperCamelCase__							,							UpperCamelCase__)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Dict        =     '''facebook/opt-350m'''

								UpperCAmelCase__  :					Union[str, Any]        =     GPTaTokenizer.from_pretrained(UpperCamelCase__)
								UpperCAmelCase__  :					List[Any]        =     TFOPTForCausalLM.from_pretrained(UpperCamelCase__)

								UpperCAmelCase__  :					str        =     '''left'''

								# use different length sentences to test batching
								UpperCAmelCase__  :					Dict        =     [
								    '''Hello, my dog is a little''',
								    '''Today, I''',
								]

								UpperCAmelCase__  :					Dict        =     tokenizer(UpperCamelCase__							,							return_tensors="""tf"""							,							padding=UpperCamelCase__)
								UpperCAmelCase__  :					List[Any]        =     inputs['''input_ids''']

								UpperCAmelCase__  :					Union[str, Any]        =     model.generate(input_ids=UpperCamelCase__							,							attention_mask=inputs["""attention_mask"""])

								UpperCAmelCase__  :					str        =     tokenizer(sentences[0]							,							return_tensors="""tf""").input_ids
								UpperCAmelCase__  :					Union[str, Any]        =     model.generate(input_ids=UpperCamelCase__)

								UpperCAmelCase__  :					List[str]        =     inputs_non_padded.shape[-1] - tf.math.reduce_sum(
								    tf.cast(inputs["""attention_mask"""][-1]							,							tf.intaa))
								UpperCAmelCase__  :					int        =     tokenizer(sentences[1]							,							return_tensors="""tf""").input_ids
								UpperCAmelCase__  :					Union[str, Any]        =     model.generate(input_ids=UpperCamelCase__							,							max_length=model.config.max_length - num_paddings)

								UpperCAmelCase__  :					Any        =     tokenizer.batch_decode(UpperCamelCase__							,							skip_special_tokens=UpperCamelCase__)
								UpperCAmelCase__  :					str        =     tokenizer.decode(output_non_padded[0]							,							skip_special_tokens=UpperCamelCase__)
								UpperCAmelCase__  :					List[str]        =     tokenizer.decode(output_padded[0]							,							skip_special_tokens=UpperCamelCase__)

								UpperCAmelCase__  :					List[str]        =     [
								    '''Hello, my dog is a little bit of a dork.\nI\'m a little bit''',
								    '''Today, I was in the middle of a conversation with a friend about the''',
								]
								self.assertListEqual(UpperCamelCase__							,							UpperCamelCase__)
								self.assertListEqual(UpperCamelCase__							,							[non_padded_sentence, padded_sentence])




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Optional[int]        =     '''facebook/opt-350m'''

								UpperCAmelCase__  :					int        =     [
								    '''Today is a beautiful day and I want to''',
								    '''In the city of San Francisco, the city''',
								    '''Paris is the capital of France and the capital''',
								    '''Computers and mobile phones have taken over the''',
								]

								UpperCAmelCase__  :					Optional[int]        =     []
								UpperCAmelCase__  :					Union[str, Any]        =     GPTaTokenizer.from_pretrained(UpperCamelCase__)
								UpperCAmelCase__  :					Union[str, Any]        =     TFOPTForCausalLM.from_pretrained(UpperCamelCase__)

								for prompt in self.prompts:
											UpperCAmelCase__  :					Tuple        =     tokenizer(UpperCamelCase__							,							return_tensors="""tf""").input_ids

											UpperCAmelCase__  :					Dict        =     model.generate(UpperCamelCase__							,							max_length=10)

											UpperCAmelCase__  :					Tuple        =     tokenizer.batch_decode(UpperCamelCase__							,							skip_special_tokens=UpperCamelCase__)
											predicted_outputs += generated_string

								self.assertListEqual(UpperCamelCase__							,							UpperCamelCase__)



'''simple docstring'''



import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
__A	=object()

# For specifying empty leaf dict `{}`
__A	=object()







def     _UpperCamelCase      (		UpperCamelCase__ ,      UpperCamelCase__					):
			UpperCAmelCase__  :					Union[str, Any]        =     tuple((re.compile(x + """$"""					) for x in qs)					)
			for i in range(len(UpperCamelCase__					) - len(UpperCamelCase__					) + 1					):
						UpperCAmelCase__  :					Optional[Any]        =     [x.match(UpperCamelCase__					) for x, y in zip(UpperCamelCase__ ,      ks[i:]					)]
						if matches and all(UpperCamelCase__					):
									return True
			return False







def     _UpperCamelCase      (		UpperCamelCase__					):
			def replace(UpperCamelCase__ ,      UpperCamelCase__					):
						for rule, replacement in rules:
									if _match(UpperCamelCase__ ,      UpperCamelCase__					):
												return replacement
						return val

			return replace







def     _UpperCamelCase      (		):
			return [
			    # embeddings
			    (("transformer", "wpe", "embedding"), P("""mp""" ,      UpperCamelCase__					)),
			    (("transformer", "wte", "embedding"), P("""mp""" ,      UpperCamelCase__					)),
			    # atention
			    (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(UpperCamelCase__ ,      """mp"""					)),
			    (("attention", "out_proj", "kernel"), P("""mp""" ,      UpperCamelCase__					)),
			    (("attention", "out_proj", "bias"), None),
			    # mlp
			    (("mlp", "c_fc", "kernel"), P(UpperCamelCase__ ,      """mp"""					)),
			    (("mlp", "c_fc", "bias"), P("""mp"""					)),
			    (("mlp", "c_proj", "kernel"), P("""mp""" ,      UpperCamelCase__					)),
			    (("mlp", "c_proj", "bias"), None),
			    # layer norms
			    ((r"ln_\d+", "bias"), None),
			    ((r"\d+", r"ln_\d+", "scale"), None),
			    (("ln_f", "bias"), None),
			    (("ln_f", "scale"), None),
			]







def     _UpperCamelCase      (		UpperCamelCase__					):
			UpperCAmelCase__  :					Optional[int]        =     _get_partition_rules()
			UpperCAmelCase__  :					Optional[int]        =     _replacement_rules(UpperCamelCase__					)
			UpperCAmelCase__  :					List[Any]        =     {k: _unmatched for k in flatten_dict(UpperCamelCase__					)}
			UpperCAmelCase__  :					Tuple        =     {k: replace(UpperCamelCase__ ,      UpperCamelCase__					) for k, v in initd.items()}
			assert _unmatched not in result.values(), "Incomplete partition spec."
			return freeze(unflatten_dict(UpperCamelCase__					)					)