



from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import (
		    TFRoFormerForCausalLM,
		    TFRoFormerForMaskedLM,
		    TFRoFormerForMultipleChoice,
		    TFRoFormerForQuestionAnswering,
		    TFRoFormerForSequenceClassification,
		    TFRoFormerForTokenClassification,
		    TFRoFormerModel,
		)
		from transformers.models.roformer.modeling_tf_roformer import (
		    TFRoFormerSelfAttention,
		    TFRoFormerSinusoidalPositionalEmbedding,
		)





class 			__magic_name__ :






		def __init__(    self   :  List[str]					,		UpperCamelCase__   :  Optional[int]					,		UpperCamelCase__   :  Any=13					,		UpperCamelCase__   :  Union[str, Any]=7					,		UpperCamelCase__   :  int=True					,		UpperCamelCase__   :  Union[str, Any]=True					,		UpperCamelCase__   :  Dict=True					,		UpperCamelCase__   :  Optional[Any]=True					,		UpperCamelCase__   :  Optional[Any]=99					,		UpperCamelCase__   :  List[str]=32					,		UpperCamelCase__   :  int=2					,		UpperCamelCase__   :  List[str]=4					,		UpperCamelCase__   :  List[str]=37					,		UpperCamelCase__   :  Optional[int]="gelu"					,		UpperCamelCase__   :  Tuple=0.1					,		UpperCamelCase__   :  int=0.1					,		UpperCamelCase__   :  int=5_12					,		UpperCamelCase__   :  Optional[int]=16					,		UpperCamelCase__   :  List[Any]=2					,		UpperCamelCase__   :  int=0.02					,		UpperCamelCase__   :  Dict=3					,		UpperCamelCase__   :  List[str]=4					,		UpperCamelCase__   :  Optional[Any]=None					,		)					->       List[Any]:
								'''simple docstring'''







								UpperCAmelCase								=  parent
								UpperCAmelCase								=  13
								UpperCAmelCase								=  7
								UpperCAmelCase								=  True
								UpperCAmelCase								=  True
								UpperCAmelCase								=  True
								UpperCAmelCase								=  True
								UpperCAmelCase								=  99
								UpperCAmelCase								=  32
								UpperCAmelCase								=  2
								UpperCAmelCase								=  4
								UpperCAmelCase								=  37
								UpperCAmelCase								=  "gelu"
								UpperCAmelCase								=  0.1
								UpperCAmelCase								=  0.1
								UpperCAmelCase								=  5_12
								UpperCAmelCase								=  16
								UpperCAmelCase								=  2
								UpperCAmelCase								=  0.02
								UpperCAmelCase								=  3
								UpperCAmelCase								=  4
								UpperCAmelCase								=  None






		def SCREAMING_SNAKE_CASE_      (    self   :  Dict   )					->       Optional[Any]:
								'''simple docstring'''







								UpperCAmelCase								=  ids_tensor([self.batch_size, self.seq_length]					,		self.vocab_size   )

								UpperCAmelCase								=  None
								if self.use_input_mask:
														UpperCAmelCase								=  random_attention_mask([self.batch_size, self.seq_length]   )

								UpperCAmelCase								=  None
								if self.use_token_type_ids:
														UpperCAmelCase								=  ids_tensor([self.batch_size, self.seq_length]					,		self.type_vocab_size   )

								UpperCAmelCase								=  None
								UpperCAmelCase								=  None
								UpperCAmelCase								=  None
								if self.use_labels:
														UpperCAmelCase								=  ids_tensor([self.batch_size]					,		self.type_sequence_label_size   )
														UpperCAmelCase								=  ids_tensor([self.batch_size, self.seq_length]					,		self.num_labels   )
														UpperCAmelCase								=  ids_tensor([self.batch_size]					,		self.num_choices   )

								UpperCAmelCase								=  RoFormerConfig(
								    vocab_size=self.vocab_size					,		hidden_size=self.hidden_size					,		num_hidden_layers=self.num_hidden_layers					,		num_attention_heads=self.num_attention_heads					,		intermediate_size=self.intermediate_size					,		hidden_act=self.hidden_act					,		hidden_dropout_prob=self.hidden_dropout_prob					,		attention_probs_dropout_prob=self.attention_probs_dropout_prob					,		max_position_embeddings=self.max_position_embeddings					,		type_vocab_size=self.type_vocab_size					,		initializer_range=self.initializer_range					,		return_dict=__a					,		)

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






		def SCREAMING_SNAKE_CASE_      (    self   :  Optional[int]					,		UpperCamelCase__   :  int					,		UpperCamelCase__   :  Tuple					,		UpperCamelCase__   :  Tuple					,		UpperCamelCase__   :  str					,		UpperCamelCase__   :  List[Any]					,		UpperCamelCase__   :  str					,		UpperCamelCase__   :  Tuple   )					->       List[str]:
								'''simple docstring'''







								UpperCAmelCase								=  TFRoFormerModel(config=__a   )
								UpperCAmelCase								=  {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}

								UpperCAmelCase								=  [input_ids, input_mask]
								UpperCAmelCase								=  model(__a   )

								UpperCAmelCase								=  model(__a   )

								self.parent.assertEqual(result.last_hidden_state.shape					,		(self.batch_size, self.seq_length, self.hidden_size)   )






		def SCREAMING_SNAKE_CASE_      (    self   :  str					,		UpperCamelCase__   :  Dict					,		UpperCamelCase__   :  str					,		UpperCamelCase__   :  Any					,		UpperCamelCase__   :  Optional[Any]					,		UpperCamelCase__   :  Dict					,		UpperCamelCase__   :  Any					,		UpperCamelCase__   :  Optional[int]   )					->       List[str]:
								'''simple docstring'''







								UpperCAmelCase								=  True
								UpperCAmelCase								=  TFRoFormerForCausalLM(config=__a   )
								UpperCAmelCase								=  {
								    "input_ids": input_ids,
								    "attention_mask": input_mask,
								    "token_type_ids": token_type_ids,
								}
								UpperCAmelCase								=  model(__a   )["logits"]
								self.parent.assertListEqual(
								    list(prediction_scores.numpy().shape   )					,		[self.batch_size, self.seq_length, self.vocab_size]   )






		def SCREAMING_SNAKE_CASE_      (    self   :  Optional[int]					,		UpperCamelCase__   :  List[Any]					,		UpperCamelCase__   :  Optional[int]					,		UpperCamelCase__   :  Dict					,		UpperCamelCase__   :  Dict					,		UpperCamelCase__   :  str					,		UpperCamelCase__   :  List[str]					,		UpperCamelCase__   :  List[str]   )					->       int:
								'''simple docstring'''







								UpperCAmelCase								=  TFRoFormerForMaskedLM(config=__a   )
								UpperCAmelCase								=  {
								    "input_ids": input_ids,
								    "attention_mask": input_mask,
								    "token_type_ids": token_type_ids,
								}
								UpperCAmelCase								=  model(__a   )
								self.parent.assertEqual(result.logits.shape					,		(self.batch_size, self.seq_length, self.vocab_size)   )






		def SCREAMING_SNAKE_CASE_      (    self   :  List[str]					,		UpperCamelCase__   :  List[str]					,		UpperCamelCase__   :  Optional[int]					,		UpperCamelCase__   :  int					,		UpperCamelCase__   :  Optional[int]					,		UpperCamelCase__   :  Union[str, Any]					,		UpperCamelCase__   :  str					,		UpperCamelCase__   :  Optional[int]   )					->       Tuple:
								'''simple docstring'''







								UpperCAmelCase								=  self.num_labels
								UpperCAmelCase								=  TFRoFormerForSequenceClassification(config=__a   )
								UpperCAmelCase								=  {
								    "input_ids": input_ids,
								    "attention_mask": input_mask,
								    "token_type_ids": token_type_ids,
								}

								UpperCAmelCase								=  model(__a   )
								self.parent.assertEqual(result.logits.shape					,		(self.batch_size, self.num_labels)   )






		def SCREAMING_SNAKE_CASE_      (    self   :  str					,		UpperCamelCase__   :  Any					,		UpperCamelCase__   :  Union[str, Any]					,		UpperCamelCase__   :  List[Any]					,		UpperCamelCase__   :  Union[str, Any]					,		UpperCamelCase__   :  Optional[Any]					,		UpperCamelCase__   :  int					,		UpperCamelCase__   :  Union[str, Any]   )					->       Optional[int]:
								'''simple docstring'''







								UpperCAmelCase								=  self.num_choices
								UpperCAmelCase								=  TFRoFormerForMultipleChoice(config=__a   )
								UpperCAmelCase								=  tf.tile(tf.expand_dims(__a					,		1   )					,		(1, self.num_choices, 1)   )
								UpperCAmelCase								=  tf.tile(tf.expand_dims(__a					,		1   )					,		(1, self.num_choices, 1)   )
								UpperCAmelCase								=  tf.tile(tf.expand_dims(__a					,		1   )					,		(1, self.num_choices, 1)   )
								UpperCAmelCase								=  {
								    "input_ids": multiple_choice_inputs_ids,
								    "attention_mask": multiple_choice_input_mask,
								    "token_type_ids": multiple_choice_token_type_ids,
								}
								UpperCAmelCase								=  model(__a   )
								self.parent.assertEqual(result.logits.shape					,		(self.batch_size, self.num_choices)   )






		def SCREAMING_SNAKE_CASE_      (    self   :  List[str]					,		UpperCamelCase__   :  Dict					,		UpperCamelCase__   :  Dict					,		UpperCamelCase__   :  Tuple					,		UpperCamelCase__   :  Optional[int]					,		UpperCamelCase__   :  Any					,		UpperCamelCase__   :  Any					,		UpperCamelCase__   :  Optional[Any]   )					->       List[Any]:
								'''simple docstring'''







								UpperCAmelCase								=  self.num_labels
								UpperCAmelCase								=  TFRoFormerForTokenClassification(config=__a   )
								UpperCAmelCase								=  {
								    "input_ids": input_ids,
								    "attention_mask": input_mask,
								    "token_type_ids": token_type_ids,
								}
								UpperCAmelCase								=  model(__a   )
								self.parent.assertEqual(result.logits.shape					,		(self.batch_size, self.seq_length, self.num_labels)   )






		def SCREAMING_SNAKE_CASE_      (    self   :  Dict					,		UpperCamelCase__   :  Optional[int]					,		UpperCamelCase__   :  Dict					,		UpperCamelCase__   :  Any					,		UpperCamelCase__   :  int					,		UpperCamelCase__   :  Any					,		UpperCamelCase__   :  Union[str, Any]					,		UpperCamelCase__   :  Tuple   )					->       str:
								'''simple docstring'''







								UpperCAmelCase								=  TFRoFormerForQuestionAnswering(config=__a   )
								UpperCAmelCase								=  {
								    "input_ids": input_ids,
								    "attention_mask": input_mask,
								    "token_type_ids": token_type_ids,
								}

								UpperCAmelCase								=  model(__a   )
								self.parent.assertEqual(result.start_logits.shape					,		(self.batch_size, self.seq_length)   )
								self.parent.assertEqual(result.end_logits.shape					,		(self.batch_size, self.seq_length)   )







		def SCREAMING_SNAKE_CASE_      (    self   :  List[Any]   )					->       Dict:
								'''simple docstring'''







								UpperCAmelCase								=  self.prepare_config_and_inputs()
								(
								    (
								    UpperCAmelCase    
								)							,		(
								    UpperCAmelCase    
								)							,		(
								    UpperCAmelCase    
								)							,		(
								    UpperCAmelCase    
								)							,		(
								    UpperCAmelCase    
								)							,		(
								    UpperCAmelCase    
								)							,		(
								    UpperCAmelCase    
								)							,		    
								)								=  config_and_inputs
								UpperCAmelCase								=  {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
								return config, inputs_dict





@require_tf
class 			__magic_name__ (				A__,	A__,	unittest.TestCase    ):
		lowercase   :							List[Any]        =(
		    (
		        TFRoFormerModel,
		        TFRoFormerForCausalLM,
		        TFRoFormerForMaskedLM,
		        TFRoFormerForQuestionAnswering,
		        TFRoFormerForSequenceClassification,
		        TFRoFormerForTokenClassification,
		        TFRoFormerForMultipleChoice,
		    )
		    if is_tf_available()
		    else ()
		)
		lowercase   :							str        =(
		    {
		        '''feature-extraction''': TFRoFormerModel,
		        '''fill-mask''': TFRoFormerForMaskedLM,
		        '''question-answering''': TFRoFormerForQuestionAnswering,
		        '''text-classification''': TFRoFormerForSequenceClassification,
		        '''text-generation''': TFRoFormerForCausalLM,
		        '''token-classification''': TFRoFormerForTokenClassification,
		        '''zero-shot''': TFRoFormerForSequenceClassification,
		    }
		    if is_tf_available()
		    else {}
		)

		lowercase   :							Optional[Any]        =False
		lowercase   :							str        =False






		def SCREAMING_SNAKE_CASE_      (    self   :  List[str]					,		UpperCamelCase__   :  str					,		UpperCamelCase__   :  Any					,		UpperCamelCase__   :  Optional[int]					,		UpperCamelCase__   :  Union[str, Any]					,		UpperCamelCase__   :  Union[str, Any]   )					->       List[str]:
								'''simple docstring'''







								if pipeline_test_casse_name == "TextGenerationPipelineTests":
														return True

								return False






		def SCREAMING_SNAKE_CASE_      (    self   :  int   )					->       Union[str, Any]:
								'''simple docstring'''







								UpperCAmelCase								=  TFRoFormerModelTester(self   )
								UpperCAmelCase								=  ConfigTester(self					,		config_class=__a					,		hidden_size=37   )






		def SCREAMING_SNAKE_CASE_      (    self   :  Any   )					->       str:
								'''simple docstring'''







								self.config_tester.run_common_tests()






		def SCREAMING_SNAKE_CASE_      (    self   :  Optional[int]   )					->       Dict:
								'''simple docstring'''







								UpperCAmelCase								=  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*__a   )






		def SCREAMING_SNAKE_CASE_      (    self   :  Dict   )					->       Any:
								'''simple docstring'''







								UpperCAmelCase								=  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*__a   )






		def SCREAMING_SNAKE_CASE_      (    self   :  Union[str, Any]   )					->       List[Any]:
								'''simple docstring'''







								UpperCAmelCase								=  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_lm_head(*__a   )






		def SCREAMING_SNAKE_CASE_      (    self   :  str   )					->       Optional[Any]:
								'''simple docstring'''







								UpperCAmelCase								=  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_multiple_choice(*__a   )






		def SCREAMING_SNAKE_CASE_      (    self   :  Tuple   )					->       List[str]:
								'''simple docstring'''







								UpperCAmelCase								=  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*__a   )






		def SCREAMING_SNAKE_CASE_      (    self   :  List[Any]   )					->       Tuple:
								'''simple docstring'''







								UpperCAmelCase								=  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*__a   )






		def SCREAMING_SNAKE_CASE_      (    self   :  List[Any]   )					->       Optional[Any]:
								'''simple docstring'''







								UpperCAmelCase								=  self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*__a   )







		@slow
		def SCREAMING_SNAKE_CASE_      (    self   :  Tuple   )					->       List[Any]:
								'''simple docstring'''







								UpperCAmelCase								=  TFRoFormerModel.from_pretrained("junnyu/roformer_chinese_base"   )
								self.assertIsNotNone(__a   )





@require_tf
class 			__magic_name__ (				unittest.TestCase    ):






		@slow
		def SCREAMING_SNAKE_CASE_      (    self   :  Union[str, Any]   )					->       str:
								'''simple docstring'''







								UpperCAmelCase								=  TFRoFormerForMaskedLM.from_pretrained("junnyu/roformer_chinese_base"   )
								UpperCAmelCase								=  tf.constant([[0, 1, 2, 3, 4, 5]]   )
								UpperCAmelCase								=  model(__a   )[0]

								# TODO Replace vocab size
								UpperCAmelCase								=  5_00_00

								UpperCAmelCase								=  [1, 6, vocab_size]
								self.assertEqual(output.shape					,		__a   )

								print(output[:, :3, :3]   )

								# TODO Replace values below with what was printed above.
								UpperCAmelCase								=  tf.constant(
								    [
								        [
								            [-0.12_05_33_41, -1.0_26_49_01, 0.29_22_19_46],
								            [-1.5_13_37_83, 0.19_74_33, 0.15_19_06_07],
								            [-5.0_13_54_03, -3.90_02_56, -0.84_03_87_64],
								        ]
								    ]   )
								tf.debugging.assert_near(output[:, :3, :3]					,		__a					,		atol=1e-4   )





@require_tf
class 			__magic_name__ (				unittest.TestCase    ):
		lowercase   :							Any        =1E-4






		def SCREAMING_SNAKE_CASE_      (    self   :  Union[str, Any]   )					->       List[str]:
								'''simple docstring'''







								UpperCAmelCase								=  tf.constant([[4, 10]]   )
								UpperCAmelCase								=  TFRoFormerSinusoidalPositionalEmbedding(num_positions=6					,		embedding_dim=6   )

								UpperCAmelCase								=  emba(input_ids.shape   )
								UpperCAmelCase								=  tf.constant(
								    [[0.00_00, 0.00_00, 0.00_00, 1.00_00, 1.00_00, 1.00_00], [0.84_15, 0.04_64, 0.00_22, 0.54_03, 0.99_89, 1.00_00]]   )

								tf.debugging.assert_near(__a					,		__a					,		atol=self.tolerance   )







		def SCREAMING_SNAKE_CASE_      (    self   :  Optional[int]   )					->       Dict:
								'''simple docstring'''







								UpperCAmelCase								=  tf.constant(
								    [
								        [0.00_00, 0.00_00, 0.00_00, 0.00_00, 0.00_00],
								        [0.84_15, 0.82_19, 0.80_20, 0.78_19, 0.76_17],
								        [0.90_93, 0.93_64, 0.95_81, 0.97_49, 0.98_70],
								    ]   )
								UpperCAmelCase								=  TFRoFormerSinusoidalPositionalEmbedding(num_positions=5_12					,		embedding_dim=5_12   )
								emba([2, 16, 5_12]   )
								UpperCAmelCase								=  emba.weight[:3, :5]

								tf.debugging.assert_near(__a					,		__a					,		atol=self.tolerance   )





@require_tf
class 			__magic_name__ (				unittest.TestCase    ):
		lowercase   :							int        =1E-4






		def SCREAMING_SNAKE_CASE_      (    self   :  Tuple   )					->       List[Any]:
								'''simple docstring'''







								UpperCAmelCase								=  tf.reshape(tf.range(2 * 12 * 16 * 64					,		dtype=tf.floataa   )					,		shape=(2, 12, 16, 64)   ) / 1_00

								UpperCAmelCase								=  -tf.reshape(tf.range(2 * 12 * 16 * 64					,		dtype=tf.floataa   )					,		shape=(2, 12, 16, 64)   ) / 1_00

								UpperCAmelCase								=  TFRoFormerSinusoidalPositionalEmbedding(num_positions=32					,		embedding_dim=64   )
								UpperCAmelCase								=  embed_positions([2, 16, 7_68]   )[None, None, :, :]

								UpperCAmelCase							,		UpperCAmelCase								=  TFRoFormerSelfAttention.apply_rotary_position_embeddings(
								    __a					,		__a					,		__a   )

								UpperCAmelCase								=  tf.constant(
								    [
								        [0.00_00, 0.01_00, 0.02_00, 0.03_00, 0.04_00, 0.05_00, 0.06_00, 0.07_00],
								        [-0.20_12, 0.88_97, 0.02_63, 0.94_01, 0.20_74, 0.94_63, 0.34_81, 0.93_43],
								        [-1.70_57, 0.62_71, -1.21_45, 1.38_97, -0.63_03, 1.76_47, -0.11_73, 1.89_85],
								        [-2.17_31, -1.63_97, -2.73_58, 0.28_54, -2.18_40, 1.71_83, -1.30_18, 2.48_71],
								        [0.27_17, -3.61_73, -2.92_06, -2.19_88, -3.66_38, 0.38_58, -2.91_55, 2.29_80],
								        [3.98_59, -2.15_80, -0.79_84, -4.49_04, -4.11_81, -2.02_52, -4.47_82, 1.12_53],
								    ]   )
								UpperCAmelCase								=  tf.constant(
								    [
								        [0.00_00, -0.01_00, -0.02_00, -0.03_00, -0.04_00, -0.05_00, -0.06_00, -0.07_00],
								        [0.20_12, -0.88_97, -0.02_63, -0.94_01, -0.20_74, -0.94_63, -0.34_81, -0.93_43],
								        [1.70_57, -0.62_71, 1.21_45, -1.38_97, 0.63_03, -1.76_47, 0.11_73, -1.89_85],
								        [2.17_31, 1.63_97, 2.73_58, -0.28_54, 2.18_40, -1.71_83, 1.30_18, -2.48_71],
								        [-0.27_17, 3.61_73, 2.92_06, 2.19_88, 3.66_38, -0.38_58, 2.91_55, -2.29_80],
								        [-3.98_59, 2.15_80, 0.79_84, 4.49_04, 4.11_81, 2.02_52, 4.47_82, -1.12_53],
								    ]   )

								tf.debugging.assert_near(query_layer[0, 0, :6, :8]					,		__a					,		atol=self.tolerance   )
								tf.debugging.assert_near(key_layer[0, 0, :6, :8]					,		__a					,		atol=self.tolerance   )





"""simple docstring"""





from collections.abc import Callable

import numpy as np



def  lowerCamelCase__						(     __snake_case,	__snake_case,	__snake_case,	__snake_case,	__snake_case      )				->						np.array:


					"""simple docstring"""





					_UpperCamelCase									=    int(np.ceil((x_end - xa) / step_size      )      )
					_UpperCamelCase									=    np.zeros((n + 1,)      )
					_UpperCamelCase									=    ya
					_UpperCamelCase									=    xa

					for k in range(__snake_case      ):
										_UpperCamelCase									=    y[k] + step_size * ode_func(__snake_case,	y[k]      )
										_UpperCamelCase									=    y[k] + (
										    (step_size / 2) * (ode_func(__snake_case,	y[k]      ) + ode_func(x + step_size,	__snake_case      ))
										)
										x += step_size

					return y


if __name__ == "__main__":
				import doctest

				doctest.testmod()






