



from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import (
							    TFRoFormerForCausalLM,
							    TFRoFormerForMaskedLM,
							    TFRoFormerForMultipleChoice,
							    TFRoFormerForQuestionAnswering,
							    TFRoFormerForSequenceClassification,
							    TFRoFormerForTokenClassification,
							    TFRoFormerModel,
							)
							from transformers.models.roformer.modeling_tf_roformer import (
							    TFRoFormerSelfAttention,
							    TFRoFormerSinusoidalPositionalEmbedding,
							)
class 					_snake_case      :




				def __init__(    self       :    Optional[int],       __lowercase       :    Any,       __lowercase       :    List[str]=13,       __lowercase       :    List[Any]=7,       __lowercase       :    Tuple=True,       __lowercase       :    Dict=True,       __lowercase       :    str=True,       __lowercase       :    Tuple=True,       __lowercase       :    Optional[int]=99,       __lowercase       :    List[Any]=32,       __lowercase       :    Optional[Any]=2,       __lowercase       :    Any=4,       __lowercase       :    Optional[int]=37,       __lowercase       :    List[str]="gelu",       __lowercase       :    Union[str, Any]=0.1,       __lowercase       :    Tuple=0.1,       __lowercase       :    Optional[Any]=512,       __lowercase       :    int=16,       __lowercase       :    List[str]=2,       __lowercase       :    List[Any]=0.02,       __lowercase       :    Dict=3,       __lowercase       :    Union[str, Any]=4,       __lowercase       :    List[Any]=None,       ):
							lowercase__       	=					parent
							lowercase__       	=					13
							lowercase__       	=					7
							lowercase__       	=					True
							lowercase__       	=					True
							lowercase__       	=					True
							lowercase__       	=					True
							lowercase__       	=					99
							lowercase__       	=					32
							lowercase__       	=					2
							lowercase__       	=					4
							lowercase__       	=					37
							lowercase__       	=					"gelu"
							lowercase__       	=					0.1
							lowercase__       	=					0.1
							lowercase__       	=					512
							lowercase__       	=					16
							lowercase__       	=					2
							lowercase__       	=					0.02
							lowercase__       	=					3
							lowercase__       	=					4
							lowercase__       	=					None




				def 				A__   (    self       :    int    ):
							lowercase__       	=					ids_tensor([self.batch_size, self.seq_length],       self.vocab_size    )

							lowercase__       	=					None
							if self.use_input_mask:
										lowercase__       	=					random_attention_mask([self.batch_size, self.seq_length]    )

							lowercase__       	=					None
							if self.use_token_type_ids:
										lowercase__       	=					ids_tensor([self.batch_size, self.seq_length],       self.type_vocab_size    )

							lowercase__       	=					None
							lowercase__       	=					None
							lowercase__       	=					None
							if self.use_labels:
										lowercase__       	=					ids_tensor([self.batch_size],       self.type_sequence_label_size    )
										lowercase__       	=					ids_tensor([self.batch_size, self.seq_length],       self.num_labels    )
										lowercase__       	=					ids_tensor([self.batch_size],       self.num_choices    )

							lowercase__       	=					RoFormerConfig(
							    vocab_size=self.vocab_size,       hidden_size=self.hidden_size,       num_hidden_layers=self.num_hidden_layers,       num_attention_heads=self.num_attention_heads,       intermediate_size=self.intermediate_size,       hidden_act=self.hidden_act,       hidden_dropout_prob=self.hidden_dropout_prob,       attention_probs_dropout_prob=self.attention_probs_dropout_prob,       max_position_embeddings=self.max_position_embeddings,       type_vocab_size=self.type_vocab_size,       initializer_range=self.initializer_range,       return_dict=lowercase_,       )

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels




				def 				A__   (    self       :    Tuple,       __lowercase       :    Any,       __lowercase       :    List[Any],       __lowercase       :    int,       __lowercase       :    Optional[int],       __lowercase       :    Any,       __lowercase       :    Optional[Any],       __lowercase       :    str    ):
							lowercase__       	=					TFRoFormerModel(config=lowercase_    )
							lowercase__       	=					{"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}

							lowercase__       	=					[input_ids, input_mask]
							lowercase__       	=					model(lowercase_    )

							lowercase__       	=					model(lowercase_    )

							self.parent.assertEqual(result.last_hidden_state.shape,       (self.batch_size, self.seq_length, self.hidden_size)    )




				def 				A__   (    self       :    Dict,       __lowercase       :    Union[str, Any],       __lowercase       :    Optional[Any],       __lowercase       :    int,       __lowercase       :    List[Any],       __lowercase       :    Tuple,       __lowercase       :    List[Any],       __lowercase       :    Tuple    ):
							lowercase__       	=					True
							lowercase__       	=					TFRoFormerForCausalLM(config=lowercase_    )
							lowercase__       	=					{
							    "input_ids": input_ids,
							    "attention_mask": input_mask,
							    "token_type_ids": token_type_ids,
							}
							lowercase__       	=					model(lowercase_    )["logits"]
							self.parent.assertListEqual(
							    list(prediction_scores.numpy().shape    ),       [self.batch_size, self.seq_length, self.vocab_size]    )




				def 				A__   (    self       :    Optional[int],       __lowercase       :    List[str],       __lowercase       :    Any,       __lowercase       :    Dict,       __lowercase       :    Dict,       __lowercase       :    int,       __lowercase       :    int,       __lowercase       :    str    ):
							lowercase__       	=					TFRoFormerForMaskedLM(config=lowercase_    )
							lowercase__       	=					{
							    "input_ids": input_ids,
							    "attention_mask": input_mask,
							    "token_type_ids": token_type_ids,
							}
							lowercase__       	=					model(lowercase_    )
							self.parent.assertEqual(result.logits.shape,       (self.batch_size, self.seq_length, self.vocab_size)    )




				def 				A__   (    self       :    Union[str, Any],       __lowercase       :    Optional[int],       __lowercase       :    Union[str, Any],       __lowercase       :    List[Any],       __lowercase       :    Optional[int],       __lowercase       :    List[str],       __lowercase       :    Tuple,       __lowercase       :    Any    ):
							lowercase__       	=					self.num_labels
							lowercase__       	=					TFRoFormerForSequenceClassification(config=lowercase_    )
							lowercase__       	=					{
							    "input_ids": input_ids,
							    "attention_mask": input_mask,
							    "token_type_ids": token_type_ids,
							}

							lowercase__       	=					model(lowercase_    )
							self.parent.assertEqual(result.logits.shape,       (self.batch_size, self.num_labels)    )




				def 				A__   (    self       :    Dict,       __lowercase       :    str,       __lowercase       :    List[Any],       __lowercase       :    Union[str, Any],       __lowercase       :    int,       __lowercase       :    Union[str, Any],       __lowercase       :    Optional[int],       __lowercase       :    Any    ):
							lowercase__       	=					self.num_choices
							lowercase__       	=					TFRoFormerForMultipleChoice(config=lowercase_    )
							lowercase__       	=					tf.tile(tf.expand_dims(lowercase_,       1    ),       (1, self.num_choices, 1)    )
							lowercase__       	=					tf.tile(tf.expand_dims(lowercase_,       1    ),       (1, self.num_choices, 1)    )
							lowercase__       	=					tf.tile(tf.expand_dims(lowercase_,       1    ),       (1, self.num_choices, 1)    )
							lowercase__       	=					{
							    "input_ids": multiple_choice_inputs_ids,
							    "attention_mask": multiple_choice_input_mask,
							    "token_type_ids": multiple_choice_token_type_ids,
							}
							lowercase__       	=					model(lowercase_    )
							self.parent.assertEqual(result.logits.shape,       (self.batch_size, self.num_choices)    )




				def 				A__   (    self       :    int,       __lowercase       :    List[str],       __lowercase       :    List[str],       __lowercase       :    List[Any],       __lowercase       :    Dict,       __lowercase       :    int,       __lowercase       :    Any,       __lowercase       :    Tuple    ):
							lowercase__       	=					self.num_labels
							lowercase__       	=					TFRoFormerForTokenClassification(config=lowercase_    )
							lowercase__       	=					{
							    "input_ids": input_ids,
							    "attention_mask": input_mask,
							    "token_type_ids": token_type_ids,
							}
							lowercase__       	=					model(lowercase_    )
							self.parent.assertEqual(result.logits.shape,       (self.batch_size, self.seq_length, self.num_labels)    )




				def 				A__   (    self       :    int,       __lowercase       :    Tuple,       __lowercase       :    Optional[Any],       __lowercase       :    Tuple,       __lowercase       :    Optional[int],       __lowercase       :    int,       __lowercase       :    Any,       __lowercase       :    List[Any]    ):
							lowercase__       	=					TFRoFormerForQuestionAnswering(config=lowercase_    )
							lowercase__       	=					{
							    "input_ids": input_ids,
							    "attention_mask": input_mask,
							    "token_type_ids": token_type_ids,
							}

							lowercase__       	=					model(lowercase_    )
							self.parent.assertEqual(result.start_logits.shape,       (self.batch_size, self.seq_length)    )
							self.parent.assertEqual(result.end_logits.shape,       (self.batch_size, self.seq_length)    )




				def 				A__   (    self       :    int    ):
							lowercase__       	=					self.prepare_config_and_inputs()
							(
							    (
							    lowercase__						
							)	,							(
							    lowercase__						
							)	,							(
							    lowercase__						
							)	,							(
							    lowercase__						
							)	,							(
							    lowercase__						
							)	,							(
							    lowercase__						
							)	,							(
							    lowercase__						
							)	,													
							)       	=					config_and_inputs
							lowercase__       	=					{"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
							return config, inputs_dict
@require_tf
class 					_snake_case      (       lowercase__     ,						lowercase__     ,						unittest.TestCase):
				UpperCamelCase__					:       str  		=(
				    (
				        TFRoFormerModel,
				        TFRoFormerForCausalLM,
				        TFRoFormerForMaskedLM,
				        TFRoFormerForQuestionAnswering,
				        TFRoFormerForSequenceClassification,
				        TFRoFormerForTokenClassification,
				        TFRoFormerForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				UpperCamelCase__					:       Dict  		=(
				    {
				        """feature-extraction""": TFRoFormerModel,
				        """fill-mask""": TFRoFormerForMaskedLM,
				        """question-answering""": TFRoFormerForQuestionAnswering,
				        """text-classification""": TFRoFormerForSequenceClassification,
				        """text-generation""": TFRoFormerForCausalLM,
				        """token-classification""": TFRoFormerForTokenClassification,
				        """zero-shot""": TFRoFormerForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)

				UpperCamelCase__					:       int  		=False
				UpperCamelCase__					:       List[Any]  		=False




				def 				A__   (    self       :    int,       __lowercase       :    Any,       __lowercase       :    List[Any],       __lowercase       :    Optional[Any],       __lowercase       :    str,       __lowercase       :    Optional[Any]    ):







							if pipeline_test_casse_name == "TextGenerationPipelineTests":
										return True

							return False




				def 				A__   (    self       :    Any    ):
							lowercase__       	=					TFRoFormerModelTester(self    )
							lowercase__       	=					ConfigTester(self,       config_class=lowercase_,       hidden_size=37    )




				def 				A__   (    self       :    Optional[int]    ):
							self.config_tester.run_common_tests()




				def 				A__   (    self       :    Union[str, Any]    ):
							lowercase__       	=					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*lowercase_    )




				def 				A__   (    self       :    int    ):
							lowercase__       	=					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*lowercase_    )




				def 				A__   (    self       :    Optional[Any]    ):
							lowercase__       	=					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_lm_head(*lowercase_    )




				def 				A__   (    self       :    Union[str, Any]    ):
							lowercase__       	=					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*lowercase_    )




				def 				A__   (    self       :    Optional[int]    ):
							lowercase__       	=					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*lowercase_    )




				def 				A__   (    self       :    Optional[Any]    ):
							lowercase__       	=					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*lowercase_    )




				def 				A__   (    self       :    Dict    ):
							lowercase__       	=					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*lowercase_    )




				@slow
				def 				A__   (    self       :    Optional[Any]    ):
							lowercase__       	=					TFRoFormerModel.from_pretrained("junnyu/roformer_chinese_base"    )
							self.assertIsNotNone(lowercase_    )
@require_tf
class 					_snake_case      (       unittest.TestCase):




				@slow
				def 				A__   (    self       :    Dict    ):
							lowercase__       	=					TFRoFormerForMaskedLM.from_pretrained("junnyu/roformer_chinese_base"    )
							lowercase__       	=					tf.constant([[0, 1, 2, 3, 4, 5]]    )
							lowercase__       	=					model(lowercase_    )[0]

							# TODO Replace vocab size
							lowercase__       	=					5_0000

							lowercase__       	=					[1, 6, vocab_size]
							self.assertEqual(output.shape,       lowercase_    )

							print(output[:, :3, :3]    )

							# TODO Replace values below with what was printed above.
							lowercase__       	=					tf.constant(
							    [
							        [
							            [-0.12053341, -1.0264901, 0.29221946],
							            [-1.5133783, 0.197433, 0.15190607],
							            [-5.0135403, -3.900256, -0.84038764],
							        ]
							    ]    )
							tf.debugging.assert_near(output[:, :3, :3],       lowercase_,       atol=1e-4    )
@require_tf
class 					_snake_case      (       unittest.TestCase):
				UpperCamelCase__					:       int  		=1E-4




				def 				A__   (    self       :    int    ):
							lowercase__       	=					tf.constant([[4, 10]]    )
							lowercase__       	=					TFRoFormerSinusoidalPositionalEmbedding(num_positions=6,       embedding_dim=6    )

							lowercase__       	=					emba(input_ids.shape    )
							lowercase__       	=					tf.constant(
							    [[0.0000, 0.0000, 0.0000, 1.0000, 1.0000, 1.0000], [0.8415, 0.0464, 0.0022, 0.5403, 0.9989, 1.0000]]    )

							tf.debugging.assert_near(lowercase_,       lowercase_,       atol=self.tolerance    )




				def 				A__   (    self       :    int    ):
							lowercase__       	=					tf.constant(
							    [
							        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
							        [0.8415, 0.8219, 0.8020, 0.7819, 0.7617],
							        [0.9093, 0.9364, 0.9581, 0.9749, 0.9870],
							    ]    )
							lowercase__       	=					TFRoFormerSinusoidalPositionalEmbedding(num_positions=512,       embedding_dim=512    )
							emba([2, 16, 512]    )
							lowercase__       	=					emba.weight[:3, :5]

							tf.debugging.assert_near(lowercase_,       lowercase_,       atol=self.tolerance    )
@require_tf
class 					_snake_case      (       unittest.TestCase):
				UpperCamelCase__					:       List[str]  		=1E-4




				def 				A__   (    self       :    List[str]    ):
							lowercase__       	=					tf.reshape(tf.range(2 * 12 * 16 * 64,       dtype=tf.floataa    ),       shape=(2, 12, 16, 64)    ) / 100

							lowercase__       	=					-tf.reshape(tf.range(2 * 12 * 16 * 64,       dtype=tf.floataa    ),       shape=(2, 12, 16, 64)    ) / 100

							lowercase__       	=					TFRoFormerSinusoidalPositionalEmbedding(num_positions=32,       embedding_dim=64    )
							lowercase__       	=					embed_positions([2, 16, 768]    )[None, None, :, :]

							lowercase__	,							lowercase__       	=					TFRoFormerSelfAttention.apply_rotary_position_embeddings(
							    lowercase_,       lowercase_,       lowercase_    )

							lowercase__       	=					tf.constant(
							    [
							        [0.0000, 0.0100, 0.0200, 0.0300, 0.0400, 0.0500, 0.0600, 0.0700],
							        [-0.2012, 0.8897, 0.0263, 0.9401, 0.2074, 0.9463, 0.3481, 0.9343],
							        [-1.7057, 0.6271, -1.2145, 1.3897, -0.6303, 1.7647, -0.1173, 1.8985],
							        [-2.1731, -1.6397, -2.7358, 0.2854, -2.1840, 1.7183, -1.3018, 2.4871],
							        [0.2717, -3.6173, -2.9206, -2.1988, -3.6638, 0.3858, -2.9155, 2.2980],
							        [3.9859, -2.1580, -0.7984, -4.4904, -4.1181, -2.0252, -4.4782, 1.1253],
							    ]    )
							lowercase__       	=					tf.constant(
							    [
							        [0.0000, -0.0100, -0.0200, -0.0300, -0.0400, -0.0500, -0.0600, -0.0700],
							        [0.2012, -0.8897, -0.0263, -0.9401, -0.2074, -0.9463, -0.3481, -0.9343],
							        [1.7057, -0.6271, 1.2145, -1.3897, 0.6303, -1.7647, 0.1173, -1.8985],
							        [2.1731, 1.6397, 2.7358, -0.2854, 2.1840, -1.7183, 1.3018, -2.4871],
							        [-0.2717, 3.6173, 2.9206, 2.1988, 3.6638, -0.3858, 2.9155, -2.2980],
							        [-3.9859, 2.1580, 0.7984, 4.4904, 4.1181, 2.0252, 4.4782, -1.1253],
							    ]    )

							tf.debugging.assert_near(query_layer[0, 0, :6, :8],       lowercase_,       atol=self.tolerance    )
							tf.debugging.assert_near(key_layer[0, 0, :6, :8],       lowercase_,       atol=self.tolerance    )





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCamelCase__  =					{
    '''configuration_longformer''': [
        '''LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''LongformerConfig''',
        '''LongformerOnnxConfig''',
    ],
    '''tokenization_longformer''': ['''LongformerTokenizer'''],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowerCamelCase__  =					['''LongformerTokenizerFast''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowerCamelCase__  =					[
       '''LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''LongformerForMaskedLM''',
       '''LongformerForMultipleChoice''',
       '''LongformerForQuestionAnswering''',
       '''LongformerForSequenceClassification''',
       '''LongformerForTokenClassification''',
       '''LongformerModel''',
       '''LongformerPreTrainedModel''',
       '''LongformerSelfAttention''',
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowerCamelCase__  =					[
       '''TF_LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''TFLongformerForMaskedLM''',
       '''TFLongformerForMultipleChoice''',
       '''TFLongformerForQuestionAnswering''',
       '''TFLongformerForSequenceClassification''',
       '''TFLongformerForTokenClassification''',
       '''TFLongformerModel''',
       '''TFLongformerPreTrainedModel''',
       '''TFLongformerSelfAttention''',
   ]


if TYPE_CHECKING:
   from .configuration_longformer import (
       LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
       LongformerConfig,
       LongformerOnnxConfig,
   )
   from .tokenization_longformer import LongformerTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_longformer_fast import LongformerTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_longformer import (
          LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          LongformerForMaskedLM,
          LongformerForMultipleChoice,
          LongformerForQuestionAnswering,
          LongformerForSequenceClassification,
          LongformerForTokenClassification,
          LongformerModel,
          LongformerPreTrainedModel,
          LongformerSelfAttention,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_longformer import (
          TF_LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFLongformerForMaskedLM,
          TFLongformerForMultipleChoice,
          TFLongformerForQuestionAnswering,
          TFLongformerForSequenceClassification,
          TFLongformerForTokenClassification,
          TFLongformerModel,
          TFLongformerPreTrainedModel,
          TFLongformerSelfAttention,
      )

else:
   import sys

   lowerCamelCase__  =					_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




