




from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
 import tensorflow as tf

 from transformers import (
     TFRoFormerForCausalLM,
     TFRoFormerForMaskedLM,
     TFRoFormerForMultipleChoice,
     TFRoFormerForQuestionAnswering,
     TFRoFormerForSequenceClassification,
     TFRoFormerForTokenClassification,
     TFRoFormerModel,
 )
 from transformers.models.roformer.modeling_tf_roformer import (
     TFRoFormerSelfAttention,
     TFRoFormerSinusoidalPositionalEmbedding,
 )

class  _lowerCamelCase						:

      """simple docstring"""

      def __init__(     self					: Optional[Any] ,       snake_case					: str ,       snake_case					: Union[str, Any]=13 ,       snake_case					: int=7 ,       snake_case					: Union[str, Any]=True ,       snake_case					: int=True ,       snake_case					: Any=True ,       snake_case					: int=True ,       snake_case					: Any=99 ,       snake_case					: List[Any]=32 ,       snake_case					: str=2 ,       snake_case					: str=4 ,       snake_case					: List[Any]=37 ,       snake_case					: str="gelu" ,       snake_case					: List[Any]=0.1 ,       snake_case					: int=0.1 ,       snake_case					: Dict=512 ,       snake_case					: List[Any]=16 ,       snake_case					: List[str]=2 ,       snake_case					: Any=0.02 ,       snake_case					: List[str]=3 ,       snake_case					: Dict=4 ,       snake_case					: Optional[int]=None ,       ):
          __UpperCamelCase  		=						parent
          __UpperCamelCase  		=						13
          __UpperCamelCase  		=						7
          __UpperCamelCase  		=						True
          __UpperCamelCase  		=						True
          __UpperCamelCase  		=						True
          __UpperCamelCase  		=						True
          __UpperCamelCase  		=						99
          __UpperCamelCase  		=						32
          __UpperCamelCase  		=						2
          __UpperCamelCase  		=						4
          __UpperCamelCase  		=						37
          __UpperCamelCase  		=						'''gelu'''
          __UpperCamelCase  		=						0.1
          __UpperCamelCase  		=						0.1
          __UpperCamelCase  		=						512
          __UpperCamelCase  		=						16
          __UpperCamelCase  		=						2
          __UpperCamelCase  		=						0.02
          __UpperCamelCase  		=						3
          __UpperCamelCase  		=						4
          __UpperCamelCase  		=						None







      def       snake_case	(     self					: Union[str, Any]				):
          __UpperCamelCase  		=						ids_tensor([self.batch_size, self.seq_length] ,       self.vocab_size				)

          __UpperCamelCase  		=						None
          if self.use_input_mask:
              __UpperCamelCase  		=						random_attention_mask([self.batch_size, self.seq_length]				)

          __UpperCamelCase  		=						None
          if self.use_token_type_ids:
              __UpperCamelCase  		=						ids_tensor([self.batch_size, self.seq_length] ,       self.type_vocab_size				)

          __UpperCamelCase  		=						None
          __UpperCamelCase  		=						None
          __UpperCamelCase  		=						None
          if self.use_labels:
              __UpperCamelCase  		=						ids_tensor([self.batch_size] ,       self.type_sequence_label_size				)
              __UpperCamelCase  		=						ids_tensor([self.batch_size, self.seq_length] ,       self.num_labels				)
              __UpperCamelCase  		=						ids_tensor([self.batch_size] ,       self.num_choices				)

          __UpperCamelCase  		=						RoFormerConfig(
              vocab_size=self.vocab_size ,       hidden_size=self.hidden_size ,       num_hidden_layers=self.num_hidden_layers ,       num_attention_heads=self.num_attention_heads ,       intermediate_size=self.intermediate_size ,       hidden_act=self.hidden_act ,       hidden_dropout_prob=self.hidden_dropout_prob ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob ,       max_position_embeddings=self.max_position_embeddings ,       type_vocab_size=self.type_vocab_size ,       initializer_range=self.initializer_range ,       return_dict=snake_case ,       )

          return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







      def       snake_case	(     self					: List[str] ,       snake_case					: int ,       snake_case					: Optional[Any] ,       snake_case					: Optional[int] ,       snake_case					: Tuple ,       snake_case					: Dict ,       snake_case					: List[str] ,       snake_case					: List[Any]				):
          __UpperCamelCase  		=						TFRoFormerModel(config=snake_case				)
          __UpperCamelCase  		=						{'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

          __UpperCamelCase  		=						[input_ids, input_mask]
          __UpperCamelCase  		=						model(snake_case				)

          __UpperCamelCase  		=						model(snake_case				)

          self.parent.assertEqual(result.last_hidden_state.shape ,       (self.batch_size, self.seq_length, self.hidden_size)				)







      def       snake_case	(     self					: Tuple ,       snake_case					: Dict ,       snake_case					: List[Any] ,       snake_case					: Any ,       snake_case					: Dict ,       snake_case					: int ,       snake_case					: List[Any] ,       snake_case					: List[str]				):
          __UpperCamelCase  		=						True
          __UpperCamelCase  		=						TFRoFormerForCausalLM(config=snake_case				)
          __UpperCamelCase  		=						{
              '''input_ids''': input_ids,
              '''attention_mask''': input_mask,
              '''token_type_ids''': token_type_ids,
          }
          __UpperCamelCase  		=						model(snake_case				)['''logits''']
          self.parent.assertListEqual(
              list(prediction_scores.numpy().shape				) ,       [self.batch_size, self.seq_length, self.vocab_size]				)







      def       snake_case	(     self					: List[Any] ,       snake_case					: Dict ,       snake_case					: Optional[Any] ,       snake_case					: List[str] ,       snake_case					: Dict ,       snake_case					: int ,       snake_case					: Union[str, Any] ,       snake_case					: Tuple				):
          __UpperCamelCase  		=						TFRoFormerForMaskedLM(config=snake_case				)
          __UpperCamelCase  		=						{
              '''input_ids''': input_ids,
              '''attention_mask''': input_mask,
              '''token_type_ids''': token_type_ids,
          }
          __UpperCamelCase  		=						model(snake_case				)
          self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.seq_length, self.vocab_size)				)







      def       snake_case	(     self					: Tuple ,       snake_case					: Union[str, Any] ,       snake_case					: List[str] ,       snake_case					: Optional[int] ,       snake_case					: Dict ,       snake_case					: Tuple ,       snake_case					: int ,       snake_case					: int				):
          __UpperCamelCase  		=						self.num_labels
          __UpperCamelCase  		=						TFRoFormerForSequenceClassification(config=snake_case				)
          __UpperCamelCase  		=						{
              '''input_ids''': input_ids,
              '''attention_mask''': input_mask,
              '''token_type_ids''': token_type_ids,
          }

          __UpperCamelCase  		=						model(snake_case				)
          self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.num_labels)				)







      def       snake_case	(     self					: Optional[Any] ,       snake_case					: int ,       snake_case					: Any ,       snake_case					: str ,       snake_case					: Dict ,       snake_case					: List[str] ,       snake_case					: Optional[Any] ,       snake_case					: Tuple				):
          __UpperCamelCase  		=						self.num_choices
          __UpperCamelCase  		=						TFRoFormerForMultipleChoice(config=snake_case				)
          __UpperCamelCase  		=						tf.tile(tf.expand_dims(snake_case ,       1				) ,       (1, self.num_choices, 1)				)
          __UpperCamelCase  		=						tf.tile(tf.expand_dims(snake_case ,       1				) ,       (1, self.num_choices, 1)				)
          __UpperCamelCase  		=						tf.tile(tf.expand_dims(snake_case ,       1				) ,       (1, self.num_choices, 1)				)
          __UpperCamelCase  		=						{
              '''input_ids''': multiple_choice_inputs_ids,
              '''attention_mask''': multiple_choice_input_mask,
              '''token_type_ids''': multiple_choice_token_type_ids,
          }
          __UpperCamelCase  		=						model(snake_case				)
          self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.num_choices)				)







      def       snake_case	(     self					: Optional[int] ,       snake_case					: Union[str, Any] ,       snake_case					: Dict ,       snake_case					: List[Any] ,       snake_case					: List[str] ,       snake_case					: Any ,       snake_case					: List[str] ,       snake_case					: int				):
          __UpperCamelCase  		=						self.num_labels
          __UpperCamelCase  		=						TFRoFormerForTokenClassification(config=snake_case				)
          __UpperCamelCase  		=						{
              '''input_ids''': input_ids,
              '''attention_mask''': input_mask,
              '''token_type_ids''': token_type_ids,
          }
          __UpperCamelCase  		=						model(snake_case				)
          self.parent.assertEqual(result.logits.shape ,       (self.batch_size, self.seq_length, self.num_labels)				)







      def       snake_case	(     self					: int ,       snake_case					: int ,       snake_case					: Dict ,       snake_case					: Optional[int] ,       snake_case					: Union[str, Any] ,       snake_case					: List[Any] ,       snake_case					: List[Any] ,       snake_case					: str				):
          __UpperCamelCase  		=						TFRoFormerForQuestionAnswering(config=snake_case				)
          __UpperCamelCase  		=						{
              '''input_ids''': input_ids,
              '''attention_mask''': input_mask,
              '''token_type_ids''': token_type_ids,
          }

          __UpperCamelCase  		=						model(snake_case				)
          self.parent.assertEqual(result.start_logits.shape ,       (self.batch_size, self.seq_length)				)
          self.parent.assertEqual(result.end_logits.shape ,       (self.batch_size, self.seq_length)				)







      def       snake_case	(     self					: List[Any]				):
          __UpperCamelCase  		=						self.prepare_config_and_inputs()
          (
              __UpperCamelCase		
          )  		=						config_and_inputs
          __UpperCamelCase  		=						{'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
          return config, inputs_dict







@require_tf
class  _lowerCamelCase						(			__A							,			__A							,			unittest.TestCase   ):

      """simple docstring"""

      lowerCAmelCase__			: Any    =					(
          (
              TFRoFormerModel,
              TFRoFormerForCausalLM,
              TFRoFormerForMaskedLM,
              TFRoFormerForQuestionAnswering,
              TFRoFormerForSequenceClassification,
              TFRoFormerForTokenClassification,
              TFRoFormerForMultipleChoice,
          )
          if is_tf_available()
          else ()
      )
      lowerCAmelCase__			: Any    =					(
          {
              "feature-extraction": TFRoFormerModel,
              "fill-mask": TFRoFormerForMaskedLM,
              "question-answering": TFRoFormerForQuestionAnswering,
              "text-classification": TFRoFormerForSequenceClassification,
              "text-generation": TFRoFormerForCausalLM,
              "token-classification": TFRoFormerForTokenClassification,
              "zero-shot": TFRoFormerForSequenceClassification,
          }
          if is_tf_available()
          else {}
      )

      lowerCAmelCase__			: Tuple    =					False
      lowerCAmelCase__			: Any    =					False







      def       snake_case	(     self					: Dict ,       snake_case					: str ,       snake_case					: List[Any] ,       snake_case					: int ,       snake_case					: Optional[int] ,       snake_case					: str				):

          if pipeline_test_casse_name == "TextGenerationPipelineTests":
              return True

          return False







      def       snake_case	(     self					: str				):
          __UpperCamelCase  		=						TFRoFormerModelTester(self				)
          __UpperCamelCase  		=						ConfigTester(self ,       config_class=snake_case ,       hidden_size=37				)







      def       snake_case	(     self					: Optional[Any]				):
          self.config_tester.run_common_tests()







      def       snake_case	(     self					: List[str]				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*snake_case				)







      def       snake_case	(     self					: List[Any]				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_masked_lm(*snake_case				)







      def       snake_case	(     self					: str				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_lm_head(*snake_case				)







      def       snake_case	(     self					: int				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_multiple_choice(*snake_case				)







      def       snake_case	(     self					: Dict				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_question_answering(*snake_case				)







      def       snake_case	(     self					: Dict				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_sequence_classification(*snake_case				)







      def       snake_case	(     self					: List[Any]				):
          __UpperCamelCase  		=						self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_token_classification(*snake_case				)







      @slow
      def       snake_case	(     self					: str				):
          __UpperCamelCase  		=						TFRoFormerModel.from_pretrained('''junnyu/roformer_chinese_base'''				)
          self.assertIsNotNone(snake_case				)







@require_tf
class  _lowerCamelCase						(			unittest.TestCase   ):

      """simple docstring"""

      @slow
      def       snake_case	(     self					: Union[str, Any]				):
          __UpperCamelCase  		=						TFRoFormerForMaskedLM.from_pretrained('''junnyu/roformer_chinese_base'''				)
          __UpperCamelCase  		=						tf.constant([[0, 1, 2, 3, 4, 5]]				)
          __UpperCamelCase  		=						model(snake_case				)[0]

          # TODO Replace vocab size
          __UpperCamelCase  		=						50000

          __UpperCamelCase  		=						[1, 6, vocab_size]
          self.assertEqual(output.shape ,       snake_case				)

          print(output[:, :3, :3]				)

          # TODO Replace values below with what was printed above.
          __UpperCamelCase  		=						tf.constant(
              [
                  [
                      [-0.12_05_33_41, -1.0_26_49_01, 0.29_22_19_46],
                      [-1.5_13_37_83, 0.19_74_33, 0.15_19_06_07],
                      [-5.0_13_54_03, -3.90_02_56, -0.84_03_87_64],
                  ]
              ]				)
          tf.debugging.assert_near(output[:, :3, :3] ,       snake_case ,       atol=1E-4				)







@require_tf
class  _lowerCamelCase						(			unittest.TestCase   ):

      """simple docstring"""

      lowerCAmelCase__			: List[str]    =					1E-4







      def       snake_case	(     self					: int				):
          __UpperCamelCase  		=						tf.constant([[4, 10]]				)
          __UpperCamelCase  		=						TFRoFormerSinusoidalPositionalEmbedding(num_positions=6 ,       embedding_dim=6				)

          __UpperCamelCase  		=						emba(input_ids.shape				)
          __UpperCamelCase  		=						tf.constant(
              [[0.00_00, 0.00_00, 0.00_00, 1.00_00, 1.00_00, 1.00_00], [0.84_15, 0.04_64, 0.00_22, 0.54_03, 0.99_89, 1.00_00]]				)

          tf.debugging.assert_near(snake_case ,       snake_case ,       atol=self.tolerance				)







      def       snake_case	(     self					: Optional[Any]				):
          __UpperCamelCase  		=						tf.constant(
              [
                  [0.00_00, 0.00_00, 0.00_00, 0.00_00, 0.00_00],
                  [0.84_15, 0.82_19, 0.80_20, 0.78_19, 0.76_17],
                  [0.90_93, 0.93_64, 0.95_81, 0.97_49, 0.98_70],
              ]				)
          __UpperCamelCase  		=						TFRoFormerSinusoidalPositionalEmbedding(num_positions=512 ,       embedding_dim=512				)
          emba([2, 16, 512]				)
          __UpperCamelCase  		=						emba.weight[:3, :5]

          tf.debugging.assert_near(snake_case ,       snake_case ,       atol=self.tolerance				)







@require_tf
class  _lowerCamelCase						(			unittest.TestCase   ):

      """simple docstring"""

      lowerCAmelCase__			: Dict    =					1E-4







      def       snake_case	(     self					: Dict				):
          __UpperCamelCase  		=						tf.reshape(tf.range(2 * 12 * 16 * 64 ,       dtype=tf.floataa				) ,       shape=(2, 12, 16, 64)				) / 100

          __UpperCamelCase  		=						-tf.reshape(tf.range(2 * 12 * 16 * 64 ,       dtype=tf.floataa				) ,       shape=(2, 12, 16, 64)				) / 100

          __UpperCamelCase  		=						TFRoFormerSinusoidalPositionalEmbedding(num_positions=32 ,       embedding_dim=64				)
          __UpperCamelCase  		=						embed_positions([2, 16, 768]				)[None, None, :, :]

          __UpperCamelCase  		=						TFRoFormerSelfAttention.apply_rotary_position_embeddings(
              snake_case ,       snake_case ,       snake_case				)

          __UpperCamelCase  		=						tf.constant(
              [
                  [0.00_00, 0.01_00, 0.02_00, 0.03_00, 0.04_00, 0.05_00, 0.06_00, 0.07_00],
                  [-0.20_12, 0.88_97, 0.02_63, 0.94_01, 0.20_74, 0.94_63, 0.34_81, 0.93_43],
                  [-1.70_57, 0.62_71, -1.21_45, 1.38_97, -0.63_03, 1.76_47, -0.11_73, 1.89_85],
                  [-2.17_31, -1.63_97, -2.73_58, 0.28_54, -2.18_40, 1.71_83, -1.30_18, 2.48_71],
                  [0.27_17, -3.61_73, -2.92_06, -2.19_88, -3.66_38, 0.38_58, -2.91_55, 2.29_80],
                  [3.98_59, -2.15_80, -0.79_84, -4.49_04, -4.11_81, -2.02_52, -4.47_82, 1.12_53],
              ]				)
          __UpperCamelCase  		=						tf.constant(
              [
                  [0.00_00, -0.01_00, -0.02_00, -0.03_00, -0.04_00, -0.05_00, -0.06_00, -0.07_00],
                  [0.20_12, -0.88_97, -0.02_63, -0.94_01, -0.20_74, -0.94_63, -0.34_81, -0.93_43],
                  [1.70_57, -0.62_71, 1.21_45, -1.38_97, 0.63_03, -1.76_47, 0.11_73, -1.89_85],
                  [2.17_31, 1.63_97, 2.73_58, -0.28_54, 2.18_40, -1.71_83, 1.30_18, -2.48_71],
                  [-0.27_17, 3.61_73, 2.92_06, 2.19_88, 3.66_38, -0.38_58, 2.91_55, -2.29_80],
                  [-3.98_59, 2.15_80, 0.79_84, 4.49_04, 4.11_81, 2.02_52, 4.47_82, -1.12_53],
              ]				)

          tf.debugging.assert_near(query_layer[0, 0, :6, :8] ,       snake_case ,       atol=self.tolerance				)
          tf.debugging.assert_near(key_layer[0, 0, :6, :8] ,       snake_case ,       atol=self.tolerance				)












from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


a_   		=     {
    "configuration_longformer": [
        "LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "LongformerConfig",
        "LongformerOnnxConfig",
    ],
    "tokenization_longformer": ["LongformerTokenizer"],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_   		=     ["LongformerTokenizerFast"]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_   		=     [
     "LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
     "LongformerForMaskedLM",
     "LongformerForMultipleChoice",
     "LongformerForQuestionAnswering",
     "LongformerForSequenceClassification",
     "LongformerForTokenClassification",
     "LongformerModel",
     "LongformerPreTrainedModel",
     "LongformerSelfAttention",
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_   		=     [
     "TF_LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
     "TFLongformerForMaskedLM",
     "TFLongformerForMultipleChoice",
     "TFLongformerForQuestionAnswering",
     "TFLongformerForSequenceClassification",
     "TFLongformerForTokenClassification",
     "TFLongformerModel",
     "TFLongformerPreTrainedModel",
     "TFLongformerSelfAttention",
 ]


if TYPE_CHECKING:
 from .configuration_longformer import (
     LONGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
     LongformerConfig,
     LongformerOnnxConfig,
 )
 from .tokenization_longformer import LongformerTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_longformer_fast import LongformerTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_longformer import (
      LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
      LongformerForMaskedLM,
      LongformerForMultipleChoice,
      LongformerForQuestionAnswering,
      LongformerForSequenceClassification,
      LongformerForTokenClassification,
      LongformerModel,
      LongformerPreTrainedModel,
      LongformerSelfAttention,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_longformer import (
      TF_LONGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFLongformerForMaskedLM,
      TFLongformerForMultipleChoice,
      TFLongformerForQuestionAnswering,
      TFLongformerForSequenceClassification,
      TFLongformerForTokenClassification,
      TFLongformerModel,
      TFLongformerPreTrainedModel,
      TFLongformerSelfAttention,
  )

else:
 import sys

 a_   		=     _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






