





from typing import Optional, Union

import torch
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithPoolingAndNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_mobilenet_va import MobileNetVaConfig


snake_case_           =     logging.get_logger(__name__)


# General docstring
snake_case_           =     """MobileNetV1Config"""

# Base docstring
snake_case_           =     """google/mobilenet_v1_1.0_224"""
snake_case_           =     [1, 1024, 7, 7]

# Image classification docstring
snake_case_           =     """google/mobilenet_v1_1.0_224"""
snake_case_           =     """tabby, tabby cat"""


snake_case_           =     [
    """google/mobilenet_v1_1.0_224""",
    """google/mobilenet_v1_0.75_192""",
    # See all MobileNetV1 models at https://huggingface.co/models?filter=mobilenet_v1
]






def lowerCamelCase__       (				snake_case_      :    Tuple			, snake_case_      :    Tuple			, snake_case_      :    Union[str, Any]=None			) ->							Any:
   __snake_case							=  {}

   if isinstance(snake_case_			, snake_case_			):
      __snake_case							=  model.mobilenet_va
   else:
      __snake_case							=  model

   __snake_case							=  '''MobilenetV1/Conv2d_0/'''
   __snake_case							=  backbone.conv_stem.convolution.weight
   __snake_case							=  backbone.conv_stem.normalization.bias
   __snake_case							=  backbone.conv_stem.normalization.weight
   __snake_case							=  backbone.conv_stem.normalization.running_mean
   __snake_case							=  backbone.conv_stem.normalization.running_var

   for i in range(13			):
      __snake_case							=  i + 1
      __snake_case							=  i * 2

      __snake_case							=  backbone.layer[pt_index]
      __snake_case							=  f"""MobilenetV1/Conv2d_{tf_index}_depthwise/"""
      __snake_case							=  pointer.convolution.weight
      __snake_case							=  pointer.normalization.bias
      __snake_case							=  pointer.normalization.weight
      __snake_case							=  pointer.normalization.running_mean
      __snake_case							=  pointer.normalization.running_var

      __snake_case							=  backbone.layer[pt_index + 1]
      __snake_case							=  f"""MobilenetV1/Conv2d_{tf_index}_pointwise/"""
      __snake_case							=  pointer.convolution.weight
      __snake_case							=  pointer.normalization.bias
      __snake_case							=  pointer.normalization.weight
      __snake_case							=  pointer.normalization.running_mean
      __snake_case							=  pointer.normalization.running_var

   if isinstance(snake_case_			, snake_case_			):
      __snake_case							=  '''MobilenetV1/Logits/Conv2d_1c_1x1/'''
      __snake_case							=  model.classifier.weight
      __snake_case							=  model.classifier.bias

   return tf_to_pt_map






def lowerCamelCase__       (				snake_case_      :    Any			, snake_case_      :    int			, snake_case_      :    List[str]			) ->							int:






   try:
      import numpy as np
      import tensorflow as tf
   except ImportError:
      logger.error(
          '''Loading a TensorFlow models in PyTorch, requires TensorFlow to be installed. Please see '''
          '''https://www.tensorflow.org/install/ for installation instructions.'''			)
      raise

   # Load weights from TF model
   __snake_case							=  tf.train.list_variables(snake_case_			)
   __snake_case							=  {}
   for name, shape in init_vars:
      logger.info(f"""Loading TF weight {name} with shape {shape}"""			)
      __snake_case							=  tf.train.load_variable(snake_case_			, snake_case_			)
      __snake_case							=  array

   # Build TF to PyTorch weights loading map
   __snake_case							=  _build_tf_to_pytorch_map(snake_case_			, snake_case_			, snake_case_			)

   for name, pointer in tf_to_pt_map.items():
      logger.info(f"""Importing {name}"""			)
      if name not in tf_weights:
         logger.info(f"""{name} not in tf pre-trained weights, skipping"""			)
         continue

      __snake_case							=  tf_weights[name]

      if "depthwise_weights" in name:
         logger.info('''Transposing depthwise'''			)
         __snake_case							=  np.transpose(snake_case_			, (2, 3, 0, 1)			)
      elif "weights" in name:
         logger.info('''Transposing'''			)
         if len(pointer.shape			) == 2:  # copying into linear layer
            __snake_case							=  array.squeeze().transpose()
         else:
            __snake_case							=  np.transpose(snake_case_			, (3, 2, 0, 1)			)

      if pointer.shape != array.shape:
         raise ValueError(f"""Pointer shape {pointer.shape} and array shape {array.shape} mismatched"""			)

      logger.info(f"""Initialize PyTorch weight {name} {array.shape}"""			)
      __snake_case							=  torch.from_numpy(snake_case_			)

      tf_weights.pop(snake_case_			, snake_case_			)
      tf_weights.pop(name + '''/RMSProp'''			, snake_case_			)
      tf_weights.pop(name + '''/RMSProp_1'''			, snake_case_			)
      tf_weights.pop(name + '''/ExponentialMovingAverage'''			, snake_case_			)

   logger.info(f"""Weights not copied to PyTorch model: {', '.join(tf_weights.keys()			)}"""			)
   return model






def lowerCamelCase__       (				snake_case_      :    List[Any]			, snake_case_      :    List[Any]			) ->							Dict:
   __snake_case							=  features.shape[-2:]
   __snake_case							=  conv_layer.stride
   __snake_case							=  conv_layer.kernel_size

   if in_height % stride_height == 0:
      __snake_case							=  max(kernel_height - stride_height			, 0			)
   else:
      __snake_case							=  max(kernel_height - (in_height % stride_height)			, 0			)

   if in_width % stride_width == 0:
      __snake_case							=  max(kernel_width - stride_width			, 0			)
   else:
      __snake_case							=  max(kernel_width - (in_width % stride_width)			, 0			)

   __snake_case							=  pad_along_width // 2
   __snake_case							=  pad_along_width - pad_left
   __snake_case							=  pad_along_height // 2
   __snake_case							=  pad_along_height - pad_top

   __snake_case							=  (pad_left, pad_right, pad_top, pad_bottom)
   return nn.functional.pad(snake_case_			, snake_case_			, '''constant'''			, 0.0			)

class 						SCREAMING_SNAKE_CASE__							(			nn.Module    ):


      def __init__(self			: Tuple							,     a__			: Dict							,     a__			: Union[str, Any]							,     a__			: List[str]							,     a__			: int							,     a__			: str = 1							,     a__			: Union[str, Any] = 1							,     a__			: Tuple = False							,     a__			: Optional[int] = True							,     a__			: List[str] = True							,     ):





         """simple docstring"""



         super().__init__()
         __snake_case							=  config

         if in_channels % groups != 0:
            raise ValueError(f"""Input channels ({in_channels}) are not divisible by {groups} groups.""" )
         if out_channels % groups != 0:
            raise ValueError(f"""Output channels ({out_channels}) are not divisible by {groups} groups.""" )

         __snake_case							=  0 if config.tf_padding else int((kernel_size - 1) / 2 )

         __snake_case							=  nn.Convad(
             in_channels=__lowerCAmelCase							,     out_channels=__lowerCAmelCase							,     kernel_size=__lowerCAmelCase							,     stride=__lowerCAmelCase							,     padding=__lowerCAmelCase							,     groups=__lowerCAmelCase							,     bias=__lowerCAmelCase							,     padding_mode='''zeros'''							,     )

         if use_normalization:
            __snake_case							=  nn.BatchNormad(
                num_features=__lowerCAmelCase							,     eps=config.layer_norm_eps							,     momentum=0.9_9_9_7							,     affine=__lowerCAmelCase							,     track_running_stats=__lowerCAmelCase							,     )
         else:
            __snake_case							=  None

         if use_activation:
            if isinstance(__lowerCAmelCase							,     __lowerCAmelCase ):
               __snake_case							=  ACTaFN[use_activation]
            elif isinstance(config.hidden_act							,     __lowerCAmelCase ):
               __snake_case							=  ACTaFN[config.hidden_act]
            else:
               __snake_case							=  config.hidden_act
         else:
            __snake_case							=  None


      def   a					(self			: Dict							,     a__			: Any ):





         """simple docstring"""



         if self.config.tf_padding:
            __snake_case							=  apply_tf_padding(__lowerCAmelCase							,     self.convolution )
         __snake_case							=  self.convolution(__lowerCAmelCase )
         if self.normalization is not None:
            __snake_case							=  self.normalization(__lowerCAmelCase )
         if self.activation is not None:
            __snake_case							=  self.activation(__lowerCAmelCase )
         return features






class 						SCREAMING_SNAKE_CASE__							(			_UpperCAmelCase    ):
      A_		:							int				=       MobileNetVaConfig
      A_		:							List[Any]				=       load_tf_weights_in_mobilenet_va
      A_		:							Tuple				=       'mobilenet_v1'
      A_		:							Optional[int]				=       'pixel_values'
      A_		:							Tuple				=       False


      def   a					(self			: Dict							,     a__			: List[Any] ):





         """simple docstring"""



         if isinstance(__lowerCAmelCase							,     (nn.Linear, nn.Convad) ):
            module.weight.data.normal_(mean=0.0							,     std=self.config.initializer_range )
            if module.bias is not None:
               module.bias.data.zero_()
         elif isinstance(__lowerCAmelCase							,     nn.BatchNormad ):
            module.bias.data.zero_()
            module.weight.data.fill_(1.0 )






snake_case_           =     r"""
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it
    as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`MobileNetV1Config`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
"""

snake_case_           =     r"""
    Args:
        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):
            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See
            [`MobileNetV1ImageProcessor.__call__`] for details.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
"""

@add_start_docstrings(
    'The bare MobileNetV1 model outputting raw hidden-states without any specific head on top.'		, _UpperCAmelCase		, )
class 						SCREAMING_SNAKE_CASE__							(			_UpperCAmelCase    ):


      def __init__(self			: Optional[int]							,     a__			: Optional[int]							,     a__			: int = True ):





         """simple docstring"""



         super().__init__(__lowerCAmelCase )
         __snake_case							=  config

         __snake_case							=  32
         __snake_case							=  max(int(depth * config.depth_multiplier )							,     config.min_depth )

         __snake_case							=  MobileNetVaConvLayer(
             __lowerCAmelCase							,     in_channels=config.num_channels							,     out_channels=__lowerCAmelCase							,     kernel_size=3							,     stride=2							,     )

         __snake_case							=  [1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 2, 1]

         __snake_case							=  nn.ModuleList()
         for i in range(13 ):
            __snake_case							=  out_channels

            if strides[i] == 2 or i == 0:
               depth *= 2
               __snake_case							=  max(int(depth * config.depth_multiplier )							,     config.min_depth )

            self.layer.append(
                MobileNetVaConvLayer(
                    __lowerCAmelCase							,     in_channels=__lowerCAmelCase							,     out_channels=__lowerCAmelCase							,     kernel_size=3							,     stride=strides[i]							,     groups=__lowerCAmelCase							,     ) )

            self.layer.append(
                MobileNetVaConvLayer(
                    __lowerCAmelCase							,     in_channels=__lowerCAmelCase							,     out_channels=__lowerCAmelCase							,     kernel_size=1							,     ) )

         __snake_case							=  nn.AdaptiveAvgPoolad((1, 1) ) if add_pooling_layer else None

         # Initialize weights and apply final processing
         self.post_init()


      def   a					(self			: str							,     a__			: Dict ):





         """simple docstring"""



         raise NotImplementedError


      @add_start_docstrings_to_model_forward(__lowerCAmelCase )
      @add_code_sample_docstrings(
          checkpoint=_CHECKPOINT_FOR_DOC							,     output_type=__lowerCAmelCase							,     config_class=_CONFIG_FOR_DOC							,     modality='''vision'''							,     expected_output=_EXPECTED_OUTPUT_SHAPE							,     )
      def   a					(self			: Optional[Any]							,     a__			: int = None							,     a__			: Dict = None							,     a__			: Tuple = None							,     ):





         """simple docstring"""



         __snake_case							=  (
             output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
         )
         __snake_case							=  return_dict if return_dict is not None else self.config.use_return_dict

         if pixel_values is None:
            raise ValueError('''You have to specify pixel_values''' )

         __snake_case							=  self.conv_stem(__lowerCAmelCase )

         __snake_case							=  () if output_hidden_states else None

         for i, layer_module in enumerate(self.layer ):
            __snake_case							=  layer_module(__lowerCAmelCase )

            if output_hidden_states:
               __snake_case							=  all_hidden_states + (hidden_states,)

         __snake_case							=  hidden_states

         if self.pooler is not None:
            __snake_case							=  torch.flatten(self.pooler(__lowerCAmelCase )							,     start_dim=1 )
         else:
            __snake_case							=  None

         if not return_dict:
            return tuple(v for v in [last_hidden_state, pooled_output, all_hidden_states] if v is not None )

         return BaseModelOutputWithPoolingAndNoAttention(
             last_hidden_state=__lowerCAmelCase							,     pooler_output=__lowerCAmelCase							,     hidden_states=__lowerCAmelCase							,     )






@add_start_docstrings(
    '\n    MobileNetV1 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for\n    ImageNet.\n    '		, _UpperCAmelCase		, )
class 						SCREAMING_SNAKE_CASE__							(			_UpperCAmelCase    ):


      def __init__(self			: Dict							,     a__			: Any ):





         """simple docstring"""



         super().__init__(__lowerCAmelCase )

         __snake_case							=  config.num_labels
         __snake_case							=  MobileNetVaModel(__lowerCAmelCase )

         __snake_case							=  self.mobilenet_va.layer[-1].convolution.out_channels

         # Classifier head
         __snake_case							=  nn.Dropout(config.classifier_dropout_prob							,     inplace=__lowerCAmelCase )
         __snake_case							=  nn.Linear(__lowerCAmelCase							,     config.num_labels ) if config.num_labels > 0 else nn.Identity()

         # Initialize weights and apply final processing
         self.post_init()


      @add_start_docstrings_to_model_forward(__lowerCAmelCase )
      @add_code_sample_docstrings(
          checkpoint=_IMAGE_CLASS_CHECKPOINT							,     output_type=__lowerCAmelCase							,     config_class=_CONFIG_FOR_DOC							,     expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT							,     )
      def   a					(self			: Optional[int]							,     a__			: Tuple = None							,     a__			: Optional[Any] = None							,     a__			: str = None							,     a__			: Any = None							,     ):





         """simple docstring"""



         __snake_case							=  return_dict if return_dict is not None else self.config.use_return_dict

         __snake_case							=  self.mobilenet_va(__lowerCAmelCase							,     output_hidden_states=__lowerCAmelCase							,     return_dict=__lowerCAmelCase )

         __snake_case							=  outputs.pooler_output if return_dict else outputs[1]

         __snake_case							=  self.classifier(self.dropout(__lowerCAmelCase ) )

         __snake_case							=  None
         if labels is not None:
            if self.config.problem_type is None:
               if self.num_labels == 1:
                  __snake_case							=  '''regression'''
               elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
                  __snake_case							=  '''single_label_classification'''
               else:
                  __snake_case							=  '''multi_label_classification'''

            if self.config.problem_type == "regression":
               __snake_case							=  MSELoss()
               if self.num_labels == 1:
                  __snake_case							=  loss_fct(logits.squeeze()							,     labels.squeeze() )
               else:
                  __snake_case							=  loss_fct(__lowerCAmelCase							,     __lowerCAmelCase )
            elif self.config.problem_type == "single_label_classification":
               __snake_case							=  CrossEntropyLoss()
               __snake_case							=  loss_fct(logits.view(-1							,     self.num_labels )							,     labels.view(-1 ) )
            elif self.config.problem_type == "multi_label_classification":
               __snake_case							=  BCEWithLogitsLoss()
               __snake_case							=  loss_fct(__lowerCAmelCase							,     __lowerCAmelCase )

         if not return_dict:
            __snake_case							=  (logits,) + outputs[2:]
            return ((loss,) + output) if loss is not None else output

         return ImageClassifierOutputWithNoAttention(
             loss=__lowerCAmelCase							,     logits=__lowerCAmelCase							,     hidden_states=outputs.hidden_states							,     )









from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import (
					    TFRoFormerForCausalLM,
					    TFRoFormerForMaskedLM,
					    TFRoFormerForMultipleChoice,
					    TFRoFormerForQuestionAnswering,
					    TFRoFormerForSequenceClassification,
					    TFRoFormerForTokenClassification,
					    TFRoFormerModel,
					)
					from transformers.models.roformer.modeling_tf_roformer import (
					    TFRoFormerSelfAttention,
					    TFRoFormerSinusoidalPositionalEmbedding,
					)
class     lowerCamelCase_					:







				def __init__(    self							,       __lowerCAmelCase							,       __lowerCAmelCase=1_3							,       __lowerCAmelCase=7							,       __lowerCAmelCase=True							,       __lowerCAmelCase=True							,       __lowerCAmelCase=True							,       __lowerCAmelCase=True							,       __lowerCAmelCase=9_9							,       __lowerCAmelCase=3_2							,       __lowerCAmelCase=2							,       __lowerCAmelCase=4							,       __lowerCAmelCase=3_7							,       __lowerCAmelCase="gelu"							,       __lowerCAmelCase=0.1							,       __lowerCAmelCase=0.1							,       __lowerCAmelCase=5_1_2							,       __lowerCAmelCase=1_6							,       __lowerCAmelCase=2							,       __lowerCAmelCase=0.02							,       __lowerCAmelCase=3							,       __lowerCAmelCase=4							,       __lowerCAmelCase=None							,       ):
								"""simple docstring"""
								__magic_name__		:Optional[int]   =   parent
								__magic_name__		:List[Any]   =   1_3
								__magic_name__		:Union[str, Any]   =   7
								__magic_name__		:Optional[Any]   =   True
								__magic_name__		:Tuple   =   True
								__magic_name__		:List[str]   =   True
								__magic_name__		:List[Any]   =   True
								__magic_name__		:int   =   9_9
								__magic_name__		:Any   =   3_2
								__magic_name__		:Union[str, Any]   =   2
								__magic_name__		:List[str]   =   4
								__magic_name__		:List[Any]   =   3_7
								__magic_name__		:Tuple   =   '''gelu'''
								__magic_name__		:Any   =   0.1
								__magic_name__		:str   =   0.1
								__magic_name__		:List[str]   =   5_1_2
								__magic_name__		:int   =   1_6
								__magic_name__		:Any   =   2
								__magic_name__		:List[Any]   =   0.02
								__magic_name__		:Optional[Any]   =   3
								__magic_name__		:Tuple   =   4
								__magic_name__		:Optional[Any]   =   None







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[int]   =   ids_tensor([self.batch_size, self.seq_length]							,       self.vocab_size  )

								__magic_name__		:str   =   None
								if self.use_input_mask:
												__magic_name__		:Optional[int]   =   random_attention_mask([self.batch_size, self.seq_length]  )

								__magic_name__		:str   =   None
								if self.use_token_type_ids:
												__magic_name__		:List[Any]   =   ids_tensor([self.batch_size, self.seq_length]							,       self.type_vocab_size  )

								__magic_name__		:Union[str, Any]   =   None
								__magic_name__		:Tuple   =   None
								__magic_name__		:str   =   None
								if self.use_labels:
												__magic_name__		:List[Any]   =   ids_tensor([self.batch_size]							,       self.type_sequence_label_size  )
												__magic_name__		:List[Any]   =   ids_tensor([self.batch_size, self.seq_length]							,       self.num_labels  )
												__magic_name__		:List[Any]   =   ids_tensor([self.batch_size]							,       self.num_choices  )

								__magic_name__		:str   =   RoFormerConfig(
								    vocab_size=self.vocab_size							,       hidden_size=self.hidden_size							,       num_hidden_layers=self.num_hidden_layers							,       num_attention_heads=self.num_attention_heads							,       intermediate_size=self.intermediate_size							,       hidden_act=self.hidden_act							,       hidden_dropout_prob=self.hidden_dropout_prob							,       attention_probs_dropout_prob=self.attention_probs_dropout_prob							,       max_position_embeddings=self.max_position_embeddings							,       type_vocab_size=self.type_vocab_size							,       initializer_range=self.initializer_range							,       return_dict=__lowerCAmelCase							,       )

								return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:int   =   TFRoFormerModel(config=__lowerCAmelCase  )
								__magic_name__		:Optional[Any]   =   {'''input_ids''': input_ids, '''attention_mask''': input_mask, '''token_type_ids''': token_type_ids}

								__magic_name__		:List[str]   =   [input_ids, input_mask]
								__magic_name__		:Any   =   model(__lowerCAmelCase  )

								__magic_name__		:List[str]   =   model(__lowerCAmelCase  )

								self.parent.assertEqual(result.last_hidden_state.shape							,       (self.batch_size, self.seq_length, self.hidden_size)  )







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   True
								__magic_name__		:List[str]   =   TFRoFormerForCausalLM(config=__lowerCAmelCase  )
								__magic_name__		:str   =   {
								    '''input_ids''': input_ids,
								    '''attention_mask''': input_mask,
								    '''token_type_ids''': token_type_ids,
								}
								__magic_name__		:Optional[Any]   =   model(__lowerCAmelCase  )['''logits''']
								self.parent.assertListEqual(
								    list(prediction_scores.numpy().shape  )							,       [self.batch_size, self.seq_length, self.vocab_size]  )







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]   =   TFRoFormerForMaskedLM(config=__lowerCAmelCase  )
								__magic_name__		:Any   =   {
								    '''input_ids''': input_ids,
								    '''attention_mask''': input_mask,
								    '''token_type_ids''': token_type_ids,
								}
								__magic_name__		:Dict   =   model(__lowerCAmelCase  )
								self.parent.assertEqual(result.logits.shape							,       (self.batch_size, self.seq_length, self.vocab_size)  )







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:int   =   self.num_labels
								__magic_name__		:str   =   TFRoFormerForSequenceClassification(config=__lowerCAmelCase  )
								__magic_name__		:Optional[int]   =   {
								    '''input_ids''': input_ids,
								    '''attention_mask''': input_mask,
								    '''token_type_ids''': token_type_ids,
								}

								__magic_name__		:str   =   model(__lowerCAmelCase  )
								self.parent.assertEqual(result.logits.shape							,       (self.batch_size, self.num_labels)  )







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]   =   self.num_choices
								__magic_name__		:Tuple   =   TFRoFormerForMultipleChoice(config=__lowerCAmelCase  )
								__magic_name__		:int   =   tf.tile(tf.expand_dims(__lowerCAmelCase							,       1  )							,       (1, self.num_choices, 1)  )
								__magic_name__		:Optional[Any]   =   tf.tile(tf.expand_dims(__lowerCAmelCase							,       1  )							,       (1, self.num_choices, 1)  )
								__magic_name__		:Union[str, Any]   =   tf.tile(tf.expand_dims(__lowerCAmelCase							,       1  )							,       (1, self.num_choices, 1)  )
								__magic_name__		:str   =   {
								    '''input_ids''': multiple_choice_inputs_ids,
								    '''attention_mask''': multiple_choice_input_mask,
								    '''token_type_ids''': multiple_choice_token_type_ids,
								}
								__magic_name__		:Tuple   =   model(__lowerCAmelCase  )
								self.parent.assertEqual(result.logits.shape							,       (self.batch_size, self.num_choices)  )







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:Optional[int]   =   self.num_labels
								__magic_name__		:Any   =   TFRoFormerForTokenClassification(config=__lowerCAmelCase  )
								__magic_name__		:str   =   {
								    '''input_ids''': input_ids,
								    '''attention_mask''': input_mask,
								    '''token_type_ids''': token_type_ids,
								}
								__magic_name__		:Dict   =   model(__lowerCAmelCase  )
								self.parent.assertEqual(result.logits.shape							,       (self.batch_size, self.seq_length, self.num_labels)  )







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:List[str]   =   TFRoFormerForQuestionAnswering(config=__lowerCAmelCase  )
								__magic_name__		:List[str]   =   {
								    '''input_ids''': input_ids,
								    '''attention_mask''': input_mask,
								    '''token_type_ids''': token_type_ids,
								}

								__magic_name__		:Union[str, Any]   =   model(__lowerCAmelCase  )
								self.parent.assertEqual(result.start_logits.shape							,       (self.batch_size, self.seq_length)  )
								self.parent.assertEqual(result.end_logits.shape							,       (self.batch_size, self.seq_length)  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]   =   self.prepare_config_and_inputs()
								(
								    (
								    __magic_name__
								)  ,					(
								    __magic_name__
								)  ,					(
								    __magic_name__
								)  ,					(
								    __magic_name__
								)  ,					(
								    __magic_name__
								)  ,					(
								    __magic_name__
								)  ,					(
								    __magic_name__
								)  ,					
								)		:Union[str, Any]   =   config_and_inputs
								__magic_name__		:Optional[Any]   =   {'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': input_mask}
								return config, inputs_dict







@require_tf
class     lowerCamelCase_					(   lowerCamelCase      ,					lowerCamelCase      ,					unittest.TestCase			):
				a__       							=	(
				    (
				        TFRoFormerModel,
				        TFRoFormerForCausalLM,
				        TFRoFormerForMaskedLM,
				        TFRoFormerForQuestionAnswering,
				        TFRoFormerForSequenceClassification,
				        TFRoFormerForTokenClassification,
				        TFRoFormerForMultipleChoice,
				    )
				    if is_tf_available()
				    else ()
				)
				a__       							=	(
				    {
				        '''feature-extraction''': TFRoFormerModel,
				        '''fill-mask''': TFRoFormerForMaskedLM,
				        '''question-answering''': TFRoFormerForQuestionAnswering,
				        '''text-classification''': TFRoFormerForSequenceClassification,
				        '''text-generation''': TFRoFormerForCausalLM,
				        '''token-classification''': TFRoFormerForTokenClassification,
				        '''zero-shot''': TFRoFormerForSequenceClassification,
				    }
				    if is_tf_available()
				    else {}
				)

				a__       							=	False
				a__       							=	False







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  ):
								"""simple docstring"""
								if pipeline_test_casse_name == "TextGenerationPipelineTests":
												return True

								return False







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[str]   =   TFRoFormerModelTester(self  )
								__magic_name__		:List[str]   =   ConfigTester(self							,       config_class=__lowerCAmelCase							,       hidden_size=3_7  )







				def       A						(    self  ):
								"""simple docstring"""
								self.config_tester.run_common_tests()







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]   =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*__lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]   =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_masked_lm(*__lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Tuple   =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_lm_head(*__lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]   =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_multiple_choice(*__lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Tuple   =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*__lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Dict   =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_sequence_classification(*__lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[int]   =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*__lowerCAmelCase  )







				@slow
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]   =   TFRoFormerModel.from_pretrained('''junnyu/roformer_chinese_base'''  )
								self.assertIsNotNone(__lowerCAmelCase  )







@require_tf
class     lowerCamelCase_					(   unittest.TestCase			):







				@slow
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   TFRoFormerForMaskedLM.from_pretrained('''junnyu/roformer_chinese_base'''  )
								__magic_name__		:Dict   =   tf.constant([[0, 1, 2, 3, 4, 5]]  )
								__magic_name__		:Optional[Any]   =   model(__lowerCAmelCase  )[0]

								# TODO Replace vocab size
								__magic_name__		:int   =   5_0_0_0_0

								__magic_name__		:Tuple   =   [1, 6, vocab_size]
								self.assertEqual(output.shape							,       __lowerCAmelCase  )

								print(output[:, :3, :3]  )

								# TODO Replace values below with what was printed above.
								__magic_name__		:Any   =   tf.constant(
								    [
								        [
								            [-0.12053341, -1.0264901, 0.29221946],
								            [-1.5133783, 0.197433, 0.15190607],
								            [-5.0135403, -3.900256, -0.84038764],
								        ]
								    ]  )
								tf.debugging.assert_near(output[:, :3, :3]							,       __lowerCAmelCase							,       atol=1E-4  )







@require_tf
class     lowerCamelCase_					(   unittest.TestCase			):
				a__       							=	1e-4







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Optional[int]   =   tf.constant([[4, 1_0]]  )
								__magic_name__		:Optional[int]   =   TFRoFormerSinusoidalPositionalEmbedding(num_positions=6							,       embedding_dim=6  )

								__magic_name__		:Optional[Any]   =   emba(input_ids.shape  )
								__magic_name__		:List[str]   =   tf.constant(
								    [[0.0000, 0.0000, 0.0000, 1.0000, 1.0000, 1.0000], [0.8415, 0.0464, 0.0022, 0.5403, 0.9989, 1.0000]]  )

								tf.debugging.assert_near(__lowerCAmelCase							,       __lowerCAmelCase							,       atol=self.tolerance  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Tuple   =   tf.constant(
								    [
								        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
								        [0.8415, 0.8219, 0.8020, 0.7819, 0.7617],
								        [0.9093, 0.9364, 0.9581, 0.9749, 0.9870],
								    ]  )
								__magic_name__		:Union[str, Any]   =   TFRoFormerSinusoidalPositionalEmbedding(num_positions=5_1_2							,       embedding_dim=5_1_2  )
								emba([2, 1_6, 5_1_2]  )
								__magic_name__		:Optional[int]   =   emba.weight[:3, :5]

								tf.debugging.assert_near(__lowerCAmelCase							,       __lowerCAmelCase							,       atol=self.tolerance  )







@require_tf
class     lowerCamelCase_					(   unittest.TestCase			):
				a__       							=	1e-4







				def       A						(    self  ):
								"""simple docstring"""
								# 2,12,16,64
								__magic_name__		:int   =   tf.reshape(tf.range(2 * 1_2 * 1_6 * 6_4							,       dtype=tf.floataa  )							,       shape=(2, 1_2, 1_6, 6_4)  ) / 1_0_0

								__magic_name__		:str   =   -tf.reshape(tf.range(2 * 1_2 * 1_6 * 6_4							,       dtype=tf.floataa  )							,       shape=(2, 1_2, 1_6, 6_4)  ) / 1_0_0

								__magic_name__		:int   =   TFRoFormerSinusoidalPositionalEmbedding(num_positions=3_2							,       embedding_dim=6_4  )
								__magic_name__		:List[str]   =   embed_positions([2, 1_6, 7_6_8]  )[None, None, :, :]

								__magic_name__  ,					__magic_name__		:Union[str, Any]   =   TFRoFormerSelfAttention.apply_rotary_position_embeddings(
								    __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  )

								__magic_name__		:Tuple   =   tf.constant(
								    [
								        [0.0000, 0.0100, 0.0200, 0.0300, 0.0400, 0.0500, 0.0600, 0.0700],
								        [-0.2012, 0.8897, 0.0263, 0.9401, 0.2074, 0.9463, 0.3481, 0.9343],
								        [-1.7057, 0.6271, -1.2145, 1.3897, -0.6303, 1.7647, -0.1173, 1.8985],
								        [-2.1731, -1.6397, -2.7358, 0.2854, -2.1840, 1.7183, -1.3018, 2.4871],
								        [0.2717, -3.6173, -2.9206, -2.1988, -3.6638, 0.3858, -2.9155, 2.2980],
								        [3.9859, -2.1580, -0.7984, -4.4904, -4.1181, -2.0252, -4.4782, 1.1253],
								    ]  )
								__magic_name__		:List[str]   =   tf.constant(
								    [
								        [0.0000, -0.0100, -0.0200, -0.0300, -0.0400, -0.0500, -0.0600, -0.0700],
								        [0.2012, -0.8897, -0.0263, -0.9401, -0.2074, -0.9463, -0.3481, -0.9343],
								        [1.7057, -0.6271, 1.2145, -1.3897, 0.6303, -1.7647, 0.1173, -1.8985],
								        [2.1731, 1.6397, 2.7358, -0.2854, 2.1840, -1.7183, 1.3018, -2.4871],
								        [-0.2717, 3.6173, 2.9206, 2.1988, 3.6638, -0.3858, 2.9155, -2.2980],
								        [-3.9859, 2.1580, 0.7984, 4.4904, 4.1181, 2.0252, 4.4782, -1.1253],
								    ]  )

								tf.debugging.assert_near(query_layer[0, 0, :6, :8]							,       __lowerCAmelCase							,       atol=self.tolerance  )
								tf.debugging.assert_near(key_layer[0, 0, :6, :8]							,       __lowerCAmelCase							,       atol=self.tolerance  )






