
from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy







@add_start_docstrings(
    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). "							,				__lowerCAmelCase							,				)
class   lowerCamelCase	(						__lowerCAmelCase						):
     UpperCAmelCase__       :							List[str]     =	RobertaConfig
     UpperCAmelCase__       :							int     =	'''roberta'''





     def __init__(self   :					List[Any]       ,					_A   :					str     )		->		List[str]:
         super().__init__(lowerCAmelCase_     )

         snake_case				  =      RobertaEmbeddings(lowerCAmelCase_     )
         self.init_weights()







@add_start_docstrings(
    "RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. "							,				__lowerCAmelCase							,				)
class   lowerCamelCase	(						__lowerCAmelCase						):
     UpperCAmelCase__       :							Optional[int]     =	RobertaConfig
     UpperCAmelCase__       :							str     =	'''roberta'''





     def __init__(self   :					Dict       ,					_A   :					str     )		->		List[str]:
         super().__init__(lowerCAmelCase_     )
         snake_case				  =      config.num_labels
         snake_case				  =      config.num_hidden_layers

         snake_case				  =      DeeRobertaModel(lowerCAmelCase_     )
         snake_case				  =      nn.Dropout(config.hidden_dropout_prob     )
         snake_case				  =      nn.Linear(config.hidden_size       ,					self.config.num_labels     )





     @add_start_docstrings_to_model_forward(lowerCAmelCase_     )
     def 	UpperCAmelCase(self   :					Optional[Any]       ,					_A   :					Optional[int]=None       ,					_A   :					Tuple=None       ,					_A   :					Tuple=None       ,					_A   :					Dict=None       ,					_A   :					Tuple=None       ,					_A   :					Optional[Any]=None       ,					_A   :					Dict=None       ,					_A   :					List[str]=-1       ,					_A   :					Optional[Any]=False       ,					)		->		List[str]:
         snake_case				  =      self.num_layers
         try:
             snake_case				  =      self.roberta(
                 lowerCAmelCase_       ,					attention_mask=lowerCAmelCase_       ,					token_type_ids=lowerCAmelCase_       ,					position_ids=lowerCAmelCase_       ,					head_mask=lowerCAmelCase_       ,					inputs_embeds=lowerCAmelCase_       ,					)

             snake_case				  =      outputs[1]

             snake_case				  =      self.dropout(lowerCAmelCase_     )
             snake_case				  =      self.classifier(lowerCAmelCase_     )
             snake_case				  =      (logits,) + outputs[2:]  # add hidden states and attention if they are here
         except HighwayException as e:
             snake_case				  =      e.message
             snake_case				  =      e.exit_layer
             snake_case				  =      outputs[0]

         if not self.training:
             snake_case				  =      entropy(lowerCAmelCase_     )
             snake_case				  =      []
             snake_case				  =      []
         if labels is not None:
             if self.num_labels == 1:
                 #  We are doing regression
                 snake_case				  =      MSELoss()
                 snake_case				  =      loss_fct(logits.view(-1     )       ,					labels.view(-1     )     )
             else:
                 snake_case				  =      CrossEntropyLoss()
                 snake_case				  =      loss_fct(logits.view(-1       ,					self.num_labels     )       ,					labels.view(-1     )     )

             # work with highway exits
             snake_case				  =      []
             for highway_exit in outputs[-1]:
                 snake_case				  =      highway_exit[0]
                 if not self.training:
                     highway_logits_all.append(lowerCAmelCase_     )
                     highway_entropy.append(highway_exit[2]     )
                 if self.num_labels == 1:
                     #  We are doing regression
                     snake_case				  =      MSELoss()
                     snake_case				  =      loss_fct(highway_logits.view(-1     )       ,					labels.view(-1     )     )
                 else:
                     snake_case				  =      CrossEntropyLoss()
                     snake_case				  =      loss_fct(highway_logits.view(-1       ,					self.num_labels     )       ,					labels.view(-1     )     )
                 highway_losses.append(lowerCAmelCase_     )

             if train_highway:
                 snake_case				  =      (sum(highway_losses[:-1]     ),) + outputs
                 # exclude the final highway, of course
             else:
                 snake_case				  =      (loss,) + outputs
         if not self.training:
             snake_case				  =      outputs + ((original_entropy, highway_entropy), exit_layer)
             if output_layer >= 0:
                 snake_case				  =      (
                     (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                 )  # use the highway of the last layer

         return outputs  # (loss), logits, (hidden_states), (attentions), entropy



from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_A						      =				logging.get_logger(__name__)

_A						      =				{
    "sail/poolformer_s12": "https://huggingface.co/sail/poolformer_s12/resolve/main/config.json",
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
}







class   lowerCamelCase	(						A_						):
     UpperCAmelCase__       :							Any     =	"poolformer"





     def __init__(self   :					Optional[int]       ,					_A   :					Optional[Any]=3       ,					_A   :					Optional[int]=1_6       ,					_A   :					Dict=1_6       ,					_A   :					Tuple=3       ,					_A   :					Tuple=4.0       ,					_A   :					int=[2, 2, 6, 2]       ,					_A   :					Dict=[6_4, 1_2_8, 3_2_0, 5_1_2]       ,					_A   :					int=[7, 3, 3, 3]       ,					_A   :					List[str]=[4, 2, 2, 2]       ,					_A   :					str=[2, 1, 1, 1]       ,					_A   :					List[Any]=4       ,					_A   :					Any=0.0       ,					_A   :					Optional[Any]="gelu"       ,					_A   :					Optional[Any]=True       ,					_A   :					List[str]=1E-5       ,					_A   :					List[str]=0.02       ,					**_A   :					str       ,					)		->		Tuple:
         snake_case				  =      num_channels
         snake_case				  =      patch_size
         snake_case				  =      stride
         snake_case				  =      padding
         snake_case				  =      pool_size
         snake_case				  =      hidden_sizes
         snake_case				  =      mlp_ratio
         snake_case				  =      depths
         snake_case				  =      patch_sizes
         snake_case				  =      strides
         snake_case				  =      num_encoder_blocks
         snake_case				  =      drop_path_rate
         snake_case				  =      hidden_act
         snake_case				  =      use_layer_scale
         snake_case				  =      layer_scale_init_value
         snake_case				  =      initializer_range
         super().__init__(**_A     )







class   lowerCamelCase	(						A_						):
     UpperCAmelCase__       :							Tuple     =	version.parse("1.11"						)





     @property
     def 	UpperCAmelCase(self   :					int     )		->		Mapping[str, Mapping[int, str]]:
         return OrderedDict(
             [
                 ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}),
             ]     )





     @property
     def 	UpperCAmelCase(self   :					int     )		->		float:
         return 2E-3

