





from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy



@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''  ,						lowerCAmelCase_  ,						)
class 							_UpperCamelCase					(	lowerCAmelCase_     ):
       _UpperCamelCase				:					str         =   RobertaConfig
       _UpperCamelCase				:					Optional[int]         =   '''roberta'''

       def __init__(		self: Optional[Any]					,   _SCREAMING_SNAKE_CASE: Any		)       -> Dict:




              """simple docstring"""


              super().__init__(_SCREAMING_SNAKE_CASE		)

              UpperCamelCase_          =  RobertaEmbeddings(_SCREAMING_SNAKE_CASE		)
              self.init_weights()



@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''  ,						lowerCAmelCase_  ,						)
class 							_UpperCamelCase					(	lowerCAmelCase_     ):
       _UpperCamelCase				:					List[Any]         =   RobertaConfig
       _UpperCamelCase				:					str         =   '''roberta'''

       def __init__(		self: Optional[Any]					,   _SCREAMING_SNAKE_CASE: str		)       -> Dict:




              """simple docstring"""


              super().__init__(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  config.num_labels
              UpperCamelCase_          =  config.num_hidden_layers

              UpperCamelCase_          =  DeeRobertaModel(_SCREAMING_SNAKE_CASE		)
              UpperCamelCase_          =  nn.Dropout(config.hidden_dropout_prob		)
              UpperCamelCase_          =  nn.Linear(config.hidden_size					,   self.config.num_labels		)



       @add_start_docstrings_to_model_forward(_SCREAMING_SNAKE_CASE		)
       def lowercase (		self: Any					,   _SCREAMING_SNAKE_CASE: Optional[Any]=None					,   _SCREAMING_SNAKE_CASE: Optional[Any]=None					,   _SCREAMING_SNAKE_CASE: Optional[int]=None					,   _SCREAMING_SNAKE_CASE: Optional[int]=None					,   _SCREAMING_SNAKE_CASE: Dict=None					,   _SCREAMING_SNAKE_CASE: Dict=None					,   _SCREAMING_SNAKE_CASE: Dict=None					,   _SCREAMING_SNAKE_CASE: Tuple=-1					,   _SCREAMING_SNAKE_CASE: int=False					,   )       -> Dict:




              """simple docstring"""


              UpperCamelCase_          =  self.num_layers
              try:
                     UpperCamelCase_          =  self.roberta(
                         _SCREAMING_SNAKE_CASE					,   attention_mask=_SCREAMING_SNAKE_CASE					,   token_type_ids=_SCREAMING_SNAKE_CASE					,   position_ids=_SCREAMING_SNAKE_CASE					,   head_mask=_SCREAMING_SNAKE_CASE					,   inputs_embeds=_SCREAMING_SNAKE_CASE					,   )

                     UpperCamelCase_          =  outputs[1]

                     UpperCamelCase_          =  self.dropout(_SCREAMING_SNAKE_CASE		)
                     UpperCamelCase_          =  self.classifier(_SCREAMING_SNAKE_CASE		)
                     UpperCamelCase_          =  (logits,) + outputs[2:]  # add hidden states and attention if they are here
              except HighwayException as e:
                     UpperCamelCase_          =  e.message
                     UpperCamelCase_          =  e.exit_layer
                     UpperCamelCase_          =  outputs[0]

              if not self.training:
                     UpperCamelCase_          =  entropy(_SCREAMING_SNAKE_CASE		)
                     UpperCamelCase_          =  []
                     UpperCamelCase_          =  []
              if labels is not None:
                     if self.num_labels == 1:
                            #  We are doing regression
                            UpperCamelCase_          =  MSELoss()
                            UpperCamelCase_          =  loss_fct(logits.view(-1		)					,   labels.view(-1		)		)
                     else:
                            UpperCamelCase_          =  CrossEntropyLoss()
                            UpperCamelCase_          =  loss_fct(logits.view(-1					,   self.num_labels		)					,   labels.view(-1		)		)

                     # work with highway exits
                     UpperCamelCase_          =  []
                     for highway_exit in outputs[-1]:
                            UpperCamelCase_          =  highway_exit[0]
                            if not self.training:
                                   highway_logits_all.append(_SCREAMING_SNAKE_CASE		)
                                   highway_entropy.append(highway_exit[2]		)
                            if self.num_labels == 1:
                                   #  We are doing regression
                                   UpperCamelCase_          =  MSELoss()
                                   UpperCamelCase_          =  loss_fct(highway_logits.view(-1		)					,   labels.view(-1		)		)
                            else:
                                   UpperCamelCase_          =  CrossEntropyLoss()
                                   UpperCamelCase_          =  loss_fct(highway_logits.view(-1					,   self.num_labels		)					,   labels.view(-1		)		)
                            highway_losses.append(_SCREAMING_SNAKE_CASE		)

                     if train_highway:
                            UpperCamelCase_          =  (sum(highway_losses[:-1]		),) + outputs
                            # exclude the final highway, of course
                     else:
                            UpperCamelCase_          =  (loss,) + outputs
              if not self.training:
                     UpperCamelCase_          =  outputs + ((original_entropy, highway_entropy), exit_layer)
                     if output_layer >= 0:
                            UpperCamelCase_          =  (
                                (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                            )  # use the highway of the last layer

              return outputs  # (loss), logits, (hidden_states), (attentions), entropy








from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCAmelCase        =				logging.get_logger(__name__)

_UpperCAmelCase        =				{'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/config.json'}



class 							_UpperCamelCase					(	lowerCAmelCase_     ):
       _UpperCamelCase				:					List[str]         =   '''openai-gpt'''
       _UpperCamelCase				:					List[str]         =   {
           '''max_position_embeddings''': '''n_positions''',
           '''hidden_size''': '''n_embd''',
           '''num_attention_heads''': '''n_head''',
           '''num_hidden_layers''': '''n_layer''',
       }

       def __init__(		self: int					,   _SCREAMING_SNAKE_CASE: int=40478					,   _SCREAMING_SNAKE_CASE: Optional[Any]=512					,   _SCREAMING_SNAKE_CASE: List[Any]=768					,   _SCREAMING_SNAKE_CASE: str=12					,   _SCREAMING_SNAKE_CASE: Any=12					,   _SCREAMING_SNAKE_CASE: Optional[int]="gelu"					,   _SCREAMING_SNAKE_CASE: int=0.1					,   _SCREAMING_SNAKE_CASE: Optional[int]=0.1					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=0.1					,   _SCREAMING_SNAKE_CASE: Any=1e-5					,   _SCREAMING_SNAKE_CASE: Tuple=0.02					,   _SCREAMING_SNAKE_CASE: str="cls_index"					,   _SCREAMING_SNAKE_CASE: Union[str, Any]=True					,   _SCREAMING_SNAKE_CASE: List[Any]=None					,   _SCREAMING_SNAKE_CASE: List[Any]=True					,   _SCREAMING_SNAKE_CASE: List[Any]=0.1					,   **_SCREAMING_SNAKE_CASE: List[Any]					,   )       -> int:




              """simple docstring"""


              UpperCamelCase_          =  vocab_size
              UpperCamelCase_          =  n_positions
              UpperCamelCase_          =  n_embd
              UpperCamelCase_          =  n_layer
              UpperCamelCase_          =  n_head
              UpperCamelCase_          =  afn
              UpperCamelCase_          =  resid_pdrop
              UpperCamelCase_          =  embd_pdrop
              UpperCamelCase_          =  attn_pdrop
              UpperCamelCase_          =  layer_norm_epsilon
              UpperCamelCase_          =  initializer_range
              UpperCamelCase_          =  summary_type
              UpperCamelCase_          =  summary_use_proj
              UpperCamelCase_          =  summary_activation
              UpperCamelCase_          =  summary_first_dropout
              UpperCamelCase_          =  summary_proj_to_labels
              super().__init__(**_SCREAMING_SNAKE_CASE		)

