
from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy





@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """ ,      _lowerCamelCase ,      )
class 						_a   (						_lowerCamelCase       ):

       """simple docstring"""





       snake_case				=RobertaConfig
       snake_case				='''roberta'''
       def __init__(					self       ,					_snake_case	):
        super().__init__(_SCREAMING_SNAKE_CASE	)

        _UpperCAmelCase 				=RobertaEmbeddings(_SCREAMING_SNAKE_CASE	)
        self.init_weights()







@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """ ,      _lowerCamelCase ,      )
class 						_a   (						_lowerCamelCase       ):

       """simple docstring"""





       snake_case				=RobertaConfig
       snake_case				='''roberta'''
       def __init__(					self       ,					_snake_case	):
        super().__init__(_SCREAMING_SNAKE_CASE	)
        _UpperCAmelCase 				=config.num_labels
        _UpperCAmelCase 				=config.num_hidden_layers

        _UpperCAmelCase 				=DeeRobertaModel(_SCREAMING_SNAKE_CASE	)
        _UpperCAmelCase 				=nn.Dropout(config.hidden_dropout_prob	)
        _UpperCAmelCase 				=nn.Linear(config.hidden_size       ,					self.config.num_labels	)

       @add_start_docstrings_to_model_forward(_SCREAMING_SNAKE_CASE	)
       def     SCREAMING_SNAKE_CASE  (					self       ,					_snake_case=None       ,					_snake_case=None       ,					_snake_case=None       ,					_snake_case=None       ,					_snake_case=None       ,					_snake_case=None       ,					_snake_case=None       ,					_snake_case=-1       ,					_snake_case=False       ,					):
        _UpperCAmelCase 				=self.num_layers
        try:
         _UpperCAmelCase 				=self.roberta(
             _SCREAMING_SNAKE_CASE       ,					attention_mask=_SCREAMING_SNAKE_CASE       ,					token_type_ids=_SCREAMING_SNAKE_CASE       ,					position_ids=_SCREAMING_SNAKE_CASE       ,					head_mask=_SCREAMING_SNAKE_CASE       ,					inputs_embeds=_SCREAMING_SNAKE_CASE       ,					)

         _UpperCAmelCase 				=outputs[1]

         _UpperCAmelCase 				=self.dropout(_SCREAMING_SNAKE_CASE	)
         _UpperCAmelCase 				=self.classifier(_SCREAMING_SNAKE_CASE	)
         _UpperCAmelCase 				=(logits,) + outputs[2:]  # add hidden states and attention if they are here
        except HighwayException as e:
         _UpperCAmelCase 				=e.message
         _UpperCAmelCase 				=e.exit_layer
         _UpperCAmelCase 				=outputs[0]

        if not self.training:
         _UpperCAmelCase 				=entropy(_SCREAMING_SNAKE_CASE	)
         _UpperCAmelCase 				=[]
         _UpperCAmelCase 				=[]
        if labels is not None:
         if self.num_labels == 1:
          #  We are doing regression
          _UpperCAmelCase 				=MSELoss()
          _UpperCAmelCase 				=loss_fct(logits.view(-1	)       ,					labels.view(-1	)	)
         else:
          _UpperCAmelCase 				=CrossEntropyLoss()
          _UpperCAmelCase 				=loss_fct(logits.view(-1       ,					self.num_labels	)       ,					labels.view(-1	)	)

         # work with highway exits
         _UpperCAmelCase 				=[]
         for highway_exit in outputs[-1]:
          _UpperCAmelCase 				=highway_exit[0]
          if not self.training:
           highway_logits_all.append(_SCREAMING_SNAKE_CASE	)
           highway_entropy.append(highway_exit[2]	)
          if self.num_labels == 1:
           #  We are doing regression
           _UpperCAmelCase 				=MSELoss()
           _UpperCAmelCase 				=loss_fct(highway_logits.view(-1	)       ,					labels.view(-1	)	)
          else:
           _UpperCAmelCase 				=CrossEntropyLoss()
           _UpperCAmelCase 				=loss_fct(highway_logits.view(-1       ,					self.num_labels	)       ,					labels.view(-1	)	)
          highway_losses.append(_SCREAMING_SNAKE_CASE	)

         if train_highway:
          _UpperCAmelCase 				=(sum(highway_losses[:-1]	),) + outputs
          # exclude the final highway, of course
         else:
          _UpperCAmelCase 				=(loss,) + outputs
        if not self.training:
         _UpperCAmelCase 				=outputs + ((original_entropy, highway_entropy), exit_layer)
         if output_layer >= 0:
          _UpperCAmelCase 				=(
              (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
          )  # use the highway of the last layer

        return outputs  # (loss), logits, (hidden_states), (attentions), entropy










"""simple docstring"""

from typing import TYPE_CHECKING

from ..utils import _LazyModule


A_						:    str				=      {
    'config': [
        'EXTERNAL_DATA_FORMAT_SIZE_LIMIT',
        'OnnxConfig',
        'OnnxConfigWithPast',
        'OnnxSeq2SeqConfigWithPast',
        'PatchingSpec',
    ],
    'convert': ['export', 'validate_model_outputs'],
    'features': ['FeaturesManager'],
    'utils': ['ParameterFormat', 'compute_serialized_parameters_size'],
}


if TYPE_CHECKING:
							from .config import (
							    EXTERNAL_DATA_FORMAT_SIZE_LIMIT,
							    OnnxConfig,
							    OnnxConfigWithPast,
							    OnnxSeqaSeqConfigWithPast,
							    PatchingSpec,
							)
							from .convert import export, validate_model_outputs
							from .features import FeaturesManager
							from .utils import ParameterFormat, compute_serialized_parameters_size

else:
							import sys

							A_						:    Optional[Any]				=      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





