





from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy


@add_start_docstrings(
    'The RoBERTa Model transformer with early exiting (DeeRoBERTa). ' ,					SCREAMING_SNAKE_CASE ,					)
class 			lowerCamelCase						(		SCREAMING_SNAKE_CASE  ):
  UpperCAmelCase		:       Optional[int]										=     RobertaConfig
  UpperCAmelCase		:       str										=     'roberta'







  def __init__(     self						:   int  ,					__snake_case						:   Dict       )    ->		Optional[int]:
        super().__init__(_UpperCamelCase       )

        _a							:  Tuple				= RobertaEmbeddings(_UpperCamelCase       )
        self.init_weights()


@add_start_docstrings(
    'RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. ' ,					SCREAMING_SNAKE_CASE ,					)
class 			lowerCamelCase						(		SCREAMING_SNAKE_CASE  ):
  UpperCAmelCase		:       Any										=     RobertaConfig
  UpperCAmelCase		:       List[Any]										=     'roberta'







  def __init__(     self						:   Optional[int]  ,					__snake_case						:   str       )    ->		List[Any]:
        super().__init__(_UpperCamelCase       )
        _a							:  Tuple				= config.num_labels
        _a							:  Dict				= config.num_hidden_layers

        _a							:  Tuple				= DeeRobertaModel(_UpperCamelCase       )
        _a							:  List[Any]				= nn.Dropout(config.hidden_dropout_prob       )
        _a							:  Optional[int]				= nn.Linear(config.hidden_size  ,					self.config.num_labels       )







  @add_start_docstrings_to_model_forward(_UpperCamelCase       )
  def      snake_case_							(     self						:   List[Any]  ,					__snake_case						:   str=None  ,					__snake_case						:   Optional[int]=None  ,					__snake_case						:   str=None  ,					__snake_case						:   Optional[Any]=None  ,					__snake_case						:   Tuple=None  ,					__snake_case						:   List[Any]=None  ,					__snake_case						:   List[Any]=None  ,					__snake_case						:   Dict=-1  ,					__snake_case						:   Any=False  ,					)    ->		Tuple:
        _a							:  Tuple				= self.num_layers
        try:
              _a							:  Dict				= self.roberta(
                  _UpperCamelCase  ,					attention_mask=_UpperCamelCase  ,					token_type_ids=_UpperCamelCase  ,					position_ids=_UpperCamelCase  ,					head_mask=_UpperCamelCase  ,					inputs_embeds=_UpperCamelCase  ,					)

              _a							:  Any				= outputs[1]

              _a							:  Tuple				= self.dropout(_UpperCamelCase       )
              _a							:  List[str]				= self.classifier(_UpperCamelCase       )
              _a							:  str				= (logits,) + outputs[2:]  # add hidden states and attention if they are here
        except HighwayException as e:
              _a							:  Dict				= e.message
              _a							:  Union[str, Any]				= e.exit_layer
              _a							:  Optional[int]				= outputs[0]

        if not self.training:
              _a							:  Any				= entropy(_UpperCamelCase       )
              _a							:  Union[str, Any]				= []
              _a							:  Optional[Any]				= []
        if labels is not None:
              if self.num_labels == 1:
                    #  We are doing regression
                    _a							:  Any				= MSELoss()
                    _a							:  Optional[Any]				= loss_fct(logits.view(-1       )  ,					labels.view(-1       )       )
              else:
                    _a							:  Tuple				= CrossEntropyLoss()
                    _a							:  Dict				= loss_fct(logits.view(-1  ,					self.num_labels       )  ,					labels.view(-1       )       )

              # work with highway exits
              _a							:  Tuple				= []
              for highway_exit in outputs[-1]:
                    _a							:  int				= highway_exit[0]
                    if not self.training:
                          highway_logits_all.append(_UpperCamelCase       )
                          highway_entropy.append(highway_exit[2]       )
                    if self.num_labels == 1:
                          #  We are doing regression
                          _a							:  Union[str, Any]				= MSELoss()
                          _a							:  Optional[int]				= loss_fct(highway_logits.view(-1       )  ,					labels.view(-1       )       )
                    else:
                          _a							:  List[str]				= CrossEntropyLoss()
                          _a							:  int				= loss_fct(highway_logits.view(-1  ,					self.num_labels       )  ,					labels.view(-1       )       )
                    highway_losses.append(_UpperCamelCase       )

              if train_highway:
                    _a							:  List[str]				= (sum(highway_losses[:-1]       ),) + outputs
                    # exclude the final highway, of course
              else:
                    _a							:  str				= (loss,) + outputs
        if not self.training:
              _a							:  Any				= outputs + ((original_entropy, highway_entropy), exit_layer)
              if output_layer >= 0:
                    _a							:  Union[str, Any]				= (
                        (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                    )  # use the highway of the last layer

        return outputs  # (loss), logits, (hidden_states), (attentions), entropy









'''simple docstring'''
def 					lowerCamelCase_ ( __UpperCamelCase					:       list							,    __UpperCamelCase					:       int							,    __UpperCamelCase					:       int = 0							,    __UpperCamelCase					:       int = 0		)					->	int:







     """simple docstring"""
     _A				=							right or len(__UpperCamelCase		) - 1
     if left > right:
          return -1
     elif list_data[left] == key:
          return left
     elif list_data[right] == key:
          return right
     else:
          return search(__UpperCamelCase							,    __UpperCamelCase							,    left + 1							,    right - 1		)


if __name__ == "__main__":
    import doctest

    doctest.testmod()





