


'''simple docstring'''






from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy






@add_start_docstrings(
    'The RoBERTa Model transformer with early exiting (DeeRoBERTa). '      ,_SCREAMING_SNAKE_CASE      ,)
class   __snake_case			(  _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase  					=   RobertaConfig
    lowercase  					=   'roberta'

    def __init__(  self       :    str       ,		lowerCamelCase       :    Any						)  ->    str:
      super().__init__(lowerCamelCase						)

      lowerCAmelCase_     : List[str]     =  RobertaEmbeddings(lowerCamelCase						)
      self.init_weights()







@add_start_docstrings(
    'RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. '      ,_SCREAMING_SNAKE_CASE      ,)
class   __snake_case			(  _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase  					=   RobertaConfig
    lowercase  					=   'roberta'

    def __init__(  self       :    Dict       ,		lowerCamelCase       :    str						)  ->    Dict:
      super().__init__(lowerCamelCase						)
      lowerCAmelCase_     : str     =  config.num_labels
      lowerCAmelCase_     : str     =  config.num_hidden_layers

      lowerCAmelCase_     : Dict     =  DeeRobertaModel(lowerCamelCase						)
      lowerCAmelCase_     : List[str]     =  nn.Dropout(config.hidden_dropout_prob						)
      lowerCAmelCase_     : Any     =  nn.Linear(config.hidden_size       ,		self.config.num_labels						)





    @add_start_docstrings_to_model_forward(lowerCamelCase						)
    def __lowercase  (  self       :    List[str]       ,		lowerCamelCase       :    Optional[int]=None       ,		lowerCamelCase       :    Union[str, Any]=None       ,		lowerCamelCase       :    int=None       ,		lowerCamelCase       :    str=None       ,		lowerCamelCase       :    str=None       ,		lowerCamelCase       :    Union[str, Any]=None       ,		lowerCamelCase       :    List[str]=None       ,		lowerCamelCase       :    Dict=-1       ,		lowerCamelCase       :    Optional[int]=False       ,		)  ->    str:
      lowerCAmelCase_     : List[str]     =  self.num_layers
      try:
        lowerCAmelCase_     : Union[str, Any]     =  self.roberta(
            lowerCamelCase       ,		attention_mask=lowerCamelCase       ,		token_type_ids=lowerCamelCase       ,		position_ids=lowerCamelCase       ,		head_mask=lowerCamelCase       ,		inputs_embeds=lowerCamelCase       ,		)

        lowerCAmelCase_     : List[str]     =  outputs[1]

        lowerCAmelCase_     : Any     =  self.dropout(lowerCamelCase						)
        lowerCAmelCase_     : int     =  self.classifier(lowerCamelCase						)
        lowerCAmelCase_     : int     =  (logits,) + outputs[2:]  # add hidden states and attention if they are here
      except HighwayException as e:
        lowerCAmelCase_     : List[str]     =  e.message
        lowerCAmelCase_     : Dict     =  e.exit_layer
        lowerCAmelCase_     : int     =  outputs[0]

      if not self.training:
        lowerCAmelCase_     : Optional[Any]     =  entropy(lowerCamelCase						)
        lowerCAmelCase_     : List[Any]     =  []
        lowerCAmelCase_     : int     =  []
      if labels is not None:
        if self.num_labels == 1:
          #  We are doing regression
          lowerCAmelCase_     : Any     =  MSELoss()
          lowerCAmelCase_     : List[Any]     =  loss_fct(logits.view(-1						)       ,		labels.view(-1						)						)
        else:
          lowerCAmelCase_     : Tuple     =  CrossEntropyLoss()
          lowerCAmelCase_     : List[Any]     =  loss_fct(logits.view(-1       ,		self.num_labels						)       ,		labels.view(-1						)						)

        # work with highway exits
        lowerCAmelCase_     : Union[str, Any]     =  []
        for highway_exit in outputs[-1]:
          lowerCAmelCase_     : str     =  highway_exit[0]
          if not self.training:
            highway_logits_all.append(lowerCamelCase						)
            highway_entropy.append(highway_exit[2]						)
          if self.num_labels == 1:
            #  We are doing regression
            lowerCAmelCase_     : str     =  MSELoss()
            lowerCAmelCase_     : List[str]     =  loss_fct(highway_logits.view(-1						)       ,		labels.view(-1						)						)
          else:
            lowerCAmelCase_     : Union[str, Any]     =  CrossEntropyLoss()
            lowerCAmelCase_     : int     =  loss_fct(highway_logits.view(-1       ,		self.num_labels						)       ,		labels.view(-1						)						)
          highway_losses.append(lowerCamelCase						)

        if train_highway:
          lowerCAmelCase_     : str     =  (sum(highway_losses[:-1]						),) + outputs
          # exclude the final highway, of course
        else:
          lowerCAmelCase_     : Any     =  (loss,) + outputs
      if not self.training:
        lowerCAmelCase_     : Any     =  outputs + ((original_entropy, highway_entropy), exit_layer)
        if output_layer >= 0:
          lowerCAmelCase_     : Optional[Any]     =  (
              (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
          )  # use the highway of the last layer

      return outputs  # (loss), logits, (hidden_states), (attentions), entropy







'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A   :    str											= logging.get_logger(__name__)

__A   :    int											= {
    "weiweishi/roc-bert-base-zh": "https://huggingface.co/weiweishi/roc-bert-base-zh/resolve/main/config.json",
}






class   __snake_case			(  _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase  					=   'roc_bert'

    def __init__(  self       :    str       ,		lowerCamelCase       :    Optional[Any]=3_05_22       ,		lowerCamelCase       :    List[Any]=7_68       ,		lowerCamelCase       :    int=12       ,		lowerCamelCase       :    str=12       ,		lowerCamelCase       :    Any=30_72       ,		lowerCamelCase       :    List[Any]="gelu"       ,		lowerCamelCase       :    Optional[int]=0.1       ,		lowerCamelCase       :    List[str]=0.1       ,		lowerCamelCase       :    int=5_12       ,		lowerCamelCase       :    List[Any]=2       ,		lowerCamelCase       :    int=0.02       ,		lowerCamelCase       :    str=1E-12       ,		lowerCamelCase       :    int=True       ,		lowerCamelCase       :    Optional[Any]=0       ,		lowerCamelCase       :    List[Any]="absolute"       ,		lowerCamelCase       :    str=None       ,		lowerCamelCase       :    str=True       ,		lowerCamelCase       :    Tuple=True       ,		lowerCamelCase       :    str=7_68       ,		lowerCamelCase       :    str=9_10       ,		lowerCamelCase       :    Tuple=5_12       ,		lowerCamelCase       :    List[str]=2_48_58       ,		lowerCamelCase       :    Dict=True       ,		**lowerCamelCase       :    Tuple       ,		)  ->    Optional[int]:
      lowerCAmelCase_     : Optional[Any]     =  vocab_size
      lowerCAmelCase_     : Any     =  max_position_embeddings
      lowerCAmelCase_     : Any     =  hidden_size
      lowerCAmelCase_     : int     =  num_hidden_layers
      lowerCAmelCase_     : Tuple     =  num_attention_heads
      lowerCAmelCase_     : List[Any]     =  intermediate_size
      lowerCAmelCase_     : Tuple     =  hidden_act
      lowerCAmelCase_     : List[Any]     =  hidden_dropout_prob
      lowerCAmelCase_     : Dict     =  attention_probs_dropout_prob
      lowerCAmelCase_     : Any     =  initializer_range
      lowerCAmelCase_     : Any     =  type_vocab_size
      lowerCAmelCase_     : Optional[int]     =  layer_norm_eps
      lowerCAmelCase_     : Tuple     =  use_cache
      lowerCAmelCase_     : List[Any]     =  enable_pronunciation
      lowerCAmelCase_     : List[Any]     =  enable_shape
      lowerCAmelCase_     : List[str]     =  pronunciation_embed_dim
      lowerCAmelCase_     : List[str]     =  pronunciation_vocab_size
      lowerCAmelCase_     : Dict     =  shape_embed_dim
      lowerCAmelCase_     : Optional[Any]     =  shape_vocab_size
      lowerCAmelCase_     : List[Any]     =  concat_input
      lowerCAmelCase_     : Any     =  position_embedding_type
      lowerCAmelCase_     : List[str]     =  classifier_dropout
      super().__init__(pad_token_id=lowerCamelCase       ,		**lowerCamelCase						)



