
from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy
@add_start_docstrings(
    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). "			,	__magic_name__			,	)
class       _lowerCAmelCase  (		__magic_name__							):

 """simple docstring"""
 SCREAMING_SNAKE_CASE_				:					List[Any]       						=RobertaConfig
 SCREAMING_SNAKE_CASE_				:					Any       						="roberta"

 def __init__(						self			:			Optional[int]					, SCREAMING_SNAKE_CASE__			:			int   ):






      """simple docstring"""



      super().__init__(SCREAMING_SNAKE_CASE__   )

      UpperCamelCase						    =					RobertaEmbeddings(SCREAMING_SNAKE_CASE__   )
      self.init_weights()




@add_start_docstrings(
    "RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. "			,	__magic_name__			,	)
class       _lowerCAmelCase  (		__magic_name__							):

 """simple docstring"""
 SCREAMING_SNAKE_CASE_				:					Dict       						=RobertaConfig
 SCREAMING_SNAKE_CASE_				:					List[Any]       						="roberta"

 def __init__(						self			:			Any					, SCREAMING_SNAKE_CASE__			:			Tuple   ):






      """simple docstring"""



      super().__init__(SCREAMING_SNAKE_CASE__   )
      UpperCamelCase						    =					config.num_labels
      UpperCamelCase						    =					config.num_hidden_layers

      UpperCamelCase						    =					DeeRobertaModel(SCREAMING_SNAKE_CASE__   )
      UpperCamelCase						    =					nn.Dropout(config.hidden_dropout_prob   )
      UpperCamelCase						    =					nn.Linear(config.hidden_size					, self.config.num_labels   )






 @add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE__   )
 def      __lowerCAmelCase  (						self			:			int					, SCREAMING_SNAKE_CASE__			:			Tuple=None					, SCREAMING_SNAKE_CASE__			:			List[Any]=None					, SCREAMING_SNAKE_CASE__			:			List[Any]=None					, SCREAMING_SNAKE_CASE__			:			Optional[int]=None					, SCREAMING_SNAKE_CASE__			:			Any=None					, SCREAMING_SNAKE_CASE__			:			Union[str, Any]=None					, SCREAMING_SNAKE_CASE__			:			List[Any]=None					, SCREAMING_SNAKE_CASE__			:			Union[str, Any]=-1					, SCREAMING_SNAKE_CASE__			:			str=False					, ):






      """simple docstring"""



      UpperCamelCase						    =					self.num_layers
      try:
           UpperCamelCase						    =					self.roberta(
               SCREAMING_SNAKE_CASE__					, attention_mask=SCREAMING_SNAKE_CASE__					, token_type_ids=SCREAMING_SNAKE_CASE__					, position_ids=SCREAMING_SNAKE_CASE__					, head_mask=SCREAMING_SNAKE_CASE__					, inputs_embeds=SCREAMING_SNAKE_CASE__					, )

           UpperCamelCase						    =					outputs[1]

           UpperCamelCase						    =					self.dropout(SCREAMING_SNAKE_CASE__   )
           UpperCamelCase						    =					self.classifier(SCREAMING_SNAKE_CASE__   )
           UpperCamelCase						    =					(logits,) + outputs[2:]  # add hidden states and attention if they are here
      except HighwayException as e:
           UpperCamelCase						    =					e.message
           UpperCamelCase						    =					e.exit_layer
           UpperCamelCase						    =					outputs[0]

      if not self.training:
           UpperCamelCase						    =					entropy(SCREAMING_SNAKE_CASE__   )
           UpperCamelCase						    =					[]
           UpperCamelCase						    =					[]
      if labels is not None:
           if self.num_labels == 1:
                #  We are doing regression
                UpperCamelCase						    =					MSELoss()
                UpperCamelCase						    =					loss_fct(logits.view(-1   )					, labels.view(-1   )   )
           else:
                UpperCamelCase						    =					CrossEntropyLoss()
                UpperCamelCase						    =					loss_fct(logits.view(-1					, self.num_labels   )					, labels.view(-1   )   )

           # work with highway exits
           UpperCamelCase						    =					[]
           for highway_exit in outputs[-1]:
                UpperCamelCase						    =					highway_exit[0]
                if not self.training:
                     highway_logits_all.append(SCREAMING_SNAKE_CASE__   )
                     highway_entropy.append(highway_exit[2]   )
                if self.num_labels == 1:
                     #  We are doing regression
                     UpperCamelCase						    =					MSELoss()
                     UpperCamelCase						    =					loss_fct(highway_logits.view(-1   )					, labels.view(-1   )   )
                else:
                     UpperCamelCase						    =					CrossEntropyLoss()
                     UpperCamelCase						    =					loss_fct(highway_logits.view(-1					, self.num_labels   )					, labels.view(-1   )   )
                highway_losses.append(SCREAMING_SNAKE_CASE__   )

           if train_highway:
                UpperCamelCase						    =					(sum(highway_losses[:-1]   ),) + outputs
                # exclude the final highway, of course
           else:
                UpperCamelCase						    =					(loss,) + outputs
      if not self.training:
           UpperCamelCase						    =					outputs + ((original_entropy, highway_entropy), exit_layer)
           if output_layer >= 0:
                UpperCamelCase						    =					(
                    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                )  # use the highway of the last layer

      return outputs  # (loss), logits, (hidden_states), (attentions), entropy




from dataclasses import dataclass
from typing import Optional, Tuple, Union

import flax
import jax.numpy as jnp
from jax import random

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .scheduling_utils_flax import FlaxSchedulerMixin
@flax.struct.dataclass
class       _lowerCAmelCase  :

 """simple docstring"""
 # setable values
 SCREAMING_SNAKE_CASE_				:					Optional[int]       						=None
 SCREAMING_SNAKE_CASE_				:					Optional[jnp.ndarray]       						=None
 SCREAMING_SNAKE_CASE_				:					Optional[jnp.ndarray]       						=None  # sigma(t_i)

 @classmethod
 def      __lowerCAmelCase  (						cls			:			Optional[Any]   ):






      """simple docstring"""



      return cls()




@dataclass
class       _lowerCAmelCase  (		__magic_name__							):

 """simple docstring"""
 SCREAMING_SNAKE_CASE_				:					jnp.ndarray
 SCREAMING_SNAKE_CASE_				:					jnp.ndarray
 SCREAMING_SNAKE_CASE_				:					KarrasVeSchedulerState




class       _lowerCAmelCase  (		__magic_name__			,	__magic_name__							):

 """simple docstring"""

 @property
 def      __lowerCAmelCase  (						self			:			Any   ):






      """simple docstring"""



      return True

 @register_to_config
 def __init__(						self			:			Dict					, SCREAMING_SNAKE_CASE__			:			float = 0.02					, SCREAMING_SNAKE_CASE__			:			float = 1_00					, SCREAMING_SNAKE_CASE__			:			float = 1.007					, SCREAMING_SNAKE_CASE__			:			float = 80					, SCREAMING_SNAKE_CASE__			:			float = 0.05					, SCREAMING_SNAKE_CASE__			:			float = 50					, ):






      """simple docstring"""



      pass

 def      __lowerCAmelCase  (						self			:			int   ):






      """simple docstring"""



      return KarrasVeSchedulerState.create()

 def      __lowerCAmelCase  (						self			:			Union[str, Any]					, SCREAMING_SNAKE_CASE__			:			KarrasVeSchedulerState					, SCREAMING_SNAKE_CASE__			:			int					, SCREAMING_SNAKE_CASE__			:			Tuple = ()   ):






      """simple docstring"""



      UpperCamelCase						    =					jnp.arange(0					, SCREAMING_SNAKE_CASE__   )[::-1].copy()
      UpperCamelCase						    =					[
          (
              self.config.sigma_max**2
              * (self.config.sigma_min**2 / self.config.sigma_max**2) ** (i / (num_inference_steps - 1))
          )
          for i in timesteps
      ]

      return state.replace(
          num_inference_steps=SCREAMING_SNAKE_CASE__					, schedule=jnp.array(SCREAMING_SNAKE_CASE__					, dtype=jnp.floataa   )					, timesteps=SCREAMING_SNAKE_CASE__					, )

 def      __lowerCAmelCase  (						self			:			List[Any]					, SCREAMING_SNAKE_CASE__			:			KarrasVeSchedulerState					, SCREAMING_SNAKE_CASE__			:			jnp.ndarray					, SCREAMING_SNAKE_CASE__			:			float					, SCREAMING_SNAKE_CASE__			:			random.KeyArray					, ):






      """simple docstring"""



      if self.config.s_min <= sigma <= self.config.s_max:
           UpperCamelCase						    =					min(self.config.s_churn / state.num_inference_steps					, 2**0.5 - 1   )
      else:
           UpperCamelCase						    =					0

      # sample eps ~ N(0, S_noise^2 * I)
      UpperCamelCase						    =					random.split(SCREAMING_SNAKE_CASE__					, num=1   )
      UpperCamelCase						    =					self.config.s_noise * random.normal(key=SCREAMING_SNAKE_CASE__					, shape=sample.shape   )
      UpperCamelCase						    =					sigma + gamma * sigma
      UpperCamelCase						    =					sample + ((sigma_hat**2 - sigma**2) ** 0.5 * eps)

      return sample_hat, sigma_hat

 def      __lowerCAmelCase  (						self			:			Optional[int]					, SCREAMING_SNAKE_CASE__			:			KarrasVeSchedulerState					, SCREAMING_SNAKE_CASE__			:			jnp.ndarray					, SCREAMING_SNAKE_CASE__			:			float					, SCREAMING_SNAKE_CASE__			:			float					, SCREAMING_SNAKE_CASE__			:			jnp.ndarray					, SCREAMING_SNAKE_CASE__			:			bool = True					, ):






      """simple docstring"""



      UpperCamelCase						    =					sample_hat + sigma_hat * model_output
      UpperCamelCase						    =					(sample_hat - pred_original_sample) / sigma_hat
      UpperCamelCase						    =					sample_hat + (sigma_prev - sigma_hat) * derivative

      if not return_dict:
           return (sample_prev, derivative, state)

      return FlaxKarrasVeOutput(prev_sample=SCREAMING_SNAKE_CASE__					, derivative=SCREAMING_SNAKE_CASE__					, state=SCREAMING_SNAKE_CASE__   )

 def      __lowerCAmelCase  (						self			:			Tuple					, SCREAMING_SNAKE_CASE__			:			KarrasVeSchedulerState					, SCREAMING_SNAKE_CASE__			:			jnp.ndarray					, SCREAMING_SNAKE_CASE__			:			float					, SCREAMING_SNAKE_CASE__			:			float					, SCREAMING_SNAKE_CASE__			:			jnp.ndarray					, SCREAMING_SNAKE_CASE__			:			jnp.ndarray					, SCREAMING_SNAKE_CASE__			:			jnp.ndarray					, SCREAMING_SNAKE_CASE__			:			bool = True					, ):






      """simple docstring"""



      UpperCamelCase						    =					sample_prev + sigma_prev * model_output
      UpperCamelCase						    =					(sample_prev - pred_original_sample) / sigma_prev
      UpperCamelCase						    =					sample_hat + (sigma_prev - sigma_hat) * (0.5 * derivative + 0.5 * derivative_corr)

      if not return_dict:
           return (sample_prev, derivative, state)

      return FlaxKarrasVeOutput(prev_sample=SCREAMING_SNAKE_CASE__					, derivative=SCREAMING_SNAKE_CASE__					, state=SCREAMING_SNAKE_CASE__   )






 def      __lowerCAmelCase  (						self			:			Any					, SCREAMING_SNAKE_CASE__			:			KarrasVeSchedulerState					, SCREAMING_SNAKE_CASE__			:			Any					, SCREAMING_SNAKE_CASE__			:			str					, SCREAMING_SNAKE_CASE__			:			int   ):






      """simple docstring"""



      raise NotImplementedError()


