






from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy




@add_start_docstrings(
    'The RoBERTa Model transformer with early exiting (DeeRoBERTa). '					,							__SCREAMING_SNAKE_CASE					,							)
class 							lowerCAmelCase__  (	__SCREAMING_SNAKE_CASE						):
					__A     :    Union[str, Any]           =						RobertaConfig
					__A     :    Tuple           =						'roberta'







					def __init__(       self      :     Dict			,		_A      :     Optional[int]):
											super().__init__(__snake_case)

											A__	:							Optional[int]						    = RobertaEmbeddings(__snake_case)
											self.init_weights()




@add_start_docstrings(
    'RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. '					,							__SCREAMING_SNAKE_CASE					,							)
class 							lowerCAmelCase__  (	__SCREAMING_SNAKE_CASE						):
					__A     :    Dict           =						RobertaConfig
					__A     :    Tuple           =						'roberta'







					def __init__(       self      :     List[str]			,		_A      :     Optional[Any]):
											super().__init__(__snake_case)
											A__	:							Any						    = config.num_labels
											A__	:							Union[str, Any]						    = config.num_hidden_layers

											A__	:							Optional[int]						    = DeeRobertaModel(__snake_case)
											A__	:							Dict						    = nn.Dropout(config.hidden_dropout_prob)
											A__	:							Optional[Any]						    = nn.Linear(config.hidden_size			,		self.config.num_labels)







					@add_start_docstrings_to_model_forward(__snake_case)
					def  _lowercase			(       self      :     int			,		_A      :     Optional[int]=None			,		_A      :     int=None			,		_A      :     Optional[Any]=None			,		_A      :     Union[str, Any]=None			,		_A      :     Any=None			,		_A      :     int=None			,		_A      :     Union[str, Any]=None			,		_A      :     Dict=-1			,		_A      :     Optional[Any]=False			,		):
											A__	:							List[str]						    = self.num_layers
											try:
																	A__	:							Tuple						    = self.roberta(
																	    __snake_case			,		attention_mask=__snake_case			,		token_type_ids=__snake_case			,		position_ids=__snake_case			,		head_mask=__snake_case			,		inputs_embeds=__snake_case			,		)

																	A__	:							int						    = outputs[1]

																	A__	:							str						    = self.dropout(__snake_case)
																	A__	:							List[Any]						    = self.classifier(__snake_case)
																	A__	:							Optional[int]						    = (logits,) + outputs[2:]  # add hidden states and attention if they are here
											except HighwayException as e:
																	A__	:							Union[str, Any]						    = e.message
																	A__	:							List[Any]						    = e.exit_layer
																	A__	:							Union[str, Any]						    = outputs[0]

											if not self.training:
																	A__	:							List[str]						    = entropy(__snake_case)
																	A__	:							Union[str, Any]						    = []
																	A__	:							Dict						    = []
											if labels is not None:
																	if self.num_labels == 1:
																							#  We are doing regression
																							A__	:							List[str]						    = MSELoss()
																							A__	:							Tuple						    = loss_fct(logits.view(-1)			,		labels.view(-1))
																	else:
																							A__	:							List[str]						    = CrossEntropyLoss()
																							A__	:							Dict						    = loss_fct(logits.view(-1			,		self.num_labels)			,		labels.view(-1))

																	# work with highway exits
																	A__	:							List[Any]						    = []
																	for highway_exit in outputs[-1]:
																							A__	:							Optional[int]						    = highway_exit[0]
																							if not self.training:
																													highway_logits_all.append(__snake_case)
																													highway_entropy.append(highway_exit[2])
																							if self.num_labels == 1:
																													#  We are doing regression
																													A__	:							Dict						    = MSELoss()
																													A__	:							int						    = loss_fct(highway_logits.view(-1)			,		labels.view(-1))
																							else:
																													A__	:							int						    = CrossEntropyLoss()
																													A__	:							List[Any]						    = loss_fct(highway_logits.view(-1			,		self.num_labels)			,		labels.view(-1))
																							highway_losses.append(__snake_case)

																	if train_highway:
																							A__	:							Dict						    = (sum(highway_losses[:-1]),) + outputs
																							# exclude the final highway, of course
																	else:
																							A__	:							Union[str, Any]						    = (loss,) + outputs
											if not self.training:
																	A__	:							List[str]						    = outputs + ((original_entropy, highway_entropy), exit_layer)
																	if output_layer >= 0:
																							A__	:							Union[str, Any]						    = (
																							    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																							)  # use the highway of the last layer

											return outputs  # (loss), logits, (hidden_states), (attentions), entropy




from collections.abc import Callable

import numpy as np
def 	snake_case__       (					__lowercase							,     __lowercase							,     __lowercase							,     __lowercase							,     __lowercase       )		->		np.array:







						"""simple docstring"""

						A__	:							Any						    = int(np.ceil((x_end - xa) / step_size       )       )
						A__	:							Union[str, Any]						    = np.zeros((n + 1,)       )
						A__	:							Any						    = ya
						A__	:							Union[str, Any]						    = xa

						for k in range(__lowercase       ):
												A__	:							Any						    = y[k] + step_size * ode_func(__lowercase							,     y[k]       )
												A__	:							Any						    = y[k] + (
												    (step_size / 2) * (ode_func(__lowercase							,     y[k]       ) + ode_func(x + step_size							,     __lowercase       ))
												)
												x += step_size

						return y


if __name__ == "__main__":
		import doctest

		doctest.testmod()