

"""simple docstring"""




from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy
@add_start_docstrings(
    "The RoBERTa Model transformer with early exiting (DeeRoBERTa). "		,    _lowerCAmelCase		,    )
class      _UpperCAmelCase   (							_lowerCAmelCase       ):
	a__		: Dict					=					RobertaConfig
	a__		: str					=					"roberta"



	def __init__(  self     :       List[str]			,    _lowercase     :       List[Any]						):
		super().__init__(_lowercase						)

		__UpperCAmelCase      		= RobertaEmbeddings(_lowercase						)
		self.init_weights()






@add_start_docstrings(
    "RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. "		,    _lowerCAmelCase		,    )
class      _UpperCAmelCase   (							_lowerCAmelCase       ):
	a__		: Optional[int]					=					RobertaConfig
	a__		: Optional[int]					=					"roberta"



	def __init__(  self     :       List[Any]			,    _lowercase     :       Union[str, Any]						):
		super().__init__(_lowercase						)
		__UpperCAmelCase      		= config.num_labels
		__UpperCAmelCase      		= config.num_hidden_layers

		__UpperCAmelCase      		= DeeRobertaModel(_lowercase						)
		__UpperCAmelCase      		= nn.Dropout(config.hidden_dropout_prob						)
		__UpperCAmelCase      		= nn.Linear(config.hidden_size			,    self.config.num_labels						)



	@add_start_docstrings_to_model_forward(_lowercase						)
	def        a  (  self     :       str			,    _lowercase     :       Tuple=None			,    _lowercase     :       Dict=None			,    _lowercase     :       Dict=None			,    _lowercase     :       Optional[int]=None			,    _lowercase     :       Optional[int]=None			,    _lowercase     :       Optional[int]=None			,    _lowercase     :       Any=None			,    _lowercase     :       Union[str, Any]=-1			,    _lowercase     :       int=False			,    ):
		__UpperCAmelCase      		= self.num_layers
		try:
			__UpperCAmelCase      		= self.roberta(
			    _lowercase			,    attention_mask=_lowercase			,    token_type_ids=_lowercase			,    position_ids=_lowercase			,    head_mask=_lowercase			,    inputs_embeds=_lowercase			,    )

			__UpperCAmelCase      		= outputs[1]

			__UpperCAmelCase      		= self.dropout(_lowercase						)
			__UpperCAmelCase      		= self.classifier(_lowercase						)
			__UpperCAmelCase      		= (logits,) + outputs[2:]  # add hidden states and attention if they are here
		except HighwayException as e:
			__UpperCAmelCase      		= e.message
			__UpperCAmelCase      		= e.exit_layer
			__UpperCAmelCase      		= outputs[0]

		if not self.training:
			__UpperCAmelCase      		= entropy(_lowercase						)
			__UpperCAmelCase      		= []
			__UpperCAmelCase      		= []
		if labels is not None:
			if self.num_labels == 1:
				#  We are doing regression
				__UpperCAmelCase      		= MSELoss()
				__UpperCAmelCase      		= loss_fct(logits.view(-1						)			,    labels.view(-1						)						)
			else:
				__UpperCAmelCase      		= CrossEntropyLoss()
				__UpperCAmelCase      		= loss_fct(logits.view(-1			,    self.num_labels						)			,    labels.view(-1						)						)

			# work with highway exits
			__UpperCAmelCase      		= []
			for highway_exit in outputs[-1]:
				__UpperCAmelCase      		= highway_exit[0]
				if not self.training:
					highway_logits_all.append(_lowercase						)
					highway_entropy.append(highway_exit[2]						)
				if self.num_labels == 1:
					#  We are doing regression
					__UpperCAmelCase      		= MSELoss()
					__UpperCAmelCase      		= loss_fct(highway_logits.view(-1						)			,    labels.view(-1						)						)
				else:
					__UpperCAmelCase      		= CrossEntropyLoss()
					__UpperCAmelCase      		= loss_fct(highway_logits.view(-1			,    self.num_labels						)			,    labels.view(-1						)						)
				highway_losses.append(_lowercase						)

			if train_highway:
				__UpperCAmelCase      		= (sum(highway_losses[:-1]						),) + outputs
				# exclude the final highway, of course
			else:
				__UpperCAmelCase      		= (loss,) + outputs
		if not self.training:
			__UpperCAmelCase      		= outputs + ((original_entropy, highway_entropy), exit_layer)
			if output_layer >= 0:
				__UpperCAmelCase      		= (
				    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
				)  # use the highway of the last layer

		return outputs  # (loss), logits, (hidden_states), (attentions), entropy






from abc import ABC, abstractmethod
from argparse import ArgumentParser
class 						UpperCAmelCase  (							UpperCAmelCase__     ):







						'''simple docstring'''





						@staticmethod
						@abstractmethod
						def 			snake_case__		(		__lowercase				:	ArgumentParser		):






							"""simple docstring"""






							raise NotImplementedError()






						@abstractmethod
						def 			snake_case__		(		self				:	Optional[Any]		):






							"""simple docstring"""






							raise NotImplementedError()
