

"""simple docstring"""






from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy







@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''					,  lowerCAmelCase_					,  )
class       snake_case__  ( lowerCAmelCase_     ):
			SCREAMING_SNAKE_CASE__     				=						RobertaConfig
			SCREAMING_SNAKE_CASE__     				=						'''roberta'''







			def __init__(			self			: List[Any]			,  lowercase			: Tuple  ):





								'''simple docstring'''



								super().__init__(lowercase  )

								UpperCAmelCase			:   Optional[int]      =				RobertaEmbeddings(lowercase  )
								self.init_weights()







@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''					,  lowerCAmelCase_					,  )
class       snake_case__  ( lowerCAmelCase_     ):
			SCREAMING_SNAKE_CASE__     				=						RobertaConfig
			SCREAMING_SNAKE_CASE__     				=						'''roberta'''







			def __init__(			self			: List[str]			,  lowercase			: Optional[int]  ):





								'''simple docstring'''



								super().__init__(lowercase  )
								UpperCAmelCase			:   Optional[Any]      =				config.num_labels
								UpperCAmelCase			:   Union[str, Any]      =				config.num_hidden_layers

								UpperCAmelCase			:   int      =				DeeRobertaModel(lowercase  )
								UpperCAmelCase			:   str      =				nn.Dropout(config.hidden_dropout_prob  )
								UpperCAmelCase			:   int      =				nn.Linear(config.hidden_size			,  self.config.num_labels  )







			@add_start_docstrings_to_model_forward(lowercase  )
			def 			__lowerCAmelCase				(			self			: Optional[int]			,  lowercase			: int=None			,  lowercase			: List[Any]=None			,  lowercase			: List[str]=None			,  lowercase			: Optional[Any]=None			,  lowercase			: Tuple=None			,  lowercase			: Dict=None			,  lowercase			: List[str]=None			,  lowercase			: Optional[int]=-1			,  lowercase			: Tuple=False			,  ):





								'''simple docstring'''



								UpperCAmelCase			:   Tuple      =				self.num_layers
								try:
													UpperCAmelCase			:   Tuple      =				self.roberta(
													    lowercase			,  attention_mask=lowercase			,  token_type_ids=lowercase			,  position_ids=lowercase			,  head_mask=lowercase			,  inputs_embeds=lowercase			,  )

													UpperCAmelCase			:   Optional[Any]      =				outputs[1]

													UpperCAmelCase			:   Optional[Any]      =				self.dropout(lowercase  )
													UpperCAmelCase			:   Any      =				self.classifier(lowercase  )
													UpperCAmelCase			:   Optional[Any]      =				(logits,) + outputs[2:]  # add hidden states and attention if they are here
								except HighwayException as e:
													UpperCAmelCase			:   str      =				e.message
													UpperCAmelCase			:   Union[str, Any]      =				e.exit_layer
													UpperCAmelCase			:   Optional[int]      =				outputs[0]

								if not self.training:
													UpperCAmelCase			:   Optional[int]      =				entropy(lowercase  )
													UpperCAmelCase			:   Dict      =				[]
													UpperCAmelCase			:   int      =				[]
								if labels is not None:
													if self.num_labels == 1:
																		#  We are doing regression
																		UpperCAmelCase			:   int      =				MSELoss()
																		UpperCAmelCase			:   Optional[Any]      =				loss_fct(logits.view(-1  )			,  labels.view(-1  )  )
													else:
																		UpperCAmelCase			:   List[Any]      =				CrossEntropyLoss()
																		UpperCAmelCase			:   str      =				loss_fct(logits.view(-1			,  self.num_labels  )			,  labels.view(-1  )  )

													# work with highway exits
													UpperCAmelCase			:   int      =				[]
													for highway_exit in outputs[-1]:
																		UpperCAmelCase			:   str      =				highway_exit[0]
																		if not self.training:
																							highway_logits_all.append(lowercase  )
																							highway_entropy.append(highway_exit[2]  )
																		if self.num_labels == 1:
																							#  We are doing regression
																							UpperCAmelCase			:   Tuple      =				MSELoss()
																							UpperCAmelCase			:   List[Any]      =				loss_fct(highway_logits.view(-1  )			,  labels.view(-1  )  )
																		else:
																							UpperCAmelCase			:   List[Any]      =				CrossEntropyLoss()
																							UpperCAmelCase			:   Optional[int]      =				loss_fct(highway_logits.view(-1			,  self.num_labels  )			,  labels.view(-1  )  )
																		highway_losses.append(lowercase  )

													if train_highway:
																		UpperCAmelCase			:   Dict      =				(sum(highway_losses[:-1]  ),) + outputs
																		# exclude the final highway, of course
													else:
																		UpperCAmelCase			:   str      =				(loss,) + outputs
								if not self.training:
													UpperCAmelCase			:   Dict      =				outputs + ((original_entropy, highway_entropy), exit_layer)
													if output_layer >= 0:
																		UpperCAmelCase			:   Any      =				(
																		    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																		)  # use the highway of the last layer

								return outputs  # (loss), logits, (hidden_states), (attentions), entropy





"""simple docstring"""






def   lowercase_	(    _lowercase       :		list							,      _lowercase       :		int							,      _lowercase       :		int = 0							,      _lowercase       :		int = 0	):




					'''simple docstring'''






					UpperCAmelCase			:   Tuple      =				right or len(_lowercase	) - 1
					if left > right:
										return -1
					elif list_data[left] == key:
										return left
					elif list_data[right] == key:
										return right
					else:
										return search(_lowercase							,      _lowercase							,      left + 1							,      right - 1	)


if __name__ == "__main__":
	import doctest

	doctest.testmod()


