

from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy



@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''						,  __magic_name__						,  )
class        _a		(__magic_name__							):


					'''simple docstring'''






					UpperCAmelCase__:  Tuple    				=			RobertaConfig
					UpperCAmelCase__:  Any    				=			'''roberta'''






					def __init__(					self       ,			A__     ):
						super().__init__(A__     )

						A__				:       Dict	  =						RobertaEmbeddings(A__     )
						self.init_weights()



@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''						,  __magic_name__						,  )
class        _a		(__magic_name__							):


					'''simple docstring'''






					UpperCAmelCase__:  List[str]    				=			RobertaConfig
					UpperCAmelCase__:  List[str]    				=			'''roberta'''






					def __init__(					self       ,			A__     ):
						super().__init__(A__     )
						A__				:       Dict	  =						config.num_labels
						A__				:       Optional[Any]	  =						config.num_hidden_layers

						A__				:       Tuple	  =						DeeRobertaModel(A__     )
						A__				:       str	  =						nn.Dropout(config.hidden_dropout_prob     )
						A__				:       int	  =						nn.Linear(config.hidden_size       ,			self.config.num_labels     )






					@add_start_docstrings_to_model_forward(A__     )
					def     __A     (					self       ,			A__=None       ,			A__=None       ,			A__=None       ,			A__=None       ,			A__=None       ,			A__=None       ,			A__=None       ,			A__=-1       ,			A__=False       ,			):
						A__				:       Optional[Any]	  =						self.num_layers
						try:
							A__				:       Optional[Any]	  =						self.roberta(
							    A__       ,			attention_mask=A__       ,			token_type_ids=A__       ,			position_ids=A__       ,			head_mask=A__       ,			inputs_embeds=A__       ,			)

							A__				:       Union[str, Any]	  =						outputs[1]

							A__				:       Union[str, Any]	  =						self.dropout(A__     )
							A__				:       Optional[int]	  =						self.classifier(A__     )
							A__				:       Union[str, Any]	  =						(logits,) + outputs[2:]  # add hidden states and attention if they are here
						except HighwayException as e:
							A__				:       Any	  =						e.message
							A__				:       str	  =						e.exit_layer
							A__				:       Dict	  =						outputs[0]

						if not self.training:
							A__				:       List[str]	  =						entropy(A__     )
							A__				:       Union[str, Any]	  =						[]
							A__				:       Any	  =						[]
						if labels is not None:
							if self.num_labels == 1:
								#  We are doing regression
								A__				:       Any	  =						MSELoss()
								A__				:       Any	  =						loss_fct(logits.view(-1     )       ,			labels.view(-1     )     )
							else:
								A__				:       Union[str, Any]	  =						CrossEntropyLoss()
								A__				:       Dict	  =						loss_fct(logits.view(-1       ,			self.num_labels     )       ,			labels.view(-1     )     )

							# work with highway exits
							A__				:       Dict	  =						[]
							for highway_exit in outputs[-1]:
								A__				:       Optional[int]	  =						highway_exit[0]
								if not self.training:
									highway_logits_all.append(A__     )
									highway_entropy.append(highway_exit[2]     )
								if self.num_labels == 1:
									#  We are doing regression
									A__				:       Union[str, Any]	  =						MSELoss()
									A__				:       Any	  =						loss_fct(highway_logits.view(-1     )       ,			labels.view(-1     )     )
								else:
									A__				:       Any	  =						CrossEntropyLoss()
									A__				:       List[Any]	  =						loss_fct(highway_logits.view(-1       ,			self.num_labels     )       ,			labels.view(-1     )     )
								highway_losses.append(A__     )

							if train_highway:
								A__				:       int	  =						(sum(highway_losses[:-1]     ),) + outputs
								# exclude the final highway, of course
							else:
								A__				:       str	  =						(loss,) + outputs
						if not self.training:
							A__				:       Union[str, Any]	  =						outputs + ((original_entropy, highway_entropy), exit_layer)
							if output_layer >= 0:
								A__				:       List[str]	  =						(
								    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
								)  # use the highway of the last layer

						return outputs  # (loss), logits, (hidden_states), (attentions), entropy






import unittest

from transformers import load_tool
from transformers.utils import is_torch_available


if is_torch_available():
	import torch

from transformers.testing_utils import require_torch

from .test_tools_common import ToolTesterMixin



@require_torch
class        _a		(unittest.TestCase						,  __magic_name__							):


					'''simple docstring'''






					def     __A     (					self     ):
						A__				:       List[str]	  =						load_tool("""text-to-speech"""     )
						self.tool.setup()






					def     __A     (					self     ):
						# SpeechT5 isn't deterministic
						torch.manual_seed(0     )
						A__				:       Tuple	  =						self.tool("""hey"""     )
						A__				:       Tuple	  =						result.to_raw()
						self.assertTrue(
						    torch.allclose(
						        resulting_tensor[:3]       ,			torch.tensor([-0.0_0_0_5_9_6_6_6_6_8_8_3_2_1_1_5_8_2_9, -0.0_0_0_3_6_5_7_6_4_0_1_9_0_7_9_5_0_6_4, -0.0_0_0_1_3_4_3_9_5_0_2_7_9_9_8_8_3_4_8_5]     )       ,			)     )






					def     __A     (					self     ):
						# SpeechT5 isn't deterministic
						torch.manual_seed(0     )
						A__				:       Any	  =						self.tool("""hey"""     )
						A__				:       Optional[Any]	  =						result.to_raw()
						self.assertTrue(
						    torch.allclose(
						        resulting_tensor[:3]       ,			torch.tensor([-0.0_0_0_5_9_6_6_6_6_8_8_3_2_1_1_5_8_2_9, -0.0_0_0_3_6_5_7_6_4_0_1_9_0_7_9_5_0_6_4, -0.0_0_0_1_3_4_3_9_5_0_2_7_9_9_8_8_3_4_8_5]     )       ,			)     )



