





from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy






@add_start_docstrings(
    'The RoBERTa Model transformer with early exiting (DeeRoBERTa). '	,   __A	,   )
class        _a(    __A    ):
		lowerCamelCase__     :Dict   				=		RobertaConfig
		lowerCamelCase__     :Any   				=		'roberta'







		def __init__(    self ,						__snake_case				)      ->      Dict:


							'''simple docstring'''

							super().__init__(__snake_case				)

							_snake_case       :					Optional[int]				       =   RobertaEmbeddings(__snake_case				)
							self.init_weights()






@add_start_docstrings(
    'RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. '	,   __A	,   )
class        _a(    __A    ):
		lowerCamelCase__     :int   				=		RobertaConfig
		lowerCamelCase__     :List[Any]   				=		'roberta'







		def __init__(    self ,						__snake_case				)      ->      Dict:


							'''simple docstring'''

							super().__init__(__snake_case				)
							_snake_case       :					List[Any]				       =   config.num_labels
							_snake_case       :					Tuple				       =   config.num_hidden_layers

							_snake_case       :					str				       =   DeeRobertaModel(__snake_case				)
							_snake_case       :					List[Any]				       =   nn.Dropout(config.hidden_dropout_prob				)
							_snake_case       :					Dict				       =   nn.Linear(config.hidden_size ,						self.config.num_labels				)







		@add_start_docstrings_to_model_forward(__snake_case				)
		def        lowercase       (    self ,						__snake_case=None ,						__snake_case=None ,						__snake_case=None ,						__snake_case=None ,						__snake_case=None ,						__snake_case=None ,						__snake_case=None ,						__snake_case=-1 ,						__snake_case=False ,						)      ->      Tuple:


							'''simple docstring'''

							_snake_case       :					Optional[int]				       =   self.num_layers
							try:
												_snake_case       :					List[str]				       =   self.roberta(
												    __snake_case ,						attention_mask=__snake_case ,						token_type_ids=__snake_case ,						position_ids=__snake_case ,						head_mask=__snake_case ,						inputs_embeds=__snake_case ,						)

												_snake_case       :					Union[str, Any]				       =   outputs[1]

												_snake_case       :					Dict				       =   self.dropout(__snake_case				)
												_snake_case       :					Dict				       =   self.classifier(__snake_case				)
												_snake_case       :					List[str]				       =   (logits,) + outputs[2:]  # add hidden states and attention if they are here
							except HighwayException as e:
												_snake_case       :					int				       =   e.message
												_snake_case       :					int				       =   e.exit_layer
												_snake_case       :					Optional[Any]				       =   outputs[0]

							if not self.training:
												_snake_case       :					int				       =   entropy(__snake_case				)
												_snake_case       :					str				       =   []
												_snake_case       :					str				       =   []
							if labels is not None:
												if self.num_labels == 1:
																	#  We are doing regression
																	_snake_case       :					Optional[Any]				       =   MSELoss()
																	_snake_case       :					int				       =   loss_fct(logits.view(-1				) ,						labels.view(-1				)				)
												else:
																	_snake_case       :					int				       =   CrossEntropyLoss()
																	_snake_case       :					Tuple				       =   loss_fct(logits.view(-1 ,						self.num_labels				) ,						labels.view(-1				)				)

												# work with highway exits
												_snake_case       :					Union[str, Any]				       =   []
												for highway_exit in outputs[-1]:
																	_snake_case       :					Optional[Any]				       =   highway_exit[0]
																	if not self.training:
																						highway_logits_all.append(__snake_case				)
																						highway_entropy.append(highway_exit[2]				)
																	if self.num_labels == 1:
																						#  We are doing regression
																						_snake_case       :					Dict				       =   MSELoss()
																						_snake_case       :					str				       =   loss_fct(highway_logits.view(-1				) ,						labels.view(-1				)				)
																	else:
																						_snake_case       :					List[str]				       =   CrossEntropyLoss()
																						_snake_case       :					int				       =   loss_fct(highway_logits.view(-1 ,						self.num_labels				) ,						labels.view(-1				)				)
																	highway_losses.append(__snake_case				)

												if train_highway:
																	_snake_case       :					str				       =   (sum(highway_losses[:-1]				),) + outputs
																	# exclude the final highway, of course
												else:
																	_snake_case       :					Any				       =   (loss,) + outputs
							if not self.training:
												_snake_case       :					str				       =   outputs + ((original_entropy, highway_entropy), exit_layer)
												if output_layer >= 0:
																	_snake_case       :					Tuple				       =   (
																	    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																	)  # use the highway of the last layer

							return outputs  # (loss), logits, (hidden_states), (attentions), entropy






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__lowerCAmelCase :str     =    {
    'configuration_nllb_moe': [
        'NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'NllbMoeConfig',
    ]
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__lowerCAmelCase :Optional[int]     =    [
							    'NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST',
							    'NllbMoeForConditionalGeneration',
							    'NllbMoeModel',
							    'NllbMoePreTrainedModel',
							    'NllbMoeTop2Router',
							    'NllbMoeSparseMLP',
							]


if TYPE_CHECKING:
							from .configuration_nllb_moe import (
							    NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP,
							    NllbMoeConfig,
							)

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_nllb_moe import (
														    NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST,
														    NllbMoeForConditionalGeneration,
														    NllbMoeModel,
														    NllbMoePreTrainedModel,
														    NllbMoeSparseMLP,
														    NllbMoeTopaRouter,
														)


else:
							import sys

							__lowerCAmelCase :int     =    _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)