





from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy

@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """    ,					a    ,					)
class       __A     (						a    ):
				__A      =				RobertaConfig
				__A      =				"""roberta"""


				def __init__(		self   , UpperCAmelCase_							):
								super().__init__(UpperCAmelCase_							)

								lowerCamelCase	 =RobertaEmbeddings(UpperCAmelCase_							)
								self.init_weights()




@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """    ,					a    ,					)
class       __A     (						a    ):
				__A      =				RobertaConfig
				__A      =				"""roberta"""


				def __init__(		self   , UpperCAmelCase_							):
								super().__init__(UpperCAmelCase_							)
								lowerCamelCase	 =config.num_labels
								lowerCamelCase	 =config.num_hidden_layers

								lowerCamelCase	 =DeeRobertaModel(UpperCAmelCase_							)
								lowerCamelCase	 =nn.Dropout(config.hidden_dropout_prob							)
								lowerCamelCase	 =nn.Linear(config.hidden_size   , self.config.num_labels							)


				@add_start_docstrings_to_model_forward(UpperCAmelCase_							)
				def _snake_case       (		self   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=-1   , UpperCAmelCase_=False   , ):
								lowerCamelCase	 =self.num_layers
								try:
												lowerCamelCase	 =self.roberta(
												    UpperCAmelCase_   , attention_mask=UpperCAmelCase_   , token_type_ids=UpperCAmelCase_   , position_ids=UpperCAmelCase_   , head_mask=UpperCAmelCase_   , inputs_embeds=UpperCAmelCase_   , )

												lowerCamelCase	 =outputs[1]

												lowerCamelCase	 =self.dropout(UpperCAmelCase_							)
												lowerCamelCase	 =self.classifier(UpperCAmelCase_							)
												lowerCamelCase	 =(logits,) + outputs[2:]  # add hidden states and attention if they are here
								except HighwayException as e:
												lowerCamelCase	 =e.message
												lowerCamelCase	 =e.exit_layer
												lowerCamelCase	 =outputs[0]

								if not self.training:
												lowerCamelCase	 =entropy(UpperCAmelCase_							)
												lowerCamelCase	 =[]
												lowerCamelCase	 =[]
								if labels is not None:
												if self.num_labels == 1:
																#  We are doing regression
																lowerCamelCase	 =MSELoss()
																lowerCamelCase	 =loss_fct(logits.view(-1							)   , labels.view(-1							)							)
												else:
																lowerCamelCase	 =CrossEntropyLoss()
																lowerCamelCase	 =loss_fct(logits.view(-1   , self.num_labels							)   , labels.view(-1							)							)

												# work with highway exits
												lowerCamelCase	 =[]
												for highway_exit in outputs[-1]:
																lowerCamelCase	 =highway_exit[0]
																if not self.training:
																				highway_logits_all.append(UpperCAmelCase_							)
																				highway_entropy.append(highway_exit[2]							)
																if self.num_labels == 1:
																				#  We are doing regression
																				lowerCamelCase	 =MSELoss()
																				lowerCamelCase	 =loss_fct(highway_logits.view(-1							)   , labels.view(-1							)							)
																else:
																				lowerCamelCase	 =CrossEntropyLoss()
																				lowerCamelCase	 =loss_fct(highway_logits.view(-1   , self.num_labels							)   , labels.view(-1							)							)
																highway_losses.append(UpperCAmelCase_							)

												if train_highway:
																lowerCamelCase	 =(sum(highway_losses[:-1]							),) + outputs
																# exclude the final highway, of course
												else:
																lowerCamelCase	 =(loss,) + outputs
								if not self.training:
												lowerCamelCase	 =outputs + ((original_entropy, highway_entropy), exit_layer)
												if output_layer >= 0:
																lowerCamelCase	 =(
																    (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
																)  # use the highway of the last layer

								return outputs  # (loss), logits, (hidden_states), (attentions), entropy












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


UpperCAmelCase__						:			List[Any]									={'''configuration_yolos''': ['''YOLOS_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''YolosConfig''', '''YolosOnnxConfig''']}

try:
							if not is_vision_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__						:			Dict									=['''YolosFeatureExtractor''']
							UpperCAmelCase__						:			List[Any]									=['''YolosImageProcessor''']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCAmelCase__						:			Optional[Any]									=[
							    '''YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''YolosForObjectDetection''',
							    '''YolosModel''',
							    '''YolosPreTrainedModel''',
							]


if TYPE_CHECKING:
							from .configuration_yolos import YOLOS_PRETRAINED_CONFIG_ARCHIVE_MAP, YolosConfig, YolosOnnxConfig

							try:
														if not is_vision_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .feature_extraction_yolos import YolosFeatureExtractor
														from .image_processing_yolos import YolosImageProcessor

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_yolos import (
														    YOLOS_PRETRAINED_MODEL_ARCHIVE_LIST,
														    YolosForObjectDetection,
														    YolosModel,
														    YolosPreTrainedModel,
														)


else:
							import sys

							UpperCAmelCase__						:			Optional[Any]									=_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





