


'''simple docstring'''



from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy


@add_start_docstrings(
    '''The RoBERTa Model transformer with early exiting (DeeRoBERTa). '''       ,   __A       ,   )
class 		_snake_case					(     __A	):
     lowerCAmelCase      :Union[str, Any]   =					RobertaConfig
     lowerCAmelCase      :List[Any]   =					'''roberta'''




     def __init__(      self							,							_lowerCamelCase):
        super().__init__(_lowerCamelCase)

        UpperCAmelCase__  :					List[str]        =     RobertaEmbeddings(_lowerCamelCase)
        self.init_weights()


@add_start_docstrings(
    '''RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. '''       ,   __A       ,   )
class 		_snake_case					(     __A	):
     lowerCAmelCase      :int   =					RobertaConfig
     lowerCAmelCase      :Optional[Any]   =					'''roberta'''




     def __init__(      self							,							_lowerCamelCase):
        super().__init__(_lowerCamelCase)
        UpperCAmelCase__  :					Optional[int]        =     config.num_labels
        UpperCAmelCase__  :					List[Any]        =     config.num_hidden_layers

        UpperCAmelCase__  :					Any        =     DeeRobertaModel(_lowerCamelCase)
        UpperCAmelCase__  :					str        =     nn.Dropout(config.hidden_dropout_prob)
        UpperCAmelCase__  :					int        =     nn.Linear(config.hidden_size							,							self.config.num_labels)




     @add_start_docstrings_to_model_forward(_lowerCamelCase)
     def 				snake_case__						(      self							,							_lowerCamelCase=None							,							_lowerCamelCase=None							,							_lowerCamelCase=None							,							_lowerCamelCase=None							,							_lowerCamelCase=None							,							_lowerCamelCase=None							,							_lowerCamelCase=None							,							_lowerCamelCase=-1							,							_lowerCamelCase=False							,							):
        UpperCAmelCase__  :					List[str]        =     self.num_layers
        try:
           UpperCAmelCase__  :					Tuple        =     self.roberta(
               _lowerCamelCase							,							attention_mask=_lowerCamelCase							,							token_type_ids=_lowerCamelCase							,							position_ids=_lowerCamelCase							,							head_mask=_lowerCamelCase							,							inputs_embeds=_lowerCamelCase							,							)

           UpperCAmelCase__  :					Optional[Any]        =     outputs[1]

           UpperCAmelCase__  :					Any        =     self.dropout(_lowerCamelCase)
           UpperCAmelCase__  :					int        =     self.classifier(_lowerCamelCase)
           UpperCAmelCase__  :					int        =     (logits,) + outputs[2:]  # add hidden states and attention if they are here
        except HighwayException as e:
           UpperCAmelCase__  :					List[str]        =     e.message
           UpperCAmelCase__  :					Optional[Any]        =     e.exit_layer
           UpperCAmelCase__  :					Any        =     outputs[0]

        if not self.training:
           UpperCAmelCase__  :					List[str]        =     entropy(_lowerCamelCase)
           UpperCAmelCase__  :					int        =     []
           UpperCAmelCase__  :					Optional[Any]        =     []
        if labels is not None:
           if self.num_labels == 1:
              #  We are doing regression
              UpperCAmelCase__  :					Dict        =     MSELoss()
              UpperCAmelCase__  :					Any        =     loss_fct(logits.view(-1)							,							labels.view(-1))
           else:
              UpperCAmelCase__  :					str        =     CrossEntropyLoss()
              UpperCAmelCase__  :					Any        =     loss_fct(logits.view(-1							,							self.num_labels)							,							labels.view(-1))

           # work with highway exits
           UpperCAmelCase__  :					List[Any]        =     []
           for highway_exit in outputs[-1]:
              UpperCAmelCase__  :					Any        =     highway_exit[0]
              if not self.training:
                 highway_logits_all.append(_lowerCamelCase)
                 highway_entropy.append(highway_exit[2])
              if self.num_labels == 1:
                 #  We are doing regression
                 UpperCAmelCase__  :					Union[str, Any]        =     MSELoss()
                 UpperCAmelCase__  :					Optional[Any]        =     loss_fct(highway_logits.view(-1)							,							labels.view(-1))
              else:
                 UpperCAmelCase__  :					Tuple        =     CrossEntropyLoss()
                 UpperCAmelCase__  :					List[Any]        =     loss_fct(highway_logits.view(-1							,							self.num_labels)							,							labels.view(-1))
              highway_losses.append(_lowerCamelCase)

           if train_highway:
              UpperCAmelCase__  :					int        =     (sum(highway_losses[:-1]),) + outputs
              # exclude the final highway, of course
           else:
              UpperCAmelCase__  :					Any        =     (loss,) + outputs
        if not self.training:
           UpperCAmelCase__  :					Union[str, Any]        =     outputs + ((original_entropy, highway_entropy), exit_layer)
           if output_layer >= 0:
              UpperCAmelCase__  :					Tuple        =     (
                  (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
              )  # use the highway of the last layer

        return outputs  # (loss), logits, (hidden_states), (attentions), entropy



'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__A	={
    'configuration_bert': ['BERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'BertConfig', 'BertOnnxConfig'],
    'tokenization_bert': ['BasicTokenizer', 'BertTokenizer', 'WordpieceTokenizer'],
}

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A	=['BertTokenizerFast']

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A	=[
		    'BERT_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'BertForMaskedLM',
		    'BertForMultipleChoice',
		    'BertForNextSentencePrediction',
		    'BertForPreTraining',
		    'BertForQuestionAnswering',
		    'BertForSequenceClassification',
		    'BertForTokenClassification',
		    'BertLayer',
		    'BertLMHeadModel',
		    'BertModel',
		    'BertPreTrainedModel',
		    'load_tf_weights_in_bert',
		]

try:
		if not is_tf_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A	=[
		    'TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'TFBertEmbeddings',
		    'TFBertForMaskedLM',
		    'TFBertForMultipleChoice',
		    'TFBertForNextSentencePrediction',
		    'TFBertForPreTraining',
		    'TFBertForQuestionAnswering',
		    'TFBertForSequenceClassification',
		    'TFBertForTokenClassification',
		    'TFBertLMHeadModel',
		    'TFBertMainLayer',
		    'TFBertModel',
		    'TFBertPreTrainedModel',
		]
try:
		if not is_tensorflow_text_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A	=['TFBertTokenizer']

try:
		if not is_flax_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A	=[
		    'FlaxBertForCausalLM',
		    'FlaxBertForMaskedLM',
		    'FlaxBertForMultipleChoice',
		    'FlaxBertForNextSentencePrediction',
		    'FlaxBertForPreTraining',
		    'FlaxBertForQuestionAnswering',
		    'FlaxBertForSequenceClassification',
		    'FlaxBertForTokenClassification',
		    'FlaxBertModel',
		    'FlaxBertPreTrainedModel',
		]

if TYPE_CHECKING:
		from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, BertConfig, BertOnnxConfig
		from .tokenization_bert import BasicTokenizer, BertTokenizer, WordpieceTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_bert_fast import BertTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_bert import (
				    BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    BertForMaskedLM,
				    BertForMultipleChoice,
				    BertForNextSentencePrediction,
				    BertForPreTraining,
				    BertForQuestionAnswering,
				    BertForSequenceClassification,
				    BertForTokenClassification,
				    BertLayer,
				    BertLMHeadModel,
				    BertModel,
				    BertPreTrainedModel,
				    load_tf_weights_in_bert,
				)

		try:
				if not is_tf_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_tf_bert import (
				    TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TFBertEmbeddings,
				    TFBertForMaskedLM,
				    TFBertForMultipleChoice,
				    TFBertForNextSentencePrediction,
				    TFBertForPreTraining,
				    TFBertForQuestionAnswering,
				    TFBertForSequenceClassification,
				    TFBertForTokenClassification,
				    TFBertLMHeadModel,
				    TFBertMainLayer,
				    TFBertModel,
				    TFBertPreTrainedModel,
				)

		try:
				if not is_tensorflow_text_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_bert_tf import TFBertTokenizer

		try:
				if not is_flax_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_flax_bert import (
				    FlaxBertForCausalLM,
				    FlaxBertForMaskedLM,
				    FlaxBertForMultipleChoice,
				    FlaxBertForNextSentencePrediction,
				    FlaxBertForPreTraining,
				    FlaxBertForQuestionAnswering,
				    FlaxBertForSequenceClassification,
				    FlaxBertForTokenClassification,
				    FlaxBertModel,
				    FlaxBertPreTrainedModel,
				)

else:
		import sys

		__A	=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)