

from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy

@add_start_docstrings(
    'The RoBERTa Model transformer with early exiting (DeeRoBERTa). '     ,				snake_case_     ,				)
class   _a					(			snake_case_				):




      """simple docstring"""

      _lowerCamelCase       :			Union[str, Any]      		=       RobertaConfig
      _lowerCamelCase       :			Dict      		=       'roberta'





      def __init__(   self							:				Tuple						,			UpperCAmelCase							:				Tuple	):
            super().__init__(UpperCAmelCase	)

            A_						  =				RobertaEmbeddings(UpperCAmelCase	)
            self.init_weights()




@add_start_docstrings(
    'RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,\n    also takes care of multi-layer training. '     ,				snake_case_     ,				)
class   _a					(			snake_case_				):




      """simple docstring"""

      _lowerCamelCase       :			str      		=       RobertaConfig
      _lowerCamelCase       :			List[str]      		=       'roberta'





      def __init__(   self							:				List[str]						,			UpperCAmelCase							:				List[Any]	):
            super().__init__(UpperCAmelCase	)
            A_						  =				config.num_labels
            A_						  =				config.num_hidden_layers

            A_						  =				DeeRobertaModel(UpperCAmelCase	)
            A_						  =				nn.Dropout(config.hidden_dropout_prob	)
            A_						  =				nn.Linear(config.hidden_size						,			self.config.num_labels	)







      @add_start_docstrings_to_model_forward(UpperCAmelCase	)
      def 		__A	(   self							:				Dict						,			UpperCAmelCase							:				Union[str, Any]=None						,			UpperCAmelCase							:				Dict=None						,			UpperCAmelCase							:				Dict=None						,			UpperCAmelCase							:				Optional[int]=None						,			UpperCAmelCase							:				Optional[Any]=None						,			UpperCAmelCase							:				int=None						,			UpperCAmelCase							:				Union[str, Any]=None						,			UpperCAmelCase							:				Optional[int]=-1						,			UpperCAmelCase							:				Optional[int]=False						,			):
            A_						  =				self.num_layers
            try:
                  A_						  =				self.roberta(
                      UpperCAmelCase						,			attention_mask=UpperCAmelCase						,			token_type_ids=UpperCAmelCase						,			position_ids=UpperCAmelCase						,			head_mask=UpperCAmelCase						,			inputs_embeds=UpperCAmelCase						,			)

                  A_						  =				outputs[1]

                  A_						  =				self.dropout(UpperCAmelCase	)
                  A_						  =				self.classifier(UpperCAmelCase	)
                  A_						  =				(logits,) + outputs[2:]  # add hidden states and attention if they are here
            except HighwayException as e:
                  A_						  =				e.message
                  A_						  =				e.exit_layer
                  A_						  =				outputs[0]

            if not self.training:
                  A_						  =				entropy(UpperCAmelCase	)
                  A_						  =				[]
                  A_						  =				[]
            if labels is not None:
                  if self.num_labels == 1:
                        #  We are doing regression
                        A_						  =				MSELoss()
                        A_						  =				loss_fct(logits.view(-1	)						,			labels.view(-1	)	)
                  else:
                        A_						  =				CrossEntropyLoss()
                        A_						  =				loss_fct(logits.view(-1						,			self.num_labels	)						,			labels.view(-1	)	)

                  # work with highway exits
                  A_						  =				[]
                  for highway_exit in outputs[-1]:
                        A_						  =				highway_exit[0]
                        if not self.training:
                              highway_logits_all.append(UpperCAmelCase	)
                              highway_entropy.append(highway_exit[2]	)
                        if self.num_labels == 1:
                              #  We are doing regression
                              A_						  =				MSELoss()
                              A_						  =				loss_fct(highway_logits.view(-1	)						,			labels.view(-1	)	)
                        else:
                              A_						  =				CrossEntropyLoss()
                              A_						  =				loss_fct(highway_logits.view(-1						,			self.num_labels	)						,			labels.view(-1	)	)
                        highway_losses.append(UpperCAmelCase	)

                  if train_highway:
                        A_						  =				(sum(highway_losses[:-1]	),) + outputs
                        # exclude the final highway, of course
                  else:
                        A_						  =				(loss,) + outputs
            if not self.training:
                  A_						  =				outputs + ((original_entropy, highway_entropy), exit_layer)
                  if output_layer >= 0:
                        A_						  =				(
                            (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                        )  # use the highway of the last layer

            return outputs  # (loss), logits, (hidden_states), (attentions), entropy







# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_SCREAMING_SNAKE_CASE   =  {
    "configuration_xmod": [
        "XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "XmodConfig",
        "XmodOnnxConfig",
    ],
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _SCREAMING_SNAKE_CASE   =  [
        "XMOD_PRETRAINED_MODEL_ARCHIVE_LIST",
        "XmodForCausalLM",
        "XmodForMaskedLM",
        "XmodForMultipleChoice",
        "XmodForQuestionAnswering",
        "XmodForSequenceClassification",
        "XmodForTokenClassification",
        "XmodModel",
        "XmodPreTrainedModel",
    ]

if TYPE_CHECKING:
    from .configuration_xmod import XMOD_PRETRAINED_CONFIG_ARCHIVE_MAP, XmodConfig, XmodOnnxConfig

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_xmod import (
            XMOD_PRETRAINED_MODEL_ARCHIVE_LIST,
            XmodForCausalLM,
            XmodForMaskedLM,
            XmodForMultipleChoice,
            XmodForQuestionAnswering,
            XmodForSequenceClassification,
            XmodForTokenClassification,
            XmodModel,
            XmodPreTrainedModel,
        )

else:
    import sys

    _SCREAMING_SNAKE_CASE   =  _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


