




'''simple docstring'''





from __future__ import absolute_import, division, print_function, unicode_literals

from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers import RobertaConfig
from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.roberta.modeling_roberta import (
    ROBERTA_INPUTS_DOCSTRING,
    ROBERTA_START_DOCSTRING,
    RobertaEmbeddings,
)

from .modeling_highway_bert import BertPreTrainedModel, DeeBertModel, HighwayException, entropy

@add_start_docstrings(
    """The RoBERTa Model transformer with early exiting (DeeRoBERTa). """   ,   __lowerCAmelCase   ,   )
class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       str       							=		RobertaConfig
     SCREAMING_SNAKE_CASE_				:       Dict       							=		"""roberta"""






     def __init__(					self   ,_SCREAMING_SNAKE_CASE       )				->     Optional[Any]:
          super().__init__(_SCREAMING_SNAKE_CASE       )

          _snake_case				= RobertaEmbeddings(_SCREAMING_SNAKE_CASE       )
          self.init_weights()







@add_start_docstrings(
    """RoBERTa Model (with early exiting - DeeRoBERTa) with a classifier on top,
    also takes care of multi-layer training. """   ,   __lowerCAmelCase   ,   )
class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       List[Any]       							=		RobertaConfig
     SCREAMING_SNAKE_CASE_				:       List[str]       							=		"""roberta"""






     def __init__(					self   ,_SCREAMING_SNAKE_CASE       )				->     str:
          super().__init__(_SCREAMING_SNAKE_CASE       )
          _snake_case				= config.num_labels
          _snake_case				= config.num_hidden_layers

          _snake_case				= DeeRobertaModel(_SCREAMING_SNAKE_CASE       )
          _snake_case				= nn.Dropout(config.hidden_dropout_prob       )
          _snake_case				= nn.Linear(config.hidden_size   ,self.config.num_labels       )






     @add_start_docstrings_to_model_forward(_SCREAMING_SNAKE_CASE       )
     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=-1   ,_SCREAMING_SNAKE_CASE=False   ,)				->     List[str]:
          _snake_case				= self.num_layers
          try:
               _snake_case				= self.roberta(
                   _SCREAMING_SNAKE_CASE   ,attention_mask=_SCREAMING_SNAKE_CASE   ,token_type_ids=_SCREAMING_SNAKE_CASE   ,position_ids=_SCREAMING_SNAKE_CASE   ,head_mask=_SCREAMING_SNAKE_CASE   ,inputs_embeds=_SCREAMING_SNAKE_CASE   ,)

               _snake_case				= outputs[1]

               _snake_case				= self.dropout(_SCREAMING_SNAKE_CASE       )
               _snake_case				= self.classifier(_SCREAMING_SNAKE_CASE       )
               _snake_case				= (logits,) + outputs[2:]  # add hidden states and attention if they are here
          except HighwayException as e:
               _snake_case				= e.message
               _snake_case				= e.exit_layer
               _snake_case				= outputs[0]

          if not self.training:
               _snake_case				= entropy(_SCREAMING_SNAKE_CASE       )
               _snake_case				= []
               _snake_case				= []
          if labels is not None:
               if self.num_labels == 1:
                    #  We are doing regression
                    _snake_case				= MSELoss()
                    _snake_case				= loss_fct(logits.view(-1       )   ,labels.view(-1       )       )
               else:
                    _snake_case				= CrossEntropyLoss()
                    _snake_case				= loss_fct(logits.view(-1   ,self.num_labels       )   ,labels.view(-1       )       )

               # work with highway exits
               _snake_case				= []
               for highway_exit in outputs[-1]:
                    _snake_case				= highway_exit[0]
                    if not self.training:
                         highway_logits_all.append(_SCREAMING_SNAKE_CASE       )
                         highway_entropy.append(highway_exit[2]       )
                    if self.num_labels == 1:
                         #  We are doing regression
                         _snake_case				= MSELoss()
                         _snake_case				= loss_fct(highway_logits.view(-1       )   ,labels.view(-1       )       )
                    else:
                         _snake_case				= CrossEntropyLoss()
                         _snake_case				= loss_fct(highway_logits.view(-1   ,self.num_labels       )   ,labels.view(-1       )       )
                    highway_losses.append(_SCREAMING_SNAKE_CASE       )

               if train_highway:
                    _snake_case				= (sum(highway_losses[:-1]       ),) + outputs
                    # exclude the final highway, of course
               else:
                    _snake_case				= (loss,) + outputs
          if not self.training:
               _snake_case				= outputs + ((original_entropy, highway_entropy), exit_layer)
               if output_layer >= 0:
                    _snake_case				= (
                        (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
                    )  # use the highway of the last layer

          return outputs  # (loss), logits, (hidden_states), (attentions), entropy








'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_		:     int          =		logging.get_logger(__name__)

UpperCamelCase_		:     Any          =		{
    '''tiiuae/falcon-40b''': '''https://huggingface.co/tiiuae/falcon-40b/resolve/main/config.json''',
    '''tiiuae/falcon-7b''': '''https://huggingface.co/tiiuae/falcon-7b/resolve/main/config.json''',
}

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       List[str]       							=		"""falcon"""
     SCREAMING_SNAKE_CASE_				:       Optional[int]       							=		["""past_key_values"""]






     def __init__(					self   ,_SCREAMING_SNAKE_CASE=65_024   ,_SCREAMING_SNAKE_CASE=4_544   ,_SCREAMING_SNAKE_CASE=32   ,_SCREAMING_SNAKE_CASE=71   ,_SCREAMING_SNAKE_CASE=1e-5   ,_SCREAMING_SNAKE_CASE=0.0_2   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=0.0   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=True   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=11   ,_SCREAMING_SNAKE_CASE=11   ,**_SCREAMING_SNAKE_CASE   ,)				->     Dict:
          _snake_case				= vocab_size
          # Backward compatibility with n_embed kwarg
          _snake_case				= kwargs.pop("n_embed"   ,_SCREAMING_SNAKE_CASE       )
          _snake_case				= hidden_size if n_embed is None else n_embed
          _snake_case				= num_hidden_layers
          _snake_case				= num_attention_heads
          _snake_case				= layer_norm_epsilon
          _snake_case				= initializer_range
          _snake_case				= use_cache
          _snake_case				= hidden_dropout
          _snake_case				= attention_dropout

          _snake_case				= bos_token_id
          _snake_case				= eos_token_id
          _snake_case				= num_attention_heads if num_kv_heads is None else num_kv_heads
          _snake_case				= alibi
          _snake_case				= new_decoder_architecture
          _snake_case				= multi_query  # Ignored when new_decoder_architecture is True
          _snake_case				= parallel_attn
          _snake_case				= bias

          super().__init__(bos_token_id=_SCREAMING_SNAKE_CASE   ,eos_token_id=_SCREAMING_SNAKE_CASE   ,**_SCREAMING_SNAKE_CASE       )






     @property
     def 							_lowercase      (					self       )				->     Union[str, Any]:
          return self.hidden_size // self.num_attention_heads






     @property
     def 							_lowercase      (					self       )				->     Optional[int]:
          return not self.alibi


