import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
       from ...tokenization_utils_base import TextInput
from ...utils import logging


a_							=      logging.get_logger(__name__)

a_							=      {"""vocab_file""": """spiece.model"""}

a_							=      {
    """vocab_file""": {
        """t5-small""": """https://huggingface.co/t5-small/resolve/main/spiece.model""",
        """t5-base""": """https://huggingface.co/t5-base/resolve/main/spiece.model""",
        """t5-large""": """https://huggingface.co/t5-large/resolve/main/spiece.model""",
        """t5-3b""": """https://huggingface.co/t5-3b/resolve/main/spiece.model""",
        """t5-11b""": """https://huggingface.co/t5-11b/resolve/main/spiece.model""",
    }
}


# TODO(PVP) - this should be removed in Transformers v5
a_							=      {
    """t5-small""": 512,
    """t5-base""": 512,
    """t5-large""": 512,
    """t5-3b""": 512,
    """t5-11b""": 512,
}

a_							=      """▁"""


class 			UpperCAmelCase__    (		snake_case					):



  """simple docstring"""






  lowerCAmelCase__		:	str    		= VOCAB_FILES_NAMES
  lowerCAmelCase__		:	List[str]    		= PRETRAINED_VOCAB_FILES_MAP
  lowerCAmelCase__		:	Tuple    		= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  lowerCAmelCase__		:	Any    		= ['input_ids', 'attention_mask']






  def __init__(				self:			List[Any]				,				__lowerCAmelCase:			Union[str, Any]				,				__lowerCAmelCase:			Union[str, Any]="</s>"				,				__lowerCAmelCase:			Optional[int]="<unk>"				,				__lowerCAmelCase:			List[Any]="<pad>"				,				__lowerCAmelCase:			Dict=100				,				__lowerCAmelCase:			Optional[int]=None				,				__lowerCAmelCase:			Optional[Dict[str, Any]] = None				,				__lowerCAmelCase:			List[Any]=True				,				**__lowerCAmelCase:			Union[str, Any]				,				) -> None:




    '''simple docstring'''




    if extra_ids > 0 and additional_special_tokens is None:
      __UpperCAmelCase						    =       [F'''<extra_id_{i}>''' for i in range(__lowerCAmelCase   )]
    elif extra_ids > 0 and additional_special_tokens is not None:
      # Check that we have the right number of extra_id special tokens
      __UpperCAmelCase						    =       len(set(filter(lambda __lowerCAmelCase   : bool("extra_id" in str(__lowerCAmelCase   )   )				,				__lowerCAmelCase   )   )   )
      if extra_tokens != extra_ids:
        raise ValueError(
            F'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
            " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
            " tokens"   )
    if legacy:
      logger.warning_once(
          F'''You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to'''
          " read the related pull request available at https://github.com/huggingface/transformers/pull/24565"   )

    __UpperCAmelCase						    =       legacy
    __UpperCAmelCase						    =       {} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        eos_token=__lowerCAmelCase				,				unk_token=__lowerCAmelCase				,				pad_token=__lowerCAmelCase				,				extra_ids=__lowerCAmelCase				,				additional_special_tokens=__lowerCAmelCase				,				sp_model_kwargs=self.sp_model_kwargs				,				legacy=__lowerCAmelCase				,				**__lowerCAmelCase				,				)

    __UpperCAmelCase						    =       vocab_file
    __UpperCAmelCase						    =       extra_ids

    __UpperCAmelCase						    =       spm.SentencePieceProcessor(**self.sp_model_kwargs   )
    self.sp_model.Load(__lowerCAmelCase   )






  @staticmethod
  def 	_UpperCAmelCase				(				__lowerCAmelCase:			int				,				__lowerCAmelCase:			Dict				,				__lowerCAmelCase:			Tuple   ) -> List[Any]:




    '''simple docstring'''




    if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
      __UpperCAmelCase						    =       TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
      if init_max_model_length is not None and init_max_model_length != max_model_length:
        return init_max_model_length
      elif init_max_model_length is None:
        warnings.warn(
            "This tokenizer was incorrectly instantiated with a model max length of"
            F''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
            " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
            " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
            F''' {pretrained_model_name_or_path} automatically truncating your input to'''
            F''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
            F''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
            " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
            " instantiate this tokenizer with `model_max_length` set to your preferred value."				,				__lowerCAmelCase				,				)

    return max_model_length






  @property
  def 	_UpperCAmelCase				(				self:			int   ) -> int:




    '''simple docstring'''




    return self.sp_model.get_piece_size() + self._extra_ids






  def 	_UpperCAmelCase				(				self:			List[str]   ) -> Optional[Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       {self.convert_ids_to_tokens(__lowerCAmelCase   ): i for i in range(self.vocab_size   )}
    vocab.update(self.added_tokens_encoder   )
    return vocab






  def 	_UpperCAmelCase				(				self:			Any				,				__lowerCAmelCase:			List[int]				,				__lowerCAmelCase:			Optional[List[int]] = None				,				__lowerCAmelCase:			bool = False   ) -> List[int]:




    '''simple docstring'''




    if already_has_special_tokens:
      return super().get_special_tokens_mask(
          token_ids_a=__lowerCAmelCase				,				token_ids_a=__lowerCAmelCase				,				already_has_special_tokens=__lowerCAmelCase   )

    # normal case: some special tokens
    if token_ids_a is None:
      return ([0] * len(__lowerCAmelCase   )) + [1]
    return ([0] * len(__lowerCAmelCase   )) + [1] + ([0] * len(__lowerCAmelCase   )) + [1]






  def 	_UpperCAmelCase				(				self:			Any   ) -> Optional[int]:




    '''simple docstring'''




    return list(
        set(filter(lambda __lowerCAmelCase   : bool(re.search(r"<extra_id_\d+>"				,				__lowerCAmelCase   )   ) is not None				,				self.additional_special_tokens   )   )   )






  def 	_UpperCAmelCase				(				self:			Dict   ) -> Optional[Any]:




    '''simple docstring'''




    return [self._convert_token_to_id(__lowerCAmelCase   ) for token in self.get_sentinel_tokens()]






  def 	_UpperCAmelCase				(				self:			Any				,				__lowerCAmelCase:			List[int]   ) -> List[int]:




    '''simple docstring'''




    if len(__lowerCAmelCase   ) > 0 and token_ids[-1] == self.eos_token_id:
      warnings.warn(
          F'''This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated'''
          " eos tokens being added."   )
      return token_ids
    else:
      return token_ids + [self.eos_token_id]






  def 	_UpperCAmelCase				(				self:			Union[str, Any]				,				__lowerCAmelCase:			List[int]				,				__lowerCAmelCase:			Optional[List[int]] = None   ) -> List[int]:




    '''simple docstring'''




    __UpperCAmelCase						    =       [self.eos_token_id]

    if token_ids_a is None:
      return len(token_ids_a + eos   ) * [0]
    return len(token_ids_a + eos + token_ids_a + eos   ) * [0]






  def 	_UpperCAmelCase				(				self:			List[Any]				,				__lowerCAmelCase:			List[int]				,				__lowerCAmelCase:			Optional[List[int]] = None   ) -> List[int]:




    '''simple docstring'''




    __UpperCAmelCase						    =       self._add_eos_if_not_present(__lowerCAmelCase   )
    if token_ids_a is None:
      return token_ids_a
    else:
      __UpperCAmelCase						    =       self._add_eos_if_not_present(__lowerCAmelCase   )
      return token_ids_a + token_ids_a






  def __getstate__(				self:			Union[str, Any]   ) -> int:




    '''simple docstring'''




    __UpperCAmelCase						    =       self.__dict__.copy()
    __UpperCAmelCase						    =       None
    return state






  def __setstate__(				self:			Any				,				__lowerCAmelCase:			List[str]   ) -> Tuple:




    '''simple docstring'''




    __UpperCAmelCase						    =       d

    # for backward compatibility
    if not hasattr(self				,				"sp_model_kwargs"   ):
      __UpperCAmelCase						    =       {}

    __UpperCAmelCase						    =       spm.SentencePieceProcessor(**self.sp_model_kwargs   )
    self.sp_model.Load(self.vocab_file   )






  def 	_UpperCAmelCase				(				self:			Tuple				,				__lowerCAmelCase:			"TextInput"				,				**__lowerCAmelCase:			Dict   ) -> List[str]:




    '''simple docstring'''




    if not self.legacy:
      __UpperCAmelCase						    =       SPIECE_UNDERLINE + text.replace(__lowerCAmelCase				,				" "   )
    return super().tokenize(__lowerCAmelCase				,				**__lowerCAmelCase   )






  def 	_UpperCAmelCase				(				self:			List[Any]				,				__lowerCAmelCase:			Dict				,				**__lowerCAmelCase:			Union[str, Any]   ) -> List[Any]:




    '''simple docstring'''




    if not self.legacy:
      __UpperCAmelCase						    =       text.startswith(__lowerCAmelCase   )
      if is_first:
        __UpperCAmelCase						    =       text[1:]

    __UpperCAmelCase						    =       self.sp_model.encode(__lowerCAmelCase				,				out_type=__lowerCAmelCase   )

    if not self.legacy and not is_first and not text.startswith(" "   ) and tokens[0].startswith(__lowerCAmelCase   ):
      __UpperCAmelCase						    =       ([tokens[0][1:]] if len(tokens[0]   ) > 1 else []) + tokens[1:]
    return tokens






  def 	_UpperCAmelCase				(				self:			Tuple				,				__lowerCAmelCase:			Optional[Any]   ) -> Union[str, Any]:




    '''simple docstring'''




    if token.startswith("<extra_id_"   ):
      __UpperCAmelCase						    =       re.match(r"<extra_id_(\d+)>"				,				__lowerCAmelCase   )
      __UpperCAmelCase						    =       int(match.group(1   )   )
      return self.vocab_size - num - 1
    return self.sp_model.piece_to_id(__lowerCAmelCase   )






  def 	_UpperCAmelCase				(				self:			Optional[Any]				,				__lowerCAmelCase:			List[Any]   ) -> int:




    '''simple docstring'''




    if index < self.sp_model.get_piece_size():
      __UpperCAmelCase						    =       self.sp_model.IdToPiece(__lowerCAmelCase   )
    else:
      __UpperCAmelCase						    =       F'''<extra_id_{self.vocab_size - 1 - index}>'''
    return token






  def 	_UpperCAmelCase				(				self:			Union[str, Any]				,				__lowerCAmelCase:			Union[str, Any]   ) -> str:




    '''simple docstring'''




    __UpperCAmelCase						    =       []
    __UpperCAmelCase						    =       ""
    __UpperCAmelCase						    =       False
    for token in tokens:
      # make sure that special tokens are not decoded using sentencepiece model
      if token in self.all_special_tokens:
        if not prev_is_special:
          out_string += " "
        out_string += self.sp_model.decode(__lowerCAmelCase   ) + token
        __UpperCAmelCase						    =       True
        __UpperCAmelCase						    =       []
      else:
        current_sub_tokens.append(__lowerCAmelCase   )
        __UpperCAmelCase						    =       False
    out_string += self.sp_model.decode(__lowerCAmelCase   )
    return out_string.strip()






  def 	_UpperCAmelCase				(				self:			str				,				__lowerCAmelCase:			str				,				__lowerCAmelCase:			Optional[str] = None   ) -> Tuple[str]:




    '''simple docstring'''




    if not os.path.isdir(__lowerCAmelCase   ):
      logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''   )
      return
    __UpperCAmelCase						    =       os.path.join(
        __lowerCAmelCase				,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )

    if os.path.abspath(self.vocab_file   ) != os.path.abspath(__lowerCAmelCase   ) and os.path.isfile(self.vocab_file   ):
      copyfile(self.vocab_file				,				__lowerCAmelCase   )
    elif not os.path.isfile(self.vocab_file   ):
      with open(__lowerCAmelCase				,				"wb"   ) as fi:
        __UpperCAmelCase						    =       self.sp_model.serialized_model_proto()
        fi.write(__lowerCAmelCase   )

    return (out_vocab_file,)




import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEmbeddings,
    BertLayer,
    BertPooler,
    BertPreTrainedModel,
)




def       __lowerCAmelCase      (						A_   : Optional[Any]					)    ->  List[str]:
  __UpperCAmelCase						    =       torch.exp(A_					)
  __UpperCAmelCase						    =       torch.sum(A_						,	dim=1					)  # sum of exp(x_i)
  __UpperCAmelCase						    =       torch.sum(x * exp_x						,	dim=1					)  # sum of x_i * exp(x_i)
  return torch.log(A_					) - B / A


class 			UpperCAmelCase__    (		nn.Module					):



  """simple docstring"""






  def __init__(				self:			Optional[Any]				,				__lowerCAmelCase:			int   ) -> List[Any]:




    '''simple docstring'''




    super().__init__()
    __UpperCAmelCase						    =       config.output_attentions
    __UpperCAmelCase						    =       config.output_hidden_states
    __UpperCAmelCase						    =       nn.ModuleList([BertLayer(__lowerCAmelCase   ) for _ in range(config.num_hidden_layers   )]   )
    __UpperCAmelCase						    =       nn.ModuleList([BertHighway(__lowerCAmelCase   ) for _ in range(config.num_hidden_layers   )]   )

    __UpperCAmelCase						    =       [-1 for _ in range(config.num_hidden_layers   )]






  def 	_UpperCAmelCase				(				self:			Optional[Any]				,				__lowerCAmelCase:			List[str]   ) -> Optional[Any]:




    '''simple docstring'''




    if (type(__lowerCAmelCase   ) is float) or (type(__lowerCAmelCase   ) is int):
      for i in range(len(self.early_exit_entropy   )   ):
        __UpperCAmelCase						    =       x
    else:
      __UpperCAmelCase						    =       x






  def 	_UpperCAmelCase				(				self:			Union[str, Any]				,				__lowerCAmelCase:			Optional[int]   ) -> Optional[Any]:




    '''simple docstring'''




    __UpperCAmelCase						    =       pooler.state_dict()
    for highway in self.highway:
      for name, param in highway.pooler.state_dict().items():
        param.copy_(loaded_model[name]   )






  def 	_UpperCAmelCase				(				self:			Dict				,				__lowerCAmelCase:			Optional[int]				,				__lowerCAmelCase:			Union[str, Any]=None				,				__lowerCAmelCase:			Any=None				,				__lowerCAmelCase:			Optional[int]=None				,				__lowerCAmelCase:			Optional[Any]=None				,				) -> List[str]:




    '''simple docstring'''




    __UpperCAmelCase						    =       ()
    __UpperCAmelCase						    =       ()
    __UpperCAmelCase						    =       ()
    for i, layer_module in enumerate(self.layer   ):
      if self.output_hidden_states:
        __UpperCAmelCase						    =       all_hidden_states + (hidden_states,)

      __UpperCAmelCase						    =       layer_module(
          __lowerCAmelCase				,				__lowerCAmelCase				,				head_mask[i]				,				__lowerCAmelCase				,				__lowerCAmelCase   )
      __UpperCAmelCase						    =       layer_outputs[0]

      if self.output_attentions:
        __UpperCAmelCase						    =       all_attentions + (layer_outputs[1],)

      __UpperCAmelCase						    =       (hidden_states,)
      if self.output_hidden_states:
        __UpperCAmelCase						    =       current_outputs + (all_hidden_states,)
      if self.output_attentions:
        __UpperCAmelCase						    =       current_outputs + (all_attentions,)

      __UpperCAmelCase						    =       self.highway[i](__lowerCAmelCase   )
      # logits, pooled_output

      if not self.training:
        __UpperCAmelCase						    =       highway_exit[0]
        __UpperCAmelCase						    =       entropy(__lowerCAmelCase   )
        __UpperCAmelCase						    =       highway_exit + (highway_entropy,)  # logits, hidden_states(?), entropy
        __UpperCAmelCase						    =       all_highway_exits + (highway_exit,)

        if highway_entropy < self.early_exit_entropy[i]:
          __UpperCAmelCase						    =       (highway_logits,) + current_outputs[1:] + (all_highway_exits,)
          raise HighwayException(__lowerCAmelCase				,				i + 1   )
      else:
        __UpperCAmelCase						    =       all_highway_exits + (highway_exit,)

        # Add last layer
    if self.output_hidden_states:
      __UpperCAmelCase						    =       all_hidden_states + (hidden_states,)

    __UpperCAmelCase						    =       (hidden_states,)
    if self.output_hidden_states:
      __UpperCAmelCase						    =       outputs + (all_hidden_states,)
    if self.output_attentions:
      __UpperCAmelCase						    =       outputs + (all_attentions,)

    __UpperCAmelCase						    =       outputs + (all_highway_exits,)
    return outputs  # last-layer hidden state, (all hidden states), (all attentions), all highway exits






@add_start_docstrings(
    'The Bert Model transformer with early exiting (DeeBERT). '		,	snake_case		,	)
class 			UpperCAmelCase__    (		snake_case					):



  """simple docstring"""






  def __init__(				self:			Union[str, Any]				,				__lowerCAmelCase:			Optional[Any]   ) -> int:




    '''simple docstring'''




    super().__init__(__lowerCAmelCase   )
    __UpperCAmelCase						    =       config

    __UpperCAmelCase						    =       BertEmbeddings(__lowerCAmelCase   )
    __UpperCAmelCase						    =       DeeBertEncoder(__lowerCAmelCase   )
    __UpperCAmelCase						    =       BertPooler(__lowerCAmelCase   )

    self.init_weights()






  def 	_UpperCAmelCase				(				self:			Any   ) -> Optional[Any]:




    '''simple docstring'''




    self.encoder.init_highway_pooler(self.pooler   )






  def 	_UpperCAmelCase				(				self:			List[str]   ) -> Tuple:




    '''simple docstring'''




    return self.embeddings.word_embeddings






  def 	_UpperCAmelCase				(				self:			Optional[Any]				,				__lowerCAmelCase:			List[str]   ) -> Dict:




    '''simple docstring'''




    __UpperCAmelCase						    =       value






  def 	_UpperCAmelCase				(				self:			Union[str, Any]				,				__lowerCAmelCase:			Optional[int]   ) -> Any:




    '''simple docstring'''




    for layer, heads in heads_to_prune.items():
      self.encoder.layer[layer].attention.prune_heads(__lowerCAmelCase   )






  @add_start_docstrings_to_model_forward(__lowerCAmelCase   )
  def 	_UpperCAmelCase				(				self:			Optional[int]				,				__lowerCAmelCase:			Dict=None				,				__lowerCAmelCase:			Tuple=None				,				__lowerCAmelCase:			List[str]=None				,				__lowerCAmelCase:			Union[str, Any]=None				,				__lowerCAmelCase:			str=None				,				__lowerCAmelCase:			Dict=None				,				__lowerCAmelCase:			Optional[int]=None				,				__lowerCAmelCase:			Tuple=None				,				) -> List[str]:




    '''simple docstring'''




    if input_ids is not None and inputs_embeds is not None:
      raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"   )
    elif input_ids is not None:
      __UpperCAmelCase						    =       input_ids.size()
    elif inputs_embeds is not None:
      __UpperCAmelCase						    =       inputs_embeds.size()[:-1]
    else:
      raise ValueError("You have to specify either input_ids or inputs_embeds"   )

    __UpperCAmelCase						    =       input_ids.device if input_ids is not None else inputs_embeds.device

    if attention_mask is None:
      __UpperCAmelCase						    =       torch.ones(__lowerCAmelCase				,				device=__lowerCAmelCase   )
    if encoder_attention_mask is None:
      __UpperCAmelCase						    =       torch.ones(__lowerCAmelCase				,				device=__lowerCAmelCase   )
    if token_type_ids is None:
      __UpperCAmelCase						    =       torch.zeros(__lowerCAmelCase				,				dtype=torch.long				,				device=__lowerCAmelCase   )

    # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
    # ourselves in which case we just need to make it broadcastable to all heads.
    __UpperCAmelCase						    =       self.get_extended_attention_mask(__lowerCAmelCase				,				__lowerCAmelCase				,				__lowerCAmelCase   )

    # If a 2D ou 3D attention mask is provided for the cross-attention
    # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
    if encoder_attention_mask.dim() == 3:
      __UpperCAmelCase						    =       encoder_attention_mask[:, None, :, :]
    if encoder_attention_mask.dim() == 2:
      __UpperCAmelCase						    =       encoder_attention_mask[:, None, None, :]

    __UpperCAmelCase						    =       encoder_extended_attention_mask.to(
        dtype=next(self.parameters()   ).dtype   )  # fp16 compatibility
    __UpperCAmelCase						    =       (1.0 - encoder_extended_attention_mask) * -10000.0

    # Prepare head mask if needed
    # 1.0 in head_mask indicate we keep the head
    # attention_probs has shape bsz x n_heads x N x N
    # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
    # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
    __UpperCAmelCase						    =       self.get_head_mask(__lowerCAmelCase				,				self.config.num_hidden_layers   )

    __UpperCAmelCase						    =       self.embeddings(
        input_ids=__lowerCAmelCase				,				position_ids=__lowerCAmelCase				,				token_type_ids=__lowerCAmelCase				,				inputs_embeds=__lowerCAmelCase   )
    __UpperCAmelCase						    =       self.encoder(
        __lowerCAmelCase				,				attention_mask=__lowerCAmelCase				,				head_mask=__lowerCAmelCase				,				encoder_hidden_states=__lowerCAmelCase				,				encoder_attention_mask=__lowerCAmelCase				,				)
    __UpperCAmelCase						    =       encoder_outputs[0]
    __UpperCAmelCase						    =       self.pooler(__lowerCAmelCase   )

    __UpperCAmelCase						    =       (
        sequence_output,
        pooled_output,
    ) + encoder_outputs[
        1:
    ]  # add hidden_states and attentions if they are here
    return outputs  # sequence_output, pooled_output, (hidden_states), (attentions), highway exits






class 			UpperCAmelCase__    (		snake_case					):



  """simple docstring"""






  def __init__(				self:			Dict				,				__lowerCAmelCase:			Dict				,				__lowerCAmelCase:			List[str]   ) -> Any:




    '''simple docstring'''




    __UpperCAmelCase						    =       message
    __UpperCAmelCase						    =       exit_layer  # start from 1!






class 			UpperCAmelCase__    (		nn.Module					):



  """simple docstring"""






  def __init__(				self:			Union[str, Any]				,				__lowerCAmelCase:			int   ) -> Optional[Any]:




    '''simple docstring'''




    super().__init__()
    __UpperCAmelCase						    =       BertPooler(__lowerCAmelCase   )
    __UpperCAmelCase						    =       nn.Dropout(config.hidden_dropout_prob   )
    __UpperCAmelCase						    =       nn.Linear(config.hidden_size				,				config.num_labels   )






  def 	_UpperCAmelCase				(				self:			Optional[int]				,				__lowerCAmelCase:			Any   ) -> Tuple:




    '''simple docstring'''




    __UpperCAmelCase						    =       encoder_outputs[0]
    __UpperCAmelCase						    =       self.pooler(__lowerCAmelCase   )
    # "return" pooler_output

    # BertModel
    __UpperCAmelCase						    =       (pooler_input, pooler_output) + encoder_outputs[1:]
    # "return" bmodel_output

    # Dropout and classification
    __UpperCAmelCase						    =       bmodel_output[1]

    __UpperCAmelCase						    =       self.dropout(__lowerCAmelCase   )
    __UpperCAmelCase						    =       self.classifier(__lowerCAmelCase   )

    return logits, pooled_output






@add_start_docstrings(
    'Bert Model (with early exiting - DeeBERT) with a classifier on top,\n    also takes care of multi-layer training. '		,	snake_case		,	)
class 			UpperCAmelCase__    (		snake_case					):



  """simple docstring"""






  def __init__(				self:			Optional[Any]				,				__lowerCAmelCase:			List[str]   ) -> str:




    '''simple docstring'''




    super().__init__(__lowerCAmelCase   )
    __UpperCAmelCase						    =       config.num_labels
    __UpperCAmelCase						    =       config.num_hidden_layers

    __UpperCAmelCase						    =       DeeBertModel(__lowerCAmelCase   )
    __UpperCAmelCase						    =       nn.Dropout(config.hidden_dropout_prob   )
    __UpperCAmelCase						    =       nn.Linear(config.hidden_size				,				self.config.num_labels   )

    self.init_weights()






  @add_start_docstrings_to_model_forward(__lowerCAmelCase   )
  def 	_UpperCAmelCase				(				self:			Any				,				__lowerCAmelCase:			Any=None				,				__lowerCAmelCase:			Optional[int]=None				,				__lowerCAmelCase:			Optional[int]=None				,				__lowerCAmelCase:			List[str]=None				,				__lowerCAmelCase:			Dict=None				,				__lowerCAmelCase:			int=None				,				__lowerCAmelCase:			str=None				,				__lowerCAmelCase:			Optional[Any]=-1				,				__lowerCAmelCase:			str=False				,				) -> Optional[int]:




    '''simple docstring'''




    __UpperCAmelCase						    =       self.num_layers
    try:
      __UpperCAmelCase						    =       self.bert(
          __lowerCAmelCase				,				attention_mask=__lowerCAmelCase				,				token_type_ids=__lowerCAmelCase				,				position_ids=__lowerCAmelCase				,				head_mask=__lowerCAmelCase				,				inputs_embeds=__lowerCAmelCase				,				)
      # sequence_output, pooled_output, (hidden_states), (attentions), highway exits

      __UpperCAmelCase						    =       outputs[1]

      __UpperCAmelCase						    =       self.dropout(__lowerCAmelCase   )
      __UpperCAmelCase						    =       self.classifier(__lowerCAmelCase   )
      __UpperCAmelCase						    =       (logits,) + outputs[2:]  # add hidden states and attention if they are here
    except HighwayException as e:
      __UpperCAmelCase						    =       e.message
      __UpperCAmelCase						    =       e.exit_layer
      __UpperCAmelCase						    =       outputs[0]

    if not self.training:
      __UpperCAmelCase						    =       entropy(__lowerCAmelCase   )
      __UpperCAmelCase						    =       []
      __UpperCAmelCase						    =       []
    if labels is not None:
      if self.num_labels == 1:
        #  We are doing regression
        __UpperCAmelCase						    =       MSELoss()
        __UpperCAmelCase						    =       loss_fct(logits.view(-1   )				,				labels.view(-1   )   )
      else:
        __UpperCAmelCase						    =       CrossEntropyLoss()
        __UpperCAmelCase						    =       loss_fct(logits.view(-1				,				self.num_labels   )				,				labels.view(-1   )   )

      # work with highway exits
      __UpperCAmelCase						    =       []
      for highway_exit in outputs[-1]:
        __UpperCAmelCase						    =       highway_exit[0]
        if not self.training:
          highway_logits_all.append(__lowerCAmelCase   )
          highway_entropy.append(highway_exit[2]   )
        if self.num_labels == 1:
          #  We are doing regression
          __UpperCAmelCase						    =       MSELoss()
          __UpperCAmelCase						    =       loss_fct(highway_logits.view(-1   )				,				labels.view(-1   )   )
        else:
          __UpperCAmelCase						    =       CrossEntropyLoss()
          __UpperCAmelCase						    =       loss_fct(highway_logits.view(-1				,				self.num_labels   )				,				labels.view(-1   )   )
        highway_losses.append(__lowerCAmelCase   )

      if train_highway:
        __UpperCAmelCase						    =       (sum(highway_losses[:-1]   ),) + outputs
        # exclude the final highway, of course
      else:
        __UpperCAmelCase						    =       (loss,) + outputs
    if not self.training:
      __UpperCAmelCase						    =       outputs + ((original_entropy, highway_entropy), exit_layer)
      if output_layer >= 0:
        __UpperCAmelCase						    =       (
            (outputs[0],) + (highway_logits_all[output_layer],) + outputs[2:]
        )  # use the highway of the last layer

    return outputs  # (loss), logits, (hidden_states), (attentions), (highway_exits)



