




import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


UpperCamelCase__ :   Any						=							logging.getLogger(__name__)







class 				lowerCAmelCase_       (					lowerCamelCase_							):






				def 		snake_case	(			self      ,snake_case__      ,snake_case__      ,snake_case__=None      ,snake_case__=None ):
							SCREAMING_SNAKE_CASE_   : Dict					  =      self.layer[current_layer](snake_case__      ,snake_case__      ,head_mask[current_layer] )

							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      layer_outputs[0]

							return hidden_states







@add_start_docstrings(
    "The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."						,     lowerCamelCase_						,     )
class 				lowerCAmelCase_       (					lowerCamelCase_							):






				def __init__(			self      ,snake_case__ ):
							super().__init__(snake_case__ )

							SCREAMING_SNAKE_CASE_   : str					  =      BertEncoderWithPabee(snake_case__ )

							self.init_weights()
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      0
							SCREAMING_SNAKE_CASE_   : str					  =      0
							SCREAMING_SNAKE_CASE_   : Optional[int]					  =      0

							SCREAMING_SNAKE_CASE_   : Any					  =      0






				def 		snake_case	(			self      ,snake_case__ ):
							SCREAMING_SNAKE_CASE_   : int					  =      threshold






				def 		snake_case	(			self      ,snake_case__ ):
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      patience






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      0
							SCREAMING_SNAKE_CASE_   : Dict					  =      0






				def 		snake_case	(			self ):
							SCREAMING_SNAKE_CASE_   : List[Any]					  =      self.inference_layers_num / self.inference_instances_num
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      (
							    F'*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='
							    F' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'
							)
							print(snake_case__ )






				@add_start_docstrings_to_model_forward(snake_case__ )
				def 		snake_case	(			self      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__=False      ,):

							if input_ids is not None and inputs_embeds is not None:
										raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time' )
							elif input_ids is not None:
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      input_ids.size()
							elif inputs_embeds is not None:
										SCREAMING_SNAKE_CASE_   : Any					  =      inputs_embeds.size()[:-1]
							else:
										raise ValueError('You have to specify either input_ids or inputs_embeds' )

							SCREAMING_SNAKE_CASE_   : Tuple					  =      input_ids.device if input_ids is not None else inputs_embeds.device

							if attention_mask is None:
										SCREAMING_SNAKE_CASE_   : str					  =      torch.ones(snake_case__      ,device=snake_case__ )
							if token_type_ids is None:
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      torch.zeros(snake_case__      ,dtype=torch.long      ,device=snake_case__ )

							# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
							# ourselves in which case we just need to make it broadcastable to all heads.
							SCREAMING_SNAKE_CASE_   : torch.Tensor					  =      self.get_extended_attention_mask(snake_case__      ,snake_case__      ,snake_case__ )

							# If a 2D ou 3D attention mask is provided for the cross-attention
							# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
							if self.config.is_decoder and encoder_hidden_states is not None:
										SCREAMING_SNAKE_CASE_  ,    SCREAMING_SNAKE_CASE_  ,    SCREAMING_SNAKE_CASE_   : List[str]					  =      encoder_hidden_states.size()
										SCREAMING_SNAKE_CASE_   : Any					  =      (encoder_batch_size, encoder_sequence_length)
										if encoder_attention_mask is None:
													SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      torch.ones(snake_case__      ,device=snake_case__ )
										SCREAMING_SNAKE_CASE_   : Any					  =      self.invert_attention_mask(snake_case__ )
							else:
										SCREAMING_SNAKE_CASE_   : int					  =      None

							# Prepare head mask if needed
							# 1.0 in head_mask indicate we keep the head
							# attention_probs has shape bsz x n_heads x N x N
							# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
							# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
							SCREAMING_SNAKE_CASE_   : List[str]					  =      self.get_head_mask(snake_case__      ,self.config.num_hidden_layers )

							SCREAMING_SNAKE_CASE_   : int					  =      self.embeddings(
							    input_ids=snake_case__      ,position_ids=snake_case__      ,token_type_ids=snake_case__      ,inputs_embeds=snake_case__ )
							SCREAMING_SNAKE_CASE_   : Dict					  =      embedding_output

							if self.training:
										SCREAMING_SNAKE_CASE_   : List[str]					  =      []
										for i in range(self.config.num_hidden_layers ):
													SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      self.encoder.adaptive_forward(
													    snake_case__      ,current_layer=snake_case__      ,attention_mask=snake_case__      ,head_mask=snake_case__ )

													SCREAMING_SNAKE_CASE_   : List[str]					  =      self.pooler(snake_case__ )
													SCREAMING_SNAKE_CASE_   : Tuple					  =      output_layers[i](output_dropout(snake_case__ ) )
													res.append(snake_case__ )
							elif self.patience == 0:  # Use all layers for inference
										SCREAMING_SNAKE_CASE_   : List[Any]					  =      self.encoder(
										    snake_case__      ,attention_mask=snake_case__      ,head_mask=snake_case__      ,encoder_hidden_states=snake_case__      ,encoder_attention_mask=snake_case__      ,)
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      self.pooler(encoder_outputs[0] )
										SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      [output_layers[self.config.num_hidden_layers - 1](snake_case__ )]
							else:
										SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      0
										SCREAMING_SNAKE_CASE_   : str					  =      None
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      0
										for i in range(self.config.num_hidden_layers ):
													calculated_layer_num += 1
													SCREAMING_SNAKE_CASE_   : List[Any]					  =      self.encoder.adaptive_forward(
													    snake_case__      ,current_layer=snake_case__      ,attention_mask=snake_case__      ,head_mask=snake_case__ )

													SCREAMING_SNAKE_CASE_   : List[str]					  =      self.pooler(snake_case__ )
													SCREAMING_SNAKE_CASE_   : int					  =      output_layers[i](snake_case__ )
													if regression:
																SCREAMING_SNAKE_CASE_   : Dict					  =      logits.detach()
																if patient_result is not None:
																			SCREAMING_SNAKE_CASE_   : List[Any]					  =      patient_result.detach()
																if (patient_result is not None) and torch.abs(patient_result - labels ) < self.regression_threshold:
																			patient_counter += 1
																else:
																			SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      0
													else:
																SCREAMING_SNAKE_CASE_   : List[Any]					  =      logits.detach().argmax(dim=1 )
																if patient_result is not None:
																			SCREAMING_SNAKE_CASE_   : str					  =      patient_result.detach().argmax(dim=1 )
																if (patient_result is not None) and torch.all(labels.eq(snake_case__ ) ):
																			patient_counter += 1
																else:
																			SCREAMING_SNAKE_CASE_   : Dict					  =      0

													SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      logits
													if patient_counter == self.patience:
																break
										SCREAMING_SNAKE_CASE_   : Any					  =      [patient_result]
										self.inference_layers_num += calculated_layer_num
										self.inference_instances_num += 1

							return res







@add_start_docstrings(
    "Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "						,     lowerCamelCase_						,     )
class 				lowerCAmelCase_       (					lowerCamelCase_							):






				def __init__(			self      ,snake_case__ ):
							super().__init__(snake_case__ )
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      config.num_labels

							SCREAMING_SNAKE_CASE_   : str					  =      BertModelWithPabee(snake_case__ )
							SCREAMING_SNAKE_CASE_   : int					  =      nn.Dropout(config.hidden_dropout_prob )
							SCREAMING_SNAKE_CASE_   : Dict					  =      nn.ModuleList(
							    [nn.Linear(config.hidden_size      ,self.config.num_labels ) for _ in range(config.num_hidden_layers )] )

							self.init_weights()






				@add_start_docstrings_to_model_forward(snake_case__ )
				def 		snake_case	(			self      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,snake_case__=None      ,):
							SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      self.bert(
							    input_ids=snake_case__      ,attention_mask=snake_case__      ,token_type_ids=snake_case__      ,position_ids=snake_case__      ,head_mask=snake_case__      ,inputs_embeds=snake_case__      ,output_dropout=self.dropout      ,output_layers=self.classifiers      ,regression=self.num_labels == 1      ,)

							SCREAMING_SNAKE_CASE_   : List[Any]					  =      (logits[-1],)

							if labels is not None:
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      None
										SCREAMING_SNAKE_CASE_   : Optional[int]					  =      0
										for ix, logits_item in enumerate(snake_case__ ):
													if self.num_labels == 1:
																#  We are doing regression
																SCREAMING_SNAKE_CASE_   : Dict					  =      MSELoss()
																SCREAMING_SNAKE_CASE_   : List[Any]					  =      loss_fct(logits_item.view(-1 )      ,labels.view(-1 ) )
													else:
																SCREAMING_SNAKE_CASE_   : str					  =      CrossEntropyLoss()
																SCREAMING_SNAKE_CASE_   : Dict					  =      loss_fct(logits_item.view(-1      ,self.num_labels )      ,labels.view(-1 ) )
													if total_loss is None:
																SCREAMING_SNAKE_CASE_   : Union[str, Any]					  =      loss
													else:
																total_loss += loss * (ix + 1)
													total_weights += ix + 1
										SCREAMING_SNAKE_CASE_   : Optional[Any]					  =      (total_loss / total_weights,) + outputs

							return outputs







'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


if is_sentencepiece_available():
      from ..ta.tokenization_ta import TaTokenizer
else:
      from ...utils.dummy_sentencepiece_objects import TaTokenizer

A__						:   int				=    TaTokenizer

if is_tokenizers_available():
      from ..ta.tokenization_ta_fast import TaTokenizerFast
else:
      from ...utils.dummy_tokenizers_objects import TaTokenizerFast

A__						:   Dict				=    TaTokenizerFast

A__						:   Dict				=    {"""configuration_mt5""": ["""MT5Config""", """MT5OnnxConfig"""]}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A__						:   Any				=    [
          """MT5EncoderModel""",
          """MT5ForConditionalGeneration""",
          """MT5ForQuestionAnswering""",
          """MT5Model""",
          """MT5PreTrainedModel""",
          """MT5Stack""",
      ]

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A__						:   str				=    ["""TFMT5EncoderModel""", """TFMT5ForConditionalGeneration""", """TFMT5Model"""]

try:
      if not is_flax_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A__						:   str				=    ["""FlaxMT5EncoderModel""", """FlaxMT5ForConditionalGeneration""", """FlaxMT5Model"""]


if TYPE_CHECKING:
      from .configuration_mta import MTaConfig, MTaOnnxConfig

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_mta import (
                MTaEncoderModel,
                MTaForConditionalGeneration,
                MTaForQuestionAnswering,
                MTaModel,
                MTaPreTrainedModel,
                MTaStack,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_mta import TFMTaEncoderModel, TFMTaForConditionalGeneration, TFMTaModel

      try:
            if not is_flax_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_flax_mta import FlaxMTaEncoderModel, FlaxMTaForConditionalGeneration, FlaxMTaModel

else:
      import sys

      A__						:   Union[str, Any]				=    _LazyModule(
          __name__,
          globals()["""__file__"""],
          _import_structure,
          extra_objects={"""MT5Tokenizer""": MTaTokenizer, """MT5TokenizerFast""": MTaTokenizerFast},
          module_spec=__spec__,
      )






