






import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


lowercase_          =		logging.getLogger(__name__)
class     SCREAMING_SNAKE_CASE       (__UpperCamelCase				):


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Union[str, Any]      ,					a      :    Optional[int]      ,					a      :    Optional[Any]      ,					a      :    Tuple=None      ,					a      :    Optional[int]=None	)->       int:





     """simple docstring"""







     lowercase__           =  self.layer[current_layer](a      ,					a      ,					head_mask[current_layer]	)

     lowercase__           =  layer_outputs[0]

     return hidden_states






@add_start_docstrings(
    'The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'      ,     __UpperCamelCase      ,     )
class     SCREAMING_SNAKE_CASE       (__UpperCamelCase				):


   def __init__(   self      :    Dict      ,					a      :    Union[str, Any]	)->       List[str]:





     """simple docstring"""







     super().__init__(a	)

     lowercase__           =  BertEncoderWithPabee(a	)

     self.init_weights()
     lowercase__           =  0
     lowercase__           =  0
     lowercase__           =  0

     lowercase__           =  0


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[Any]      ,					a      :    Tuple	)->       Tuple:





     """simple docstring"""







     lowercase__           =  threshold


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    str	)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  patience


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[Any]	)->       List[Any]:





     """simple docstring"""







     lowercase__           =  0
     lowercase__           =  0


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int	)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  self.inference_layers_num / self.inference_instances_num
     lowercase__           =  (
         f"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
         f""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
     )
     print(a	)


   @add_start_docstrings_to_model_forward(a	)
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    str=None      ,					a      :    Tuple=None      ,					a      :    str=None      ,					a      :    int=None      ,					a      :    List[Any]=None      ,					a      :    Optional[Any]=None      ,					a      :    List[str]=None      ,					a      :    Optional[int]=None      ,					a      :    Tuple=None      ,					a      :    str=None      ,					a      :    Optional[int]=False      ,					)->       Union[str, Any]:





     """simple docstring"""







     if input_ids is not None and inputs_embeds is not None:
       raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time'	)
     elif input_ids is not None:
       lowercase__           =  input_ids.size()
     elif inputs_embeds is not None:
       lowercase__           =  inputs_embeds.size()[:-1]
     else:
       raise ValueError('You have to specify either input_ids or inputs_embeds'	)

     lowercase__           =  input_ids.device if input_ids is not None else inputs_embeds.device

     if attention_mask is None:
       lowercase__           =  torch.ones(a      ,					device=a	)
     if token_type_ids is None:
       lowercase__           =  torch.zeros(a      ,					dtype=torch.long      ,					device=a	)

     # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
     # ourselves in which case we just need to make it broadcastable to all heads.
     lowercase__           =  self.get_extended_attention_mask(a      ,					a      ,					a	)

     # If a 2D ou 3D attention mask is provided for the cross-attention
     # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
     if self.config.is_decoder and encoder_hidden_states is not None:
       lowercase__           =  encoder_hidden_states.size()
       lowercase__           =  (encoder_batch_size, encoder_sequence_length)
       if encoder_attention_mask is None:
         lowercase__           =  torch.ones(a      ,					device=a	)
       lowercase__           =  self.invert_attention_mask(a	)
     else:
       lowercase__           =  None

     # Prepare head mask if needed
     # 1.0 in head_mask indicate we keep the head
     # attention_probs has shape bsz x n_heads x N x N
     # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
     # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
     lowercase__           =  self.get_head_mask(a      ,					self.config.num_hidden_layers	)

     lowercase__           =  self.embeddings(
         input_ids=a      ,					position_ids=a      ,					token_type_ids=a      ,					inputs_embeds=a	)
     lowercase__           =  embedding_output

     if self.training:
       lowercase__           =  []
       for i in range(self.config.num_hidden_layers	):
         lowercase__           =  self.encoder.adaptive_forward(
             a      ,					current_layer=a      ,					attention_mask=a      ,					head_mask=a	)

         lowercase__           =  self.pooler(a	)
         lowercase__           =  output_layers[i](output_dropout(a	)	)
         res.append(a	)
     elif self.patience == 0:  # Use all layers for inference
       lowercase__           =  self.encoder(
           a      ,					attention_mask=a      ,					head_mask=a      ,					encoder_hidden_states=a      ,					encoder_attention_mask=a      ,					)
       lowercase__           =  self.pooler(encoder_outputs[0]	)
       lowercase__           =  [output_layers[self.config.num_hidden_layers - 1](a	)]
     else:
       lowercase__           =  0
       lowercase__           =  None
       lowercase__           =  0
       for i in range(self.config.num_hidden_layers	):
         calculated_layer_num += 1
         lowercase__           =  self.encoder.adaptive_forward(
             a      ,					current_layer=a      ,					attention_mask=a      ,					head_mask=a	)

         lowercase__           =  self.pooler(a	)
         lowercase__           =  output_layers[i](a	)
         if regression:
           lowercase__           =  logits.detach()
           if patient_result is not None:
             lowercase__           =  patient_result.detach()
           if (patient_result is not None) and torch.abs(patient_result - labels	) < self.regression_threshold:
             patient_counter += 1
           else:
             lowercase__           =  0
         else:
           lowercase__           =  logits.detach().argmax(dim=1	)
           if patient_result is not None:
             lowercase__           =  patient_result.detach().argmax(dim=1	)
           if (patient_result is not None) and torch.all(labels.eq(a	)	):
             patient_counter += 1
           else:
             lowercase__           =  0

         lowercase__           =  logits
         if patient_counter == self.patience:
           break
       lowercase__           =  [patient_result]
       self.inference_layers_num += calculated_layer_num
       self.inference_instances_num += 1

     return res






@add_start_docstrings(
    'Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '      ,     __UpperCamelCase      ,     )
class     SCREAMING_SNAKE_CASE       (__UpperCamelCase				):


   def __init__(   self      :    Any      ,					a      :    Union[str, Any]	)->       List[str]:





     """simple docstring"""







     super().__init__(a	)
     lowercase__           =  config.num_labels

     lowercase__           =  BertModelWithPabee(a	)
     lowercase__           =  nn.Dropout(config.hidden_dropout_prob	)
     lowercase__           =  nn.ModuleList(
         [nn.Linear(config.hidden_size      ,					self.config.num_labels	) for _ in range(config.num_hidden_layers	)]	)

     self.init_weights()


   @add_start_docstrings_to_model_forward(a	)
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    int=None      ,					a      :    int=None      ,					a      :    Tuple=None      ,					a      :    Any=None      ,					a      :    Union[str, Any]=None      ,					a      :    Union[str, Any]=None      ,					a      :    Tuple=None      ,					)->       Dict:





     """simple docstring"""







     lowercase__           =  self.bert(
         input_ids=a      ,					attention_mask=a      ,					token_type_ids=a      ,					position_ids=a      ,					head_mask=a      ,					inputs_embeds=a      ,					output_dropout=self.dropout      ,					output_layers=self.classifiers      ,					regression=self.num_labels == 1      ,					)

     lowercase__           =  (logits[-1],)

     if labels is not None:
       lowercase__           =  None
       lowercase__           =  0
       for ix, logits_item in enumerate(a	):
         if self.num_labels == 1:
           #  We are doing regression
           lowercase__           =  MSELoss()
           lowercase__           =  loss_fct(logits_item.view(-1	)      ,					labels.view(-1	)	)
         else:
           lowercase__           =  CrossEntropyLoss()
           lowercase__           =  loss_fct(logits_item.view(-1      ,					self.num_labels	)      ,					labels.view(-1	)	)
         if total_loss is None:
           lowercase__           =  loss
         else:
           total_loss += loss * (ix + 1)
         total_weights += ix + 1
       lowercase__           =  (total_loss / total_weights,) + outputs

     return outputs









from typing import TYPE_CHECKING

from ..utils import _LazyModule


lowercase_          =		{
    """config""": [
        """EXTERNAL_DATA_FORMAT_SIZE_LIMIT""",
        """OnnxConfig""",
        """OnnxConfigWithPast""",
        """OnnxSeq2SeqConfigWithPast""",
        """PatchingSpec""",
    ],
    """convert""": ["""export""", """validate_model_outputs"""],
    """features""": ["""FeaturesManager"""],
    """utils""": ["""ParameterFormat""", """compute_serialized_parameters_size"""],
}


if TYPE_CHECKING:
      from .config import (
          EXTERNAL_DATA_FORMAT_SIZE_LIMIT,
          OnnxConfig,
          OnnxConfigWithPast,
          OnnxSeqaSeqConfigWithPast,
          PatchingSpec,
      )
      from .convert import export, validate_model_outputs
      from .features import FeaturesManager
      from .utils import ParameterFormat, compute_serialized_parameters_size

else:
      import sys

      lowercase_          =		_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


