

"""simple docstring"""




import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


SCREAMING_SNAKE_CASE_     :			List[str]	  =			logging.getLogger(__name__)


class 		_A       (			__a					):





      def 					_lowerCamelCase  (  self    ,						SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None		)   ->					str:
         lowerCamelCase__						=			self.layer[current_layer](SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__    ,						head_mask[current_layer]		)

         lowerCamelCase__						=			layer_outputs[0]

         return hidden_states






@add_start_docstrings(
    'The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'				,       __a				,       )
class 		_A       (			__a					):





      def __init__(  self    ,						SCREAMING_SNAKE_CASE__		)   ->					List[Any]:
         super().__init__(SCREAMING_SNAKE_CASE__		)

         lowerCamelCase__						=			BertEncoderWithPabee(SCREAMING_SNAKE_CASE__		)

         self.init_weights()
         lowerCamelCase__						=			0
         lowerCamelCase__						=			0
         lowerCamelCase__						=			0

         lowerCamelCase__						=			0





      def 					_lowerCamelCase  (  self    ,						SCREAMING_SNAKE_CASE__		)   ->					Any:
         lowerCamelCase__						=			threshold





      def 					_lowerCamelCase  (  self    ,						SCREAMING_SNAKE_CASE__		)   ->					Dict:
         lowerCamelCase__						=			patience





      def 					_lowerCamelCase  (  self		)   ->					Optional[Any]:
         lowerCamelCase__						=			0
         lowerCamelCase__						=			0





      def 					_lowerCamelCase  (  self		)   ->					Dict:
         lowerCamelCase__						=			self.inference_layers_num / self.inference_instances_num
         lowerCamelCase__						=			(
             f'*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='
             f' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'
         )
         print(SCREAMING_SNAKE_CASE__		)





      @add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE__		)
      def 					_lowerCamelCase  (  self    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=False    ,						)   ->					Union[str, Any]:

         if input_ids is not None and inputs_embeds is not None:
            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"		)
         elif input_ids is not None:
            lowerCamelCase__						=			input_ids.size()
         elif inputs_embeds is not None:
            lowerCamelCase__						=			inputs_embeds.size()[:-1]
         else:
            raise ValueError("You have to specify either input_ids or inputs_embeds"		)

         lowerCamelCase__						=			input_ids.device if input_ids is not None else inputs_embeds.device

         if attention_mask is None:
            lowerCamelCase__						=			torch.ones(SCREAMING_SNAKE_CASE__    ,						device=SCREAMING_SNAKE_CASE__		)
         if token_type_ids is None:
            lowerCamelCase__						=			torch.zeros(SCREAMING_SNAKE_CASE__    ,						dtype=torch.long    ,						device=SCREAMING_SNAKE_CASE__		)

         # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
         # ourselves in which case we just need to make it broadcastable to all heads.
         lowerCamelCase__						=			self.get_extended_attention_mask(SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__    ,						SCREAMING_SNAKE_CASE__		)

         # If a 2D ou 3D attention mask is provided for the cross-attention
         # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
         if self.config.is_decoder and encoder_hidden_states is not None:
            lowerCamelCase__					,       lowerCamelCase__					,       lowerCamelCase__						=			encoder_hidden_states.size()
            lowerCamelCase__						=			(encoder_batch_size, encoder_sequence_length)
            if encoder_attention_mask is None:
               lowerCamelCase__						=			torch.ones(SCREAMING_SNAKE_CASE__    ,						device=SCREAMING_SNAKE_CASE__		)
            lowerCamelCase__						=			self.invert_attention_mask(SCREAMING_SNAKE_CASE__		)
         else:
            lowerCamelCase__						=			None

         # Prepare head mask if needed
         # 1.0 in head_mask indicate we keep the head
         # attention_probs has shape bsz x n_heads x N x N
         # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
         # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
         lowerCamelCase__						=			self.get_head_mask(SCREAMING_SNAKE_CASE__    ,						self.config.num_hidden_layers		)

         lowerCamelCase__						=			self.embeddings(
             input_ids=SCREAMING_SNAKE_CASE__    ,						position_ids=SCREAMING_SNAKE_CASE__    ,						token_type_ids=SCREAMING_SNAKE_CASE__    ,						inputs_embeds=SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						=			embedding_output

         if self.training:
            lowerCamelCase__						=			[]
            for i in range(self.config.num_hidden_layers		):
               lowerCamelCase__						=			self.encoder.adaptive_forward(
                   SCREAMING_SNAKE_CASE__    ,						current_layer=SCREAMING_SNAKE_CASE__    ,						attention_mask=SCREAMING_SNAKE_CASE__    ,						head_mask=SCREAMING_SNAKE_CASE__		)

               lowerCamelCase__						=			self.pooler(SCREAMING_SNAKE_CASE__		)
               lowerCamelCase__						=			output_layers[i](output_dropout(SCREAMING_SNAKE_CASE__		)		)
               res.append(SCREAMING_SNAKE_CASE__		)
         elif self.patience == 0:  # Use all layers for inference
            lowerCamelCase__						=			self.encoder(
                SCREAMING_SNAKE_CASE__    ,						attention_mask=SCREAMING_SNAKE_CASE__    ,						head_mask=SCREAMING_SNAKE_CASE__    ,						encoder_hidden_states=SCREAMING_SNAKE_CASE__    ,						encoder_attention_mask=SCREAMING_SNAKE_CASE__    ,						)
            lowerCamelCase__						=			self.pooler(encoder_outputs[0]		)
            lowerCamelCase__						=			[output_layers[self.config.num_hidden_layers - 1](SCREAMING_SNAKE_CASE__		)]
         else:
            lowerCamelCase__						=			0
            lowerCamelCase__						=			None
            lowerCamelCase__						=			0
            for i in range(self.config.num_hidden_layers		):
               calculated_layer_num += 1
               lowerCamelCase__						=			self.encoder.adaptive_forward(
                   SCREAMING_SNAKE_CASE__    ,						current_layer=SCREAMING_SNAKE_CASE__    ,						attention_mask=SCREAMING_SNAKE_CASE__    ,						head_mask=SCREAMING_SNAKE_CASE__		)

               lowerCamelCase__						=			self.pooler(SCREAMING_SNAKE_CASE__		)
               lowerCamelCase__						=			output_layers[i](SCREAMING_SNAKE_CASE__		)
               if regression:
                  lowerCamelCase__						=			logits.detach()
                  if patient_result is not None:
                     lowerCamelCase__						=			patient_result.detach()
                  if (patient_result is not None) and torch.abs(patient_result - labels		) < self.regression_threshold:
                     patient_counter += 1
                  else:
                     lowerCamelCase__						=			0
               else:
                  lowerCamelCase__						=			logits.detach().argmax(dim=1		)
                  if patient_result is not None:
                     lowerCamelCase__						=			patient_result.detach().argmax(dim=1		)
                  if (patient_result is not None) and torch.all(labels.eq(SCREAMING_SNAKE_CASE__		)		):
                     patient_counter += 1
                  else:
                     lowerCamelCase__						=			0

               lowerCamelCase__						=			logits
               if patient_counter == self.patience:
                  break
            lowerCamelCase__						=			[patient_result]
            self.inference_layers_num += calculated_layer_num
            self.inference_instances_num += 1

         return res






@add_start_docstrings(
    'Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '				,       __a				,       )
class 		_A       (			__a					):





      def __init__(  self    ,						SCREAMING_SNAKE_CASE__		)   ->					Dict:
         super().__init__(SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						=			config.num_labels

         lowerCamelCase__						=			BertModelWithPabee(SCREAMING_SNAKE_CASE__		)
         lowerCamelCase__						=			nn.Dropout(config.hidden_dropout_prob		)
         lowerCamelCase__						=			nn.ModuleList(
             [nn.Linear(config.hidden_size    ,						self.config.num_labels		) for _ in range(config.num_hidden_layers		)]		)

         self.init_weights()





      @add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE__		)
      def 					_lowerCamelCase  (  self    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						SCREAMING_SNAKE_CASE__=None    ,						)   ->					int:
         lowerCamelCase__						=			self.bert(
             input_ids=SCREAMING_SNAKE_CASE__    ,						attention_mask=SCREAMING_SNAKE_CASE__    ,						token_type_ids=SCREAMING_SNAKE_CASE__    ,						position_ids=SCREAMING_SNAKE_CASE__    ,						head_mask=SCREAMING_SNAKE_CASE__    ,						inputs_embeds=SCREAMING_SNAKE_CASE__    ,						output_dropout=self.dropout    ,						output_layers=self.classifiers    ,						regression=self.num_labels == 1    ,						)

         lowerCamelCase__						=			(logits[-1],)

         if labels is not None:
            lowerCamelCase__						=			None
            lowerCamelCase__						=			0
            for ix, logits_item in enumerate(SCREAMING_SNAKE_CASE__		):
               if self.num_labels == 1:
                  #  We are doing regression
                  lowerCamelCase__						=			MSELoss()
                  lowerCamelCase__						=			loss_fct(logits_item.view(-1		)    ,						labels.view(-1		)		)
               else:
                  lowerCamelCase__						=			CrossEntropyLoss()
                  lowerCamelCase__						=			loss_fct(logits_item.view(-1    ,						self.num_labels		)    ,						labels.view(-1		)		)
               if total_loss is None:
                  lowerCamelCase__						=			loss
               else:
                  total_loss += loss * (ix + 1)
               total_weights += ix + 1
            lowerCamelCase__						=			(total_loss / total_weights,) + outputs

         return outputs







"""simple docstring"""




import numpy as np



def 	UpperCAmelCase__   (				A__   )      ->      np.array:




   """simple docstring"""
   return 1 / (1 + np.exp(-vector   ))


if __name__ == "__main__":
     import doctest

     doctest.testmod()




