





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


UpperCAmelCase__						:			Any									=logging.getLogger(__name__)

class       __A     (						a    ):


				def _snake_case       (		self   , UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_=None   , UpperCAmelCase_=None							):
								lowerCamelCase	 =self.layer[current_layer](UpperCAmelCase_   , UpperCAmelCase_   , head_mask[current_layer]							)

								lowerCamelCase	 =layer_outputs[0]

								return hidden_states




@add_start_docstrings(
    """The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."""    ,					a    ,					)
class       __A     (						a    ):


				def __init__(		self   , UpperCAmelCase_							):
								super().__init__(UpperCAmelCase_							)

								lowerCamelCase	 =BertEncoderWithPabee(UpperCAmelCase_							)

								self.init_weights()
								lowerCamelCase	 =0
								lowerCamelCase	 =0
								lowerCamelCase	 =0

								lowerCamelCase	 =0


				def _snake_case       (		self   , UpperCAmelCase_							):
								lowerCamelCase	 =threshold


				def _snake_case       (		self   , UpperCAmelCase_							):
								lowerCamelCase	 =patience


				def _snake_case       (		self							):
								lowerCamelCase	 =0
								lowerCamelCase	 =0


				def _snake_case       (		self							):
								lowerCamelCase	 =self.inference_layers_num / self.inference_instances_num
								lowerCamelCase	 =(
								    f"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
								    f""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
								)
								print(UpperCAmelCase_							)


				@add_start_docstrings_to_model_forward(UpperCAmelCase_							)
				def _snake_case       (		self   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=False   , ):

								if input_ids is not None and inputs_embeds is not None:
												raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""							)
								elif input_ids is not None:
												lowerCamelCase	 =input_ids.size()
								elif inputs_embeds is not None:
												lowerCamelCase	 =inputs_embeds.size()[:-1]
								else:
												raise ValueError("""You have to specify either input_ids or inputs_embeds"""							)

								lowerCamelCase	 =input_ids.device if input_ids is not None else inputs_embeds.device

								if attention_mask is None:
												lowerCamelCase	 =torch.ones(UpperCAmelCase_   , device=UpperCAmelCase_							)
								if token_type_ids is None:
												lowerCamelCase	 =torch.zeros(UpperCAmelCase_   , dtype=torch.long   , device=UpperCAmelCase_							)

								# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
								# ourselves in which case we just need to make it broadcastable to all heads.
								lowerCamelCase	 =self.get_extended_attention_mask(UpperCAmelCase_   , UpperCAmelCase_   , UpperCAmelCase_							)

								# If a 2D ou 3D attention mask is provided for the cross-attention
								# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
								if self.config.is_decoder and encoder_hidden_states is not None:
												lowerCamelCase ,  lowerCamelCase ,  lowerCamelCase	 =encoder_hidden_states.size()
												lowerCamelCase	 =(encoder_batch_size, encoder_sequence_length)
												if encoder_attention_mask is None:
																lowerCamelCase	 =torch.ones(UpperCAmelCase_   , device=UpperCAmelCase_							)
												lowerCamelCase	 =self.invert_attention_mask(UpperCAmelCase_							)
								else:
												lowerCamelCase	 =None

								# Prepare head mask if needed
								# 1.0 in head_mask indicate we keep the head
								# attention_probs has shape bsz x n_heads x N x N
								# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
								# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
								lowerCamelCase	 =self.get_head_mask(UpperCAmelCase_   , self.config.num_hidden_layers							)

								lowerCamelCase	 =self.embeddings(
								    input_ids=UpperCAmelCase_   , position_ids=UpperCAmelCase_   , token_type_ids=UpperCAmelCase_   , inputs_embeds=UpperCAmelCase_							)
								lowerCamelCase	 =embedding_output

								if self.training:
												lowerCamelCase	 =[]
												for i in range(self.config.num_hidden_layers							):
																lowerCamelCase	 =self.encoder.adaptive_forward(
																    UpperCAmelCase_   , current_layer=UpperCAmelCase_   , attention_mask=UpperCAmelCase_   , head_mask=UpperCAmelCase_							)

																lowerCamelCase	 =self.pooler(UpperCAmelCase_							)
																lowerCamelCase	 =output_layers[i](output_dropout(UpperCAmelCase_							)							)
																res.append(UpperCAmelCase_							)
								elif self.patience == 0:  # Use all layers for inference
												lowerCamelCase	 =self.encoder(
												    UpperCAmelCase_   , attention_mask=UpperCAmelCase_   , head_mask=UpperCAmelCase_   , encoder_hidden_states=UpperCAmelCase_   , encoder_attention_mask=UpperCAmelCase_   , )
												lowerCamelCase	 =self.pooler(encoder_outputs[0]							)
												lowerCamelCase	 =[output_layers[self.config.num_hidden_layers - 1](UpperCAmelCase_							)]
								else:
												lowerCamelCase	 =0
												lowerCamelCase	 =None
												lowerCamelCase	 =0
												for i in range(self.config.num_hidden_layers							):
																calculated_layer_num += 1
																lowerCamelCase	 =self.encoder.adaptive_forward(
																    UpperCAmelCase_   , current_layer=UpperCAmelCase_   , attention_mask=UpperCAmelCase_   , head_mask=UpperCAmelCase_							)

																lowerCamelCase	 =self.pooler(UpperCAmelCase_							)
																lowerCamelCase	 =output_layers[i](UpperCAmelCase_							)
																if regression:
																				lowerCamelCase	 =logits.detach()
																				if patient_result is not None:
																								lowerCamelCase	 =patient_result.detach()
																				if (patient_result is not None) and torch.abs(patient_result - labels							) < self.regression_threshold:
																								patient_counter += 1
																				else:
																								lowerCamelCase	 =0
																else:
																				lowerCamelCase	 =logits.detach().argmax(dim=1							)
																				if patient_result is not None:
																								lowerCamelCase	 =patient_result.detach().argmax(dim=1							)
																				if (patient_result is not None) and torch.all(labels.eq(UpperCAmelCase_							)							):
																								patient_counter += 1
																				else:
																								lowerCamelCase	 =0

																lowerCamelCase	 =logits
																if patient_counter == self.patience:
																				break
												lowerCamelCase	 =[patient_result]
												self.inference_layers_num += calculated_layer_num
												self.inference_instances_num += 1

								return res




@add_start_docstrings(
    """Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. """    ,					a    ,					)
class       __A     (						a    ):


				def __init__(		self   , UpperCAmelCase_							):
								super().__init__(UpperCAmelCase_							)
								lowerCamelCase	 =config.num_labels

								lowerCamelCase	 =BertModelWithPabee(UpperCAmelCase_							)
								lowerCamelCase	 =nn.Dropout(config.hidden_dropout_prob							)
								lowerCamelCase	 =nn.ModuleList(
								    [nn.Linear(config.hidden_size   , self.config.num_labels							) for _ in range(config.num_hidden_layers							)]							)

								self.init_weights()


				@add_start_docstrings_to_model_forward(UpperCAmelCase_							)
				def _snake_case       (		self   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , UpperCAmelCase_=None   , ):
								lowerCamelCase	 =self.bert(
								    input_ids=UpperCAmelCase_   , attention_mask=UpperCAmelCase_   , token_type_ids=UpperCAmelCase_   , position_ids=UpperCAmelCase_   , head_mask=UpperCAmelCase_   , inputs_embeds=UpperCAmelCase_   , output_dropout=self.dropout   , output_layers=self.classifiers   , regression=self.num_labels == 1   , )

								lowerCamelCase	 =(logits[-1],)

								if labels is not None:
												lowerCamelCase	 =None
												lowerCamelCase	 =0
												for ix, logits_item in enumerate(UpperCAmelCase_							):
																if self.num_labels == 1:
																				#  We are doing regression
																				lowerCamelCase	 =MSELoss()
																				lowerCamelCase	 =loss_fct(logits_item.view(-1							)   , labels.view(-1							)							)
																else:
																				lowerCamelCase	 =CrossEntropyLoss()
																				lowerCamelCase	 =loss_fct(logits_item.view(-1   , self.num_labels							)   , labels.view(-1							)							)
																if total_loss is None:
																				lowerCamelCase	 =loss
																else:
																				total_loss += loss * (ix + 1)
																total_weights += ix + 1
												lowerCamelCase	 =(total_loss / total_weights,) + outputs

								return outputs












from abc import ABC, abstractmethod
from argparse import ArgumentParser

class       __A     (						a    ):


				@staticmethod
				@abstractmethod
				def _snake_case       (		UpperCAmelCase_							):
								raise NotImplementedError()


				@abstractmethod
				def _snake_case       (		self							):
								raise NotImplementedError()





