




"""simple docstring"""




import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


a_        =       logging.getLogger(__name__)

class   UpperCAmelCase_    (			snake_case		):
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_=None				,						UpperCamelCase_=None       )						->       Optional[Any]:
					__lowercase			:  Tuple								=       self.layer[current_layer](UpperCamelCase_				,						UpperCamelCase_				,						head_mask[current_layer]       )

					__lowercase			:  Any								=       layer_outputs[0]

					return hidden_states


@add_start_docstrings(
    "The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."   ,					snake_case   ,					)
class   UpperCAmelCase_    (			snake_case		):
		def __init__(						self				,						UpperCamelCase_       )						->       int:
					super().__init__(UpperCamelCase_       )

					__lowercase			:  Optional[Any]								=       BertEncoderWithPabee(UpperCamelCase_       )

					self.init_weights()
					__lowercase			:  str								=       0
					__lowercase			:  Optional[Any]								=       0
					__lowercase			:  Optional[int]								=       0

					__lowercase			:  int								=       0
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Dict:
					__lowercase			:  Tuple								=       threshold
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Union[str, Any]:
					__lowercase			:  Optional[int]								=       patience
		def 	_lowerCamelCase							(						self       )						->       List[str]:
					__lowercase			:  Tuple								=       0
					__lowercase			:  Tuple								=       0
		def 	_lowerCamelCase							(						self       )						->       List[Any]:
					__lowercase			:  Optional[int]								=       self.inference_layers_num / self.inference_instances_num
					__lowercase			:  int								=       (
					    F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
					    F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
					)
					print(UpperCamelCase_       )
		@add_start_docstrings_to_model_forward(UpperCamelCase_       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=False				,						)						->       Union[str, Any]:

					if input_ids is not None and inputs_embeds is not None:
								raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''       )
					elif input_ids is not None:
								__lowercase			:  Tuple								=       input_ids.size()
					elif inputs_embeds is not None:
								__lowercase			:  List[Any]								=       inputs_embeds.size()[:-1]
					else:
								raise ValueError('''You have to specify either input_ids or inputs_embeds'''       )

					__lowercase			:  int								=       input_ids.device if input_ids is not None else inputs_embeds.device

					if attention_mask is None:
								__lowercase			:  Dict								=       torch.ones(UpperCamelCase_				,						device=UpperCamelCase_       )
					if token_type_ids is None:
								__lowercase			:  int								=       torch.zeros(UpperCamelCase_				,						dtype=torch.long				,						device=UpperCamelCase_       )

					# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
					# ourselves in which case we just need to make it broadcastable to all heads.
					__lowercase			:  torch.Tensor								=       self.get_extended_attention_mask(UpperCamelCase_				,						UpperCamelCase_				,						UpperCamelCase_       )

					# If a 2D ou 3D attention mask is provided for the cross-attention
					# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
					if self.config.is_decoder and encoder_hidden_states is not None:
								__lowercase			,__lowercase			,__lowercase			:  Optional[int]								=       encoder_hidden_states.size()
								__lowercase			:  Any								=       (encoder_batch_size, encoder_sequence_length)
								if encoder_attention_mask is None:
											__lowercase			:  List[str]								=       torch.ones(UpperCamelCase_				,						device=UpperCamelCase_       )
								__lowercase			:  Tuple								=       self.invert_attention_mask(UpperCamelCase_       )
					else:
								__lowercase			:  Tuple								=       None

					# Prepare head mask if needed
					# 1.0 in head_mask indicate we keep the head
					# attention_probs has shape bsz x n_heads x N x N
					# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
					# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
					__lowercase			:  Optional[int]								=       self.get_head_mask(UpperCamelCase_				,						self.config.num_hidden_layers       )

					__lowercase			:  Optional[int]								=       self.embeddings(
					    input_ids=UpperCamelCase_				,						position_ids=UpperCamelCase_				,						token_type_ids=UpperCamelCase_				,						inputs_embeds=UpperCamelCase_       )
					__lowercase			:  Union[str, Any]								=       embedding_output

					if self.training:
								__lowercase			:  List[Any]								=       []
								for i in range(self.config.num_hidden_layers       ):
											__lowercase			:  str								=       self.encoder.adaptive_forward(
											    UpperCamelCase_				,						current_layer=UpperCamelCase_				,						attention_mask=UpperCamelCase_				,						head_mask=UpperCamelCase_       )

											__lowercase			:  int								=       self.pooler(UpperCamelCase_       )
											__lowercase			:  str								=       output_layers[i](output_dropout(UpperCamelCase_       )       )
											res.append(UpperCamelCase_       )
					elif self.patience == 0:  # Use all layers for inference
								__lowercase			:  int								=       self.encoder(
								    UpperCamelCase_				,						attention_mask=UpperCamelCase_				,						head_mask=UpperCamelCase_				,						encoder_hidden_states=UpperCamelCase_				,						encoder_attention_mask=UpperCamelCase_				,						)
								__lowercase			:  Optional[Any]								=       self.pooler(encoder_outputs[0]       )
								__lowercase			:  int								=       [output_layers[self.config.num_hidden_layers - 1](UpperCamelCase_       )]
					else:
								__lowercase			:  Optional[int]								=       0
								__lowercase			:  Union[str, Any]								=       None
								__lowercase			:  int								=       0
								for i in range(self.config.num_hidden_layers       ):
											calculated_layer_num += 1
											__lowercase			:  Tuple								=       self.encoder.adaptive_forward(
											    UpperCamelCase_				,						current_layer=UpperCamelCase_				,						attention_mask=UpperCamelCase_				,						head_mask=UpperCamelCase_       )

											__lowercase			:  Dict								=       self.pooler(UpperCamelCase_       )
											__lowercase			:  Optional[int]								=       output_layers[i](UpperCamelCase_       )
											if regression:
														__lowercase			:  Any								=       logits.detach()
														if patient_result is not None:
																	__lowercase			:  List[str]								=       patient_result.detach()
														if (patient_result is not None) and torch.abs(patient_result - labels       ) < self.regression_threshold:
																	patient_counter += 1
														else:
																	__lowercase			:  int								=       0
											else:
														__lowercase			:  List[str]								=       logits.detach().argmax(dim=1       )
														if patient_result is not None:
																	__lowercase			:  Optional[Any]								=       patient_result.detach().argmax(dim=1       )
														if (patient_result is not None) and torch.all(labels.eq(UpperCamelCase_       )       ):
																	patient_counter += 1
														else:
																	__lowercase			:  Tuple								=       0

											__lowercase			:  Union[str, Any]								=       logits
											if patient_counter == self.patience:
														break
								__lowercase			:  Optional[int]								=       [patient_result]
								self.inference_layers_num += calculated_layer_num
								self.inference_instances_num += 1

					return res


@add_start_docstrings(
    "Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "   ,					snake_case   ,					)
class   UpperCAmelCase_    (			snake_case		):
		def __init__(						self				,						UpperCamelCase_       )						->       Optional[Any]:
					super().__init__(UpperCamelCase_       )
					__lowercase			:  List[Any]								=       config.num_labels

					__lowercase			:  int								=       BertModelWithPabee(UpperCamelCase_       )
					__lowercase			:  int								=       nn.Dropout(config.hidden_dropout_prob       )
					__lowercase			:  Union[str, Any]								=       nn.ModuleList(
					    [nn.Linear(config.hidden_size				,						self.config.num_labels       ) for _ in range(config.num_hidden_layers       )]       )

					self.init_weights()
		@add_start_docstrings_to_model_forward(UpperCamelCase_       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						UpperCamelCase_=None				,						)						->       int:
					__lowercase			:  Union[str, Any]								=       self.bert(
					    input_ids=UpperCamelCase_				,						attention_mask=UpperCamelCase_				,						token_type_ids=UpperCamelCase_				,						position_ids=UpperCamelCase_				,						head_mask=UpperCamelCase_				,						inputs_embeds=UpperCamelCase_				,						output_dropout=self.dropout				,						output_layers=self.classifiers				,						regression=self.num_labels == 1				,						)

					__lowercase			:  List[str]								=       (logits[-1],)

					if labels is not None:
								__lowercase			:  Any								=       None
								__lowercase			:  Optional[int]								=       0
								for ix, logits_item in enumerate(UpperCamelCase_       ):
											if self.num_labels == 1:
														#  We are doing regression
														__lowercase			:  Any								=       MSELoss()
														__lowercase			:  Any								=       loss_fct(logits_item.view(-1       )				,						labels.view(-1       )       )
											else:
														__lowercase			:  str								=       CrossEntropyLoss()
														__lowercase			:  Dict								=       loss_fct(logits_item.view(-1				,						self.num_labels       )				,						labels.view(-1       )       )
											if total_loss is None:
														__lowercase			:  List[str]								=       loss
											else:
														total_loss += loss * (ix + 1)
											total_weights += ix + 1
								__lowercase			:  Union[str, Any]								=       (total_loss / total_weights,) + outputs

					return outputs











__a:     List[Any]   					=					{
    '''Pillow''': '''Pillow''',
    '''accelerate''': '''accelerate>=0.11.0''',
    '''compel''': '''compel==0.1.8''',
    '''black''': '''black~=23.1''',
    '''datasets''': '''datasets''',
    '''filelock''': '''filelock''',
    '''flax''': '''flax>=0.4.1''',
    '''hf-doc-builder''': '''hf-doc-builder>=0.3.0''',
    '''huggingface-hub''': '''huggingface-hub>=0.13.2''',
    '''requests-mock''': '''requests-mock==1.10.0''',
    '''importlib_metadata''': '''importlib_metadata''',
    '''invisible-watermark''': '''invisible-watermark''',
    '''isort''': '''isort>=5.5.4''',
    '''jax''': '''jax>=0.2.8,!=0.3.2''',
    '''jaxlib''': '''jaxlib>=0.1.65''',
    '''Jinja2''': '''Jinja2''',
    '''k-diffusion''': '''k-diffusion>=0.0.12''',
    '''torchsde''': '''torchsde''',
    '''note_seq''': '''note_seq''',
    '''librosa''': '''librosa''',
    '''numpy''': '''numpy''',
    '''omegaconf''': '''omegaconf''',
    '''parameterized''': '''parameterized''',
    '''protobuf''': '''protobuf>=3.20.3,<4''',
    '''pytest''': '''pytest''',
    '''pytest-timeout''': '''pytest-timeout''',
    '''pytest-xdist''': '''pytest-xdist''',
    '''ruff''': '''ruff>=0.0.241''',
    '''safetensors''': '''safetensors''',
    '''sentencepiece''': '''sentencepiece>=0.1.91,!=0.1.92''',
    '''scipy''': '''scipy''',
    '''onnx''': '''onnx''',
    '''regex''': '''regex!=2019.12.17''',
    '''requests''': '''requests''',
    '''tensorboard''': '''tensorboard''',
    '''torch''': '''torch>=1.4''',
    '''torchvision''': '''torchvision''',
    '''transformers''': '''transformers>=4.25.1''',
    '''urllib3''': '''urllib3<=2.0.0''',
}