





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


A_												=     logging.getLogger(__name__)

class SCREAMING_SNAKE_CASE_ (					lowercase_   ):





					"""simple docstring"""


					def 					__magic_name__ ( self	,  _lowerCAmelCase	,  _lowerCAmelCase	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	):
										lowerCamelCase__       =       self.layer[current_layer](_lowerCAmelCase	,  _lowerCAmelCase	,  head_mask[current_layer]	)

										lowerCamelCase__       =       layer_outputs[0]

										return hidden_states

@add_start_docstrings(
    "The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."							,       lowercase_							,       )
class SCREAMING_SNAKE_CASE_ (					lowercase_   ):





					"""simple docstring"""


					def __init__( self	,  _lowerCAmelCase	):
										super().__init__(_lowerCAmelCase	)

										lowerCamelCase__       =       BertEncoderWithPabee(_lowerCAmelCase	)

										self.init_weights()
										lowerCamelCase__       =       0
										lowerCamelCase__       =       0
										lowerCamelCase__       =       0

										lowerCamelCase__       =       0





					def 					__magic_name__ ( self	,  _lowerCAmelCase	):
										lowerCamelCase__       =       threshold





					def 					__magic_name__ ( self	,  _lowerCAmelCase	):
										lowerCamelCase__       =       patience





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       0
										lowerCamelCase__       =       0





					def 					__magic_name__ ( self	):
										lowerCamelCase__       =       self.inference_layers_num / self.inference_instances_num
										lowerCamelCase__       =       (
										    F"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
										    F" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
										)
										print(_lowerCAmelCase	)






					@add_start_docstrings_to_model_forward(_lowerCAmelCase	)
					def 					__magic_name__ ( self	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=False	,  ):

										if input_ids is not None and inputs_embeds is not None:
															raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"	)
										elif input_ids is not None:
															lowerCamelCase__       =       input_ids.size()
										elif inputs_embeds is not None:
															lowerCamelCase__       =       inputs_embeds.size()[:-1]
										else:
															raise ValueError("You have to specify either input_ids or inputs_embeds"	)

										lowerCamelCase__       =       input_ids.device if input_ids is not None else inputs_embeds.device

										if attention_mask is None:
															lowerCamelCase__       =       torch.ones(_lowerCAmelCase	,  device=_lowerCAmelCase	)
										if token_type_ids is None:
															lowerCamelCase__       =       torch.zeros(_lowerCAmelCase	,  dtype=torch.long	,  device=_lowerCAmelCase	)

										# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
										# ourselves in which case we just need to make it broadcastable to all heads.
										lowerCamelCase__       =       self.get_extended_attention_mask(_lowerCAmelCase	,  _lowerCAmelCase	,  _lowerCAmelCase	)

										# If a 2D ou 3D attention mask is provided for the cross-attention
										# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
										if self.config.is_decoder and encoder_hidden_states is not None:
															lowerCamelCase__    ,		lowerCamelCase__    ,		lowerCamelCase__       =       encoder_hidden_states.size()
															lowerCamelCase__       =       (encoder_batch_size, encoder_sequence_length)
															if encoder_attention_mask is None:
																				lowerCamelCase__       =       torch.ones(_lowerCAmelCase	,  device=_lowerCAmelCase	)
															lowerCamelCase__       =       self.invert_attention_mask(_lowerCAmelCase	)
										else:
															lowerCamelCase__       =       None

										# Prepare head mask if needed
										# 1.0 in head_mask indicate we keep the head
										# attention_probs has shape bsz x n_heads x N x N
										# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
										# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
										lowerCamelCase__       =       self.get_head_mask(_lowerCAmelCase	,  self.config.num_hidden_layers	)

										lowerCamelCase__       =       self.embeddings(
										    input_ids=_lowerCAmelCase	,  position_ids=_lowerCAmelCase	,  token_type_ids=_lowerCAmelCase	,  inputs_embeds=_lowerCAmelCase	)
										lowerCamelCase__       =       embedding_output

										if self.training:
															lowerCamelCase__       =       []
															for i in range(self.config.num_hidden_layers	):
																				lowerCamelCase__       =       self.encoder.adaptive_forward(
																				    _lowerCAmelCase	,  current_layer=_lowerCAmelCase	,  attention_mask=_lowerCAmelCase	,  head_mask=_lowerCAmelCase	)

																				lowerCamelCase__       =       self.pooler(_lowerCAmelCase	)
																				lowerCamelCase__       =       output_layers[i](output_dropout(_lowerCAmelCase	)	)
																				res.append(_lowerCAmelCase	)
										elif self.patience == 0:  # Use all layers for inference
															lowerCamelCase__       =       self.encoder(
															    _lowerCAmelCase	,  attention_mask=_lowerCAmelCase	,  head_mask=_lowerCAmelCase	,  encoder_hidden_states=_lowerCAmelCase	,  encoder_attention_mask=_lowerCAmelCase	,  )
															lowerCamelCase__       =       self.pooler(encoder_outputs[0]	)
															lowerCamelCase__       =       [output_layers[self.config.num_hidden_layers - 1](_lowerCAmelCase	)]
										else:
															lowerCamelCase__       =       0
															lowerCamelCase__       =       None
															lowerCamelCase__       =       0
															for i in range(self.config.num_hidden_layers	):
																				calculated_layer_num += 1
																				lowerCamelCase__       =       self.encoder.adaptive_forward(
																				    _lowerCAmelCase	,  current_layer=_lowerCAmelCase	,  attention_mask=_lowerCAmelCase	,  head_mask=_lowerCAmelCase	)

																				lowerCamelCase__       =       self.pooler(_lowerCAmelCase	)
																				lowerCamelCase__       =       output_layers[i](_lowerCAmelCase	)
																				if regression:
																									lowerCamelCase__       =       logits.detach()
																									if patient_result is not None:
																														lowerCamelCase__       =       patient_result.detach()
																									if (patient_result is not None) and torch.abs(patient_result - labels	) < self.regression_threshold:
																														patient_counter += 1
																									else:
																														lowerCamelCase__       =       0
																				else:
																									lowerCamelCase__       =       logits.detach().argmax(dim=1	)
																									if patient_result is not None:
																														lowerCamelCase__       =       patient_result.detach().argmax(dim=1	)
																									if (patient_result is not None) and torch.all(labels.eq(_lowerCAmelCase	)	):
																														patient_counter += 1
																									else:
																														lowerCamelCase__       =       0

																				lowerCamelCase__       =       logits
																				if patient_counter == self.patience:
																									break
															lowerCamelCase__       =       [patient_result]
															self.inference_layers_num += calculated_layer_num
															self.inference_instances_num += 1

										return res

@add_start_docstrings(
    "Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "							,       lowercase_							,       )
class SCREAMING_SNAKE_CASE_ (					lowercase_   ):





					"""simple docstring"""


					def __init__( self	,  _lowerCAmelCase	):
										super().__init__(_lowerCAmelCase	)
										lowerCamelCase__       =       config.num_labels

										lowerCamelCase__       =       BertModelWithPabee(_lowerCAmelCase	)
										lowerCamelCase__       =       nn.Dropout(config.hidden_dropout_prob	)
										lowerCamelCase__       =       nn.ModuleList(
										    [nn.Linear(config.hidden_size	,  self.config.num_labels	) for _ in range(config.num_hidden_layers	)]	)

										self.init_weights()






					@add_start_docstrings_to_model_forward(_lowerCAmelCase	)
					def 					__magic_name__ ( self	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  _lowerCAmelCase=None	,  ):
										lowerCamelCase__       =       self.bert(
										    input_ids=_lowerCAmelCase	,  attention_mask=_lowerCAmelCase	,  token_type_ids=_lowerCAmelCase	,  position_ids=_lowerCAmelCase	,  head_mask=_lowerCAmelCase	,  inputs_embeds=_lowerCAmelCase	,  output_dropout=self.dropout	,  output_layers=self.classifiers	,  regression=self.num_labels == 1	,  )

										lowerCamelCase__       =       (logits[-1],)

										if labels is not None:
															lowerCamelCase__       =       None
															lowerCamelCase__       =       0
															for ix, logits_item in enumerate(_lowerCAmelCase	):
																				if self.num_labels == 1:
																									#  We are doing regression
																									lowerCamelCase__       =       MSELoss()
																									lowerCamelCase__       =       loss_fct(logits_item.view(-1	)	,  labels.view(-1	)	)
																				else:
																									lowerCamelCase__       =       CrossEntropyLoss()
																									lowerCamelCase__       =       loss_fct(logits_item.view(-1	,  self.num_labels	)	,  labels.view(-1	)	)
																				if total_loss is None:
																									lowerCamelCase__       =       loss
																				else:
																									total_loss += loss * (ix + 1)
																				total_weights += ix + 1
															lowerCamelCase__       =       (total_loss / total_weights,) + outputs

										return outputs








A_												=     {str(digit): digit**5 for digit in range(1_0)}





def    __UpperCamelCase      (    a) ->int:
					return sum(DIGITS_FIFTH_POWER[digit] for digit in str(a))





def    __UpperCamelCase      (    ) ->int:
					return sum(
					    number
					    for number in range(1000,     1000000)
					    if number == digits_fifth_powers_sum(a))


if __name__ == "__main__":
						print(solution())

