





"""simple docstring"""



import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


a_					=     logging.getLogger(__name__)






class 						__snake_case   (						SCREAMING_SNAKE_CASE__	):





 """simple docstring"""


 def    UpperCamelCase__(       self   ,		__lowerCamelCase   ,		__lowerCamelCase   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None    ):



     '''simple docstring'''




     __A       :		int							=	self.layer[current_layer](snake_case__   ,		snake_case__   ,		head_mask[current_layer]    )

     __A       :		Optional[int]							=	layer_outputs[0]

     return hidden_states






@add_start_docstrings(
    """The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."""				,				SCREAMING_SNAKE_CASE__				,				)
class 						__snake_case   (						SCREAMING_SNAKE_CASE__	):





 """simple docstring"""


 def __init__(       self   ,		__lowerCamelCase    ):



     '''simple docstring'''




     super().__init__(snake_case__    )

     __A       :		Dict							=	BertEncoderWithPabee(snake_case__    )

     self.init_weights()
     __A       :		str							=	0
     __A       :		List[str]							=	0
     __A       :		int							=	0

     __A       :		Union[str, Any]							=	0

 def    UpperCamelCase__(       self   ,		__lowerCamelCase    ):



     '''simple docstring'''




     __A       :		Dict							=	threshold

 def    UpperCamelCase__(       self   ,		__lowerCamelCase    ):



     '''simple docstring'''




     __A       :		Optional[int]							=	patience

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		List[Any]							=	0
     __A       :		str							=	0

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Optional[int]							=	self.inference_layers_num / self.inference_instances_num
     __A       :		Union[str, Any]							=	(
         F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
         F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
     )
     print(snake_case__    )







 @add_start_docstrings_to_model_forward(snake_case__    )
 def    UpperCamelCase__(       self   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=False   ,		):



     '''simple docstring'''




     if input_ids is not None and inputs_embeds is not None:
         raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''    )
     elif input_ids is not None:
         __A       :		Tuple							=	input_ids.size()
     elif inputs_embeds is not None:
         __A       :		Dict							=	inputs_embeds.size()[:-1]
     else:
         raise ValueError('''You have to specify either input_ids or inputs_embeds'''    )

     __A       :		Tuple							=	input_ids.device if input_ids is not None else inputs_embeds.device

     if attention_mask is None:
         __A       :		Tuple							=	torch.ones(snake_case__   ,		device=snake_case__    )
     if token_type_ids is None:
         __A       :		List[Any]							=	torch.zeros(snake_case__   ,		dtype=torch.long   ,		device=snake_case__    )

     # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
     # ourselves in which case we just need to make it broadcastable to all heads.
     __A       :		int							=	self.get_extended_attention_mask(snake_case__   ,		snake_case__   ,		snake_case__    )

     # If a 2D ou 3D attention mask is provided for the cross-attention
     # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
     if self.config.is_decoder and encoder_hidden_states is not None:
         __A					,			__A					,			__A       :		str							=	encoder_hidden_states.size()
         __A       :		Dict							=	(encoder_batch_size, encoder_sequence_length)
         if encoder_attention_mask is None:
             __A       :		List[str]							=	torch.ones(snake_case__   ,		device=snake_case__    )
         __A       :		int							=	self.invert_attention_mask(snake_case__    )
     else:
         __A       :		Union[str, Any]							=	None

     # Prepare head mask if needed
     # 1.0 in head_mask indicate we keep the head
     # attention_probs has shape bsz x n_heads x N x N
     # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
     # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
     __A       :		Optional[int]							=	self.get_head_mask(snake_case__   ,		self.config.num_hidden_layers    )

     __A       :		Optional[int]							=	self.embeddings(
         input_ids=snake_case__   ,		position_ids=snake_case__   ,		token_type_ids=snake_case__   ,		inputs_embeds=snake_case__    )
     __A       :		int							=	embedding_output

     if self.training:
         __A       :		List[Any]							=	[]
         for i in range(self.config.num_hidden_layers    ):
             __A       :		List[str]							=	self.encoder.adaptive_forward(
                 snake_case__   ,		current_layer=snake_case__   ,		attention_mask=snake_case__   ,		head_mask=snake_case__    )

             __A       :		List[str]							=	self.pooler(snake_case__    )
             __A       :		Tuple							=	output_layers[i](output_dropout(snake_case__    )    )
             res.append(snake_case__    )
     elif self.patience == 0:  # Use all layers for inference
         __A       :		Any							=	self.encoder(
             snake_case__   ,		attention_mask=snake_case__   ,		head_mask=snake_case__   ,		encoder_hidden_states=snake_case__   ,		encoder_attention_mask=snake_case__   ,		)
         __A       :		Union[str, Any]							=	self.pooler(encoder_outputs[0]    )
         __A       :		Dict							=	[output_layers[self.config.num_hidden_layers - 1](snake_case__    )]
     else:
         __A       :		Dict							=	0
         __A       :		int							=	None
         __A       :		Union[str, Any]							=	0
         for i in range(self.config.num_hidden_layers    ):
             calculated_layer_num += 1
             __A       :		Tuple							=	self.encoder.adaptive_forward(
                 snake_case__   ,		current_layer=snake_case__   ,		attention_mask=snake_case__   ,		head_mask=snake_case__    )

             __A       :		List[str]							=	self.pooler(snake_case__    )
             __A       :		Tuple							=	output_layers[i](snake_case__    )
             if regression:
                 __A       :		Any							=	logits.detach()
                 if patient_result is not None:
                     __A       :		Optional[int]							=	patient_result.detach()
                 if (patient_result is not None) and torch.abs(patient_result - labels    ) < self.regression_threshold:
                     patient_counter += 1
                 else:
                     __A       :		Union[str, Any]							=	0
             else:
                 __A       :		List[Any]							=	logits.detach().argmax(dim=1    )
                 if patient_result is not None:
                     __A       :		Union[str, Any]							=	patient_result.detach().argmax(dim=1    )
                 if (patient_result is not None) and torch.all(labels.eq(snake_case__    )    ):
                     patient_counter += 1
                 else:
                     __A       :		List[str]							=	0

             __A       :		Union[str, Any]							=	logits
             if patient_counter == self.patience:
                 break
         __A       :		Tuple							=	[patient_result]
         self.inference_layers_num += calculated_layer_num
         self.inference_instances_num += 1

     return res






@add_start_docstrings(
    """Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. """				,				SCREAMING_SNAKE_CASE__				,				)
class 						__snake_case   (						SCREAMING_SNAKE_CASE__	):





 """simple docstring"""


 def __init__(       self   ,		__lowerCamelCase    ):



     '''simple docstring'''




     super().__init__(snake_case__    )
     __A       :		List[Any]							=	config.num_labels

     __A       :		List[Any]							=	BertModelWithPabee(snake_case__    )
     __A       :		Dict							=	nn.Dropout(config.hidden_dropout_prob    )
     __A       :		Union[str, Any]							=	nn.ModuleList(
         [nn.Linear(config.hidden_size   ,		self.config.num_labels    ) for _ in range(config.num_hidden_layers    )]    )

     self.init_weights()







 @add_start_docstrings_to_model_forward(snake_case__    )
 def    UpperCamelCase__(       self   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		__lowerCamelCase=None   ,		):



     '''simple docstring'''




     __A       :		List[str]							=	self.bert(
         input_ids=snake_case__   ,		attention_mask=snake_case__   ,		token_type_ids=snake_case__   ,		position_ids=snake_case__   ,		head_mask=snake_case__   ,		inputs_embeds=snake_case__   ,		output_dropout=self.dropout   ,		output_layers=self.classifiers   ,		regression=self.num_labels == 1   ,		)

     __A       :		Optional[Any]							=	(logits[-1],)

     if labels is not None:
         __A       :		int							=	None
         __A       :		Tuple							=	0
         for ix, logits_item in enumerate(snake_case__    ):
             if self.num_labels == 1:
                 #  We are doing regression
                 __A       :		Tuple							=	MSELoss()
                 __A       :		List[Any]							=	loss_fct(logits_item.view(-1    )   ,		labels.view(-1    )    )
             else:
                 __A       :		Union[str, Any]							=	CrossEntropyLoss()
                 __A       :		Union[str, Any]							=	loss_fct(logits_item.view(-1   ,		self.num_labels    )   ,		labels.view(-1    )    )
             if total_loss is None:
                 __A       :		Tuple							=	loss
             else:
                 total_loss += loss * (ix + 1)
             total_weights += ix + 1
         __A       :		Tuple							=	(total_loss / total_weights,) + outputs

     return outputs










SCREAMING_SNAKE_CASE:     Optional[int]									= {str(digit): digit**5 for digit in range(1_0)}


def 					_a							(					lowerCAmelCase )->	int:
		return sum(DIGITS_FIFTH_POWER[digit] for digit in str(lowerCAmelCase ) )


def 					_a							(					)->	int:
		return sum(
		    number
		    for number in range(1000    ,						1000000 )
		    if number == digits_fifth_powers_sum(lowerCAmelCase ) )


if __name__ == "__main__":
						print(solution())