




import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


SCREAMING_SNAKE_CASE:     Optional[Any]									= logging.getLogger(__name__)



class    lowercase_		(SCREAMING_SNAKE_CASE__ ):



						def 	__a					(			self	:   str					,							snake_case__	:   str					,							snake_case__	:   int					,							snake_case__	:   int=None					,							snake_case__	:   Tuple=None  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.layer[current_layer](snake_case__					,							snake_case__					,							head_mask[current_layer]  )

								SCREAMING_SNAKE_CASE_					      =							layer_outputs[0]

								return hidden_states



@add_start_docstrings(
    "The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."							,	SCREAMING_SNAKE_CASE__							,	)
class    lowercase_		(SCREAMING_SNAKE_CASE__ ):



						def __init__(			self	:   Union[str, Any]					,							snake_case__	:   List[Any]  ):

								"""simple docstring"""

								super().__init__(snake_case__  )

								SCREAMING_SNAKE_CASE_					      =							BertEncoderWithPabee(snake_case__  )

								self.init_weights()
								SCREAMING_SNAKE_CASE_					      =							0
								SCREAMING_SNAKE_CASE_					      =							0
								SCREAMING_SNAKE_CASE_					      =							0

								SCREAMING_SNAKE_CASE_					      =							0



						def 	__a					(			self	:   Dict					,							snake_case__	:   Optional[Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							threshold



						def 	__a					(			self	:   Tuple					,							snake_case__	:   List[Any]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							patience



						def 	__a					(			self	:   List[str]  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							0
								SCREAMING_SNAKE_CASE_					      =							0



						def 	__a					(			self	:   str  ):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.inference_layers_num / self.inference_instances_num
								SCREAMING_SNAKE_CASE_					      =							(
								    f'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
								    f''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
								)
								print(snake_case__  )






						@add_start_docstrings_to_model_forward(snake_case__  )
						def 	__a					(			self	:   Optional[Any]					,							snake_case__	:   Dict=None					,							snake_case__	:   Union[str, Any]=None					,							snake_case__	:   int=None					,							snake_case__	:   List[Any]=None					,							snake_case__	:   Union[str, Any]=None					,							snake_case__	:   Dict=None					,							snake_case__	:   Any=None					,							snake_case__	:   List[Any]=None					,							snake_case__	:   List[str]=None					,							snake_case__	:   str=None					,							snake_case__	:   Union[str, Any]=False					,							):

								"""simple docstring"""

								if input_ids is not None and inputs_embeds is not None:
										raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time'  )
								elif input_ids is not None:
										SCREAMING_SNAKE_CASE_					      =							input_ids.size()
								elif inputs_embeds is not None:
										SCREAMING_SNAKE_CASE_					      =							inputs_embeds.size()[:-1]
								else:
										raise ValueError('You have to specify either input_ids or inputs_embeds'  )

								SCREAMING_SNAKE_CASE_					      =							input_ids.device if input_ids is not None else inputs_embeds.device

								if attention_mask is None:
										SCREAMING_SNAKE_CASE_					      =							torch.ones(snake_case__					,							device=snake_case__  )
								if token_type_ids is None:
										SCREAMING_SNAKE_CASE_					      =							torch.zeros(snake_case__					,							dtype=torch.long					,							device=snake_case__  )

								# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
								# ourselves in which case we just need to make it broadcastable to all heads.
								SCREAMING_SNAKE_CASE_					      =							self.get_extended_attention_mask(snake_case__					,							snake_case__					,							snake_case__  )

								# If a 2D ou 3D attention mask is provided for the cross-attention
								# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
								if self.config.is_decoder and encoder_hidden_states is not None:
										SCREAMING_SNAKE_CASE_      ,				SCREAMING_SNAKE_CASE_      ,				SCREAMING_SNAKE_CASE_					      =							encoder_hidden_states.size()
										SCREAMING_SNAKE_CASE_					      =							(encoder_batch_size, encoder_sequence_length)
										if encoder_attention_mask is None:
												SCREAMING_SNAKE_CASE_					      =							torch.ones(snake_case__					,							device=snake_case__  )
										SCREAMING_SNAKE_CASE_					      =							self.invert_attention_mask(snake_case__  )
								else:
										SCREAMING_SNAKE_CASE_					      =							None

								# Prepare head mask if needed
								# 1.0 in head_mask indicate we keep the head
								# attention_probs has shape bsz x n_heads x N x N
								# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
								# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
								SCREAMING_SNAKE_CASE_					      =							self.get_head_mask(snake_case__					,							self.config.num_hidden_layers  )

								SCREAMING_SNAKE_CASE_					      =							self.embeddings(
								    input_ids=snake_case__					,							position_ids=snake_case__					,							token_type_ids=snake_case__					,							inputs_embeds=snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							embedding_output

								if self.training:
										SCREAMING_SNAKE_CASE_					      =							[]
										for i in range(self.config.num_hidden_layers  ):
												SCREAMING_SNAKE_CASE_					      =							self.encoder.adaptive_forward(
												    snake_case__					,							current_layer=snake_case__					,							attention_mask=snake_case__					,							head_mask=snake_case__  )

												SCREAMING_SNAKE_CASE_					      =							self.pooler(snake_case__  )
												SCREAMING_SNAKE_CASE_					      =							output_layers[i](output_dropout(snake_case__  )  )
												res.append(snake_case__  )
								elif self.patience == 0:  # Use all layers for inference
										SCREAMING_SNAKE_CASE_					      =							self.encoder(
										    snake_case__					,							attention_mask=snake_case__					,							head_mask=snake_case__					,							encoder_hidden_states=snake_case__					,							encoder_attention_mask=snake_case__					,							)
										SCREAMING_SNAKE_CASE_					      =							self.pooler(encoder_outputs[0]  )
										SCREAMING_SNAKE_CASE_					      =							[output_layers[self.config.num_hidden_layers - 1](snake_case__  )]
								else:
										SCREAMING_SNAKE_CASE_					      =							0
										SCREAMING_SNAKE_CASE_					      =							None
										SCREAMING_SNAKE_CASE_					      =							0
										for i in range(self.config.num_hidden_layers  ):
												calculated_layer_num += 1
												SCREAMING_SNAKE_CASE_					      =							self.encoder.adaptive_forward(
												    snake_case__					,							current_layer=snake_case__					,							attention_mask=snake_case__					,							head_mask=snake_case__  )

												SCREAMING_SNAKE_CASE_					      =							self.pooler(snake_case__  )
												SCREAMING_SNAKE_CASE_					      =							output_layers[i](snake_case__  )
												if regression:
														SCREAMING_SNAKE_CASE_					      =							logits.detach()
														if patient_result is not None:
																SCREAMING_SNAKE_CASE_					      =							patient_result.detach()
														if (patient_result is not None) and torch.abs(patient_result - labels  ) < self.regression_threshold:
																patient_counter += 1
														else:
																SCREAMING_SNAKE_CASE_					      =							0
												else:
														SCREAMING_SNAKE_CASE_					      =							logits.detach().argmax(dim=1  )
														if patient_result is not None:
																SCREAMING_SNAKE_CASE_					      =							patient_result.detach().argmax(dim=1  )
														if (patient_result is not None) and torch.all(labels.eq(snake_case__  )  ):
																patient_counter += 1
														else:
																SCREAMING_SNAKE_CASE_					      =							0

												SCREAMING_SNAKE_CASE_					      =							logits
												if patient_counter == self.patience:
														break
										SCREAMING_SNAKE_CASE_					      =							[patient_result]
										self.inference_layers_num += calculated_layer_num
										self.inference_instances_num += 1

								return res



@add_start_docstrings(
    "Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "							,	SCREAMING_SNAKE_CASE__							,	)
class    lowercase_		(SCREAMING_SNAKE_CASE__ ):



						def __init__(			self	:   Tuple					,							snake_case__	:   Optional[int]  ):

								"""simple docstring"""

								super().__init__(snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							config.num_labels

								SCREAMING_SNAKE_CASE_					      =							BertModelWithPabee(snake_case__  )
								SCREAMING_SNAKE_CASE_					      =							nn.Dropout(config.hidden_dropout_prob  )
								SCREAMING_SNAKE_CASE_					      =							nn.ModuleList(
								    [nn.Linear(config.hidden_size					,							self.config.num_labels  ) for _ in range(config.num_hidden_layers  )]  )

								self.init_weights()






						@add_start_docstrings_to_model_forward(snake_case__  )
						def 	__a					(			self	:   int					,							snake_case__	:   str=None					,							snake_case__	:   Optional[int]=None					,							snake_case__	:   Dict=None					,							snake_case__	:   str=None					,							snake_case__	:   str=None					,							snake_case__	:   List[Any]=None					,							snake_case__	:   int=None					,							):

								"""simple docstring"""

								SCREAMING_SNAKE_CASE_					      =							self.bert(
								    input_ids=snake_case__					,							attention_mask=snake_case__					,							token_type_ids=snake_case__					,							position_ids=snake_case__					,							head_mask=snake_case__					,							inputs_embeds=snake_case__					,							output_dropout=self.dropout					,							output_layers=self.classifiers					,							regression=self.num_labels == 1					,							)

								SCREAMING_SNAKE_CASE_					      =							(logits[-1],)

								if labels is not None:
										SCREAMING_SNAKE_CASE_					      =							None
										SCREAMING_SNAKE_CASE_					      =							0
										for ix, logits_item in enumerate(snake_case__  ):
												if self.num_labels == 1:
														#  We are doing regression
														SCREAMING_SNAKE_CASE_					      =							MSELoss()
														SCREAMING_SNAKE_CASE_					      =							loss_fct(logits_item.view(-1  )					,							labels.view(-1  )  )
												else:
														SCREAMING_SNAKE_CASE_					      =							CrossEntropyLoss()
														SCREAMING_SNAKE_CASE_					      =							loss_fct(logits_item.view(-1					,							self.num_labels  )					,							labels.view(-1  )  )
												if total_loss is None:
														SCREAMING_SNAKE_CASE_					      =							loss
												else:
														total_loss += loss * (ix + 1)
												total_weights += ix + 1
										SCREAMING_SNAKE_CASE_					      =							(total_loss / total_weights,) + outputs

								return outputs





SCREAMING_SNAKE_CASE:     Optional[int]									= {str(digit): digit**5 for digit in range(1_0)}


def 					_a							(					lowerCAmelCase )->	int:
		return sum(DIGITS_FIFTH_POWER[digit] for digit in str(lowerCAmelCase ) )


def 					_a							(					)->	int:
		return sum(
		    number
		    for number in range(1000    ,						1000000 )
		    if number == digits_fifth_powers_sum(lowerCAmelCase ) )


if __name__ == "__main__":
						print(solution())