






import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


a__       =     logging.getLogger(__name__)


class    _lowerCAmelCase			(		lowercase_      ):



				"""simple docstring"""



				def 					__magic_name__							(      self    :    Optional[int]			,		UpperCamelCase__    :    List[Any]			,		UpperCamelCase__    :    Tuple			,		UpperCamelCase__    :    Optional[int]=None			,		UpperCamelCase__    :    str=None):






											'''simple docstring'''







											snake_case__	  =       self.layer[current_layer](UpperCamelCase__			,		UpperCamelCase__			,		head_mask[current_layer])

											snake_case__	  =       layer_outputs[0]

											return hidden_states


@add_start_docstrings(
    '''The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''     ,      lowercase_     ,      )
class    _lowerCAmelCase			(		lowercase_      ):



				"""simple docstring"""



				def __init__(      self    :    Optional[int]			,		UpperCamelCase__    :    int):






											'''simple docstring'''







											super().__init__(UpperCamelCase__)

											snake_case__	  =       BertEncoderWithPabee(UpperCamelCase__)

											self.init_weights()
											snake_case__	  =       0
											snake_case__	  =       0
											snake_case__	  =       0

											snake_case__	  =       0



				def 					__magic_name__							(      self    :    Dict			,		UpperCamelCase__    :    Union[str, Any]):






											'''simple docstring'''







											snake_case__	  =       threshold



				def 					__magic_name__							(      self    :    Tuple			,		UpperCamelCase__    :    int):






											'''simple docstring'''







											snake_case__	  =       patience



				def 					__magic_name__							(      self    :    str):






											'''simple docstring'''







											snake_case__	  =       0
											snake_case__	  =       0



				def 					__magic_name__							(      self    :    Union[str, Any]):






											'''simple docstring'''







											snake_case__	  =       self.inference_layers_num / self.inference_instances_num
											snake_case__	  =       (
											    F'''*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='''
											    F''' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'''
											)
											print(UpperCamelCase__)



				@add_start_docstrings_to_model_forward(UpperCamelCase__)
				def 					__magic_name__							(      self    :    Tuple			,		UpperCamelCase__    :    Union[str, Any]=None			,		UpperCamelCase__    :    Tuple=None			,		UpperCamelCase__    :    Tuple=None			,		UpperCamelCase__    :    str=None			,		UpperCamelCase__    :    str=None			,		UpperCamelCase__    :    List[Any]=None			,		UpperCamelCase__    :    List[Any]=None			,		UpperCamelCase__    :    List[str]=None			,		UpperCamelCase__    :    Dict=None			,		UpperCamelCase__    :    Optional[Any]=None			,		UpperCamelCase__    :    Tuple=False			,		):






											'''simple docstring'''







											if input_ids is not None and inputs_embeds is not None:
																		raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time""")
											elif input_ids is not None:
																		snake_case__	  =       input_ids.size()
											elif inputs_embeds is not None:
																		snake_case__	  =       inputs_embeds.size()[:-1]
											else:
																		raise ValueError("""You have to specify either input_ids or inputs_embeds""")

											snake_case__	  =       input_ids.device if input_ids is not None else inputs_embeds.device

											if attention_mask is None:
																		snake_case__	  =       torch.ones(UpperCamelCase__			,		device=UpperCamelCase__)
											if token_type_ids is None:
																		snake_case__	  =       torch.zeros(UpperCamelCase__			,		dtype=torch.long			,		device=UpperCamelCase__)

											# We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
											# ourselves in which case we just need to make it broadcastable to all heads.
											snake_case__	  =       self.get_extended_attention_mask(UpperCamelCase__			,		UpperCamelCase__			,		UpperCamelCase__)

											# If a 2D ou 3D attention mask is provided for the cross-attention
											# we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
											if self.config.is_decoder and encoder_hidden_states is not None:
																		snake_case__  , snake_case__  , snake_case__	  =       encoder_hidden_states.size()
																		snake_case__	  =       (encoder_batch_size, encoder_sequence_length)
																		if encoder_attention_mask is None:
																									snake_case__	  =       torch.ones(UpperCamelCase__			,		device=UpperCamelCase__)
																		snake_case__	  =       self.invert_attention_mask(UpperCamelCase__)
											else:
																		snake_case__	  =       None

											# Prepare head mask if needed
											# 1.0 in head_mask indicate we keep the head
											# attention_probs has shape bsz x n_heads x N x N
											# input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
											# and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
											snake_case__	  =       self.get_head_mask(UpperCamelCase__			,		self.config.num_hidden_layers)

											snake_case__	  =       self.embeddings(
											    input_ids=UpperCamelCase__			,		position_ids=UpperCamelCase__			,		token_type_ids=UpperCamelCase__			,		inputs_embeds=UpperCamelCase__)
											snake_case__	  =       embedding_output

											if self.training:
																		snake_case__	  =       []
																		for i in range(self.config.num_hidden_layers):
																									snake_case__	  =       self.encoder.adaptive_forward(
																									    UpperCamelCase__			,		current_layer=UpperCamelCase__			,		attention_mask=UpperCamelCase__			,		head_mask=UpperCamelCase__)

																									snake_case__	  =       self.pooler(UpperCamelCase__)
																									snake_case__	  =       output_layers[i](output_dropout(UpperCamelCase__))
																									res.append(UpperCamelCase__)
											elif self.patience == 0:  # Use all layers for inference
																		snake_case__	  =       self.encoder(
																		    UpperCamelCase__			,		attention_mask=UpperCamelCase__			,		head_mask=UpperCamelCase__			,		encoder_hidden_states=UpperCamelCase__			,		encoder_attention_mask=UpperCamelCase__			,		)
																		snake_case__	  =       self.pooler(encoder_outputs[0])
																		snake_case__	  =       [output_layers[self.config.num_hidden_layers - 1](UpperCamelCase__)]
											else:
																		snake_case__	  =       0
																		snake_case__	  =       None
																		snake_case__	  =       0
																		for i in range(self.config.num_hidden_layers):
																									calculated_layer_num += 1
																									snake_case__	  =       self.encoder.adaptive_forward(
																									    UpperCamelCase__			,		current_layer=UpperCamelCase__			,		attention_mask=UpperCamelCase__			,		head_mask=UpperCamelCase__)

																									snake_case__	  =       self.pooler(UpperCamelCase__)
																									snake_case__	  =       output_layers[i](UpperCamelCase__)
																									if regression:
																																snake_case__	  =       logits.detach()
																																if patient_result is not None:
																																							snake_case__	  =       patient_result.detach()
																																if (patient_result is not None) and torch.abs(patient_result - labels) < self.regression_threshold:
																																							patient_counter += 1
																																else:
																																							snake_case__	  =       0
																									else:
																																snake_case__	  =       logits.detach().argmax(dim=1)
																																if patient_result is not None:
																																							snake_case__	  =       patient_result.detach().argmax(dim=1)
																																if (patient_result is not None) and torch.all(labels.eq(UpperCamelCase__)):
																																							patient_counter += 1
																																else:
																																							snake_case__	  =       0

																									snake_case__	  =       logits
																									if patient_counter == self.patience:
																																break
																		snake_case__	  =       [patient_result]
																		self.inference_layers_num += calculated_layer_num
																		self.inference_instances_num += 1

											return res


@add_start_docstrings(
    '''Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. '''     ,      lowercase_     ,      )
class    _lowerCAmelCase			(		lowercase_      ):



				"""simple docstring"""



				def __init__(      self    :    Dict			,		UpperCamelCase__    :    str):






											'''simple docstring'''







											super().__init__(UpperCamelCase__)
											snake_case__	  =       config.num_labels

											snake_case__	  =       BertModelWithPabee(UpperCamelCase__)
											snake_case__	  =       nn.Dropout(config.hidden_dropout_prob)
											snake_case__	  =       nn.ModuleList(
											    [nn.Linear(config.hidden_size			,		self.config.num_labels) for _ in range(config.num_hidden_layers)])

											self.init_weights()



				@add_start_docstrings_to_model_forward(UpperCamelCase__)
				def 					__magic_name__							(      self    :    Dict			,		UpperCamelCase__    :    Dict=None			,		UpperCamelCase__    :    Tuple=None			,		UpperCamelCase__    :    int=None			,		UpperCamelCase__    :    int=None			,		UpperCamelCase__    :    Union[str, Any]=None			,		UpperCamelCase__    :    Dict=None			,		UpperCamelCase__    :    Union[str, Any]=None			,		):






											'''simple docstring'''







											snake_case__	  =       self.bert(
											    input_ids=UpperCamelCase__			,		attention_mask=UpperCamelCase__			,		token_type_ids=UpperCamelCase__			,		position_ids=UpperCamelCase__			,		head_mask=UpperCamelCase__			,		inputs_embeds=UpperCamelCase__			,		output_dropout=self.dropout			,		output_layers=self.classifiers			,		regression=self.num_labels == 1			,		)

											snake_case__	  =       (logits[-1],)

											if labels is not None:
																		snake_case__	  =       None
																		snake_case__	  =       0
																		for ix, logits_item in enumerate(UpperCamelCase__):
																									if self.num_labels == 1:
																																#  We are doing regression
																																snake_case__	  =       MSELoss()
																																snake_case__	  =       loss_fct(logits_item.view(-1)			,		labels.view(-1))
																									else:
																																snake_case__	  =       CrossEntropyLoss()
																																snake_case__	  =       loss_fct(logits_item.view(-1			,		self.num_labels)			,		labels.view(-1))
																									if total_loss is None:
																																snake_case__	  =       loss
																									else:
																																total_loss += loss * (ix + 1)
																									total_weights += ix + 1
																		snake_case__	  =       (total_loss / total_weights,) + outputs

											return outputs








import math
from collections.abc import Iterator
from itertools import takewhile

def _UpperCAmelCase   (	a						:	int ):

							if 1 < number < 4:
														# 2 and 3 are primes
														return True
							elif number < 2 or number % 2 == 0 or number % 3 == 0:
														# Negatives, 0, 1, all even numbers, all multiples of 3 are not primes
														return False

							# All primes number are in format of 6k +/- 1
							for i in range(5  ,					int(math.sqrt(a ) + 1 )  ,					6 ):
														if number % i == 0 or number % (i + 2) == 0:
																					return False
							return True

def _UpperCAmelCase   (	):
							snake_case__	  =       2
							while True:
														if is_prime(a ):
																					yield num
														num += 1

def _UpperCAmelCase   (	a						:	int = 200_0000 ):
							return sum(takewhile(lambda a : x < n  ,					prime_generator() ) )


if __name__ == "__main__":
	print(F'''{solution() = }''')
