




'''simple docstring'''



UpperCAmelCase_							:	List[Any]	     =		{
    0: '''0''',
    1: '''1''',
    2: '''2''',
    3: '''3''',
    4: '''4''',
    5: '''5''',
    6: '''6''',
    7: '''7''',
    8: '''8''',
    9: '''9''',
    1_0: '''a''',
    1_1: '''b''',
    1_2: '''c''',
    1_3: '''d''',
    1_4: '''e''',
    1_5: '''f''',
}







def       _UpperCamelCase			(_lowerCamelCase				:  float						)-> str:




     '''simple docstring'''





     assert type(_lowerCamelCase						) in (int, float) and decimal == int(_lowerCamelCase						)
     __snake_case            =	int(_lowerCamelCase						)
     __snake_case            =	''''''
     __snake_case            =	False
     if decimal < 0:
          __snake_case            =	True
          decimal *= -1
     while decimal > 0:
          __snake_case       ,     __snake_case            =	divmod(_lowerCamelCase ,						16						)
          __snake_case            =	values[remainder] + hexadecimal
     __snake_case            =	'''0x''' + hexadecimal
     if negative:
          __snake_case            =	'''-''' + hexadecimal
     return hexadecimal


if __name__ == "__main__":
     import doctest

     doctest.testmod()














import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


UpperCamelCase    							=      logging.getLogger(__name__)







class 		_A						( UpperCAmelCase_ ):

  def    a					(    self						:	Tuple   ,				lowerCamelCase__						:	Optional[int]   ,				lowerCamelCase__						:	Union[str, Any]   ,				lowerCamelCase__						:	str=None   ,				lowerCamelCase__						:	int=None			):
     """simple docstring"""

     __UpperCamelCase	:			Optional[Any]					=					self.layer[current_layer](lowerCamelCase__   ,				lowerCamelCase__   ,				head_mask[current_layer]			)

     __UpperCamelCase	:			Optional[Any]					=					layer_outputs[0]

     return hidden_states







@add_start_docstrings(
    '''The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''	,				UpperCAmelCase_	,				)
class 		_A						( UpperCAmelCase_ ):

  def __init__(    self						:	Tuple   ,				lowerCamelCase__						:	Tuple			):
     """simple docstring"""

     super().__init__(lowerCamelCase__			)

     __UpperCamelCase	:			List[str]					=					BertEncoderWithPabee(lowerCamelCase__			)

     self.init_weights()
     __UpperCamelCase	:			Optional[int]					=					0
     __UpperCamelCase	:			Tuple					=					0
     __UpperCamelCase	:			Any					=					0

     __UpperCamelCase	:			Optional[Any]					=					0

  def    a					(    self						:	Union[str, Any]   ,				lowerCamelCase__						:	Any			):
     """simple docstring"""

     __UpperCamelCase	:			List[Any]					=					threshold

  def    a					(    self						:	Optional[int]   ,				lowerCamelCase__						:	Union[str, Any]			):
     """simple docstring"""

     __UpperCamelCase	:			int					=					patience

  def    a					(    self						:	Optional[int]			):
     """simple docstring"""

     __UpperCamelCase	:			int					=					0
     __UpperCamelCase	:			Tuple					=					0

  def    a					(    self						:	int			):
     """simple docstring"""

     __UpperCamelCase	:			int					=					self.inference_layers_num / self.inference_instances_num
     __UpperCamelCase	:			str					=					(
         f'*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='
         f' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'
     )
     print(lowerCamelCase__			)

  @add_start_docstrings_to_model_forward(lowerCamelCase__			)
  def    a					(    self						:	Any   ,				lowerCamelCase__						:	List[Any]=None   ,				lowerCamelCase__						:	List[str]=None   ,				lowerCamelCase__						:	List[Any]=None   ,				lowerCamelCase__						:	Optional[Any]=None   ,				lowerCamelCase__						:	List[str]=None   ,				lowerCamelCase__						:	str=None   ,				lowerCamelCase__						:	List[Any]=None   ,				lowerCamelCase__						:	Any=None   ,				lowerCamelCase__						:	List[str]=None   ,				lowerCamelCase__						:	List[str]=None   ,				lowerCamelCase__						:	str=False   ,				):
     """simple docstring"""

     if input_ids is not None and inputs_embeds is not None:
        raise ValueError("""You cannot specify both input_ids and inputs_embeds at the same time"""			)
     elif input_ids is not None:
        __UpperCamelCase	:			Optional[Any]					=					input_ids.size()
     elif inputs_embeds is not None:
        __UpperCamelCase	:			Tuple					=					inputs_embeds.size()[:-1]
     else:
        raise ValueError("""You have to specify either input_ids or inputs_embeds"""			)

     __UpperCamelCase	:			Tuple					=					input_ids.device if input_ids is not None else inputs_embeds.device

     if attention_mask is None:
        __UpperCamelCase	:			Optional[Any]					=					torch.ones(lowerCamelCase__   ,				device=lowerCamelCase__			)
     if token_type_ids is None:
        __UpperCamelCase	:			str					=					torch.zeros(lowerCamelCase__   ,				dtype=torch.long   ,				device=lowerCamelCase__			)

     # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
     # ourselves in which case we just need to make it broadcastable to all heads.
     __UpperCamelCase	:			torch.Tensor					=					self.get_extended_attention_mask(lowerCamelCase__   ,				lowerCamelCase__   ,				lowerCamelCase__			)

     # If a 2D ou 3D attention mask is provided for the cross-attention
     # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
     if self.config.is_decoder and encoder_hidden_states is not None:
        __UpperCamelCase	,     __UpperCamelCase	,     __UpperCamelCase	:			Optional[Any]					=					encoder_hidden_states.size()
        __UpperCamelCase	:			Union[str, Any]					=					(encoder_batch_size, encoder_sequence_length)
        if encoder_attention_mask is None:
           __UpperCamelCase	:			Union[str, Any]					=					torch.ones(lowerCamelCase__   ,				device=lowerCamelCase__			)
        __UpperCamelCase	:			Optional[int]					=					self.invert_attention_mask(lowerCamelCase__			)
     else:
        __UpperCamelCase	:			str					=					None

     # Prepare head mask if needed
     # 1.0 in head_mask indicate we keep the head
     # attention_probs has shape bsz x n_heads x N x N
     # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
     # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
     __UpperCamelCase	:			int					=					self.get_head_mask(lowerCamelCase__   ,				self.config.num_hidden_layers			)

     __UpperCamelCase	:			Any					=					self.embeddings(
         input_ids=lowerCamelCase__   ,				position_ids=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__   ,				inputs_embeds=lowerCamelCase__			)
     __UpperCamelCase	:			Optional[int]					=					embedding_output

     if self.training:
        __UpperCamelCase	:			List[Any]					=					[]
        for i in range(self.config.num_hidden_layers			):
           __UpperCamelCase	:			List[Any]					=					self.encoder.adaptive_forward(
               lowerCamelCase__   ,				current_layer=lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				head_mask=lowerCamelCase__			)

           __UpperCamelCase	:			Tuple					=					self.pooler(lowerCamelCase__			)
           __UpperCamelCase	:			str					=					output_layers[i](output_dropout(lowerCamelCase__			)			)
           res.append(lowerCamelCase__			)
     elif self.patience == 0:  # Use all layers for inference
        __UpperCamelCase	:			Union[str, Any]					=					self.encoder(
            lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				head_mask=lowerCamelCase__   ,				encoder_hidden_states=lowerCamelCase__   ,				encoder_attention_mask=lowerCamelCase__   ,				)
        __UpperCamelCase	:			Optional[int]					=					self.pooler(encoder_outputs[0]			)
        __UpperCamelCase	:			Union[str, Any]					=					[output_layers[self.config.num_hidden_layers - 1](lowerCamelCase__			)]
     else:
        __UpperCamelCase	:			Union[str, Any]					=					0
        __UpperCamelCase	:			Optional[Any]					=					None
        __UpperCamelCase	:			List[Any]					=					0
        for i in range(self.config.num_hidden_layers			):
           calculated_layer_num += 1
           __UpperCamelCase	:			Union[str, Any]					=					self.encoder.adaptive_forward(
               lowerCamelCase__   ,				current_layer=lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				head_mask=lowerCamelCase__			)

           __UpperCamelCase	:			str					=					self.pooler(lowerCamelCase__			)
           __UpperCamelCase	:			Any					=					output_layers[i](lowerCamelCase__			)
           if regression:
              __UpperCamelCase	:			Optional[int]					=					logits.detach()
              if patient_result is not None:
                 __UpperCamelCase	:			Dict					=					patient_result.detach()
              if (patient_result is not None) and torch.abs(patient_result - labels			) < self.regression_threshold:
                 patient_counter += 1
              else:
                 __UpperCamelCase	:			List[str]					=					0
           else:
              __UpperCamelCase	:			Optional[int]					=					logits.detach().argmax(dim=1			)
              if patient_result is not None:
                 __UpperCamelCase	:			Union[str, Any]					=					patient_result.detach().argmax(dim=1			)
              if (patient_result is not None) and torch.all(labels.eq(lowerCamelCase__			)			):
                 patient_counter += 1
              else:
                 __UpperCamelCase	:			str					=					0

           __UpperCamelCase	:			Union[str, Any]					=					logits
           if patient_counter == self.patience:
              break
        __UpperCamelCase	:			List[str]					=					[patient_result]
        self.inference_layers_num += calculated_layer_num
        self.inference_instances_num += 1

     return res







@add_start_docstrings(
    '''Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. '''	,				UpperCAmelCase_	,				)
class 		_A						( UpperCAmelCase_ ):

  def __init__(    self						:	Any   ,				lowerCamelCase__						:	Union[str, Any]			):
     """simple docstring"""

     super().__init__(lowerCamelCase__			)
     __UpperCamelCase	:			Dict					=					config.num_labels

     __UpperCamelCase	:			List[Any]					=					BertModelWithPabee(lowerCamelCase__			)
     __UpperCamelCase	:			Optional[int]					=					nn.Dropout(config.hidden_dropout_prob			)
     __UpperCamelCase	:			List[Any]					=					nn.ModuleList(
         [nn.Linear(config.hidden_size   ,				self.config.num_labels			) for _ in range(config.num_hidden_layers			)]			)

     self.init_weights()

  @add_start_docstrings_to_model_forward(lowerCamelCase__			)
  def    a					(    self						:	List[Any]   ,				lowerCamelCase__						:	Union[str, Any]=None   ,				lowerCamelCase__						:	Dict=None   ,				lowerCamelCase__						:	Union[str, Any]=None   ,				lowerCamelCase__						:	List[Any]=None   ,				lowerCamelCase__						:	Optional[Any]=None   ,				lowerCamelCase__						:	Optional[int]=None   ,				lowerCamelCase__						:	Dict=None   ,				):
     """simple docstring"""

     __UpperCamelCase	:			str					=					self.bert(
         input_ids=lowerCamelCase__   ,				attention_mask=lowerCamelCase__   ,				token_type_ids=lowerCamelCase__   ,				position_ids=lowerCamelCase__   ,				head_mask=lowerCamelCase__   ,				inputs_embeds=lowerCamelCase__   ,				output_dropout=self.dropout   ,				output_layers=self.classifiers   ,				regression=self.num_labels == 1   ,				)

     __UpperCamelCase	:			List[str]					=					(logits[-1],)

     if labels is not None:
        __UpperCamelCase	:			Tuple					=					None
        __UpperCamelCase	:			Union[str, Any]					=					0
        for ix, logits_item in enumerate(lowerCamelCase__			):
           if self.num_labels == 1:
              #  We are doing regression
              __UpperCamelCase	:			Union[str, Any]					=					MSELoss()
              __UpperCamelCase	:			Optional[Any]					=					loss_fct(logits_item.view(-1			)   ,				labels.view(-1			)			)
           else:
              __UpperCamelCase	:			Dict					=					CrossEntropyLoss()
              __UpperCamelCase	:			Dict					=					loss_fct(logits_item.view(-1   ,				self.num_labels			)   ,				labels.view(-1			)			)
           if total_loss is None:
              __UpperCamelCase	:			Dict					=					loss
           else:
              total_loss += loss * (ix + 1)
           total_weights += ix + 1
        __UpperCamelCase	:			Tuple					=					(total_loss / total_weights,) + outputs

     return outputs



