





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


__A     =    logging.getLogger(__name__)



class        _SCREAMING_SNAKE_CASE						(     lowerCamelCase__   ):







 '''simple docstring'''

 def 							SCREAMING_SNAKE_CASE_ (self    :       List[str]	,			UpperCAmelCase_    :       Tuple	,			UpperCAmelCase_    :       List[Any]	,			UpperCAmelCase_    :       Dict=None	,			UpperCAmelCase_    :       Optional[int]=None)	->Dict:



   '''simple docstring'''



   lowerCamelCase__:				Optional[int]        =self.layer[current_layer](lowercase__	,			lowercase__	,			head_mask[current_layer])

   lowerCamelCase__:				Union[str, Any]        =layer_outputs[0]

   return hidden_states



@add_start_docstrings(
    "The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."   ,							lowerCamelCase__   ,							)
class        _SCREAMING_SNAKE_CASE						(     lowerCamelCase__   ):







 '''simple docstring'''

 def __init__(self    :       List[str]	,			UpperCAmelCase_    :       Tuple)	->Any:



   '''simple docstring'''



   super().__init__(lowercase__)

   lowerCamelCase__:				Tuple        =BertEncoderWithPabee(lowercase__)

   self.init_weights()
   lowerCamelCase__:				str        =0
   lowerCamelCase__:				Dict        =0
   lowerCamelCase__:				Tuple        =0

   lowerCamelCase__:				Tuple        =0

 def 							SCREAMING_SNAKE_CASE_ (self    :       List[str]	,			UpperCAmelCase_    :       Union[str, Any])	->str:



   '''simple docstring'''



   lowerCamelCase__:				List[str]        =threshold

 def 							SCREAMING_SNAKE_CASE_ (self    :       Tuple	,			UpperCAmelCase_    :       Union[str, Any])	->List[str]:



   '''simple docstring'''



   lowerCamelCase__:				Tuple        =patience

 def 							SCREAMING_SNAKE_CASE_ (self    :       int)	->List[str]:



   '''simple docstring'''



   lowerCamelCase__:				Union[str, Any]        =0
   lowerCamelCase__:				Optional[Any]        =0

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[int])	->int:



   '''simple docstring'''



   lowerCamelCase__:				Union[str, Any]        =self.inference_layers_num / self.inference_instances_num
   lowerCamelCase__:				Dict        =(
       F"""*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="""
       F""" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"""
   )
   print(lowercase__)







 @add_start_docstrings_to_model_forward(lowercase__)
 def 							SCREAMING_SNAKE_CASE_ (self    :       Dict	,			UpperCAmelCase_    :       int=None	,			UpperCAmelCase_    :       int=None	,			UpperCAmelCase_    :       Optional[Any]=None	,			UpperCAmelCase_    :       Dict=None	,			UpperCAmelCase_    :       Optional[int]=None	,			UpperCAmelCase_    :       List[str]=None	,			UpperCAmelCase_    :       Union[str, Any]=None	,			UpperCAmelCase_    :       Tuple=None	,			UpperCAmelCase_    :       List[Any]=None	,			UpperCAmelCase_    :       Union[str, Any]=None	,			UpperCAmelCase_    :       Optional[int]=False	,			)	->Any:



   '''simple docstring'''



   if input_ids is not None and inputs_embeds is not None:
     raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
   elif input_ids is not None:
     lowerCamelCase__:				Any        =input_ids.size()
   elif inputs_embeds is not None:
     lowerCamelCase__:				int        =inputs_embeds.size()[:-1]
   else:
     raise ValueError("You have to specify either input_ids or inputs_embeds")

   lowerCamelCase__:				Dict        =input_ids.device if input_ids is not None else inputs_embeds.device

   if attention_mask is None:
     lowerCamelCase__:				List[str]        =torch.ones(lowercase__	,			device=lowercase__)
   if token_type_ids is None:
     lowerCamelCase__:				str        =torch.zeros(lowercase__	,			dtype=torch.long	,			device=lowercase__)

   # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
   # ourselves in which case we just need to make it broadcastable to all heads.
   lowerCamelCase__:				Dict        =self.get_extended_attention_mask(lowercase__	,			lowercase__	,			lowercase__)

   # If a 2D ou 3D attention mask is provided for the cross-attention
   # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
   if self.config.is_decoder and encoder_hidden_states is not None:
     lowerCamelCase__  , lowerCamelCase__  , lowerCamelCase__:				List[Any]        =encoder_hidden_states.size()
     lowerCamelCase__:				Union[str, Any]        =(encoder_batch_size, encoder_sequence_length)
     if encoder_attention_mask is None:
       lowerCamelCase__:				Any        =torch.ones(lowercase__	,			device=lowercase__)
     lowerCamelCase__:				List[Any]        =self.invert_attention_mask(lowercase__)
   else:
     lowerCamelCase__:				Dict        =None

   # Prepare head mask if needed
   # 1.0 in head_mask indicate we keep the head
   # attention_probs has shape bsz x n_heads x N x N
   # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
   # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
   lowerCamelCase__:				List[Any]        =self.get_head_mask(lowercase__	,			self.config.num_hidden_layers)

   lowerCamelCase__:				List[str]        =self.embeddings(
       input_ids=lowercase__	,			position_ids=lowercase__	,			token_type_ids=lowercase__	,			inputs_embeds=lowercase__)
   lowerCamelCase__:				str        =embedding_output

   if self.training:
     lowerCamelCase__:				Any        =[]
     for i in range(self.config.num_hidden_layers):
       lowerCamelCase__:				List[str]        =self.encoder.adaptive_forward(
           lowercase__	,			current_layer=lowercase__	,			attention_mask=lowercase__	,			head_mask=lowercase__)

       lowerCamelCase__:				Any        =self.pooler(lowercase__)
       lowerCamelCase__:				Union[str, Any]        =output_layers[i](output_dropout(lowercase__))
       res.append(lowercase__)
   elif self.patience == 0:  # Use all layers for inference
     lowerCamelCase__:				List[Any]        =self.encoder(
         lowercase__	,			attention_mask=lowercase__	,			head_mask=lowercase__	,			encoder_hidden_states=lowercase__	,			encoder_attention_mask=lowercase__	,			)
     lowerCamelCase__:				Any        =self.pooler(encoder_outputs[0])
     lowerCamelCase__:				List[Any]        =[output_layers[self.config.num_hidden_layers - 1](lowercase__)]
   else:
     lowerCamelCase__:				Optional[int]        =0
     lowerCamelCase__:				str        =None
     lowerCamelCase__:				List[Any]        =0
     for i in range(self.config.num_hidden_layers):
       calculated_layer_num += 1
       lowerCamelCase__:				Tuple        =self.encoder.adaptive_forward(
           lowercase__	,			current_layer=lowercase__	,			attention_mask=lowercase__	,			head_mask=lowercase__)

       lowerCamelCase__:				int        =self.pooler(lowercase__)
       lowerCamelCase__:				Any        =output_layers[i](lowercase__)
       if regression:
         lowerCamelCase__:				Union[str, Any]        =logits.detach()
         if patient_result is not None:
           lowerCamelCase__:				Optional[int]        =patient_result.detach()
         if (patient_result is not None) and torch.abs(patient_result - labels) < self.regression_threshold:
           patient_counter += 1
         else:
           lowerCamelCase__:				str        =0
       else:
         lowerCamelCase__:				Any        =logits.detach().argmax(dim=1)
         if patient_result is not None:
           lowerCamelCase__:				List[Any]        =patient_result.detach().argmax(dim=1)
         if (patient_result is not None) and torch.all(labels.eq(lowercase__)):
           patient_counter += 1
         else:
           lowerCamelCase__:				str        =0

       lowerCamelCase__:				Optional[int]        =logits
       if patient_counter == self.patience:
         break
     lowerCamelCase__:				List[str]        =[patient_result]
     self.inference_layers_num += calculated_layer_num
     self.inference_instances_num += 1

   return res



@add_start_docstrings(
    "Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "   ,							lowerCamelCase__   ,							)
class        _SCREAMING_SNAKE_CASE						(     lowerCamelCase__   ):







 '''simple docstring'''

 def __init__(self    :       Union[str, Any]	,			UpperCAmelCase_    :       Optional[int])	->Dict:



   '''simple docstring'''



   super().__init__(lowercase__)
   lowerCamelCase__:				Optional[int]        =config.num_labels

   lowerCamelCase__:				Optional[int]        =BertModelWithPabee(lowercase__)
   lowerCamelCase__:				Union[str, Any]        =nn.Dropout(config.hidden_dropout_prob)
   lowerCamelCase__:				Optional[int]        =nn.ModuleList(
       [nn.Linear(config.hidden_size	,			self.config.num_labels) for _ in range(config.num_hidden_layers)])

   self.init_weights()







 @add_start_docstrings_to_model_forward(lowercase__)
 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[int]	,			UpperCAmelCase_    :       str=None	,			UpperCAmelCase_    :       Any=None	,			UpperCAmelCase_    :       str=None	,			UpperCAmelCase_    :       Optional[Any]=None	,			UpperCAmelCase_    :       Optional[Any]=None	,			UpperCAmelCase_    :       Optional[int]=None	,			UpperCAmelCase_    :       int=None	,			)	->Any:



   '''simple docstring'''



   lowerCamelCase__:				Tuple        =self.bert(
       input_ids=lowercase__	,			attention_mask=lowercase__	,			token_type_ids=lowercase__	,			position_ids=lowercase__	,			head_mask=lowercase__	,			inputs_embeds=lowercase__	,			output_dropout=self.dropout	,			output_layers=self.classifiers	,			regression=self.num_labels == 1	,			)

   lowerCamelCase__:				int        =(logits[-1],)

   if labels is not None:
     lowerCamelCase__:				Union[str, Any]        =None
     lowerCamelCase__:				str        =0
     for ix, logits_item in enumerate(lowercase__):
       if self.num_labels == 1:
         #  We are doing regression
         lowerCamelCase__:				List[str]        =MSELoss()
         lowerCamelCase__:				List[str]        =loss_fct(logits_item.view(-1)	,			labels.view(-1))
       else:
         lowerCamelCase__:				Tuple        =CrossEntropyLoss()
         lowerCamelCase__:				Dict        =loss_fct(logits_item.view(-1	,			self.num_labels)	,			labels.view(-1))
       if total_loss is None:
         lowerCamelCase__:				Dict        =loss
       else:
         total_loss += loss * (ix + 1)
       total_weights += ix + 1
     lowerCamelCase__:				List[str]        =(total_loss / total_weights,) + outputs

   return outputs








import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


__A     =    get_logger(__name__)



class        _SCREAMING_SNAKE_CASE						(     enum.Enum   ):







 '''simple docstring'''
 lowercase_		=      "all_checks"
 lowercase_		=      "basic_checks"
 lowercase_		=      "no_checks"



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''





def 					lowerCAmelCase_			(	__a     ,   __a     ,   __a=None		)					->		Optional[int]:







  """simple docstring"""





  if expected_checksums is None:
    logger.info("Unable to verify checksums."		)
    return
  if len(set(__a		) - set(__a		)		) > 0:
    raise ExpectedMoreDownloadedFiles(str(set(__a		) - set(__a		)		)		)
  if len(set(__a		) - set(__a		)		) > 0:
    raise UnexpectedDownloadedFile(str(set(__a		) - set(__a		)		)		)
  lowerCamelCase__:				List[Any]        =[url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
  lowerCamelCase__:				Union[str, Any]        =" for " + verification_name if verification_name is not None else ""
  if len(__a		) > 0:
    raise NonMatchingChecksumError(
        F"""Checksums didn't match{for_verification_name}:\n"""
        F"""{bad_urls}\n"""
        "Set `verification_mode='no_checks'` to skip checksums verification and ignore this error"		)
  logger.info("All the checksums matched successfully" + for_verification_name		)



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''





def 					lowerCAmelCase_			(	__a     ,   __a		)					->		Any:







  """simple docstring"""





  if expected_splits is None:
    logger.info("Unable to verify splits sizes."		)
    return
  if len(set(__a		) - set(__a		)		) > 0:
    raise ExpectedMoreSplits(str(set(__a		) - set(__a		)		)		)
  if len(set(__a		) - set(__a		)		) > 0:
    raise UnexpectedSplits(str(set(__a		) - set(__a		)		)		)
  lowerCamelCase__:				Optional[int]        =[
      {"expected": expected_splits[name], "recorded": recorded_splits[name]}
      for name in expected_splits
      if expected_splits[name].num_examples != recorded_splits[name].num_examples
  ]
  if len(__a		) > 0:
    raise NonMatchingSplitsSizesError(str(__a		)		)
  logger.info("All the splits matched successfully."		)





def 					lowerCAmelCase_			(	__a     ,   __a = True		)					->		dict:







  """simple docstring"""





  if record_checksum:
    lowerCamelCase__:				str        =shaaaa()
    with open(__a     ,   "rb"		) as f:
      for chunk in iter(lambda: f.read(1 << 20		)     ,   b""		):
        m.update(__a		)
      lowerCamelCase__:				Dict        =m.hexdigest()
  else:
    lowerCamelCase__:				List[str]        =None
  return {"num_bytes": os.path.getsize(__a		), "checksum": checksum}






def 					lowerCAmelCase_			(	__a		)					->		int:







  """simple docstring"""





  if dataset_size and config.IN_MEMORY_MAX_SIZE:
    return dataset_size < config.IN_MEMORY_MAX_SIZE
  else:
    return False

