


'''simple docstring'''






import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


lowercase				=logging.getLogger(__name__)






class    __magic_name__				(						lowerCAmelCase							):






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case    ,				snake_case=None    ,				snake_case=None)		->    Tuple:




          '''simple docstring'''
          _UpperCAmelCase  :  Tuple				       =self.layer[current_layer](snake_case    ,				snake_case    ,				head_mask[current_layer])

          _UpperCAmelCase  :  List[str]				       =layer_outputs[0]

          return hidden_states






@add_start_docstrings(
    "The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."   ,lowerCAmelCase   ,)
class    __magic_name__				(						lowerCAmelCase							):






       def __init__(     self    ,				snake_case)		->    int:




          '''simple docstring'''
          super().__init__(snake_case)

          _UpperCAmelCase  :  Tuple				       =BertEncoderWithPabee(snake_case)

          self.init_weights()
          _UpperCAmelCase  :  List[str]				       =0
          _UpperCAmelCase  :  Any				       =0
          _UpperCAmelCase  :  Dict				       =0

          _UpperCAmelCase  :  int				       =0






       def      lowerCAmelCase						(     self    ,				snake_case)		->    str:




          '''simple docstring'''
          _UpperCAmelCase  :  Dict				       =threshold






       def      lowerCAmelCase						(     self    ,				snake_case)		->    str:




          '''simple docstring'''
          _UpperCAmelCase  :  int				       =patience






       def      lowerCAmelCase						(     self)		->    int:




          '''simple docstring'''
          _UpperCAmelCase  :  Union[str, Any]				       =0
          _UpperCAmelCase  :  Dict				       =0






       def      lowerCAmelCase						(     self)		->    Dict:




          '''simple docstring'''
          _UpperCAmelCase  :  Dict				       =self.inference_layers_num / self.inference_instances_num
          _UpperCAmelCase  :  Any				       =(
              f"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
              f" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
          )
          print(snake_case)






       @add_start_docstrings_to_model_forward(snake_case)
       def      lowerCAmelCase						(     self    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case=False    ,				)		->    List[str]:




          '''simple docstring'''

          if input_ids is not None and inputs_embeds is not None:
             raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time')
          elif input_ids is not None:
             _UpperCAmelCase  :  Tuple				       =input_ids.size()
          elif inputs_embeds is not None:
             _UpperCAmelCase  :  Optional[int]				       =inputs_embeds.size()[:-1]
          else:
             raise ValueError('You have to specify either input_ids or inputs_embeds')

          _UpperCAmelCase  :  Union[str, Any]				       =input_ids.device if input_ids is not None else inputs_embeds.device

          if attention_mask is None:
             _UpperCAmelCase  :  Dict				       =torch.ones(snake_case    ,				device=snake_case)
          if token_type_ids is None:
             _UpperCAmelCase  :  List[str]				       =torch.zeros(snake_case    ,				dtype=torch.long    ,				device=snake_case)

          # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
          # ourselves in which case we just need to make it broadcastable to all heads.
          _UpperCAmelCase  :  torch.Tensor				       =self.get_extended_attention_mask(snake_case    ,				snake_case    ,				snake_case)

          # If a 2D ou 3D attention mask is provided for the cross-attention
          # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
          if self.config.is_decoder and encoder_hidden_states is not None:
             _UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase  :  Optional[int]				       =encoder_hidden_states.size()
             _UpperCAmelCase  :  Tuple				       =(encoder_batch_size, encoder_sequence_length)
             if encoder_attention_mask is None:
                _UpperCAmelCase  :  Dict				       =torch.ones(snake_case    ,				device=snake_case)
             _UpperCAmelCase  :  Optional[int]				       =self.invert_attention_mask(snake_case)
          else:
             _UpperCAmelCase  :  Tuple				       =None

          # Prepare head mask if needed
          # 1.0 in head_mask indicate we keep the head
          # attention_probs has shape bsz x n_heads x N x N
          # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
          # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
          _UpperCAmelCase  :  List[Any]				       =self.get_head_mask(snake_case    ,				self.config.num_hidden_layers)

          _UpperCAmelCase  :  Optional[int]				       =self.embeddings(
              input_ids=snake_case    ,				position_ids=snake_case    ,				token_type_ids=snake_case    ,				inputs_embeds=snake_case)
          _UpperCAmelCase  :  List[Any]				       =embedding_output

          if self.training:
             _UpperCAmelCase  :  Tuple				       =[]
             for i in range(self.config.num_hidden_layers):
                _UpperCAmelCase  :  Tuple				       =self.encoder.adaptive_forward(
                    snake_case    ,				current_layer=snake_case    ,				attention_mask=snake_case    ,				head_mask=snake_case)

                _UpperCAmelCase  :  Union[str, Any]				       =self.pooler(snake_case)
                _UpperCAmelCase  :  Optional[int]				       =output_layers[i](output_dropout(snake_case))
                res.append(snake_case)
          elif self.patience == 0:  # Use all layers for inference
             _UpperCAmelCase  :  List[Any]				       =self.encoder(
                 snake_case    ,				attention_mask=snake_case    ,				head_mask=snake_case    ,				encoder_hidden_states=snake_case    ,				encoder_attention_mask=snake_case    ,				)
             _UpperCAmelCase  :  List[Any]				       =self.pooler(encoder_outputs[0])
             _UpperCAmelCase  :  List[str]				       =[output_layers[self.config.num_hidden_layers - 1](snake_case)]
          else:
             _UpperCAmelCase  :  Tuple				       =0
             _UpperCAmelCase  :  List[str]				       =None
             _UpperCAmelCase  :  List[str]				       =0
             for i in range(self.config.num_hidden_layers):
                calculated_layer_num += 1
                _UpperCAmelCase  :  Union[str, Any]				       =self.encoder.adaptive_forward(
                    snake_case    ,				current_layer=snake_case    ,				attention_mask=snake_case    ,				head_mask=snake_case)

                _UpperCAmelCase  :  Union[str, Any]				       =self.pooler(snake_case)
                _UpperCAmelCase  :  Dict				       =output_layers[i](snake_case)
                if regression:
                   _UpperCAmelCase  :  int				       =logits.detach()
                   if patient_result is not None:
                      _UpperCAmelCase  :  Optional[int]				       =patient_result.detach()
                   if (patient_result is not None) and torch.abs(patient_result - labels) < self.regression_threshold:
                      patient_counter += 1
                   else:
                      _UpperCAmelCase  :  int				       =0
                else:
                   _UpperCAmelCase  :  List[Any]				       =logits.detach().argmax(dim=1)
                   if patient_result is not None:
                      _UpperCAmelCase  :  str				       =patient_result.detach().argmax(dim=1)
                   if (patient_result is not None) and torch.all(labels.eq(snake_case)):
                      patient_counter += 1
                   else:
                      _UpperCAmelCase  :  str				       =0

                _UpperCAmelCase  :  Any				       =logits
                if patient_counter == self.patience:
                   break
             _UpperCAmelCase  :  Optional[Any]				       =[patient_result]
             self.inference_layers_num += calculated_layer_num
             self.inference_instances_num += 1

          return res






@add_start_docstrings(
    "Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "   ,lowerCAmelCase   ,)
class    __magic_name__				(						lowerCAmelCase							):






       def __init__(     self    ,				snake_case)		->    Dict:




          '''simple docstring'''
          super().__init__(snake_case)
          _UpperCAmelCase  :  Dict				       =config.num_labels

          _UpperCAmelCase  :  List[Any]				       =BertModelWithPabee(snake_case)
          _UpperCAmelCase  :  Optional[int]				       =nn.Dropout(config.hidden_dropout_prob)
          _UpperCAmelCase  :  int				       =nn.ModuleList(
              [nn.Linear(config.hidden_size    ,				self.config.num_labels) for _ in range(config.num_hidden_layers)])

          self.init_weights()






       @add_start_docstrings_to_model_forward(snake_case)
       def      lowerCAmelCase						(     self    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				snake_case=None    ,				)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  List[str]				       =self.bert(
              input_ids=snake_case    ,				attention_mask=snake_case    ,				token_type_ids=snake_case    ,				position_ids=snake_case    ,				head_mask=snake_case    ,				inputs_embeds=snake_case    ,				output_dropout=self.dropout    ,				output_layers=self.classifiers    ,				regression=self.num_labels == 1    ,				)

          _UpperCAmelCase  :  List[str]				       =(logits[-1],)

          if labels is not None:
             _UpperCAmelCase  :  Union[str, Any]				       =None
             _UpperCAmelCase  :  Union[str, Any]				       =0
             for ix, logits_item in enumerate(snake_case):
                if self.num_labels == 1:
                   #  We are doing regression
                   _UpperCAmelCase  :  List[str]				       =MSELoss()
                   _UpperCAmelCase  :  Dict				       =loss_fct(logits_item.view(-1)    ,				labels.view(-1))
                else:
                   _UpperCAmelCase  :  List[str]				       =CrossEntropyLoss()
                   _UpperCAmelCase  :  Union[str, Any]				       =loss_fct(logits_item.view(-1    ,				self.num_labels)    ,				labels.view(-1))
                if total_loss is None:
                   _UpperCAmelCase  :  Dict				       =loss
                else:
                   total_loss += loss * (ix + 1)
                total_weights += ix + 1
             _UpperCAmelCase  :  List[str]				       =(total_loss / total_weights,) + outputs

          return outputs





'''simple docstring'''






import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text





def lowerCamelCase__     (		__lowerCamelCase			:				List[str]				):







   '''simple docstring'''




   _UpperCAmelCase  :  int				       =[]
   for line in lines:
      _UpperCAmelCase  :  Dict				       =re.sub(R'#.*'			,       ''			,       __lowerCamelCase				)  # remove comments
      if line:
         filtered_lines.append(__lowerCamelCase				)
   _UpperCAmelCase  :  str				       ='\n'.join(__lowerCamelCase				)

   # Make a hash from all this code
   _UpperCAmelCase  :  str				       =full_str.encode('utf-8'				)
   return shaaaa(__lowerCamelCase				).hexdigest()


# get importable module names and hash for caching
lowercase				={
    'csv': (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    'json': (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    'pandas': (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    'parquet': (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    'arrow': (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    'text': (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    'imagefolder': (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    'audiofolder': (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
lowercase				={
    '.csv': ('csv', {}),
    '.tsv': ('csv', {'sep': '\t'}),
    '.json': ('json', {}),
    '.jsonl': ('json', {}),
    '.parquet': ('parquet', {}),
    '.arrow': ('arrow', {}),
    '.txt': ('text', {}),
}
_EXTENSION_TO_MODULE.update({ext: ('imagefolder', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('imagefolder', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ('audiofolder', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('audiofolder', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
lowercase				={'imagefolder', 'audiofolder'}

# Used to filter data files based on extensions given a module name
lowercase				={}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
 _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append('.zip')




_MODULE_TO_EXTENSIONS["audiofolder"].append('.zip')

