





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


a__: Dict							   =						logging.getLogger(__name__)




class        SCREAMING_SNAKE_CASE__					(		UpperCamelCase__ ):



 def 							UpperCamelCase     (				self,__lowerCamelCase,__lowerCamelCase,__lowerCamelCase=None,__lowerCamelCase=None					):
        A__		   =		self.layer[current_layer](__lowerCamelCase,__lowerCamelCase,head_mask[current_layer]					)

        A__		   =		layer_outputs[0]

        return hidden_states




@add_start_docstrings(
    '''The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''							,   UpperCamelCase__							,   )
class        SCREAMING_SNAKE_CASE__					(		UpperCamelCase__ ):



 def __init__(				self,__lowerCamelCase					):
        super().__init__(__lowerCamelCase					)

        A__		   =		BertEncoderWithPabee(__lowerCamelCase					)

        self.init_weights()
        A__		   =		0
        A__		   =		0
        A__		   =		0

        A__		   =		0



 def 							UpperCamelCase     (				self,__lowerCamelCase					):
        A__		   =		threshold



 def 							UpperCamelCase     (				self,__lowerCamelCase					):
        A__		   =		patience



 def 							UpperCamelCase     (				self					):
        A__		   =		0
        A__		   =		0



 def 							UpperCamelCase     (				self					):
        A__		   =		self.inference_layers_num / self.inference_instances_num
        A__		   =		(
            f"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
            f" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
        )
        print(__lowerCamelCase					)




 @add_start_docstrings_to_model_forward(__lowerCamelCase					)
 def 							UpperCamelCase     (				self,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=False,):

        if input_ids is not None and inputs_embeds is not None:
               raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time'''					)
        elif input_ids is not None:
               A__		   =		input_ids.size()
        elif inputs_embeds is not None:
               A__		   =		inputs_embeds.size()[:-1]
        else:
               raise ValueError('''You have to specify either input_ids or inputs_embeds'''					)

        A__		   =		input_ids.device if input_ids is not None else inputs_embeds.device

        if attention_mask is None:
               A__		   =		torch.ones(__lowerCamelCase,device=__lowerCamelCase					)
        if token_type_ids is None:
               A__		   =		torch.zeros(__lowerCamelCase,dtype=torch.long,device=__lowerCamelCase					)

        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        A__		   =		self.get_extended_attention_mask(__lowerCamelCase,__lowerCamelCase,__lowerCamelCase					)

        # If a 2D ou 3D attention mask is provided for the cross-attention
        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
        if self.config.is_decoder and encoder_hidden_states is not None:
               A__   ,      A__   ,      A__		   =		encoder_hidden_states.size()
               A__		   =		(encoder_batch_size, encoder_sequence_length)
               if encoder_attention_mask is None:
                      A__		   =		torch.ones(__lowerCamelCase,device=__lowerCamelCase					)
               A__		   =		self.invert_attention_mask(__lowerCamelCase					)
        else:
               A__		   =		None

        # Prepare head mask if needed
        # 1.0 in head_mask indicate we keep the head
        # attention_probs has shape bsz x n_heads x N x N
        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
        A__		   =		self.get_head_mask(__lowerCamelCase,self.config.num_hidden_layers					)

        A__		   =		self.embeddings(
            input_ids=__lowerCamelCase,position_ids=__lowerCamelCase,token_type_ids=__lowerCamelCase,inputs_embeds=__lowerCamelCase					)
        A__		   =		embedding_output

        if self.training:
               A__		   =		[]
               for i in range(self.config.num_hidden_layers					):
                      A__		   =		self.encoder.adaptive_forward(
                          __lowerCamelCase,current_layer=__lowerCamelCase,attention_mask=__lowerCamelCase,head_mask=__lowerCamelCase					)

                      A__		   =		self.pooler(__lowerCamelCase					)
                      A__		   =		output_layers[i](output_dropout(__lowerCamelCase					)					)
                      res.append(__lowerCamelCase					)
        elif self.patience == 0:  # Use all layers for inference
               A__		   =		self.encoder(
                   __lowerCamelCase,attention_mask=__lowerCamelCase,head_mask=__lowerCamelCase,encoder_hidden_states=__lowerCamelCase,encoder_attention_mask=__lowerCamelCase,)
               A__		   =		self.pooler(encoder_outputs[0]					)
               A__		   =		[output_layers[self.config.num_hidden_layers - 1](__lowerCamelCase					)]
        else:
               A__		   =		0
               A__		   =		None
               A__		   =		0
               for i in range(self.config.num_hidden_layers					):
                      calculated_layer_num += 1
                      A__		   =		self.encoder.adaptive_forward(
                          __lowerCamelCase,current_layer=__lowerCamelCase,attention_mask=__lowerCamelCase,head_mask=__lowerCamelCase					)

                      A__		   =		self.pooler(__lowerCamelCase					)
                      A__		   =		output_layers[i](__lowerCamelCase					)
                      if regression:
                             A__		   =		logits.detach()
                             if patient_result is not None:
                                    A__		   =		patient_result.detach()
                             if (patient_result is not None) and torch.abs(patient_result - labels					) < self.regression_threshold:
                                    patient_counter += 1
                             else:
                                    A__		   =		0
                      else:
                             A__		   =		logits.detach().argmax(dim=1					)
                             if patient_result is not None:
                                    A__		   =		patient_result.detach().argmax(dim=1					)
                             if (patient_result is not None) and torch.all(labels.eq(__lowerCamelCase					)					):
                                    patient_counter += 1
                             else:
                                    A__		   =		0

                      A__		   =		logits
                      if patient_counter == self.patience:
                             break
               A__		   =		[patient_result]
               self.inference_layers_num += calculated_layer_num
               self.inference_instances_num += 1

        return res




@add_start_docstrings(
    '''Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks. '''							,   UpperCamelCase__							,   )
class        SCREAMING_SNAKE_CASE__					(		UpperCamelCase__ ):



 def __init__(				self,__lowerCamelCase					):
        super().__init__(__lowerCamelCase					)
        A__		   =		config.num_labels

        A__		   =		BertModelWithPabee(__lowerCamelCase					)
        A__		   =		nn.Dropout(config.hidden_dropout_prob					)
        A__		   =		nn.ModuleList(
            [nn.Linear(config.hidden_size,self.config.num_labels					) for _ in range(config.num_hidden_layers					)]					)

        self.init_weights()




 @add_start_docstrings_to_model_forward(__lowerCamelCase					)
 def 							UpperCamelCase     (				self,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=None,):
        A__		   =		self.bert(
            input_ids=__lowerCamelCase,attention_mask=__lowerCamelCase,token_type_ids=__lowerCamelCase,position_ids=__lowerCamelCase,head_mask=__lowerCamelCase,inputs_embeds=__lowerCamelCase,output_dropout=self.dropout,output_layers=self.classifiers,regression=self.num_labels == 1,)

        A__		   =		(logits[-1],)

        if labels is not None:
               A__		   =		None
               A__		   =		0
               for ix, logits_item in enumerate(__lowerCamelCase					):
                      if self.num_labels == 1:
                             #  We are doing regression
                             A__		   =		MSELoss()
                             A__		   =		loss_fct(logits_item.view(-1					),labels.view(-1					)					)
                      else:
                             A__		   =		CrossEntropyLoss()
                             A__		   =		loss_fct(logits_item.view(-1,self.num_labels					),labels.view(-1					)					)
                      if total_loss is None:
                             A__		   =		loss
                      else:
                             total_loss += loss * (ix + 1)
                      total_weights += ix + 1
               A__		   =		(total_loss / total_weights,) + outputs

        return outputs












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a__: List[Any]							   =						{
    'configuration_x_clip': [
        'XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'XCLIPConfig',
        'XCLIPTextConfig',
        'XCLIPVisionConfig',
    ],
    'processing_x_clip': ['XCLIPProcessor'],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a__: Any							   =						[
           'XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST',
           'XCLIPModel',
           'XCLIPPreTrainedModel',
           'XCLIPTextModel',
           'XCLIPVisionModel',
       ]

if TYPE_CHECKING:
       from .configuration_x_clip import (
           XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
           XCLIPConfig,
           XCLIPTextConfig,
           XCLIPVisionConfig,
       )
       from .processing_x_clip import XCLIPProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_x_clip import (
                  XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
                  XCLIPModel,
                  XCLIPPreTrainedModel,
                  XCLIPTextModel,
                  XCLIPVisionModel,
              )

else:
       import sys

       a__: Optional[Any]							   =						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





