'''simple docstring'''




import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


snake_case	=      logging.getLogger(__name__)





class 							SCREAMING_SNAKE_CASE (					UpperCAmelCase_			):
  """simple docstring"""







  def 	a (	self :       Tuple       ,       __lowerCAmelCase :       Optional[int]       ,       __lowerCAmelCase :       Union[str, Any]       ,       __lowerCAmelCase :       str=None       ,       __lowerCAmelCase :       int=None       ):


        """simple docstring"""



        _lowerCAmelCase		 = self.layer[current_layer](lowerCamelCase__       ,       lowerCamelCase__       ,       head_mask[current_layer]       )

        _lowerCAmelCase		 = layer_outputs[0]

        return hidden_states





@add_start_docstrings(
    "The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top."						,    UpperCAmelCase_						,    )
class 							SCREAMING_SNAKE_CASE (					UpperCAmelCase_			):
  """simple docstring"""







  def __init__(	self :       Tuple       ,       __lowerCAmelCase :       Tuple       ):


        """simple docstring"""



        super().__init__(lowerCamelCase__       )

        _lowerCAmelCase		 = BertEncoderWithPabee(lowerCamelCase__       )

        self.init_weights()
        _lowerCAmelCase		 = 0
        _lowerCAmelCase		 = 0
        _lowerCAmelCase		 = 0

        _lowerCAmelCase		 = 0



  def 	a (	self :       Union[str, Any]       ,       __lowerCAmelCase :       Any       ):


        """simple docstring"""



        _lowerCAmelCase		 = threshold



  def 	a (	self :       Optional[int]       ,       __lowerCAmelCase :       Union[str, Any]       ):


        """simple docstring"""



        _lowerCAmelCase		 = patience



  def 	a (	self :       Optional[int]       ):


        """simple docstring"""



        _lowerCAmelCase		 = 0
        _lowerCAmelCase		 = 0



  def 	a (	self :       int       ):


        """simple docstring"""



        _lowerCAmelCase		 = self.inference_layers_num / self.inference_instances_num
        _lowerCAmelCase		 = (
            F"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
            F" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
        )
        print(lowerCamelCase__       )



  @add_start_docstrings_to_model_forward(lowerCamelCase__       )
  def 	a (	self :       Any       ,       __lowerCAmelCase :       List[Any]=None       ,       __lowerCAmelCase :       List[str]=None       ,       __lowerCAmelCase :       List[Any]=None       ,       __lowerCAmelCase :       Optional[Any]=None       ,       __lowerCAmelCase :       List[str]=None       ,       __lowerCAmelCase :       str=None       ,       __lowerCAmelCase :       List[Any]=None       ,       __lowerCAmelCase :       Any=None       ,       __lowerCAmelCase :       List[str]=None       ,       __lowerCAmelCase :       List[str]=None       ,       __lowerCAmelCase :       str=False       ,       ):


        """simple docstring"""



        if input_ids is not None and inputs_embeds is not None:
              raise ValueError('You cannot specify both input_ids and inputs_embeds at the same time'       )
        elif input_ids is not None:
              _lowerCAmelCase		 = input_ids.size()
        elif inputs_embeds is not None:
              _lowerCAmelCase		 = inputs_embeds.size()[:-1]
        else:
              raise ValueError('You have to specify either input_ids or inputs_embeds'       )

        _lowerCAmelCase		 = input_ids.device if input_ids is not None else inputs_embeds.device

        if attention_mask is None:
              _lowerCAmelCase		 = torch.ones(lowerCamelCase__       ,       device=lowerCamelCase__       )
        if token_type_ids is None:
              _lowerCAmelCase		 = torch.zeros(lowerCamelCase__       ,       dtype=torch.long       ,       device=lowerCamelCase__       )

        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        _lowerCAmelCase		 = self.get_extended_attention_mask(lowerCamelCase__       ,       lowerCamelCase__       ,       lowerCamelCase__       )

        # If a 2D ou 3D attention mask is provided for the cross-attention
        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
        if self.config.is_decoder and encoder_hidden_states is not None:
              _lowerCAmelCase		 = encoder_hidden_states.size()
              _lowerCAmelCase		 = (encoder_batch_size, encoder_sequence_length)
              if encoder_attention_mask is None:
                    _lowerCAmelCase		 = torch.ones(lowerCamelCase__       ,       device=lowerCamelCase__       )
              _lowerCAmelCase		 = self.invert_attention_mask(lowerCamelCase__       )
        else:
              _lowerCAmelCase		 = None

        # Prepare head mask if needed
        # 1.0 in head_mask indicate we keep the head
        # attention_probs has shape bsz x n_heads x N x N
        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
        _lowerCAmelCase		 = self.get_head_mask(lowerCamelCase__       ,       self.config.num_hidden_layers       )

        _lowerCAmelCase		 = self.embeddings(
            input_ids=lowerCamelCase__       ,       position_ids=lowerCamelCase__       ,       token_type_ids=lowerCamelCase__       ,       inputs_embeds=lowerCamelCase__       )
        _lowerCAmelCase		 = embedding_output

        if self.training:
              _lowerCAmelCase		 = []
              for i in range(self.config.num_hidden_layers       ):
                    _lowerCAmelCase		 = self.encoder.adaptive_forward(
                        lowerCamelCase__       ,       current_layer=lowerCamelCase__       ,       attention_mask=lowerCamelCase__       ,       head_mask=lowerCamelCase__       )

                    _lowerCAmelCase		 = self.pooler(lowerCamelCase__       )
                    _lowerCAmelCase		 = output_layers[i](output_dropout(lowerCamelCase__       )       )
                    res.append(lowerCamelCase__       )
        elif self.patience == 0:  # Use all layers for inference
              _lowerCAmelCase		 = self.encoder(
                  lowerCamelCase__       ,       attention_mask=lowerCamelCase__       ,       head_mask=lowerCamelCase__       ,       encoder_hidden_states=lowerCamelCase__       ,       encoder_attention_mask=lowerCamelCase__       ,       )
              _lowerCAmelCase		 = self.pooler(encoder_outputs[0]       )
              _lowerCAmelCase		 = [output_layers[self.config.num_hidden_layers - 1](lowerCamelCase__       )]
        else:
              _lowerCAmelCase		 = 0
              _lowerCAmelCase		 = None
              _lowerCAmelCase		 = 0
              for i in range(self.config.num_hidden_layers       ):
                    calculated_layer_num += 1
                    _lowerCAmelCase		 = self.encoder.adaptive_forward(
                        lowerCamelCase__       ,       current_layer=lowerCamelCase__       ,       attention_mask=lowerCamelCase__       ,       head_mask=lowerCamelCase__       )

                    _lowerCAmelCase		 = self.pooler(lowerCamelCase__       )
                    _lowerCAmelCase		 = output_layers[i](lowerCamelCase__       )
                    if regression:
                          _lowerCAmelCase		 = logits.detach()
                          if patient_result is not None:
                                _lowerCAmelCase		 = patient_result.detach()
                          if (patient_result is not None) and torch.abs(patient_result - labels       ) < self.regression_threshold:
                                patient_counter += 1
                          else:
                                _lowerCAmelCase		 = 0
                    else:
                          _lowerCAmelCase		 = logits.detach().argmax(dim=1       )
                          if patient_result is not None:
                                _lowerCAmelCase		 = patient_result.detach().argmax(dim=1       )
                          if (patient_result is not None) and torch.all(labels.eq(lowerCamelCase__       )       ):
                                patient_counter += 1
                          else:
                                _lowerCAmelCase		 = 0

                    _lowerCAmelCase		 = logits
                    if patient_counter == self.patience:
                          break
              _lowerCAmelCase		 = [patient_result]
              self.inference_layers_num += calculated_layer_num
              self.inference_instances_num += 1

        return res





@add_start_docstrings(
    "Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "						,    UpperCAmelCase_						,    )
class 							SCREAMING_SNAKE_CASE (					UpperCAmelCase_			):
  """simple docstring"""







  def __init__(	self :       Any       ,       __lowerCAmelCase :       Union[str, Any]       ):


        """simple docstring"""



        super().__init__(lowerCamelCase__       )
        _lowerCAmelCase		 = config.num_labels

        _lowerCAmelCase		 = BertModelWithPabee(lowerCamelCase__       )
        _lowerCAmelCase		 = nn.Dropout(config.hidden_dropout_prob       )
        _lowerCAmelCase		 = nn.ModuleList(
            [nn.Linear(config.hidden_size       ,       self.config.num_labels       ) for _ in range(config.num_hidden_layers       )]       )

        self.init_weights()



  @add_start_docstrings_to_model_forward(lowerCamelCase__       )
  def 	a (	self :       List[Any]       ,       __lowerCAmelCase :       Union[str, Any]=None       ,       __lowerCAmelCase :       Dict=None       ,       __lowerCAmelCase :       Union[str, Any]=None       ,       __lowerCAmelCase :       List[Any]=None       ,       __lowerCAmelCase :       Optional[Any]=None       ,       __lowerCAmelCase :       Optional[int]=None       ,       __lowerCAmelCase :       Dict=None       ,       ):


        """simple docstring"""



        _lowerCAmelCase		 = self.bert(
            input_ids=lowerCamelCase__       ,       attention_mask=lowerCamelCase__       ,       token_type_ids=lowerCamelCase__       ,       position_ids=lowerCamelCase__       ,       head_mask=lowerCamelCase__       ,       inputs_embeds=lowerCamelCase__       ,       output_dropout=self.dropout       ,       output_layers=self.classifiers       ,       regression=self.num_labels == 1       ,       )

        _lowerCAmelCase		 = (logits[-1],)

        if labels is not None:
              _lowerCAmelCase		 = None
              _lowerCAmelCase		 = 0
              for ix, logits_item in enumerate(lowerCamelCase__       ):
                    if self.num_labels == 1:
                          #  We are doing regression
                          _lowerCAmelCase		 = MSELoss()
                          _lowerCAmelCase		 = loss_fct(logits_item.view(-1       )       ,       labels.view(-1       )       )
                    else:
                          _lowerCAmelCase		 = CrossEntropyLoss()
                          _lowerCAmelCase		 = loss_fct(logits_item.view(-1       ,       self.num_labels       )       ,       labels.view(-1       )       )
                    if total_loss is None:
                          _lowerCAmelCase		 = loss
                    else:
                          total_loss += loss * (ix + 1)
                    total_weights += ix + 1
              _lowerCAmelCase		 = (total_loss / total_weights,) + outputs

        return outputs













from typing import List, Optional, Union

from ...image_utils import ImageInput
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType







class 		_A						( UpperCAmelCase_ ):
  lowercase_						:			Tuple						=					['''image_processor''', '''tokenizer''']
  lowercase_						:			str						=					'''BlipImageProcessor'''
  lowercase_						:			str						=					('''BertTokenizer''', '''BertTokenizerFast''')

  def __init__(    self						:	List[str]   ,				lowerCamelCase__						:	str   ,				lowerCamelCase__						:	Union[str, Any]			):
     """simple docstring"""

     __UpperCamelCase	:			Optional[int]					=					False
     super().__init__(lowerCamelCase__   ,				lowerCamelCase__			)
     __UpperCamelCase	:			Tuple					=					self.image_processor

  def __call__(    self						:	Optional[Any]   ,				lowerCamelCase__						:	ImageInput = None   ,				lowerCamelCase__						:	Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None   ,				lowerCamelCase__						:	bool = True   ,				lowerCamelCase__						:	Union[bool, str, PaddingStrategy] = False   ,				lowerCamelCase__						:	Union[bool, str, TruncationStrategy] = None   ,				lowerCamelCase__						:	Optional[int] = None   ,				lowerCamelCase__						:	int = 0   ,				lowerCamelCase__						:	Optional[int] = None   ,				lowerCamelCase__						:	Optional[bool] = None   ,				lowerCamelCase__						:	bool = False   ,				lowerCamelCase__						:	bool = False   ,				lowerCamelCase__						:	bool = False   ,				lowerCamelCase__						:	bool = False   ,				lowerCamelCase__						:	bool = False   ,				lowerCamelCase__						:	bool = True   ,				lowerCamelCase__						:	Optional[Union[str, TensorType]] = None   ,				**lowerCamelCase__						:	int   ,				):
     """simple docstring"""

     if images is None and text is None:
        raise ValueError("""You have to specify either images or text."""			)

     # Get only text
     if images is None:
        __UpperCamelCase	:			Optional[int]					=					self.tokenizer
        __UpperCamelCase	:			Dict					=					self.tokenizer(
            text=lowerCamelCase__   ,				add_special_tokens=lowerCamelCase__   ,				padding=lowerCamelCase__   ,				truncation=lowerCamelCase__   ,				max_length=lowerCamelCase__   ,				stride=lowerCamelCase__   ,				pad_to_multiple_of=lowerCamelCase__   ,				return_attention_mask=lowerCamelCase__   ,				return_overflowing_tokens=lowerCamelCase__   ,				return_special_tokens_mask=lowerCamelCase__   ,				return_offsets_mapping=lowerCamelCase__   ,				return_token_type_ids=lowerCamelCase__   ,				return_length=lowerCamelCase__   ,				verbose=lowerCamelCase__   ,				return_tensors=lowerCamelCase__   ,				**lowerCamelCase__   ,				)
        return text_encoding

     # add pixel_values
     __UpperCamelCase	:			str					=					self.image_processor(lowerCamelCase__   ,				return_tensors=lowerCamelCase__			)

     if text is not None:
        __UpperCamelCase	:			Optional[int]					=					self.tokenizer(
            text=lowerCamelCase__   ,				add_special_tokens=lowerCamelCase__   ,				padding=lowerCamelCase__   ,				truncation=lowerCamelCase__   ,				max_length=lowerCamelCase__   ,				stride=lowerCamelCase__   ,				pad_to_multiple_of=lowerCamelCase__   ,				return_attention_mask=lowerCamelCase__   ,				return_overflowing_tokens=lowerCamelCase__   ,				return_special_tokens_mask=lowerCamelCase__   ,				return_offsets_mapping=lowerCamelCase__   ,				return_token_type_ids=lowerCamelCase__   ,				return_length=lowerCamelCase__   ,				verbose=lowerCamelCase__   ,				return_tensors=lowerCamelCase__   ,				**lowerCamelCase__   ,				)
     else:
        __UpperCamelCase	:			Optional[Any]					=					None

     if text_encoding is not None:
        encoding_image_processor.update(lowerCamelCase__			)

     return encoding_image_processor

  def    a					(    self						:	List[str]   ,				*lowerCamelCase__						:	Any   ,				**lowerCamelCase__						:	List[Any]			):
     """simple docstring"""

     return self.tokenizer.batch_decode(*lowerCamelCase__   ,				**lowerCamelCase__			)

  def    a					(    self						:	List[str]   ,				*lowerCamelCase__						:	int   ,				**lowerCamelCase__						:	Tuple			):
     """simple docstring"""

     return self.tokenizer.decode(*lowerCamelCase__   ,				**lowerCamelCase__			)

  @property
  def    a					(    self						:	Any			):
     """simple docstring"""

     __UpperCamelCase	:			str					=					self.tokenizer.model_input_names
     __UpperCamelCase	:			str					=					self.image_processor.model_input_names
     return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names			)			)



