




import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


a_      						=   logging.getLogger(__name__)

class        _lowerCamelCase  (	_SCREAMING_SNAKE_CASE      ):

      """simple docstring"""

      def 					snake_case		(   self    :       Optional[int]       ,				snake_case    :       Dict       ,				snake_case    :       Union[str, Any]       ,				snake_case    :       List[str]=None       ,				snake_case    :       List[str]=None ):
          if current_layer == 0:
              __UpperCamelCase     						=  self.embedding_hidden_mapping_in(snake_case )
          else:
              __UpperCamelCase     						=  hidden_states[0]

          __UpperCamelCase     						=  int(self.config.num_hidden_layers / self.config.num_hidden_groups )

          # Index of the hidden group
          __UpperCamelCase     						=  int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups) )

          __UpperCamelCase     						=  self.albert_layer_groups[group_idx](
              snake_case       ,				snake_case       ,				head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]       ,				)
          __UpperCamelCase     						=  layer_group_output[0]

          return (hidden_states,)







@add_start_docstrings(
    "The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top."							,	_SCREAMING_SNAKE_CASE							,	)
class        _lowerCamelCase  (	_SCREAMING_SNAKE_CASE      ):

      """simple docstring"""

      def __init__(   self    :       Any       ,				snake_case    :       List[Any] ):
          super().__init__(snake_case )

          __UpperCamelCase     						=  AlbertTransformerWithPabee(snake_case )

          self.init_weights()
          __UpperCamelCase     						=  0
          __UpperCamelCase     						=  0
          __UpperCamelCase     						=  0

          __UpperCamelCase     						=  0







      def 					snake_case		(   self    :       List[Any]       ,				snake_case    :       int ):
          __UpperCamelCase     						=  threshold







      def 					snake_case		(   self    :       Any       ,				snake_case    :       str ):
          __UpperCamelCase     						=  patience







      def 					snake_case		(   self    :       Dict ):
          __UpperCamelCase     						=  0
          __UpperCamelCase     						=  0







      def 					snake_case		(   self    :       Any ):
          __UpperCamelCase     						=  self.inference_layers_num / self.inference_instances_num
          __UpperCamelCase     						=  (
              F"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
              F" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
          )
          print(snake_case )







      @add_start_docstrings_to_model_forward(snake_case )
      def 					snake_case		(   self    :       Tuple       ,				snake_case    :       List[str]=None       ,				snake_case    :       Dict=None       ,				snake_case    :       Optional[Any]=None       ,				snake_case    :       List[str]=None       ,				snake_case    :       Optional[int]=None       ,				snake_case    :       str=None       ,				snake_case    :       Any=None       ,				snake_case    :       Optional[Any]=None       ,				snake_case    :       List[Any]=False       ,				):

          if input_ids is not None and inputs_embeds is not None:
              raise ValueError('''You cannot specify both input_ids and inputs_embeds at the same time''' )
          elif input_ids is not None:
              __UpperCamelCase     						=  input_ids.size()
          elif inputs_embeds is not None:
              __UpperCamelCase     						=  inputs_embeds.size()[:-1]
          else:
              raise ValueError('''You have to specify either input_ids or inputs_embeds''' )

          __UpperCamelCase     						=  input_ids.device if input_ids is not None else inputs_embeds.device

          if attention_mask is None:
              __UpperCamelCase     						=  torch.ones(snake_case       ,				device=snake_case )
          if token_type_ids is None:
              __UpperCamelCase     						=  torch.zeros(snake_case       ,				dtype=torch.long       ,				device=snake_case )

          __UpperCamelCase     						=  attention_mask.unsqueeze(1 ).unsqueeze(2 )
          __UpperCamelCase     						=  extended_attention_mask.to(dtype=self.dtype )  # fp16 compatibility
          __UpperCamelCase     						=  (1.0 - extended_attention_mask) * -1_00_00.0
          __UpperCamelCase     						=  self.get_head_mask(snake_case       ,				self.config.num_hidden_layers )

          __UpperCamelCase     						=  self.embeddings(
              snake_case       ,				position_ids=snake_case       ,				token_type_ids=snake_case       ,				inputs_embeds=snake_case )
          __UpperCamelCase     						=  embedding_output

          if self.training:
              __UpperCamelCase     						=  []
              for i in range(self.config.num_hidden_layers ):
                  __UpperCamelCase     						=  self.encoder.adaptive_forward(
                      snake_case       ,				current_layer=snake_case       ,				attention_mask=snake_case       ,				head_mask=snake_case       ,				)

                  __UpperCamelCase     						=  self.pooler_activation(self.pooler(encoder_outputs[0][:, 0] ) )
                  __UpperCamelCase     						=  output_layers[i](output_dropout(snake_case ) )
                  res.append(snake_case )
          elif self.patience == 0:  # Use all layers for inference
              __UpperCamelCase     						=  self.encoder(snake_case       ,				snake_case       ,				head_mask=snake_case )
              __UpperCamelCase     						=  self.pooler_activation(self.pooler(encoder_outputs[0][:, 0] ) )
              __UpperCamelCase     						=  [output_layers[self.config.num_hidden_layers - 1](snake_case )]
          else:
              __UpperCamelCase     						=  0
              __UpperCamelCase     						=  None
              __UpperCamelCase     						=  0
              for i in range(self.config.num_hidden_layers ):
                  calculated_layer_num += 1
                  __UpperCamelCase     						=  self.encoder.adaptive_forward(
                      snake_case       ,				current_layer=snake_case       ,				attention_mask=snake_case       ,				head_mask=snake_case       ,				)

                  __UpperCamelCase     						=  self.pooler_activation(self.pooler(encoder_outputs[0][:, 0] ) )
                  __UpperCamelCase     						=  output_layers[i](snake_case )
                  if regression:
                      __UpperCamelCase     						=  logits.detach()
                      if patient_result is not None:
                          __UpperCamelCase     						=  patient_result.detach()
                      if (patient_result is not None) and torch.abs(patient_result - labels ) < self.regression_threshold:
                          patient_counter += 1
                      else:
                          __UpperCamelCase     						=  0
                  else:
                      __UpperCamelCase     						=  logits.detach().argmax(dim=1 )
                      if patient_result is not None:
                          __UpperCamelCase     						=  patient_result.detach().argmax(dim=1 )
                      if (patient_result is not None) and torch.all(labels.eq(snake_case ) ):
                          patient_counter += 1
                      else:
                          __UpperCamelCase     						=  0

                  __UpperCamelCase     						=  logits
                  if patient_counter == self.patience:
                      break
              __UpperCamelCase     						=  [patient_result]
              self.inference_layers_num += calculated_layer_num
              self.inference_instances_num += 1

          return res







@add_start_docstrings(
    "Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "							,	_SCREAMING_SNAKE_CASE							,	)
class        _lowerCamelCase  (	_SCREAMING_SNAKE_CASE      ):

      """simple docstring"""

      def __init__(   self    :       Optional[int]       ,				snake_case    :       int ):
          super().__init__(snake_case )
          __UpperCamelCase     						=  config.num_labels

          __UpperCamelCase     						=  AlbertModelWithPabee(snake_case )
          __UpperCamelCase     						=  nn.Dropout(config.classifier_dropout_prob )
          __UpperCamelCase     						=  nn.ModuleList(
              [nn.Linear(config.hidden_size       ,				self.config.num_labels ) for _ in range(config.num_hidden_layers )] )

          self.init_weights()







      @add_start_docstrings_to_model_forward(snake_case )
      def 					snake_case		(   self    :       int       ,				snake_case    :       Optional[int]=None       ,				snake_case    :       Any=None       ,				snake_case    :       Tuple=None       ,				snake_case    :       List[str]=None       ,				snake_case    :       Optional[int]=None       ,				snake_case    :       str=None       ,				snake_case    :       List[Any]=None       ,				):
          __UpperCamelCase     						=  self.albert(
              input_ids=snake_case       ,				attention_mask=snake_case       ,				token_type_ids=snake_case       ,				position_ids=snake_case       ,				head_mask=snake_case       ,				inputs_embeds=snake_case       ,				output_dropout=self.dropout       ,				output_layers=self.classifiers       ,				regression=self.num_labels == 1       ,				)

          __UpperCamelCase     						=  (logits[-1],)

          if labels is not None:
              __UpperCamelCase     						=  None
              __UpperCamelCase     						=  0
              for ix, logits_item in enumerate(snake_case ):
                  if self.num_labels == 1:
                      #  We are doing regression
                      __UpperCamelCase     						=  MSELoss()
                      __UpperCamelCase     						=  loss_fct(logits_item.view(-1 )       ,				labels.view(-1 ) )
                  else:
                      __UpperCamelCase     						=  CrossEntropyLoss()
                      __UpperCamelCase     						=  loss_fct(logits_item.view(-1       ,				self.num_labels )       ,				labels.view(-1 ) )
                  if total_loss is None:
                      __UpperCamelCase     						=  loss
                  else:
                      total_loss += loss * (ix + 1)
                  total_weights += ix + 1
              __UpperCamelCase     						=  (total_loss / total_weights,) + outputs

          return outputs












from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import center_crop, normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_DEFAULT_MEAN,
    IMAGENET_DEFAULT_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
 import PIL


a_      						=   logging.get_logger(__name__)

class        _lowerCamelCase  (	_SCREAMING_SNAKE_CASE      ):

      """simple docstring"""

      lowerCAmelCase__	:       Union[str, Any]			=    ["pixel_values"]







      def __init__(   self    :       Union[str, Any]       ,				snake_case    :       bool = True       ,				snake_case    :       Dict[str, int] = None       ,				snake_case    :       bool = True       ,				snake_case    :       Dict[str, int] = None       ,				snake_case    :       PILImageResampling = PILImageResampling.BICUBIC       ,				snake_case    :       bool = True       ,				snake_case    :       Union[int, float] = 1 / 255       ,				snake_case    :       bool = True       ,				snake_case    :       Optional[Union[float, List[float]]] = None       ,				snake_case    :       Optional[Union[float, List[float]]] = None       ,				**snake_case    :       Union[str, Any]       ,				):
          super().__init__(**snake_case )
          __UpperCamelCase     						=  crop_size if crop_size is not None else {'''height''': 256, '''width''': 256}
          __UpperCamelCase     						=  get_size_dict(snake_case       ,				param_name='''crop_size''' )
          __UpperCamelCase     						=  size if size is not None else {'''height''': 224, '''width''': 224}
          __UpperCamelCase     						=  get_size_dict(snake_case )

          __UpperCamelCase     						=  do_center_crop
          __UpperCamelCase     						=  crop_size
          __UpperCamelCase     						=  do_resize
          __UpperCamelCase     						=  size
          __UpperCamelCase     						=  resample
          __UpperCamelCase     						=  do_rescale
          __UpperCamelCase     						=  rescale_factor
          __UpperCamelCase     						=  do_normalize
          __UpperCamelCase     						=  image_mean if image_mean is not None else IMAGENET_DEFAULT_MEAN
          __UpperCamelCase     						=  image_std if image_std is not None else IMAGENET_DEFAULT_STD







      def 					snake_case		(   self    :       Optional[Any]       ,				snake_case    :       np.ndarray       ,				snake_case    :       Dict[str, int]       ,				snake_case    :       Optional[int] = None       ,				snake_case    :       Optional[Union[str, ChannelDimension]] = None       ,				**snake_case    :       List[Any]       ,				):
          __UpperCamelCase     						=  self.size if size is None else size
          __UpperCamelCase     						=  get_size_dict(snake_case )
          __UpperCamelCase     						=  get_size_dict(snake_case       ,				param_name='''crop_size''' )

          __UpperCamelCase						,					__UpperCamelCase     						=  get_image_size(snake_case )
          __UpperCamelCase     						=  min(snake_case       ,				snake_case )
          __UpperCamelCase     						=  (size['''height'''] / crop_size['''height''']) * min_dim
          __UpperCamelCase     						=  (size['''width'''] / crop_size['''width''']) * min_dim
          return center_crop(snake_case       ,				size=(cropped_height, cropped_width)       ,				data_format=snake_case       ,				**snake_case )







      def 					snake_case		(   self    :       Union[str, Any]       ,				snake_case    :       np.ndarray       ,				snake_case    :       Dict[str, int]       ,				snake_case    :       PILImageResampling = PIL.Image.BICUBIC       ,				snake_case    :       Optional[Union[str, ChannelDimension]] = None       ,				**snake_case    :       List[str]       ,				):
          __UpperCamelCase     						=  get_size_dict(snake_case )
          if "height" not in size or "width" not in size:
              raise ValueError(F"The size dictionary must contain the keys 'height' and 'width'. Got {size.keys()}" )
          return resize(
              snake_case       ,				size=(size['''height'''], size['''width'''])       ,				resample=snake_case       ,				data_format=snake_case       ,				**snake_case )







      def 					snake_case		(   self    :       Union[str, Any]       ,				snake_case    :       np.ndarray       ,				snake_case    :       Union[int, float]       ,				snake_case    :       Optional[Union[str, ChannelDimension]] = None       ,				**snake_case    :       Optional[int]       ,				):
          return rescale(snake_case       ,				scale=snake_case       ,				data_format=snake_case       ,				**snake_case )







      def 					snake_case		(   self    :       Optional[int]       ,				snake_case    :       np.ndarray       ,				snake_case    :       Union[float, List[float]]       ,				snake_case    :       Union[float, List[float]]       ,				snake_case    :       Optional[Union[str, ChannelDimension]] = None       ,				**snake_case    :       Any       ,				):
          return normalize(snake_case       ,				mean=snake_case       ,				std=snake_case       ,				data_format=snake_case       ,				**snake_case )







      def 					snake_case		(   self    :       List[Any]       ,				snake_case    :       ImageInput       ,				snake_case    :       Optional[bool] = None       ,				snake_case    :       Optional[Dict[str, int]] = None       ,				snake_case    :       Optional[bool] = None       ,				snake_case    :       Optional[Dict[str, int]] = None       ,				snake_case    :       PILImageResampling = None       ,				snake_case    :       Optional[bool] = None       ,				snake_case    :       Optional[float] = None       ,				snake_case    :       Optional[bool] = None       ,				snake_case    :       Optional[Union[float, List[float]]] = None       ,				snake_case    :       Optional[Union[float, List[float]]] = None       ,				snake_case    :       Optional[Union[str, TensorType]] = None       ,				snake_case    :       ChannelDimension = ChannelDimension.FIRST       ,				**snake_case    :       Tuple       ,				):
          __UpperCamelCase     						=  do_center_crop if do_center_crop is not None else self.do_center_crop
          __UpperCamelCase     						=  crop_size if crop_size is not None else self.crop_size
          __UpperCamelCase     						=  get_size_dict(snake_case       ,				param_name='''crop_size''' )
          __UpperCamelCase     						=  do_resize if do_resize is not None else self.do_resize
          __UpperCamelCase     						=  size if size is not None else self.size
          __UpperCamelCase     						=  get_size_dict(snake_case )
          __UpperCamelCase     						=  resample if resample is not None else self.resample
          __UpperCamelCase     						=  do_rescale if do_rescale is not None else self.do_rescale
          __UpperCamelCase     						=  rescale_factor if rescale_factor is not None else self.rescale_factor
          __UpperCamelCase     						=  do_normalize if do_normalize is not None else self.do_normalize
          __UpperCamelCase     						=  image_mean if image_mean is not None else self.image_mean
          __UpperCamelCase     						=  image_std if image_std is not None else self.image_std

          __UpperCamelCase     						=  make_list_of_images(snake_case )

          if not valid_images(snake_case ):
              raise ValueError(
                  '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
                  '''torch.Tensor, tf.Tensor or jax.ndarray.''' )

          if do_center_crop and crop_size is None:
              raise ValueError('''If `do_center_crop` is set to `True`, `crop_size` must be provided.''' )

          if do_resize and size is None:
              raise ValueError('''Size must be specified if do_resize is True.''' )

          if do_rescale and rescale_factor is None:
              raise ValueError('''Rescale factor must be specified if do_rescale is True.''' )

          if do_normalize and (image_mean is None or image_std is None):
              raise ValueError('''Image mean and image standard deviation must be specified if do_normalize is True.''' )

          # All transformations expect numpy arrays.
          __UpperCamelCase     						=  [to_numpy_array(snake_case ) for image in images]

          if do_center_crop:
              __UpperCamelCase     						=  [self.center_crop(snake_case       ,				snake_case       ,				size=snake_case ) for image in images]

          if do_resize:
              __UpperCamelCase     						=  [self.resize(image=snake_case       ,				size=snake_case       ,				resample=snake_case ) for image in images]

          if do_rescale:
              __UpperCamelCase     						=  [self.rescale(image=snake_case       ,				scale=snake_case ) for image in images]

          if do_normalize:
              __UpperCamelCase     						=  [self.normalize(image=snake_case       ,				mean=snake_case       ,				std=snake_case ) for image in images]

          __UpperCamelCase     						=  [to_channel_dimension_format(snake_case       ,				snake_case ) for image in images]

          __UpperCamelCase     						=  {'''pixel_values''': images}
          return BatchFeature(data=snake_case       ,				tensor_type=snake_case )






