




import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.albert.modeling_albert import (
    ALBERT_INPUTS_DOCSTRING,
    ALBERT_START_DOCSTRING,
    AlbertModel,
    AlbertPreTrainedModel,
    AlbertTransformer,
)


UpperCamelCase								=     logging.getLogger(__name__)





class  __lowerCamelCase       (    UpperCamelCase__    ):




    """simple docstring"""

    def    a      (       self     :     Union[str, Any]   ,	SCREAMING_SNAKE_CASE__     :     Tuple   ,	SCREAMING_SNAKE_CASE__     :     List[Any]   ,	SCREAMING_SNAKE_CASE__     :     str=None   ,	SCREAMING_SNAKE_CASE__     :     Optional[int]=None   )      ->       Optional[Any]:
         if current_layer == 0:
              lowerCAmelCase__		= self.embedding_hidden_mapping_in(SCREAMING_SNAKE_CASE__   )
         else:
              lowerCAmelCase__		= hidden_states[0]

         lowerCAmelCase__		= int(self.config.num_hidden_layers / self.config.num_hidden_groups   )

         # Index of the hidden group
         lowerCAmelCase__		= int(current_layer / (self.config.num_hidden_layers / self.config.num_hidden_groups)   )

         lowerCAmelCase__		= self.albert_layer_groups[group_idx](
             SCREAMING_SNAKE_CASE__   ,	SCREAMING_SNAKE_CASE__   ,	head_mask[group_idx * layers_per_group : (group_idx + 1) * layers_per_group]   ,	)
         lowerCAmelCase__		= layer_group_output[0]

         return (hidden_states,)






@add_start_docstrings(
    "The bare ALBERT Model transformer with PABEE outputting raw hidden-states without any specific head on top."	,				UpperCamelCase__	,				)
class  __lowerCamelCase       (    UpperCamelCase__    ):




    """simple docstring"""

    def __init__(       self     :     Any   ,	SCREAMING_SNAKE_CASE__     :     List[str]   )      ->       Optional[Any]:
         super().__init__(SCREAMING_SNAKE_CASE__   )

         lowerCAmelCase__		= AlbertTransformerWithPabee(SCREAMING_SNAKE_CASE__   )

         self.init_weights()
         lowerCAmelCase__		= 0
         lowerCAmelCase__		= 0
         lowerCAmelCase__		= 0

         lowerCAmelCase__		= 0


    def    a      (       self     :     Optional[Any]   ,	SCREAMING_SNAKE_CASE__     :     Optional[int]   )      ->       Tuple:
         lowerCAmelCase__		= threshold


    def    a      (       self     :     Any   ,	SCREAMING_SNAKE_CASE__     :     Any   )      ->       Tuple:
         lowerCAmelCase__		= patience


    def    a      (       self     :     Any   )      ->       Any:
         lowerCAmelCase__		= 0
         lowerCAmelCase__		= 0


    def    a      (       self     :     List[str]   )      ->       int:
         lowerCAmelCase__		= self.inference_layers_num / self.inference_instances_num
         lowerCAmelCase__		= (
             f'*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ='
             f' {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***'
         )
         print(SCREAMING_SNAKE_CASE__   )


    @add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE__   )
    def    a      (       self     :     Optional[int]   ,	SCREAMING_SNAKE_CASE__     :     List[Any]=None   ,	SCREAMING_SNAKE_CASE__     :     Union[str, Any]=None   ,	SCREAMING_SNAKE_CASE__     :     Optional[Any]=None   ,	SCREAMING_SNAKE_CASE__     :     Union[str, Any]=None   ,	SCREAMING_SNAKE_CASE__     :     Dict=None   ,	SCREAMING_SNAKE_CASE__     :     Tuple=None   ,	SCREAMING_SNAKE_CASE__     :     int=None   ,	SCREAMING_SNAKE_CASE__     :     str=None   ,	SCREAMING_SNAKE_CASE__     :     Optional[Any]=False   ,	)      ->       List[str]:

         if input_ids is not None and inputs_embeds is not None:
              raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"   )
         elif input_ids is not None:
              lowerCAmelCase__		= input_ids.size()
         elif inputs_embeds is not None:
              lowerCAmelCase__		= inputs_embeds.size()[:-1]
         else:
              raise ValueError("You have to specify either input_ids or inputs_embeds"   )

         lowerCAmelCase__		= input_ids.device if input_ids is not None else inputs_embeds.device

         if attention_mask is None:
              lowerCAmelCase__		= torch.ones(SCREAMING_SNAKE_CASE__   ,	device=SCREAMING_SNAKE_CASE__   )
         if token_type_ids is None:
              lowerCAmelCase__		= torch.zeros(SCREAMING_SNAKE_CASE__   ,	dtype=torch.long   ,	device=SCREAMING_SNAKE_CASE__   )

         lowerCAmelCase__		= attention_mask.unsqueeze(1   ).unsqueeze(2   )
         lowerCAmelCase__		= extended_attention_mask.to(dtype=self.dtype   )  # fp16 compatibility
         lowerCAmelCase__		= (1.0 - extended_attention_mask) * -10_000.0
         lowerCAmelCase__		= self.get_head_mask(SCREAMING_SNAKE_CASE__   ,	self.config.num_hidden_layers   )

         lowerCAmelCase__		= self.embeddings(
             SCREAMING_SNAKE_CASE__   ,	position_ids=SCREAMING_SNAKE_CASE__   ,	token_type_ids=SCREAMING_SNAKE_CASE__   ,	inputs_embeds=SCREAMING_SNAKE_CASE__   )
         lowerCAmelCase__		= embedding_output

         if self.training:
              lowerCAmelCase__		= []
              for i in range(self.config.num_hidden_layers   ):
                   lowerCAmelCase__		= self.encoder.adaptive_forward(
                       SCREAMING_SNAKE_CASE__   ,	current_layer=SCREAMING_SNAKE_CASE__   ,	attention_mask=SCREAMING_SNAKE_CASE__   ,	head_mask=SCREAMING_SNAKE_CASE__   ,	)

                   lowerCAmelCase__		= self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]   )   )
                   lowerCAmelCase__		= output_layers[i](output_dropout(SCREAMING_SNAKE_CASE__   )   )
                   res.append(SCREAMING_SNAKE_CASE__   )
         elif self.patience == 0:  # Use all layers for inference
              lowerCAmelCase__		= self.encoder(SCREAMING_SNAKE_CASE__   ,	SCREAMING_SNAKE_CASE__   ,	head_mask=SCREAMING_SNAKE_CASE__   )
              lowerCAmelCase__		= self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]   )   )
              lowerCAmelCase__		= [output_layers[self.config.num_hidden_layers - 1](SCREAMING_SNAKE_CASE__   )]
         else:
              lowerCAmelCase__		= 0
              lowerCAmelCase__		= None
              lowerCAmelCase__		= 0
              for i in range(self.config.num_hidden_layers   ):
                   calculated_layer_num += 1
                   lowerCAmelCase__		= self.encoder.adaptive_forward(
                       SCREAMING_SNAKE_CASE__   ,	current_layer=SCREAMING_SNAKE_CASE__   ,	attention_mask=SCREAMING_SNAKE_CASE__   ,	head_mask=SCREAMING_SNAKE_CASE__   ,	)

                   lowerCAmelCase__		= self.pooler_activation(self.pooler(encoder_outputs[0][:, 0]   )   )
                   lowerCAmelCase__		= output_layers[i](SCREAMING_SNAKE_CASE__   )
                   if regression:
                        lowerCAmelCase__		= logits.detach()
                        if patient_result is not None:
                             lowerCAmelCase__		= patient_result.detach()
                        if (patient_result is not None) and torch.abs(patient_result - labels   ) < self.regression_threshold:
                             patient_counter += 1
                        else:
                             lowerCAmelCase__		= 0
                   else:
                        lowerCAmelCase__		= logits.detach().argmax(dim=1   )
                        if patient_result is not None:
                             lowerCAmelCase__		= patient_result.detach().argmax(dim=1   )
                        if (patient_result is not None) and torch.all(labels.eq(SCREAMING_SNAKE_CASE__   )   ):
                             patient_counter += 1
                        else:
                             lowerCAmelCase__		= 0

                   lowerCAmelCase__		= logits
                   if patient_counter == self.patience:
                        break
              lowerCAmelCase__		= [patient_result]
              self.inference_layers_num += calculated_layer_num
              self.inference_instances_num += 1

         return res






@add_start_docstrings(
    "Albert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. "	,				UpperCamelCase__	,				)
class  __lowerCamelCase       (    UpperCamelCase__    ):




    """simple docstring"""

    def __init__(       self     :     Union[str, Any]   ,	SCREAMING_SNAKE_CASE__     :     Optional[Any]   )      ->       Tuple:
         super().__init__(SCREAMING_SNAKE_CASE__   )
         lowerCAmelCase__		= config.num_labels

         lowerCAmelCase__		= AlbertModelWithPabee(SCREAMING_SNAKE_CASE__   )
         lowerCAmelCase__		= nn.Dropout(config.classifier_dropout_prob   )
         lowerCAmelCase__		= nn.ModuleList(
             [nn.Linear(config.hidden_size   ,	self.config.num_labels   ) for _ in range(config.num_hidden_layers   )]   )

         self.init_weights()


    @add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE__   )
    def    a      (       self     :     Optional[Any]   ,	SCREAMING_SNAKE_CASE__     :     Union[str, Any]=None   ,	SCREAMING_SNAKE_CASE__     :     Union[str, Any]=None   ,	SCREAMING_SNAKE_CASE__     :     List[Any]=None   ,	SCREAMING_SNAKE_CASE__     :     str=None   ,	SCREAMING_SNAKE_CASE__     :     str=None   ,	SCREAMING_SNAKE_CASE__     :     Tuple=None   ,	SCREAMING_SNAKE_CASE__     :     List[Any]=None   ,	)      ->       Tuple:
         lowerCAmelCase__		= self.albert(
             input_ids=SCREAMING_SNAKE_CASE__   ,	attention_mask=SCREAMING_SNAKE_CASE__   ,	token_type_ids=SCREAMING_SNAKE_CASE__   ,	position_ids=SCREAMING_SNAKE_CASE__   ,	head_mask=SCREAMING_SNAKE_CASE__   ,	inputs_embeds=SCREAMING_SNAKE_CASE__   ,	output_dropout=self.dropout   ,	output_layers=self.classifiers   ,	regression=self.num_labels == 1   ,	)

         lowerCAmelCase__		= (logits[-1],)

         if labels is not None:
              lowerCAmelCase__		= None
              lowerCAmelCase__		= 0
              for ix, logits_item in enumerate(SCREAMING_SNAKE_CASE__   ):
                   if self.num_labels == 1:
                        #  We are doing regression
                        lowerCAmelCase__		= MSELoss()
                        lowerCAmelCase__		= loss_fct(logits_item.view(-1   )   ,	labels.view(-1   )   )
                   else:
                        lowerCAmelCase__		= CrossEntropyLoss()
                        lowerCAmelCase__		= loss_fct(logits_item.view(-1   ,	self.num_labels   )   ,	labels.view(-1   )   )
                   if total_loss is None:
                        lowerCAmelCase__		= loss
                   else:
                        total_loss += loss * (ix + 1)
                   total_weights += ix + 1
              lowerCAmelCase__		= (total_loss / total_weights,) + outputs

         return outputs






from ..utils import (
    OptionalDependencyNotAvailable,
    is_flax_available,
    is_invisible_watermark_available,
    is_k_diffusion_available,
    is_librosa_available,
    is_note_seq_available,
    is_onnx_available,
    is_torch_available,
    is_transformers_available,
)


try:
     if not is_torch_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_pt_objects import *  # noqa F403
else:
     from .consistency_models import ConsistencyModelPipeline
     from .dance_diffusion import DanceDiffusionPipeline
     from .ddim import DDIMPipeline
     from .ddpm import DDPMPipeline
     from .dit import DiTPipeline
     from .latent_diffusion import LDMSuperResolutionPipeline
     from .latent_diffusion_uncond import LDMPipeline
     from .pipeline_utils import AudioPipelineOutput, DiffusionPipeline, ImagePipelineOutput
     from .pndm import PNDMPipeline
     from .repaint import RePaintPipeline
     from .score_sde_ve import ScoreSdeVePipeline
     from .stochastic_karras_ve import KarrasVePipeline

try:
     if not (is_torch_available() and is_librosa_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_torch_and_librosa_objects import *  # noqa F403
else:
     from .audio_diffusion import AudioDiffusionPipeline, Mel

try:
     if not (is_torch_available() and is_transformers_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_torch_and_transformers_objects import *  # noqa F403
else:
     from .alt_diffusion import AltDiffusionImgaImgPipeline, AltDiffusionPipeline
     from .audioldm import AudioLDMPipeline
     from .controlnet import (
         StableDiffusionControlNetImgaImgPipeline,
         StableDiffusionControlNetInpaintPipeline,
         StableDiffusionControlNetPipeline,
     )
     from .deepfloyd_if import (
         IFImgaImgPipeline,
         IFImgaImgSuperResolutionPipeline,
         IFInpaintingPipeline,
         IFInpaintingSuperResolutionPipeline,
         IFPipeline,
         IFSuperResolutionPipeline,
     )
     from .kandinsky import (
         KandinskyImgaImgPipeline,
         KandinskyInpaintPipeline,
         KandinskyPipeline,
         KandinskyPriorPipeline,
     )
     from .kandinskya_a import (
         KandinskyVaaControlnetImgaImgPipeline,
         KandinskyVaaControlnetPipeline,
         KandinskyVaaImgaImgPipeline,
         KandinskyVaaInpaintPipeline,
         KandinskyVaaPipeline,
         KandinskyVaaPriorEmbaEmbPipeline,
         KandinskyVaaPriorPipeline,
     )
     from .latent_diffusion import LDMTextToImagePipeline
     from .paint_by_example import PaintByExamplePipeline
     from .semantic_stable_diffusion import SemanticStableDiffusionPipeline
     from .shap_e import ShapEImgaImgPipeline, ShapEPipeline
     from .stable_diffusion import (
         CycleDiffusionPipeline,
         StableDiffusionAttendAndExcitePipeline,
         StableDiffusionDepthaImgPipeline,
         StableDiffusionDiffEditPipeline,
         StableDiffusionImageVariationPipeline,
         StableDiffusionImgaImgPipeline,
         StableDiffusionInpaintPipeline,
         StableDiffusionInpaintPipelineLegacy,
         StableDiffusionInstructPixaPixPipeline,
         StableDiffusionLatentUpscalePipeline,
         StableDiffusionLDMaDPipeline,
         StableDiffusionModelEditingPipeline,
         StableDiffusionPanoramaPipeline,
         StableDiffusionParadigmsPipeline,
         StableDiffusionPipeline,
         StableDiffusionPixaPixZeroPipeline,
         StableDiffusionSAGPipeline,
         StableDiffusionUpscalePipeline,
         StableUnCLIPImgaImgPipeline,
         StableUnCLIPPipeline,
     )
     from .stable_diffusion_safe import StableDiffusionPipelineSafe
     from .text_to_video_synthesis import TextToVideoSDPipeline, TextToVideoZeroPipeline, VideoToVideoSDPipeline
     from .unclip import UnCLIPImageVariationPipeline, UnCLIPPipeline
     from .unidiffuser import ImageTextPipelineOutput, UniDiffuserModel, UniDiffuserPipeline, UniDiffuserTextDecoder
     from .versatile_diffusion import (
         VersatileDiffusionDualGuidedPipeline,
         VersatileDiffusionImageVariationPipeline,
         VersatileDiffusionPipeline,
         VersatileDiffusionTextToImagePipeline,
     )
     from .vq_diffusion import VQDiffusionPipeline


try:
     if not (is_torch_available() and is_transformers_available() and is_invisible_watermark_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_torch_and_transformers_and_invisible_watermark_objects import *  # noqa F403
else:
     from .stable_diffusion_xl import StableDiffusionXLImgaImgPipeline, StableDiffusionXLPipeline

try:
     if not is_onnx_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_onnx_objects import *  # noqa F403
else:
     from .onnx_utils import OnnxRuntimeModel

try:
     if not (is_torch_available() and is_transformers_available() and is_onnx_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_torch_and_transformers_and_onnx_objects import *  # noqa F403
else:
     from .stable_diffusion import (
         OnnxStableDiffusionImgaImgPipeline,
         OnnxStableDiffusionInpaintPipeline,
         OnnxStableDiffusionInpaintPipelineLegacy,
         OnnxStableDiffusionPipeline,
         OnnxStableDiffusionUpscalePipeline,
         StableDiffusionOnnxPipeline,
     )

try:
     if not (is_torch_available() and is_transformers_available() and is_k_diffusion_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_torch_and_transformers_and_k_diffusion_objects import *  # noqa F403
else:
     from .stable_diffusion import StableDiffusionKDiffusionPipeline

try:
     if not is_flax_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_flax_objects import *  # noqa F403
else:
     from .pipeline_flax_utils import FlaxDiffusionPipeline


try:
     if not (is_flax_available() and is_transformers_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_flax_and_transformers_objects import *  # noqa F403
else:
     from .controlnet import FlaxStableDiffusionControlNetPipeline
     from .stable_diffusion import (
         FlaxStableDiffusionImgaImgPipeline,
         FlaxStableDiffusionInpaintPipeline,
         FlaxStableDiffusionPipeline,
     )
try:
     if not (is_transformers_available() and is_torch_available() and is_note_seq_available()):
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     from ..utils.dummy_transformers_and_torch_and_note_seq_objects import *  # noqa F403
else:
     from .spectrogram_diffusion import MidiProcessor, SpectrogramDiffusionPipeline
