





import logging

import torch
from torch import nn
from torch.nn import CrossEntropyLoss, MSELoss

from transformers.file_utils import add_start_docstrings, add_start_docstrings_to_model_forward
from transformers.models.bert.modeling_bert import (
    BERT_INPUTS_DOCSTRING,
    BERT_START_DOCSTRING,
    BertEncoder,
    BertModel,
    BertPreTrainedModel,
)


_lowerCAmelCase				:   Optional[int]						       =						logging.getLogger(__name__)


class        __magic_name__	(				A_				):





     """simple docstring"""



     def     SCREAMING_SNAKE_CASE		(				self							:int      ,   snake_case							:Tuple      ,   snake_case							:List[Any]      ,   snake_case							:Tuple=None      ,   snake_case							:Any=None	):





        '''simple docstring'''




        A_       :					int        =      self.layer[current_layer](snake_case__      ,   snake_case__      ,   head_mask[current_layer]	)

        A_       :					Dict        =      layer_outputs[0]

        return hidden_states




@add_start_docstrings(
    '''The bare Bert Model transformer with PABEE outputting raw hidden-states without any specific head on top.'''     ,					A_     ,					)
class        __magic_name__	(				A_				):





     """simple docstring"""



     def __init__(				self							:int      ,   snake_case							:List[Any]	):





        '''simple docstring'''




        super().__init__(snake_case__	)

        A_       :					List[Any]        =      BertEncoderWithPabee(snake_case__	)

        self.init_weights()
        A_       :					Optional[int]        =      0
        A_       :					str        =      0
        A_       :					str        =      0

        A_       :					Optional[int]        =      0



     def     SCREAMING_SNAKE_CASE		(				self							:Optional[Any]      ,   snake_case							:Dict	):





        '''simple docstring'''




        A_       :					int        =      threshold



     def     SCREAMING_SNAKE_CASE		(				self							:List[Any]      ,   snake_case							:Optional[int]	):





        '''simple docstring'''




        A_       :					Union[str, Any]        =      patience



     def     SCREAMING_SNAKE_CASE		(				self							:str	):





        '''simple docstring'''




        A_       :					List[str]        =      0
        A_       :					str        =      0



     def     SCREAMING_SNAKE_CASE		(				self							:Optional[Any]	):





        '''simple docstring'''




        A_       :					Dict        =      self.inference_layers_num / self.inference_instances_num
        A_       :					Union[str, Any]        =      (
            f"*** Patience = {self.patience} Avg. Inference Layers = {avg_inf_layers:.2f} Speed Up ="
            f" {1 - avg_inf_layers / self.config.num_hidden_layers:.2f} ***"
        )
        print(snake_case__	)





     @add_start_docstrings_to_model_forward(snake_case__	)
     def     SCREAMING_SNAKE_CASE		(				self							:Tuple      ,   snake_case							:str=None      ,   snake_case							:List[str]=None      ,   snake_case							:List[str]=None      ,   snake_case							:Union[str, Any]=None      ,   snake_case							:List[Any]=None      ,   snake_case							:str=None      ,   snake_case							:Union[str, Any]=None      ,   snake_case							:int=None      ,   snake_case							:List[Any]=None      ,   snake_case							:Dict=None      ,   snake_case							:Tuple=False      ,   ):





        '''simple docstring'''




        if input_ids is not None and inputs_embeds is not None:
           raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time"	)
        elif input_ids is not None:
           A_       :					List[str]        =      input_ids.size()
        elif inputs_embeds is not None:
           A_       :					Union[str, Any]        =      inputs_embeds.size()[:-1]
        else:
           raise ValueError("You have to specify either input_ids or inputs_embeds"	)

        A_       :					Union[str, Any]        =      input_ids.device if input_ids is not None else inputs_embeds.device

        if attention_mask is None:
           A_       :					Optional[Any]        =      torch.ones(snake_case__      ,   device=snake_case__	)
        if token_type_ids is None:
           A_       :					int        =      torch.zeros(snake_case__      ,   dtype=torch.long      ,   device=snake_case__	)

        # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length]
        # ourselves in which case we just need to make it broadcastable to all heads.
        A_       :					torch.Tensor        =      self.get_extended_attention_mask(snake_case__      ,   snake_case__      ,   snake_case__	)

        # If a 2D ou 3D attention mask is provided for the cross-attention
        # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length]
        if self.config.is_decoder and encoder_hidden_states is not None:
           A_       :					Dict        =      encoder_hidden_states.size()
           A_       :					Tuple        =      (encoder_batch_size, encoder_sequence_length)
           if encoder_attention_mask is None:
              A_       :					int        =      torch.ones(snake_case__      ,   device=snake_case__	)
           A_       :					List[str]        =      self.invert_attention_mask(snake_case__	)
        else:
           A_       :					str        =      None

        # Prepare head mask if needed
        # 1.0 in head_mask indicate we keep the head
        # attention_probs has shape bsz x n_heads x N x N
        # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads]
        # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length]
        A_       :					str        =      self.get_head_mask(snake_case__      ,   self.config.num_hidden_layers	)

        A_       :					List[str]        =      self.embeddings(
            input_ids=snake_case__      ,   position_ids=snake_case__      ,   token_type_ids=snake_case__      ,   inputs_embeds=snake_case__	)
        A_       :					int        =      embedding_output

        if self.training:
           A_       :					Tuple        =      []
           for i in range(self.config.num_hidden_layers	):
              A_       :					int        =      self.encoder.adaptive_forward(
                  snake_case__      ,   current_layer=snake_case__      ,   attention_mask=snake_case__      ,   head_mask=snake_case__	)

              A_       :					Optional[int]        =      self.pooler(snake_case__	)
              A_       :					Optional[int]        =      output_layers[i](output_dropout(snake_case__	)	)
              res.append(snake_case__	)
        elif self.patience == 0:  # Use all layers for inference
           A_       :					Dict        =      self.encoder(
               snake_case__      ,   attention_mask=snake_case__      ,   head_mask=snake_case__      ,   encoder_hidden_states=snake_case__      ,   encoder_attention_mask=snake_case__      ,   )
           A_       :					Optional[int]        =      self.pooler(encoder_outputs[0]	)
           A_       :					List[Any]        =      [output_layers[self.config.num_hidden_layers - 1](snake_case__	)]
        else:
           A_       :					List[str]        =      0
           A_       :					int        =      None
           A_       :					Any        =      0
           for i in range(self.config.num_hidden_layers	):
              calculated_layer_num += 1
              A_       :					List[str]        =      self.encoder.adaptive_forward(
                  snake_case__      ,   current_layer=snake_case__      ,   attention_mask=snake_case__      ,   head_mask=snake_case__	)

              A_       :					Any        =      self.pooler(snake_case__	)
              A_       :					List[str]        =      output_layers[i](snake_case__	)
              if regression:
                 A_       :					Tuple        =      logits.detach()
                 if patient_result is not None:
                    A_       :					Optional[int]        =      patient_result.detach()
                 if (patient_result is not None) and torch.abs(patient_result - labels	) < self.regression_threshold:
                    patient_counter += 1
                 else:
                    A_       :					Optional[Any]        =      0
              else:
                 A_       :					List[Any]        =      logits.detach().argmax(dim=1	)
                 if patient_result is not None:
                    A_       :					List[Any]        =      patient_result.detach().argmax(dim=1	)
                 if (patient_result is not None) and torch.all(labels.eq(snake_case__	)	):
                    patient_counter += 1
                 else:
                    A_       :					Any        =      0

              A_       :					int        =      logits
              if patient_counter == self.patience:
                 break
           A_       :					Optional[int]        =      [patient_result]
           self.inference_layers_num += calculated_layer_num
           self.inference_instances_num += 1

        return res




@add_start_docstrings(
    '''Bert Model transformer with PABEE and a sequence classification/regression head on top (a linear layer on top of\n    the pooled output) e.g. for GLUE tasks. '''     ,					A_     ,					)
class        __magic_name__	(				A_				):





     """simple docstring"""



     def __init__(				self							:Optional[int]      ,   snake_case							:Dict	):





        '''simple docstring'''




        super().__init__(snake_case__	)
        A_       :					Any        =      config.num_labels

        A_       :					Dict        =      BertModelWithPabee(snake_case__	)
        A_       :					Dict        =      nn.Dropout(config.hidden_dropout_prob	)
        A_       :					Union[str, Any]        =      nn.ModuleList(
            [nn.Linear(config.hidden_size      ,   self.config.num_labels	) for _ in range(config.num_hidden_layers	)]	)

        self.init_weights()





     @add_start_docstrings_to_model_forward(snake_case__	)
     def     SCREAMING_SNAKE_CASE		(				self							:Optional[Any]      ,   snake_case							:List[Any]=None      ,   snake_case							:Optional[Any]=None      ,   snake_case							:Optional[int]=None      ,   snake_case							:Optional[int]=None      ,   snake_case							:Dict=None      ,   snake_case							:Dict=None      ,   snake_case							:Optional[int]=None      ,   ):





        '''simple docstring'''




        A_       :					Optional[int]        =      self.bert(
            input_ids=snake_case__      ,   attention_mask=snake_case__      ,   token_type_ids=snake_case__      ,   position_ids=snake_case__      ,   head_mask=snake_case__      ,   inputs_embeds=snake_case__      ,   output_dropout=self.dropout      ,   output_layers=self.classifiers      ,   regression=self.num_labels == 1      ,   )

        A_       :					Any        =      (logits[-1],)

        if labels is not None:
           A_       :					List[Any]        =      None
           A_       :					List[Any]        =      0
           for ix, logits_item in enumerate(snake_case__	):
              if self.num_labels == 1:
                 #  We are doing regression
                 A_       :					Optional[Any]        =      MSELoss()
                 A_       :					Tuple        =      loss_fct(logits_item.view(-1	)      ,   labels.view(-1	)	)
              else:
                 A_       :					Optional[Any]        =      CrossEntropyLoss()
                 A_       :					str        =      loss_fct(logits_item.view(-1      ,   self.num_labels	)      ,   labels.view(-1	)	)
              if total_loss is None:
                 A_       :					Tuple        =      loss
              else:
                 total_loss += loss * (ix + 1)
              total_weights += ix + 1
           A_       :					Any        =      (total_loss / total_weights,) + outputs

        return outputs


from typing import List, Optional, Union

import torch

from ...models import UNetaDConditionModel, VQModel
from ...pipelines import DiffusionPipeline
from ...pipelines.pipeline_utils import ImagePipelineOutput
from ...schedulers import DDPMScheduler
from ...utils import (
    is_accelerate_available,
    is_accelerate_version,
    logging,
    randn_tensor,
    replace_example_docstring,
)


__lowerCamelCase				=			logging.get_logger(__name__)  # pylint: disable=invalid-name

__lowerCamelCase				=			"""
    Examples:
        ```py
        >>> from diffusers import KandinskyV22Pipeline, KandinskyV22PriorPipeline
        >>> import torch

        >>> pipe_prior = KandinskyV22PriorPipeline.from_pretrained(\"kandinsky-community/kandinsky-2-2-prior\")
        >>> pipe_prior.to(\"cuda\")
        >>> prompt = \"red cat, 4k photo\"
        >>> out = pipe_prior(prompt)
        >>> image_emb = out.image_embeds
        >>> zero_image_emb = out.negative_image_embeds
        >>> pipe = KandinskyV22Pipeline.from_pretrained(\"kandinsky-community/kandinsky-2-2-decoder\")
        >>> pipe.to(\"cuda\")
        >>> image = pipe(
        ...     image_embeds=image_emb,
        ...     negative_image_embeds=zero_image_emb,
        ...     height=768,
        ...     width=768,
        ...     num_inference_steps=50,
        ... ).images
        >>> image[0].save(\"cat.png\")
        ```
"""


def      UpperCamelCase   (       __lowerCamelCase	:					Union[str, Any]							,   __lowerCamelCase	:					Dict							,   __lowerCamelCase	:					int=8       ):
       snake_case			:  Optional[int]     							= height // scale_factor**2
       if height % scale_factor**2 != 0:
              new_height += 1
       snake_case			:  Union[str, Any]     							= width // scale_factor**2
       if width % scale_factor**2 != 0:
              new_width += 1
       return new_height * scale_factor, new_width * scale_factor






class     UpperCAmelCase						(		A_					):



   def __init__(self						:   int       ,		snake_case__						:   UNetaDConditionModel       ,		snake_case__						:   DDPMScheduler       ,		snake_case__						:   VQModel       ,		)						->      Tuple:


          '''simple docstring'''


          super().__init__()

          self.register_modules(
              unet=snake_case__       ,		scheduler=snake_case__       ,		movq=snake_case__       ,		)
          snake_case			:  List[Any]     							= 2 ** (len(self.movq.config.block_out_channels					) - 1)



   def 							_SCREAMING_SNAKE_CASE  (self						:   Optional[Any]       ,		snake_case__						:   Optional[Any]       ,		snake_case__						:   Optional[Any]       ,		snake_case__						:   Optional[Any]       ,		snake_case__						:   int       ,		snake_case__						:   List[Any]       ,		snake_case__						:   List[str]					)						->      Optional[Any]:


          '''simple docstring'''


          if latents is None:
                 snake_case			:  int     							= randn_tensor(snake_case__       ,		generator=snake_case__       ,		device=snake_case__       ,		dtype=snake_case__					)
          else:
                 if latents.shape != shape:
                        raise ValueError(f"""Unexpected latents shape, got {latents.shape}, expected {shape}"""					)
                 snake_case			:  Tuple     							= latents.to(snake_case__					)

          snake_case			:  int     							= latents * scheduler.init_noise_sigma
          return latents



   def 							_SCREAMING_SNAKE_CASE  (self						:   str       ,		snake_case__						:   Optional[Any]=0					)						->      Tuple:


          '''simple docstring'''


          if is_accelerate_available():
                 from accelerate import cpu_offload
          else:
                 raise ImportError("Please install accelerate via `pip install accelerate`"					)

          snake_case			:  Optional[int]     							= torch.device(f"""cuda:{gpu_id}"""					)

          snake_case			:  List[str]     							= [
              self.unet,
              self.movq,
          ]
          for cpu_offloaded_model in models:
                 if cpu_offloaded_model is not None:
                        cpu_offload(snake_case__       ,		snake_case__					)



   def 							_SCREAMING_SNAKE_CASE  (self						:   Tuple       ,		snake_case__						:   Union[str, Any]=0					)						->      Union[str, Any]:


          '''simple docstring'''


          if is_accelerate_available() and is_accelerate_version(">="       ,		"0.17.0.dev0"					):
                 from accelerate import cpu_offload_with_hook
          else:
                 raise ImportError("`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher."					)

          snake_case			:  Union[str, Any]     							= torch.device(f"""cuda:{gpu_id}"""					)

          if self.device.type != "cpu":
                 self.to("cpu"       ,		silence_dtype_warnings=snake_case__					)
                 torch.cuda.empty_cache()  # otherwise we don't see the memory savings (but they probably exist)

          snake_case			:  Tuple     							= None
          for cpu_offloaded_model in [self.unet, self.movq]:
                 snake_case							,   snake_case			:  int     							= cpu_offload_with_hook(snake_case__       ,		snake_case__       ,		prev_module_hook=snake_case__					)

          # We'll offload the last model manually.
          snake_case			:  Tuple     							= hook



   @property
   # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._execution_device
   def 							_SCREAMING_SNAKE_CASE  (self						:   List[str]					)						->      str:


          '''simple docstring'''


          if not hasattr(self.unet       ,		"_hf_hook"					):
                 return self.device
          for module in self.unet.modules():
                 if (
                     hasattr(snake_case__       ,		"_hf_hook"					)
                     and hasattr(module._hf_hook       ,		"execution_device"					)
                     and module._hf_hook.execution_device is not None
                 ):
                        return torch.device(module._hf_hook.execution_device					)
          return self.device



   @torch.no_grad()
   @replace_example_docstring(snake_case__					)
   def __call__(self						:   int       ,		snake_case__						:   Union[torch.FloatTensor, List[torch.FloatTensor]]       ,		snake_case__						:   Union[torch.FloatTensor, List[torch.FloatTensor]]       ,		snake_case__						:   int = 5_12       ,		snake_case__						:   int = 5_12       ,		snake_case__						:   int = 1_00       ,		snake_case__						:   float = 4.0       ,		snake_case__						:   int = 1       ,		snake_case__						:   Optional[Union[torch.Generator, List[torch.Generator]]] = None       ,		snake_case__						:   Optional[torch.FloatTensor] = None       ,		snake_case__						:   Optional[str] = "pil"       ,		snake_case__						:   bool = True       ,		)						->      Dict:


          '''simple docstring'''


          snake_case			:  Optional[Any]     							= self._execution_device

          snake_case			:  List[str]     							= guidance_scale > 1.0

          if isinstance(snake_case__       ,		snake_case__					):
                 snake_case			:  Optional[int]     							= torch.cat(snake_case__       ,		dim=0					)
          snake_case			:  Optional[int]     							= image_embeds.shape[0] * num_images_per_prompt
          if isinstance(snake_case__       ,		snake_case__					):
                 snake_case			:  List[Any]     							= torch.cat(snake_case__       ,		dim=0					)

          if do_classifier_free_guidance:
                 snake_case			:  str     							= image_embeds.repeat_interleave(snake_case__       ,		dim=0					)
                 snake_case			:  Optional[int]     							= negative_image_embeds.repeat_interleave(snake_case__       ,		dim=0					)

          snake_case			:  Optional[Any]     							= torch.cat([negative_image_embeds, image_embeds]       ,		dim=0					).to(dtype=self.unet.dtype       ,		device=snake_case__					)

          self.scheduler.set_timesteps(snake_case__       ,		device=snake_case__					)
          snake_case			:  Optional[int]     							= self.scheduler.timesteps

          snake_case			:  int     							= self.unet.config.in_channels

          snake_case							,   snake_case			:  Any     							= downscale_height_and_width(snake_case__       ,		snake_case__       ,		self.movq_scale_factor					)

          # create initial latent
          snake_case			:  List[Any]     							= self.prepare_latents(
              (batch_size, num_channels_latents, height, width)       ,		image_embeds.dtype       ,		snake_case__       ,		snake_case__       ,		snake_case__       ,		self.scheduler       ,		)

          for i, t in enumerate(self.progress_bar(snake_case__					)					):
                 # expand the latents if we are doing classifier free guidance
                 snake_case			:  List[str]     							= torch.cat([latents] * 2					) if do_classifier_free_guidance else latents

                 snake_case			:  Tuple     							= {"image_embeds": image_embeds}
                 snake_case			:  str     							= self.unet(
                     sample=snake_case__       ,		timestep=snake_case__       ,		encoder_hidden_states=snake_case__       ,		added_cond_kwargs=snake_case__       ,		return_dict=snake_case__       ,		)[0]

                 if do_classifier_free_guidance:
                        snake_case							,   snake_case			:  List[Any]     							= noise_pred.split(latents.shape[1]       ,		dim=1					)
                        snake_case							,   snake_case			:  Any     							= noise_pred.chunk(2					)
                        snake_case							,   snake_case			:  Union[str, Any]     							= variance_pred.chunk(2					)
                        snake_case			:  Optional[Any]     							= noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
                        snake_case			:  str     							= torch.cat([noise_pred, variance_pred_text]       ,		dim=1					)

                 if not (
                     hasattr(self.scheduler.config       ,		"variance_type"					)
                     and self.scheduler.config.variance_type in ["learned", "learned_range"]
                 ):
                        snake_case							,   snake_case			:  List[Any]     							= noise_pred.split(latents.shape[1]       ,		dim=1					)

                 # compute the previous noisy sample x_t -> x_t-1
                 snake_case			:  Tuple     							= self.scheduler.step(
                     snake_case__       ,		snake_case__       ,		snake_case__       ,		generator=snake_case__       ,		)[0]
          # post-processing
          snake_case			:  Optional[Any]     							= self.movq.decode(snake_case__       ,		force_not_quantize=snake_case__					)["sample"]

          if output_type not in ["pt", "np", "pil"]:
                 raise ValueError(f"""Only the output types `pt`, `pil` and `np` are supported not output_type={output_type}"""					)

          if output_type in ["np", "pil"]:
                 snake_case			:  Union[str, Any]     							= image * 0.5 + 0.5
                 snake_case			:  int     							= image.clamp(0       ,		1					)
                 snake_case			:  Optional[int]     							= image.cpu().permute(0       ,		2       ,		3       ,		1					).float().numpy()

          if output_type == "pil":
                 snake_case			:  Any     							= self.numpy_to_pil(snake_case__					)

          if not return_dict:
                 return (image,)

          return ImagePipelineOutput(images=snake_case__					)





