
'''simple docstring'''


from dataclasses import dataclass
from typing import Dict, Optional, Union

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .attention_processor import AttentionProcessor, AttnProcessor
from .embeddings import TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin


@dataclass
class 						lowercase__	(  lowercase_   ):






   '''simple docstring'''







   _snake_case		   =							42




class 						lowercase__	(  lowercase_,					lowercase_   ):






   '''simple docstring'''







   @register_to_config
   def __init__(			self ,   lowerCamelCase__ = 3_2 ,   lowerCamelCase__ = 6_4 ,   lowerCamelCase__ = 2_0 ,   lowerCamelCase__ = 7_6_8 ,   lowerCamelCase__=7_7 ,   lowerCamelCase__=4 ,   lowerCamelCase__ = 0.0 ,   lowerCamelCase__ = "silu" ,   lowerCamelCase__ = None ,   lowerCamelCase__ = None ,   lowerCamelCase__ = "linear" ,   lowerCamelCase__ = "prd" ,   lowerCamelCase__ = None ,   lowerCamelCase__ = None ,   lowerCamelCase__ = None ,   ):

      '''simple docstring'''







      super().__init__()
      UpperCamelCase       =  num_attention_heads
      UpperCamelCase       =  attention_head_dim
      UpperCamelCase       =  num_attention_heads * attention_head_dim
      UpperCamelCase       =  additional_embeddings

      UpperCamelCase       =  time_embed_dim or inner_dim
      UpperCamelCase       =  embedding_proj_dim or embedding_dim
      UpperCamelCase       =  clip_embed_dim or embedding_dim

      UpperCamelCase       =  Timesteps(lowerCamelCase_ ,   lowerCamelCase_ ,   0   )
      UpperCamelCase       =  TimestepEmbedding(lowerCamelCase_ ,   lowerCamelCase_ ,   out_dim=lowerCamelCase_ ,   act_fn=lowerCamelCase_   )

      UpperCamelCase       =  nn.Linear(lowerCamelCase_ ,   lowerCamelCase_   )

      if embedding_proj_norm_type is None:
         UpperCamelCase       =  None
      elif embedding_proj_norm_type == "layer":
         UpperCamelCase       =  nn.LayerNorm(lowerCamelCase_   )
      else:
         raise ValueError(f'unsupported embedding_proj_norm_type: {embedding_proj_norm_type}'   )

      UpperCamelCase       =  nn.Linear(lowerCamelCase_ ,   lowerCamelCase_   )

      if encoder_hid_proj_type is None:
         UpperCamelCase       =  None
      elif encoder_hid_proj_type == "linear":
         UpperCamelCase       =  nn.Linear(lowerCamelCase_ ,   lowerCamelCase_   )
      else:
         raise ValueError(f'unsupported encoder_hid_proj_type: {encoder_hid_proj_type}'   )

      UpperCamelCase       =  nn.Parameter(torch.zeros(1 ,   num_embeddings + additional_embeddings ,   lowerCamelCase_   )   )

      if added_emb_type == "prd":
         UpperCamelCase       =  nn.Parameter(torch.zeros(1 ,   1 ,   lowerCamelCase_   )   )
      elif added_emb_type is None:
         UpperCamelCase       =  None
      else:
         raise ValueError(
             f'`added_emb_type`: {added_emb_type} is not supported. Make sure to choose one of `\'prd\'` or `None`.'   )

      UpperCamelCase       =  nn.ModuleList(
          [
              BasicTransformerBlock(
                  lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_ ,   dropout=lowerCamelCase_ ,   activation_fn='''gelu''' ,   attention_bias=lowerCamelCase_ ,   )
              for d in range(lowerCamelCase_   )
          ]   )

      if norm_in_type == "layer":
         UpperCamelCase       =  nn.LayerNorm(lowerCamelCase_   )
      elif norm_in_type is None:
         UpperCamelCase       =  None
      else:
         raise ValueError(f'Unsupported norm_in_type: {norm_in_type}.'   )

      UpperCamelCase       =  nn.LayerNorm(lowerCamelCase_   )

      UpperCamelCase       =  nn.Linear(lowerCamelCase_ ,   lowerCamelCase_   )

      UpperCamelCase       =  torch.full(
          [num_embeddings + additional_embeddings, num_embeddings + additional_embeddings] ,   -1_0_0_0_0.0   )
      causal_attention_mask.triu_(1   )
      UpperCamelCase       =  causal_attention_mask[None, ...]
      self.register_buffer('''causal_attention_mask''' ,   lowerCamelCase_ ,   persistent=lowerCamelCase_   )

      UpperCamelCase       =  nn.Parameter(torch.zeros(1 ,   lowerCamelCase_   )   )
      UpperCamelCase       =  nn.Parameter(torch.zeros(1 ,   lowerCamelCase_   )   )







   @property
   # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors
   def       UpperCAmelCase      (			self   ):

      '''simple docstring'''







      UpperCamelCase       =  {}

      def fn_recursive_add_processors(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   ):
         if hasattr(lowerCamelCase_ ,   '''set_processor'''   ):
            UpperCamelCase       =  module.processor

         for sub_name, child in module.named_children():
            fn_recursive_add_processors(f'{name}.{sub_name}' ,   lowerCamelCase_ ,   lowerCamelCase_   )

         return processors

      for name, module in self.named_children():
         fn_recursive_add_processors(lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_   )

      return processors







   def       UpperCAmelCase      (			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  len(self.attn_processors.keys()   )

      if isinstance(lowerCamelCase_ ,   lowerCamelCase_   ) and len(lowerCamelCase_   ) != count:
         raise ValueError(
             f'A dict of processors was passed, but the number of processors {len(lowerCamelCase_   )} does not match the'
             f' number of attention layers: {count}. Please make sure to pass {count} processor classes.'   )

      def fn_recursive_attn_processor(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   ):
         if hasattr(lowerCamelCase_ ,   '''set_processor'''   ):
            if not isinstance(lowerCamelCase_ ,   lowerCamelCase_   ):
               module.set_processor(lowerCamelCase_   )
            else:
               module.set_processor(processor.pop(f'{name}.processor'   )   )

         for sub_name, child in module.named_children():
            fn_recursive_attn_processor(f'{name}.{sub_name}' ,   lowerCamelCase_ ,   lowerCamelCase_   )

      for name, module in self.named_children():
         fn_recursive_attn_processor(lowerCamelCase_ ,   lowerCamelCase_ ,   lowerCamelCase_   )







   def       UpperCAmelCase      (			self   ):

      '''simple docstring'''







      self.set_attn_processor(AttnProcessor()   )







   def       UpperCAmelCase      (			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ = None ,   lowerCamelCase__ = None ,   lowerCamelCase__ = True ,   ):

      '''simple docstring'''







      UpperCamelCase       =  hidden_states.shape[0]

      UpperCamelCase       =  timestep
      if not torch.is_tensor(lowerCamelCase_   ):
         UpperCamelCase       =  torch.tensor([timesteps] ,   dtype=torch.long ,   device=hidden_states.device   )
      elif torch.is_tensor(lowerCamelCase_   ) and len(timesteps.shape   ) == 0:
         UpperCamelCase       =  timesteps[None].to(hidden_states.device   )

      # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
      UpperCamelCase       =  timesteps * torch.ones(lowerCamelCase_ ,   dtype=timesteps.dtype ,   device=timesteps.device   )

      UpperCamelCase       =  self.time_proj(lowerCamelCase_   )

      # timesteps does not contain any weights and will always return f32 tensors
      # but time_embedding might be fp16, so we need to cast here.
      UpperCamelCase       =  timesteps_projected.to(dtype=self.dtype   )
      UpperCamelCase       =  self.time_embedding(lowerCamelCase_   )

      if self.embedding_proj_norm is not None:
         UpperCamelCase       =  self.embedding_proj_norm(lowerCamelCase_   )

      UpperCamelCase       =  self.embedding_proj(lowerCamelCase_   )
      if self.encoder_hidden_states_proj is not None and encoder_hidden_states is not None:
         UpperCamelCase       =  self.encoder_hidden_states_proj(lowerCamelCase_   )
      elif self.encoder_hidden_states_proj is not None and encoder_hidden_states is None:
         raise ValueError('''`encoder_hidden_states_proj` requires `encoder_hidden_states` to be set'''   )

      UpperCamelCase       =  self.proj_in(lowerCamelCase_   )

      UpperCamelCase       =  self.positional_embedding.to(hidden_states.dtype   )

      UpperCamelCase       =  []
      UpperCamelCase       =  0

      if encoder_hidden_states is not None:
         additional_embeds.append(lowerCamelCase_   )
         additional_embeddings_len += encoder_hidden_states.shape[1]

      if len(proj_embeddings.shape   ) == 2:
         UpperCamelCase       =  proj_embeddings[:, None, :]

      if len(hidden_states.shape   ) == 2:
         UpperCamelCase       =  hidden_states[:, None, :]

      UpperCamelCase       =  additional_embeds + [
          proj_embeddings,
          time_embeddings[:, None, :],
          hidden_states,
      ]

      if self.prd_embedding is not None:
         UpperCamelCase       =  self.prd_embedding.to(hidden_states.dtype   ).expand(lowerCamelCase_ ,   -1 ,   -1   )
         additional_embeds.append(lowerCamelCase_   )

      UpperCamelCase       =  torch.cat(
          lowerCamelCase_ ,   dim=1 ,   )

      # Allow positional_embedding to not include the `addtional_embeddings` and instead pad it with zeros for these additional tokens
      UpperCamelCase       =  additional_embeddings_len + proj_embeddings.shape[1] + 1
      if positional_embeddings.shape[1] < hidden_states.shape[1]:
         UpperCamelCase       =  F.pad(
             lowerCamelCase_ ,   (
                 0,
                 0,
                 additional_embeddings_len,
                 self.prd_embedding.shape[1] if self.prd_embedding is not None else 0,
             ) ,   value=0.0 ,   )

      UpperCamelCase       =  hidden_states + positional_embeddings

      if attention_mask is not None:
         UpperCamelCase       =  (1 - attention_mask.to(hidden_states.dtype   )) * -1_0_0_0_0.0
         UpperCamelCase       =  F.pad(lowerCamelCase_ ,   (0, self.additional_embeddings) ,   value=0.0   )
         UpperCamelCase       =  (attention_mask[:, None, :] + self.causal_attention_mask).to(hidden_states.dtype   )
         UpperCamelCase       =  attention_mask.repeat_interleave(self.config.num_attention_heads ,   dim=0   )

      if self.norm_in is not None:
         UpperCamelCase       =  self.norm_in(lowerCamelCase_   )

      for block in self.transformer_blocks:
         UpperCamelCase       =  block(lowerCamelCase_ ,   attention_mask=lowerCamelCase_   )

      UpperCamelCase       =  self.norm_out(lowerCamelCase_   )

      if self.prd_embedding is not None:
         UpperCamelCase       =  hidden_states[:, -1]
      else:
         UpperCamelCase       =  hidden_states[:, additional_embeddings_len:]

      UpperCamelCase       =  self.proj_to_clip_embeddings(lowerCamelCase_   )

      if not return_dict:
         return (predicted_image_embedding,)

      return PriorTransformerOutput(predicted_image_embedding=lowerCamelCase_   )







   def       UpperCAmelCase      (			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  (prior_latents * self.clip_std) + self.clip_mean
      return prior_latents







'''simple docstring'''


import os
from typing import List, Optional, Union

from ...image_processing_utils import BatchFeature
from ...image_utils import ImageInput
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType
from ..auto import AutoTokenizer


class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   _snake_case		   =							['''image_processor''', '''tokenizer''']
   _snake_case		   =							'''BlipImageProcessor'''
   _snake_case		   =							'''AutoTokenizer'''







   def __init__(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   ):

      '''simple docstring'''







      super().__init__(lowerCamelCase__ ,   lowerCamelCase__   )

      # add QFormer tokenizer
      UpperCamelCase       =  qformer_tokenizer







   def __call__(			self ,   lowerCamelCase__ = None ,   lowerCamelCase__ = None ,   lowerCamelCase__ = True ,   lowerCamelCase__ = False ,   lowerCamelCase__ = None ,   lowerCamelCase__ = None ,   lowerCamelCase__ = 0 ,   lowerCamelCase__ = None ,   lowerCamelCase__ = None ,   lowerCamelCase__ = False ,   lowerCamelCase__ = False ,   lowerCamelCase__ = False ,   lowerCamelCase__ = False ,   lowerCamelCase__ = False ,   lowerCamelCase__ = True ,   lowerCamelCase__ = None ,   **lowerCamelCase__ ,   ):

      '''simple docstring'''







      if images is None and text is None:
         raise ValueError('''You have to specify at least images or text.'''   )

      UpperCamelCase       =  BatchFeature()

      if text is not None:
         UpperCamelCase       =  self.tokenizer(
             text=lowerCamelCase__ ,   add_special_tokens=lowerCamelCase__ ,   padding=lowerCamelCase__ ,   truncation=lowerCamelCase__ ,   max_length=lowerCamelCase__ ,   stride=lowerCamelCase__ ,   pad_to_multiple_of=lowerCamelCase__ ,   return_attention_mask=lowerCamelCase__ ,   return_overflowing_tokens=lowerCamelCase__ ,   return_special_tokens_mask=lowerCamelCase__ ,   return_offsets_mapping=lowerCamelCase__ ,   return_token_type_ids=lowerCamelCase__ ,   return_length=lowerCamelCase__ ,   verbose=lowerCamelCase__ ,   return_tensors=lowerCamelCase__ ,   **lowerCamelCase__ ,   )
         encoding.update(lowerCamelCase__   )
         UpperCamelCase       =  self.qformer_tokenizer(
             text=lowerCamelCase__ ,   add_special_tokens=lowerCamelCase__ ,   padding=lowerCamelCase__ ,   truncation=lowerCamelCase__ ,   max_length=lowerCamelCase__ ,   stride=lowerCamelCase__ ,   pad_to_multiple_of=lowerCamelCase__ ,   return_attention_mask=lowerCamelCase__ ,   return_overflowing_tokens=lowerCamelCase__ ,   return_special_tokens_mask=lowerCamelCase__ ,   return_offsets_mapping=lowerCamelCase__ ,   return_token_type_ids=lowerCamelCase__ ,   return_length=lowerCamelCase__ ,   verbose=lowerCamelCase__ ,   return_tensors=lowerCamelCase__ ,   **lowerCamelCase__ ,   )
         UpperCamelCase       =  qformer_text_encoding.pop('''input_ids'''   )
         UpperCamelCase       =  qformer_text_encoding.pop('''attention_mask'''   )

      if images is not None:
         UpperCamelCase       =  self.image_processor(lowerCamelCase__ ,   return_tensors=lowerCamelCase__   )
         encoding.update(lowerCamelCase__   )

      return encoding







   def       UpperCAmelCase				(			self ,   *lowerCamelCase__ ,   **lowerCamelCase__   ):

      '''simple docstring'''







      return self.tokenizer.batch_decode(*lowerCamelCase__ ,   **lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   *lowerCamelCase__ ,   **lowerCamelCase__   ):

      '''simple docstring'''







      return self.tokenizer.decode(*lowerCamelCase__ ,   **lowerCamelCase__   )







   @property
   # Copied from transformers.models.blip.processing_blip.BlipProcessor.model_input_names
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.tokenizer.model_input_names
      UpperCamelCase       =  self.image_processor.model_input_names
      return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names   )   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   **lowerCamelCase__   ):

      '''simple docstring'''







      if os.path.isfile(lowerCamelCase__   ):
         raise ValueError(f'Provided path ({save_directory}) should be a directory, not a file'   )
      os.makedirs(lowerCamelCase__ ,   exist_ok=lowerCamelCase__   )
      UpperCamelCase       =  os.path.join(lowerCamelCase__ ,   '''qformer_tokenizer'''   )
      self.qformer_tokenizer.save_pretrained(lowerCamelCase__   )
      return super().save_pretrained(lowerCamelCase__ ,   **lowerCamelCase__   )







   @classmethod
   def       UpperCAmelCase				(			cls ,   lowerCamelCase__ ,   **lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  AutoTokenizer.from_pretrained(lowerCamelCase__ ,   subfolder='''qformer_tokenizer'''   )
      UpperCamelCase       =  cls._get_arguments_from_pretrained(lowerCamelCase__ ,   **lowerCamelCase__   )
      args.append(lowerCamelCase__   )
      return cls(*lowerCamelCase__   )





