





"""simple docstring"""





from dataclasses import dataclass
from typing import Dict, Optional, Union

import torch
import torch.nn.functional as F
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .attention import BasicTransformerBlock
from .attention_processor import AttentionProcessor, AttnProcessor
from .embeddings import TimestepEmbedding, Timesteps
from .modeling_utils import ModelMixin




@dataclass
class 							snake_case							(			__UpperCAmelCase ):

   '''simple docstring'''



   _A     :       torch.FloatTensor




class 							snake_case							(			__UpperCAmelCase      ,  __UpperCAmelCase ):

   '''simple docstring'''



   @register_to_config
   def __init__(							self       :      Union[str, Any]						,				__lowercase       :      int = 32						,				__lowercase       :      int = 64						,				__lowercase       :      int = 20						,				__lowercase       :      int = 768						,				__lowercase       :      Optional[int]=77						,				__lowercase       :      Union[str, Any]=4						,				__lowercase       :      float = 0.0						,				__lowercase       :      str = "silu"						,				__lowercase       :      Optional[str] = None						,				__lowercase       :      Optional[str] = None						,				__lowercase       :      Optional[str] = "linear"						,				__lowercase       :      Optional[str] = "prd"						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      Optional[int] = None						,				__lowercase       :      Optional[int] = None						,				):


     '''simple docstring'''




     super().__init__()
     __UpperCAmelCase    :						int       =						num_attention_heads
     __UpperCAmelCase    :						List[Any]       =						attention_head_dim
     __UpperCAmelCase    :						int       =						num_attention_heads * attention_head_dim
     __UpperCAmelCase    :						List[str]       =						additional_embeddings

     __UpperCAmelCase    :						Optional[int]       =						time_embed_dim or inner_dim
     __UpperCAmelCase    :						Tuple       =						embedding_proj_dim or embedding_dim
     __UpperCAmelCase    :						Dict       =						clip_embed_dim or embedding_dim

     __UpperCAmelCase    :						Dict       =						Timesteps(__lowercase						,				__lowercase						,				0 )
     __UpperCAmelCase    :						List[str]       =						TimestepEmbedding(__lowercase						,				__lowercase						,				out_dim=__lowercase						,				act_fn=__lowercase )

     __UpperCAmelCase    :						Any       =						nn.Linear(__lowercase						,				__lowercase )

     if embedding_proj_norm_type is None:
       __UpperCAmelCase    :						Dict       =						None
     elif embedding_proj_norm_type == "layer":
       __UpperCAmelCase    :						Any       =						nn.LayerNorm(__lowercase )
     else:
       raise ValueError(f'''unsupported embedding_proj_norm_type: {embedding_proj_norm_type}''' )

     __UpperCAmelCase    :						List[Any]       =						nn.Linear(__lowercase						,				__lowercase )

     if encoder_hid_proj_type is None:
       __UpperCAmelCase    :						int       =						None
     elif encoder_hid_proj_type == "linear":
       __UpperCAmelCase    :						int       =						nn.Linear(__lowercase						,				__lowercase )
     else:
       raise ValueError(f'''unsupported encoder_hid_proj_type: {encoder_hid_proj_type}''' )

     __UpperCAmelCase    :						str       =						nn.Parameter(torch.zeros(1						,				num_embeddings + additional_embeddings						,				__lowercase ) )

     if added_emb_type == "prd":
       __UpperCAmelCase    :						Optional[int]       =						nn.Parameter(torch.zeros(1						,				1						,				__lowercase ) )
     elif added_emb_type is None:
       __UpperCAmelCase    :						str       =						None
     else:
       raise ValueError(
           f'''`added_emb_type`: {added_emb_type} is not supported. Make sure to choose one of `\'prd\'` or `None`.''' )

     __UpperCAmelCase    :						Union[str, Any]       =						nn.ModuleList(
         [
             BasicTransformerBlock(
                 __lowercase						,				__lowercase						,				__lowercase						,				dropout=__lowercase						,				activation_fn='''gelu'''						,				attention_bias=__lowercase						,				)
             for d in range(__lowercase )
         ] )

     if norm_in_type == "layer":
       __UpperCAmelCase    :						Optional[Any]       =						nn.LayerNorm(__lowercase )
     elif norm_in_type is None:
       __UpperCAmelCase    :						List[Any]       =						None
     else:
       raise ValueError(f'''Unsupported norm_in_type: {norm_in_type}.''' )

     __UpperCAmelCase    :						str       =						nn.LayerNorm(__lowercase )

     __UpperCAmelCase    :						List[Any]       =						nn.Linear(__lowercase						,				__lowercase )

     __UpperCAmelCase    :						List[Any]       =						torch.full(
         [num_embeddings + additional_embeddings, num_embeddings + additional_embeddings]						,				-10_000.0 )
     causal_attention_mask.triu_(1 )
     __UpperCAmelCase    :						Optional[Any]       =						causal_attention_mask[None, ...]
     self.register_buffer('''causal_attention_mask'''						,				__lowercase						,				persistent=__lowercase )

     __UpperCAmelCase    :						Any       =						nn.Parameter(torch.zeros(1						,				__lowercase ) )
     __UpperCAmelCase    :						List[str]       =						nn.Parameter(torch.zeros(1						,				__lowercase ) )

   @property
   # Copied from diffusers.models.unet_2d_condition.UNet2DConditionModel.attn_processors
   def       A_					(							self       :      Optional[int] ):


     '''simple docstring'''




     __UpperCAmelCase    :						Optional[int]       =						{}

     def fn_recursive_add_processors(__lowercase       :      str						,				__lowercase       :      torch.nn.Module						,				__lowercase       :      Dict[str, AttentionProcessor] ):
       if hasattr(__lowercase						,				'''set_processor''' ):
         __UpperCAmelCase    :						Optional[Any]       =						module.processor

       for sub_name, child in module.named_children():
         fn_recursive_add_processors(f'''{name}.{sub_name}'''						,				__lowercase						,				__lowercase )

       return processors

     for name, module in self.named_children():
       fn_recursive_add_processors(__lowercase						,				__lowercase						,				__lowercase )

     return processors

   def       A_					(							self       :      Any						,				__lowercase       :      Union[AttentionProcessor, Dict[str, AttentionProcessor]] ):


     '''simple docstring'''




     __UpperCAmelCase    :						Tuple       =						len(self.attn_processors.keys() )

     if isinstance(__lowercase						,				__lowercase ) and len(__lowercase ) != count:
       raise ValueError(
           f'''A dict of processors was passed, but the number of processors {len(__lowercase )} does not match the'''
           f''' number of attention layers: {count}. Please make sure to pass {count} processor classes.''' )

     def fn_recursive_attn_processor(__lowercase       :      str						,				__lowercase       :      torch.nn.Module						,				__lowercase       :      int ):
       if hasattr(__lowercase						,				'''set_processor''' ):
         if not isinstance(__lowercase						,				__lowercase ):
           module.set_processor(__lowercase )
         else:
           module.set_processor(processor.pop(f'''{name}.processor''' ) )

       for sub_name, child in module.named_children():
         fn_recursive_attn_processor(f'''{name}.{sub_name}'''						,				__lowercase						,				__lowercase )

     for name, module in self.named_children():
       fn_recursive_attn_processor(__lowercase						,				__lowercase						,				__lowercase )

   def       A_					(							self       :      Optional[Any] ):


     '''simple docstring'''




     self.set_attn_processor(AttnProcessor() )

   def       A_					(							self       :      Optional[int]						,				__lowercase       :      Dict						,				__lowercase       :      Union[torch.Tensor, float, int]						,				__lowercase       :      torch.FloatTensor						,				__lowercase       :      Optional[torch.FloatTensor] = None						,				__lowercase       :      Optional[torch.BoolTensor] = None						,				__lowercase       :      bool = True						,				):


     '''simple docstring'''




     __UpperCAmelCase    :						List[Any]       =						hidden_states.shape[0]

     __UpperCAmelCase    :						Any       =						timestep
     if not torch.is_tensor(__lowercase ):
       __UpperCAmelCase    :						Union[str, Any]       =						torch.tensor([timesteps]						,				dtype=torch.long						,				device=hidden_states.device )
     elif torch.is_tensor(__lowercase ) and len(timesteps.shape ) == 0:
       __UpperCAmelCase    :						Union[str, Any]       =						timesteps[None].to(hidden_states.device )

     # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
     __UpperCAmelCase    :						List[str]       =						timesteps * torch.ones(__lowercase						,				dtype=timesteps.dtype						,				device=timesteps.device )

     __UpperCAmelCase    :						List[str]       =						self.time_proj(__lowercase )

     # timesteps does not contain any weights and will always return f32 tensors
     # but time_embedding might be fp16, so we need to cast here.
     __UpperCAmelCase    :						Any       =						timesteps_projected.to(dtype=self.dtype )
     __UpperCAmelCase    :						List[str]       =						self.time_embedding(__lowercase )

     if self.embedding_proj_norm is not None:
       __UpperCAmelCase    :						Dict       =						self.embedding_proj_norm(__lowercase )

     __UpperCAmelCase    :						Optional[int]       =						self.embedding_proj(__lowercase )
     if self.encoder_hidden_states_proj is not None and encoder_hidden_states is not None:
       __UpperCAmelCase    :						Dict       =						self.encoder_hidden_states_proj(__lowercase )
     elif self.encoder_hidden_states_proj is not None and encoder_hidden_states is None:
       raise ValueError('''`encoder_hidden_states_proj` requires `encoder_hidden_states` to be set''' )

     __UpperCAmelCase    :						int       =						self.proj_in(__lowercase )

     __UpperCAmelCase    :						Tuple       =						self.positional_embedding.to(hidden_states.dtype )

     __UpperCAmelCase    :						Optional[Any]       =						[]
     __UpperCAmelCase    :						str       =						0

     if encoder_hidden_states is not None:
       additional_embeds.append(__lowercase )
       additional_embeddings_len += encoder_hidden_states.shape[1]

     if len(proj_embeddings.shape ) == 2:
       __UpperCAmelCase    :						str       =						proj_embeddings[:, None, :]

     if len(hidden_states.shape ) == 2:
       __UpperCAmelCase    :						Optional[int]       =						hidden_states[:, None, :]

     __UpperCAmelCase    :						str       =						additional_embeds + [
         proj_embeddings,
         time_embeddings[:, None, :],
         hidden_states,
     ]

     if self.prd_embedding is not None:
       __UpperCAmelCase    :						Dict       =						self.prd_embedding.to(hidden_states.dtype ).expand(__lowercase						,				-1						,				-1 )
       additional_embeds.append(__lowercase )

     __UpperCAmelCase    :						Optional[Any]       =						torch.cat(
         __lowercase						,				dim=1						,				)

     # Allow positional_embedding to not include the `addtional_embeddings` and instead pad it with zeros for these additional tokens
     __UpperCAmelCase    :						Optional[int]       =						additional_embeddings_len + proj_embeddings.shape[1] + 1
     if positional_embeddings.shape[1] < hidden_states.shape[1]:
       __UpperCAmelCase    :						List[str]       =						F.pad(
           __lowercase						,				(
               0,
               0,
               additional_embeddings_len,
               self.prd_embedding.shape[1] if self.prd_embedding is not None else 0,
           )						,				value=0.0						,				)

     __UpperCAmelCase    :						Dict       =						hidden_states + positional_embeddings

     if attention_mask is not None:
       __UpperCAmelCase    :						Optional[Any]       =						(1 - attention_mask.to(hidden_states.dtype )) * -10_000.0
       __UpperCAmelCase    :						Optional[Any]       =						F.pad(__lowercase						,				(0, self.additional_embeddings)						,				value=0.0 )
       __UpperCAmelCase    :						Dict       =						(attention_mask[:, None, :] + self.causal_attention_mask).to(hidden_states.dtype )
       __UpperCAmelCase    :						Tuple       =						attention_mask.repeat_interleave(self.config.num_attention_heads						,				dim=0 )

     if self.norm_in is not None:
       __UpperCAmelCase    :						str       =						self.norm_in(__lowercase )

     for block in self.transformer_blocks:
       __UpperCAmelCase    :						Dict       =						block(__lowercase						,				attention_mask=__lowercase )

     __UpperCAmelCase    :						Any       =						self.norm_out(__lowercase )

     if self.prd_embedding is not None:
       __UpperCAmelCase    :						List[Any]       =						hidden_states[:, -1]
     else:
       __UpperCAmelCase    :						List[str]       =						hidden_states[:, additional_embeddings_len:]

     __UpperCAmelCase    :						Optional[int]       =						self.proj_to_clip_embeddings(__lowercase )

     if not return_dict:
       return (predicted_image_embedding,)

     return PriorTransformerOutput(predicted_image_embedding=__lowercase )







   def       A_					(							self       :      List[Any]						,				__lowercase       :      Optional[int] ):


     '''simple docstring'''




     __UpperCAmelCase    :						List[str]       =						(prior_latents * self.clip_std) + self.clip_mean
     return prior_latents






"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__				:int				   =       logging.get_logger(__name__)

lowercase__				:List[str]				   =       {
    'EleutherAI/gpt-neox-20b': 'https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json',
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       List[Any]         =       'gpt_neox'

      def __init__(							self       :      Union[str, Any]						,				__lowercase       :      Any=50_432						,				__lowercase       :      int=6_144						,				__lowercase       :      List[str]=44						,				__lowercase       :      List[Any]=64						,				__lowercase       :      int=24_576						,				__lowercase       :      List[str]="gelu"						,				__lowercase       :      Optional[int]=0.2_5						,				__lowercase       :      Any=10_000						,				__lowercase       :      Union[str, Any]=0.0						,				__lowercase       :      List[str]=0.0						,				__lowercase       :      Union[str, Any]=0.1						,				__lowercase       :      Optional[int]=2_048						,				__lowercase       :      Union[str, Any]=0.0_2						,				__lowercase       :      Optional[int]=1e-5						,				__lowercase       :      str=True						,				__lowercase       :      str=0						,				__lowercase       :      Tuple=2						,				__lowercase       :      Tuple=False						,				__lowercase       :      int=True						,				__lowercase       :      Any=None						,				**__lowercase       :      List[str]						,				):


        '''simple docstring'''




        super().__init__(bos_token_id=__lowercase						,				eos_token_id=__lowercase						,				**__lowercase )
        __UpperCAmelCase    :						int       =						vocab_size
        __UpperCAmelCase    :						Union[str, Any]       =						max_position_embeddings
        __UpperCAmelCase    :						List[str]       =						hidden_size
        __UpperCAmelCase    :						Tuple       =						num_hidden_layers
        __UpperCAmelCase    :						Any       =						num_attention_heads
        __UpperCAmelCase    :						Union[str, Any]       =						intermediate_size
        __UpperCAmelCase    :						str       =						hidden_act
        __UpperCAmelCase    :						str       =						rotary_pct
        __UpperCAmelCase    :						str       =						rotary_emb_base
        __UpperCAmelCase    :						Dict       =						attention_dropout
        __UpperCAmelCase    :						str       =						hidden_dropout
        __UpperCAmelCase    :						Any       =						classifier_dropout
        __UpperCAmelCase    :						Tuple       =						initializer_range
        __UpperCAmelCase    :						Any       =						layer_norm_eps
        __UpperCAmelCase    :						Optional[int]       =						use_cache
        __UpperCAmelCase    :						List[Any]       =						tie_word_embeddings
        __UpperCAmelCase    :						Any       =						use_parallel_residual
        __UpperCAmelCase    :						List[str]       =						rope_scaling
        self._rope_scaling_validation()

        if self.hidden_size % self.num_attention_heads != 0:
          raise ValueError(
              '''The hidden size is not divisble by the number of attention heads! Make sure to update them!''' )







      def       A_					(							self       :      int ):


        '''simple docstring'''




        if self.rope_scaling is None:
          return

        if not isinstance(self.rope_scaling						,				__lowercase ) or len(self.rope_scaling ) != 2:
          raise ValueError(
              '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
              f'''got {self.rope_scaling}''' )
        __UpperCAmelCase    :						Union[str, Any]       =						self.rope_scaling.get('''type'''						,				__lowercase )
        __UpperCAmelCase    :						List[Any]       =						self.rope_scaling.get('''factor'''						,				__lowercase )
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
          raise ValueError(
              f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}''' )
        if rope_scaling_factor is None or not isinstance(__lowercase						,				__lowercase ) or rope_scaling_factor <= 1.0:
          raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}''' )