





'''simple docstring'''






from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings
@maybe_allow_in_graph
class 					__snake_case  (					nn.Module							):






   '''simple docstring'''





   def __init__(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE=0.0	,		__SCREAMING_SNAKE_CASE = None	,		__SCREAMING_SNAKE_CASE = "geglu"	,		__SCREAMING_SNAKE_CASE = None	,		__SCREAMING_SNAKE_CASE = False	,		__SCREAMING_SNAKE_CASE = False	,		__SCREAMING_SNAKE_CASE = False	,		__SCREAMING_SNAKE_CASE = False	,		__SCREAMING_SNAKE_CASE = True	,		__SCREAMING_SNAKE_CASE = "layer_norm"	,		__SCREAMING_SNAKE_CASE = False	,		):
    super().__init__()
    snake_case__     :					int    =  only_cross_attention

    snake_case__     :					Tuple    =  (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
    snake_case__     :					Union[str, Any]    =  (num_embeds_ada_norm is not None) and norm_type == "ada_norm"

    if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
     raise ValueError(
         f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
         f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."       )

    # Define 3 blocks. Each block has its own normalization layer.
    # 1. Self-Attn
    if self.use_ada_layer_norm:
     snake_case__     :					List[Any]    =  AdaLayerNorm(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )
    elif self.use_ada_layer_norm_zero:
     snake_case__     :					List[Any]    =  AdaLayerNormZero(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )
    else:
     snake_case__     :					Tuple    =  nn.LayerNorm(__SCREAMING_SNAKE_CASE	,		elementwise_affine=__SCREAMING_SNAKE_CASE       )
    snake_case__     :					List[str]    =  Attention(
        query_dim=__SCREAMING_SNAKE_CASE	,		heads=__SCREAMING_SNAKE_CASE	,		dim_head=__SCREAMING_SNAKE_CASE	,		dropout=__SCREAMING_SNAKE_CASE	,		bias=__SCREAMING_SNAKE_CASE	,		cross_attention_dim=cross_attention_dim if only_cross_attention else None	,		upcast_attention=__SCREAMING_SNAKE_CASE	,		)

    # 2. Cross-Attn
    if cross_attention_dim is not None or double_self_attention:
     # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
     # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
     # the second cross attention block.
     snake_case__     :					Optional[int]    =  (
         AdaLayerNorm(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )
         if self.use_ada_layer_norm
         else nn.LayerNorm(__SCREAMING_SNAKE_CASE	,		elementwise_affine=__SCREAMING_SNAKE_CASE       )
     )
     snake_case__     :					str    =  Attention(
         query_dim=__SCREAMING_SNAKE_CASE	,		cross_attention_dim=cross_attention_dim if not double_self_attention else None	,		heads=__SCREAMING_SNAKE_CASE	,		dim_head=__SCREAMING_SNAKE_CASE	,		dropout=__SCREAMING_SNAKE_CASE	,		bias=__SCREAMING_SNAKE_CASE	,		upcast_attention=__SCREAMING_SNAKE_CASE	,		)  # is self-attn if encoder_hidden_states is none
    else:
     snake_case__     :					Optional[Any]    =  None
     snake_case__     :					Tuple    =  None

    # 3. Feed-forward
    snake_case__     :					Dict    =  nn.LayerNorm(__SCREAMING_SNAKE_CASE	,		elementwise_affine=__SCREAMING_SNAKE_CASE       )
    snake_case__     :					Optional[int]    =  FeedForward(__SCREAMING_SNAKE_CASE	,		dropout=__SCREAMING_SNAKE_CASE	,		activation_fn=__SCREAMING_SNAKE_CASE	,		final_dropout=__SCREAMING_SNAKE_CASE       )

    # let chunk size default to None
    snake_case__     :					Tuple    =  None
    snake_case__     :					List[Any]    =  0

   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       ):
    snake_case__     :					Tuple    =  chunk_size
    snake_case__     :					Any    =  dim


   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE = None	,		__SCREAMING_SNAKE_CASE = None	,		__SCREAMING_SNAKE_CASE = None	,		__SCREAMING_SNAKE_CASE = None	,		__SCREAMING_SNAKE_CASE = None	,		__SCREAMING_SNAKE_CASE = None	,		):




    if self.use_ada_layer_norm:
     snake_case__     :					List[str]    =  self.norma(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )
    elif self.use_ada_layer_norm_zero:
     snake_case__     :					Optional[int]    =  self.norma(
         __SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		hidden_dtype=hidden_states.dtype       )
    else:
     snake_case__     :					Optional[int]    =  self.norma(__SCREAMING_SNAKE_CASE       )

    snake_case__     :					Union[str, Any]    =  cross_attention_kwargs if cross_attention_kwargs is not None else {}

    snake_case__     :					Dict    =  self.attna(
        __SCREAMING_SNAKE_CASE	,		encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None	,		attention_mask=__SCREAMING_SNAKE_CASE	,		**__SCREAMING_SNAKE_CASE	,		)
    if self.use_ada_layer_norm_zero:
     snake_case__     :					Optional[int]    =  gate_msa.unsqueeze(1       ) * attn_output
    snake_case__     :					Any    =  attn_output + hidden_states

    # 2. Cross-Attention
    if self.attna is not None:
     snake_case__     :					str    =  (
         self.norma(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       ) if self.use_ada_layer_norm else self.norma(__SCREAMING_SNAKE_CASE       )
     )

     snake_case__     :					Dict    =  self.attna(
         __SCREAMING_SNAKE_CASE	,		encoder_hidden_states=__SCREAMING_SNAKE_CASE	,		attention_mask=__SCREAMING_SNAKE_CASE	,		**__SCREAMING_SNAKE_CASE	,		)
     snake_case__     :					List[Any]    =  attn_output + hidden_states

    # 3. Feed-forward
    snake_case__     :					List[str]    =  self.norma(__SCREAMING_SNAKE_CASE       )

    if self.use_ada_layer_norm_zero:
     snake_case__     :					Optional[Any]    =  norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

    if self._chunk_size is not None:
     # "feed_forward_chunk_size" can be used to save memory
     if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
      raise ValueError(
          f"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."       )

     snake_case__     :					str    =  norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
     snake_case__     :					Union[str, Any]    =  torch.cat(
         [self.ff(__SCREAMING_SNAKE_CASE       ) for hid_slice in norm_hidden_states.chunk(__SCREAMING_SNAKE_CASE	,		dim=self._chunk_dim       )]	,		dim=self._chunk_dim	,		)
    else:
     snake_case__     :					Optional[int]    =  self.ff(__SCREAMING_SNAKE_CASE       )

    if self.use_ada_layer_norm_zero:
     snake_case__     :					Optional[int]    =  gate_mlp.unsqueeze(1       ) * ff_output

    snake_case__     :					List[Any]    =  ff_output + hidden_states

    return hidden_states



class 					__snake_case  (					nn.Module							):






   '''simple docstring'''





   def __init__(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE = None	,		__SCREAMING_SNAKE_CASE = 4	,		__SCREAMING_SNAKE_CASE = 0.0	,		__SCREAMING_SNAKE_CASE = "geglu"	,		__SCREAMING_SNAKE_CASE = False	,		):
    super().__init__()
    snake_case__     :					Optional[Any]    =  int(dim * mult       )
    snake_case__     :					Any    =  dim_out if dim_out is not None else dim

    if activation_fn == "gelu":
     snake_case__     :					Any    =  GELU(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )
    if activation_fn == "gelu-approximate":
     snake_case__     :					int    =  GELU(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		approximate="""tanh"""       )
    elif activation_fn == "geglu":
     snake_case__     :					Any    =  GEGLU(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )
    elif activation_fn == "geglu-approximate":
     snake_case__     :					Tuple    =  ApproximateGELU(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )

    snake_case__     :					Union[str, Any]    =  nn.ModuleList([]       )
    # project in
    self.net.append(__SCREAMING_SNAKE_CASE       )
    # project dropout
    self.net.append(nn.Dropout(__SCREAMING_SNAKE_CASE       )       )
    # project out
    self.net.append(nn.Linear(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )       )
    # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
    if final_dropout:
     self.net.append(nn.Dropout(__SCREAMING_SNAKE_CASE       )       )


   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE       ):




    for module in self.net:
     snake_case__     :					Optional[int]    =  module(__SCREAMING_SNAKE_CASE       )
    return hidden_states



class 					__snake_case  (					nn.Module							):






   '''simple docstring'''





   def __init__(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE = "none"       ):
    super().__init__()
    snake_case__     :					Any    =  nn.Linear(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )
    snake_case__     :					Union[str, Any]    =  approximate

   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE       ):




    if gate.device.type != "mps":
     return F.gelu(__SCREAMING_SNAKE_CASE	,		approximate=self.approximate       )
    # mps: gelu is not implemented for float16
    return F.gelu(gate.to(dtype=torch.floataa       )	,		approximate=self.approximate       ).to(dtype=gate.dtype       )


   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE       ):
    snake_case__     :					List[str]    =  self.proj(__SCREAMING_SNAKE_CASE       )
    snake_case__     :					List[Any]    =  self.gelu(__SCREAMING_SNAKE_CASE       )
    return hidden_states



class 					__snake_case  (					nn.Module							):






   '''simple docstring'''





   def __init__(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       ):
    super().__init__()
    snake_case__     :					int    =  nn.Linear(__SCREAMING_SNAKE_CASE	,		dim_out * 2       )

   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE       ):




    if gate.device.type != "mps":
     return F.gelu(__SCREAMING_SNAKE_CASE       )
    # mps: gelu is not implemented for float16
    return F.gelu(gate.to(dtype=torch.floataa       )       ).to(dtype=gate.dtype       )


   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE       ):
    snake_case__     :					Dict    =  self.proj(__SCREAMING_SNAKE_CASE       ).chunk(2	,		dim=-1       )
    return hidden_states * self.gelu(__SCREAMING_SNAKE_CASE       )



class 					__snake_case  (					nn.Module							):






   '''simple docstring'''





   def __init__(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       ):
    super().__init__()
    snake_case__     :					Any    =  nn.Linear(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )


   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE       ):
    snake_case__     :					Optional[int]    =  self.proj(__SCREAMING_SNAKE_CASE       )
    return x * torch.sigmoid(1.702 * x       )



class 					__snake_case  (					nn.Module							):






   '''simple docstring'''





   def __init__(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       ):
    super().__init__()
    snake_case__     :					Any    =  nn.Embedding(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )
    snake_case__     :					Union[str, Any]    =  nn.SiLU()
    snake_case__     :					List[Any]    =  nn.Linear(__SCREAMING_SNAKE_CASE	,		embedding_dim * 2       )
    snake_case__     :					Dict    =  nn.LayerNorm(__SCREAMING_SNAKE_CASE	,		elementwise_affine=__SCREAMING_SNAKE_CASE       )


   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       ):
    snake_case__     :					Optional[int]    =  self.linear(self.silu(self.emb(__SCREAMING_SNAKE_CASE       )       )       )
    snake_case__     :					List[Any]    =  torch.chunk(__SCREAMING_SNAKE_CASE	,		2       )
    snake_case__     :					Tuple    =  self.norm(__SCREAMING_SNAKE_CASE       ) * (1 + scale) + shift
    return x



class 					__snake_case  (					nn.Module							):






   '''simple docstring'''





   def __init__(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       ):
    super().__init__()

    snake_case__     :					Dict    =  CombinedTimestepLabelEmbeddings(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       )

    snake_case__     :					Union[str, Any]    =  nn.SiLU()
    snake_case__     :					int    =  nn.Linear(__SCREAMING_SNAKE_CASE	,		6 * embedding_dim	,		bias=__SCREAMING_SNAKE_CASE       )
    snake_case__     :					Optional[Any]    =  nn.LayerNorm(__SCREAMING_SNAKE_CASE	,		elementwise_affine=__SCREAMING_SNAKE_CASE	,		eps=1e-6       )


   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE=None       ):
    snake_case__     :					Any    =  self.linear(self.silu(self.emb(__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		hidden_dtype=__SCREAMING_SNAKE_CASE       )       )       )
    snake_case__     :					int    =  emb.chunk(6	,		dim=1       )
    snake_case__     :					List[Any]    =  self.norm(__SCREAMING_SNAKE_CASE       ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
    return x, gate_msa, shift_mlp, scale_mlp, gate_mlp



class 					__snake_case  (					nn.Module							):






   '''simple docstring'''





   def __init__(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE = None	,		__SCREAMING_SNAKE_CASE = 1e-5       ):
    super().__init__()
    snake_case__     :					Any    =  num_groups
    snake_case__     :					Any    =  eps

    if act_fn is None:
     snake_case__     :					str    =  None
    else:
     snake_case__     :					Union[str, Any]    =  get_activation(__SCREAMING_SNAKE_CASE       )

    snake_case__     :					int    =  nn.Linear(__SCREAMING_SNAKE_CASE	,		out_dim * 2       )


   def 		__UpperCamelCase						(							self	,		__SCREAMING_SNAKE_CASE	,		__SCREAMING_SNAKE_CASE       ):




    if self.act:
     snake_case__     :					Optional[int]    =  self.act(__SCREAMING_SNAKE_CASE       )
    snake_case__     :					Optional[Any]    =  self.linear(__SCREAMING_SNAKE_CASE       )
    snake_case__     :					Optional[Any]    =  emb[:, :, None, None]
    snake_case__     :					Any    =  emb.chunk(2	,		dim=1       )

    snake_case__     :					List[str]    =  F.group_norm(__SCREAMING_SNAKE_CASE	,		self.num_groups	,		eps=self.eps       )
    snake_case__     :					str    =  x * (1 + scale) + shift
    return x











from __future__ import annotations

from cmath import sqrt

def   __snake_case    (       _lowerCAmelCase			:			int					,					_lowerCAmelCase			:			int					,					_lowerCAmelCase			:			int						)			->    tuple[complex, complex]:

   if a == 0:
      raise ValueError("Coefficient 'a' must not be zero."						)
   A_       :					Tuple        =      b * b - 4 * a * c

   A_       :					Any        =      (-b + sqrt(_lowerCAmelCase						)) / (2 * a)
   A_       :					Union[str, Any]        =      (-b - sqrt(_lowerCAmelCase						)) / (2 * a)

   return (
       root_a.real if not root_a.imag else root_a,
       root_a.real if not root_a.imag else root_a,
   )

def   __snake_case    (       )			->    List[Any]:
   A_   ,    A_       :					Optional[Any]        =      quadratic_roots(a=5					,					b=6					,					c=1						)
   print(f"The solutions are: {solutiona} and {solutiona}"						)


if __name__ == "__main__":
 main()
