

from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings



@maybe_allow_in_graph
class 							_UpperCAmelCase			( nn.Module      ):
  '''simple docstring'''







  def __init__(			self						:			List[Any]		,    UpperCamelCase__						:			int		,    UpperCamelCase__						:			int		,    UpperCamelCase__						:			int		,    UpperCamelCase__						:			Optional[Any]=0.0		,    UpperCamelCase__						:			Optional[int] = None		,    UpperCamelCase__						:			str = "geglu"		,    UpperCamelCase__						:			Optional[int] = None		,    UpperCamelCase__						:			bool = False		,    UpperCamelCase__						:			bool = False		,    UpperCamelCase__						:			bool = False		,    UpperCamelCase__						:			bool = False		,    UpperCamelCase__						:			bool = True		,    UpperCamelCase__						:			str = "layer_norm"		,    UpperCamelCase__						:			bool = False		,    ):
        super().__init__()
        A 			=					only_cross_attention

        A 			=					(num_embeds_ada_norm is not None) and norm_type == """ada_norm_zero"""
        A 			=					(num_embeds_ada_norm is not None) and norm_type == """ada_norm"""

        if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
              raise ValueError(
                  f'''`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'''
                  f''' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'''			)

        # Define 3 blocks. Each block has its own normalization layer.
        # 1. Self-Attn
        if self.use_ada_layer_norm:
              A 			=					AdaLayerNorm(snake_case_		,    snake_case_			)
        elif self.use_ada_layer_norm_zero:
              A 			=					AdaLayerNormZero(snake_case_		,    snake_case_			)
        else:
              A 			=					nn.LayerNorm(snake_case_		,    elementwise_affine=snake_case_			)
        A 			=					Attention(
            query_dim=snake_case_		,    heads=snake_case_		,    dim_head=snake_case_		,    dropout=snake_case_		,    bias=snake_case_		,    cross_attention_dim=cross_attention_dim if only_cross_attention else None		,    upcast_attention=snake_case_		,    )

        # 2. Cross-Attn
        if cross_attention_dim is not None or double_self_attention:
              # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
              # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
              # the second cross attention block.
              A 			=					(
                  AdaLayerNorm(snake_case_		,    snake_case_			)
                  if self.use_ada_layer_norm
                  else nn.LayerNorm(snake_case_		,    elementwise_affine=snake_case_			)
              )
              A 			=					Attention(
                  query_dim=snake_case_		,    cross_attention_dim=cross_attention_dim if not double_self_attention else None		,    heads=snake_case_		,    dim_head=snake_case_		,    dropout=snake_case_		,    bias=snake_case_		,    upcast_attention=snake_case_		,    )  # is self-attn if encoder_hidden_states is none
        else:
              A 			=					None
              A 			=					None

        # 3. Feed-forward
        A 			=					nn.LayerNorm(snake_case_		,    elementwise_affine=snake_case_			)
        A 			=					FeedForward(snake_case_		,    dropout=snake_case_		,    activation_fn=snake_case_		,    final_dropout=snake_case_			)

        # let chunk size default to None
        A 			=					None
        A 			=					0







  def 						UpperCamelCase							(			self						:			Optional[Any]		,    UpperCamelCase__						:			Optional[int]		,    UpperCamelCase__						:			int			):
        # Sets chunk feed-forward
        A 			=					chunk_size
        A 			=					dim







  def 						UpperCamelCase							(			self						:			Tuple		,    UpperCamelCase__						:			torch.FloatTensor		,    UpperCamelCase__						:			Optional[torch.FloatTensor] = None		,    UpperCamelCase__						:			Optional[torch.FloatTensor] = None		,    UpperCamelCase__						:			Optional[torch.FloatTensor] = None		,    UpperCamelCase__						:			Optional[torch.LongTensor] = None		,    UpperCamelCase__						:			Dict[str, Any] = None		,    UpperCamelCase__						:			Optional[torch.LongTensor] = None		,    ):
        # Notice that normalization is always applied before the real computation in the following blocks.
        # 1. Self-Attention
        if self.use_ada_layer_norm:
              A 			=					self.norma(snake_case_		,    snake_case_			)
        elif self.use_ada_layer_norm_zero:
              A 			=					self.norma(
                  snake_case_		,    snake_case_		,    snake_case_		,    hidden_dtype=hidden_states.dtype			)
        else:
              A 			=					self.norma(snake_case_			)

        A 			=					cross_attention_kwargs if cross_attention_kwargs is not None else {}

        A 			=					self.attna(
            snake_case_		,    encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None		,    attention_mask=snake_case_		,    **snake_case_		,    )
        if self.use_ada_layer_norm_zero:
              A 			=					gate_msa.unsqueeze(1			) * attn_output
        A 			=					attn_output + hidden_states

        # 2. Cross-Attention
        if self.attna is not None:
              A 			=					(
                  self.norma(snake_case_		,    snake_case_			) if self.use_ada_layer_norm else self.norma(snake_case_			)
              )

              A 			=					self.attna(
                  snake_case_		,    encoder_hidden_states=snake_case_		,    attention_mask=snake_case_		,    **snake_case_		,    )
              A 			=					attn_output + hidden_states

        # 3. Feed-forward
        A 			=					self.norma(snake_case_			)

        if self.use_ada_layer_norm_zero:
              A 			=					norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

        if self._chunk_size is not None:
              # "feed_forward_chunk_size" can be used to save memory
              if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
                    raise ValueError(
                        f'''`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'''			)

              A 			=					norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
              A 			=					torch.cat(
                  [self.ff(snake_case_			) for hid_slice in norm_hidden_states.chunk(snake_case_		,    dim=self._chunk_dim			)]		,    dim=self._chunk_dim		,    )
        else:
              A 			=					self.ff(snake_case_			)

        if self.use_ada_layer_norm_zero:
              A 			=					gate_mlp.unsqueeze(1			) * ff_output

        A 			=					ff_output + hidden_states

        return hidden_states



class 							_UpperCAmelCase			( nn.Module      ):
  '''simple docstring'''







  def __init__(			self						:			Union[str, Any]		,    UpperCamelCase__						:			int		,    UpperCamelCase__						:			Optional[int] = None		,    UpperCamelCase__						:			int = 4		,    UpperCamelCase__						:			float = 0.0		,    UpperCamelCase__						:			str = "geglu"		,    UpperCamelCase__						:			bool = False		,    ):
        super().__init__()
        A 			=					int(dim * mult			)
        A 			=					dim_out if dim_out is not None else dim

        if activation_fn == "gelu":
              A 			=					GELU(snake_case_		,    snake_case_			)
        if activation_fn == "gelu-approximate":
              A 			=					GELU(snake_case_		,    snake_case_		,    approximate='tanh'			)
        elif activation_fn == "geglu":
              A 			=					GEGLU(snake_case_		,    snake_case_			)
        elif activation_fn == "geglu-approximate":
              A 			=					ApproximateGELU(snake_case_		,    snake_case_			)

        A 			=					nn.ModuleList([]			)
        # project in
        self.net.append(snake_case_			)
        # project dropout
        self.net.append(nn.Dropout(snake_case_			)			)
        # project out
        self.net.append(nn.Linear(snake_case_		,    snake_case_			)			)
        # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
        if final_dropout:
              self.net.append(nn.Dropout(snake_case_			)			)







  def 						UpperCamelCase							(			self						:			str		,    UpperCamelCase__						:			int			):
        for module in self.net:
              A 			=					module(snake_case_			)
        return hidden_states



class 							_UpperCAmelCase			( nn.Module      ):
  '''simple docstring'''







  def __init__(			self						:			Optional[Any]		,    UpperCamelCase__						:			int		,    UpperCamelCase__						:			int		,    UpperCamelCase__						:			str = "none"			):
        super().__init__()
        A 			=					nn.Linear(snake_case_		,    snake_case_			)
        A 			=					approximate







  def 						UpperCamelCase							(			self						:			Optional[Any]		,    UpperCamelCase__						:			Optional[int]			):
        if gate.device.type != "mps":
              return F.gelu(snake_case_		,    approximate=self.approximate			)
        # mps: gelu is not implemented for float16
        return F.gelu(gate.to(dtype=torch.floataa			)		,    approximate=self.approximate			).to(dtype=gate.dtype			)







  def 						UpperCamelCase							(			self						:			Any		,    UpperCamelCase__						:			Optional[int]			):
        A 			=					self.proj(snake_case_			)
        A 			=					self.gelu(snake_case_			)
        return hidden_states



class 							_UpperCAmelCase			( nn.Module      ):
  '''simple docstring'''







  def __init__(			self						:			List[Any]		,    UpperCamelCase__						:			int		,    UpperCamelCase__						:			int			):
        super().__init__()
        A 			=					nn.Linear(snake_case_		,    dim_out * 2			)







  def 						UpperCamelCase							(			self						:			Optional[Any]		,    UpperCamelCase__						:			str			):
        if gate.device.type != "mps":
              return F.gelu(snake_case_			)
        # mps: gelu is not implemented for float16
        return F.gelu(gate.to(dtype=torch.floataa			)			).to(dtype=gate.dtype			)







  def 						UpperCamelCase							(			self						:			Optional[int]		,    UpperCamelCase__						:			Dict			):
        A 			=					self.proj(snake_case_			).chunk(2		,    dim=-1			)
        return hidden_states * self.gelu(snake_case_			)



class 							_UpperCAmelCase			( nn.Module      ):
  '''simple docstring'''







  def __init__(			self						:			Optional[Any]		,    UpperCamelCase__						:			int		,    UpperCamelCase__						:			int			):
        super().__init__()
        A 			=					nn.Linear(snake_case_		,    snake_case_			)







  def 						UpperCamelCase							(			self						:			Union[str, Any]		,    UpperCamelCase__						:			Tuple			):
        A 			=					self.proj(snake_case_			)
        return x * torch.sigmoid(1.702 * x			)



class 							_UpperCAmelCase			( nn.Module      ):
  '''simple docstring'''







  def __init__(			self						:			List[str]		,    UpperCamelCase__						:			Optional[Any]		,    UpperCamelCase__						:			List[Any]			):
        super().__init__()
        A 			=					nn.Embedding(snake_case_		,    snake_case_			)
        A 			=					nn.SiLU()
        A 			=					nn.Linear(snake_case_		,    embedding_dim * 2			)
        A 			=					nn.LayerNorm(snake_case_		,    elementwise_affine=snake_case_			)







  def 						UpperCamelCase							(			self						:			int		,    UpperCamelCase__						:			Union[str, Any]		,    UpperCamelCase__						:			Optional[int]			):
        A 			=					self.linear(self.silu(self.emb(snake_case_			)			)			)
        A 			=					torch.chunk(snake_case_		,    2			)
        A 			=					self.norm(snake_case_			) * (1 + scale) + shift
        return x



class 							_UpperCAmelCase			( nn.Module      ):
  '''simple docstring'''







  def __init__(			self						:			List[Any]		,    UpperCamelCase__						:			str		,    UpperCamelCase__						:			Optional[int]			):
        super().__init__()

        A 			=					CombinedTimestepLabelEmbeddings(snake_case_		,    snake_case_			)

        A 			=					nn.SiLU()
        A 			=					nn.Linear(snake_case_		,    6 * embedding_dim		,    bias=snake_case_			)
        A 			=					nn.LayerNorm(snake_case_		,    elementwise_affine=snake_case_		,    eps=1e-6			)







  def 						UpperCamelCase							(			self						:			Any		,    UpperCamelCase__						:			Tuple		,    UpperCamelCase__						:			List[Any]		,    UpperCamelCase__						:			Tuple		,    UpperCamelCase__						:			Optional[int]=None			):
        A 			=					self.linear(self.silu(self.emb(snake_case_		,    snake_case_		,    hidden_dtype=snake_case_			)			)			)
        A 			=					emb.chunk(6		,    dim=1			)
        A 			=					self.norm(snake_case_			) * (1 + scale_msa[:, None]) + shift_msa[:, None]
        return x, gate_msa, shift_mlp, scale_mlp, gate_mlp



class 							_UpperCAmelCase			( nn.Module      ):
  '''simple docstring'''







  def __init__(			self						:			Any		,    UpperCamelCase__						:			int		,    UpperCamelCase__						:			int		,    UpperCamelCase__						:			int		,    UpperCamelCase__						:			Optional[str] = None		,    UpperCamelCase__						:			float = 1e-5			):
        super().__init__()
        A 			=					num_groups
        A 			=					eps

        if act_fn is None:
              A 			=					None
        else:
              A 			=					get_activation(snake_case_			)

        A 			=					nn.Linear(snake_case_		,    out_dim * 2			)







  def 						UpperCamelCase							(			self						:			List[str]		,    UpperCamelCase__						:			str		,    UpperCamelCase__						:			str			):
        if self.act:
              A 			=					self.act(snake_case_			)
        A 			=					self.linear(snake_case_			)
        A 			=					emb[:, :, None, None]
        A 			=					emb.chunk(2		,    dim=1			)

        A 			=					F.group_norm(snake_case_		,    self.num_groups		,    eps=self.eps			)
        A 			=					x * (1 + scale) + shift
        return x





lowerCamelCase_    :			Tuple    =	{
    """meter""": """m""",
    """kilometer""": """km""",
    """megametre""": """Mm""",
    """gigametre""": """Gm""",
    """terametre""": """Tm""",
    """petametre""": """Pm""",
    """exametre""": """Em""",
    """zettametre""": """Zm""",
    """yottametre""": """Ym""",
}
# Exponent of the factor(meter)
lowerCamelCase_    :			Union[str, Any]    =	{
    """m""": 0,
    """km""": 3,
    """Mm""": 6,
    """Gm""": 9,
    """Tm""": 12,
    """Pm""": 15,
    """Em""": 18,
    """Zm""": 21,
    """Ym""": 24,
}







def 							A__       (    lowerCamelCase  ,     lowerCamelCase  ,     lowerCamelCase    )	->						float:
				UpperCamelCase_:				Any						=    from_type.lower().strip("""s"""    )
				UpperCamelCase_:				int						=    to_type.lower().strip("""s"""    )

				UpperCamelCase_:				Any						=    UNIT_SYMBOL.get(lowerCamelCase  ,     lowerCamelCase    )
				UpperCamelCase_:				str						=    UNIT_SYMBOL.get(lowerCamelCase  ,     lowerCamelCase    )

				if from_sanitized not in METRIC_CONVERSION:
								UpperCamelCase_:				Optional[int]						=    (
								    F'''Invalid \'from_type\' value: {from_type!r}.\n'''
								    F'''Conversion abbreviations are: {", ".join(lowerCamelCase    )}'''
								)
								raise ValueError(lowerCamelCase    )
				if to_sanitized not in METRIC_CONVERSION:
								UpperCamelCase_:				Dict						=    (
								    F'''Invalid \'to_type\' value: {to_type!r}.\n'''
								    F'''Conversion abbreviations are: {", ".join(lowerCamelCase    )}'''
								)
								raise ValueError(lowerCamelCase    )
				UpperCamelCase_:				Union[str, Any]						=    METRIC_CONVERSION[from_sanitized]
				UpperCamelCase_:				str						=    METRIC_CONVERSION[to_sanitized]
				UpperCamelCase_:				Tuple						=    1

				if from_exponent > to_exponent:
								UpperCamelCase_:				Union[str, Any]						=    from_exponent - to_exponent
				else:
								UpperCamelCase_:				Dict						=    -(to_exponent - from_exponent)

				return value * pow(10  ,     lowerCamelCase    )


if __name__ == "__main__":
							from doctest import testmod

							testmod()






