
"""simple docstring"""




from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings

@maybe_allow_in_graph
class 						a__		( nn.Module		):
   def __init__(     self      ,       _a      ,       _a      ,       _a      ,       _a=0.0      ,       _a = None      ,       _a = "geglu"      ,       _a = None      ,       _a = False      ,       _a = False      ,       _a = False      ,       _a = False      ,       _a = True      ,       _a = "layer_norm"      ,       _a = False      ,       ):
          super().__init__()
          lowercase       :							Optional[int]									=				only_cross_attention

          lowercase       :							Optional[int]									=				(num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
          lowercase       :							Any									=				(num_embeds_ada_norm is not None) and norm_type == "ada_norm"

          if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
                 raise ValueError(
                     f"""`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"""
                     f""" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."""							)

          # Define 3 blocks. Each block has its own normalization layer.
          # 1. Self-Attn
          if self.use_ada_layer_norm:
                 lowercase       :							Dict									=				AdaLayerNorm(_a      ,       _a							)
          elif self.use_ada_layer_norm_zero:
                 lowercase       :							Tuple									=				AdaLayerNormZero(_a      ,       _a							)
          else:
                 lowercase       :							List[Any]									=				nn.LayerNorm(_a      ,       elementwise_affine=_a							)
          lowercase       :							List[str]									=				Attention(
              query_dim=_a      ,       heads=_a      ,       dim_head=_a      ,       dropout=_a      ,       bias=_a      ,       cross_attention_dim=cross_attention_dim if only_cross_attention else None      ,       upcast_attention=_a      ,       )

          # 2. Cross-Attn
          if cross_attention_dim is not None or double_self_attention:
                 # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
                 # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
                 # the second cross attention block.
                 lowercase       :							Tuple									=				(
                     AdaLayerNorm(_a      ,       _a							)
                     if self.use_ada_layer_norm
                     else nn.LayerNorm(_a      ,       elementwise_affine=_a							)
                 )
                 lowercase       :							List[str]									=				Attention(
                     query_dim=_a      ,       cross_attention_dim=cross_attention_dim if not double_self_attention else None      ,       heads=_a      ,       dim_head=_a      ,       dropout=_a      ,       bias=_a      ,       upcast_attention=_a      ,       )  # is self-attn if encoder_hidden_states is none
          else:
                 lowercase       :							Union[str, Any]									=				None
                 lowercase       :							Tuple									=				None

          # 3. Feed-forward
          lowercase       :							str									=				nn.LayerNorm(_a      ,       elementwise_affine=_a							)
          lowercase       :							Optional[Any]									=				FeedForward(_a      ,       dropout=_a      ,       activation_fn=_a      ,       final_dropout=_a							)

          # let chunk size default to None
          lowercase       :							int									=				None
          lowercase       :							int									=				0
   def      __magic_name__     (     self      ,       _a      ,       _a							):
          # Sets chunk feed-forward
          lowercase       :							int									=				chunk_size
          lowercase       :							List[str]									=				dim

   def      __magic_name__     (     self      ,       _a      ,       _a = None      ,       _a = None      ,       _a = None      ,       _a = None      ,       _a = None      ,       _a = None      ,       ):
          # Notice that normalization is always applied before the real computation in the following blocks.
          # 1. Self-Attention
          if self.use_ada_layer_norm:
                 lowercase       :							Optional[int]									=				self.norma(_a      ,       _a							)
          elif self.use_ada_layer_norm_zero:
                 lowercase						,       lowercase						,       lowercase						,       lowercase						,       lowercase       :							List[Any]									=				self.norma(
                     _a      ,       _a      ,       _a      ,       hidden_dtype=hidden_states.dtype							)
          else:
                 lowercase       :							List[Any]									=				self.norma(_a							)

          lowercase       :							Optional[int]									=				cross_attention_kwargs if cross_attention_kwargs is not None else {}

          lowercase       :							List[str]									=				self.attna(
              _a      ,       encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None      ,       attention_mask=_a      ,       **_a      ,       )
          if self.use_ada_layer_norm_zero:
                 lowercase       :							str									=				gate_msa.unsqueeze(1							) * attn_output
          lowercase       :							List[Any]									=				attn_output + hidden_states

          # 2. Cross-Attention
          if self.attna is not None:
                 lowercase       :							int									=				(
                     self.norma(_a      ,       _a							) if self.use_ada_layer_norm else self.norma(_a							)
                 )

                 lowercase       :							List[Any]									=				self.attna(
                     _a      ,       encoder_hidden_states=_a      ,       attention_mask=_a      ,       **_a      ,       )
                 lowercase       :							Any									=				attn_output + hidden_states

          # 3. Feed-forward
          lowercase       :							Dict									=				self.norma(_a							)

          if self.use_ada_layer_norm_zero:
                 lowercase       :							Any									=				norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

          if self._chunk_size is not None:
                 # "feed_forward_chunk_size" can be used to save memory
                 if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
                        raise ValueError(
                            f"""`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."""							)

                 lowercase       :							List[str]									=				norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
                 lowercase       :							Any									=				torch.cat(
                     [self.ff(_a							) for hid_slice in norm_hidden_states.chunk(_a      ,       dim=self._chunk_dim							)]      ,       dim=self._chunk_dim      ,       )
          else:
                 lowercase       :							Optional[Any]									=				self.ff(_a							)

          if self.use_ada_layer_norm_zero:
                 lowercase       :							Optional[int]									=				gate_mlp.unsqueeze(1							) * ff_output

          lowercase       :							List[str]									=				ff_output + hidden_states

          return hidden_states


class 						a__		( nn.Module		):
   def __init__(     self      ,       _a      ,       _a = None      ,       _a = 4      ,       _a = 0.0      ,       _a = "geglu"      ,       _a = False      ,       ):
          super().__init__()
          lowercase       :							Dict									=				int(dim * mult							)
          lowercase       :							Any									=				dim_out if dim_out is not None else dim

          if activation_fn == "gelu":
                 lowercase       :							Optional[Any]									=				GELU(_a      ,       _a							)
          if activation_fn == "gelu-approximate":
                 lowercase       :							str									=				GELU(_a      ,       _a      ,       approximate="tanh"							)
          elif activation_fn == "geglu":
                 lowercase       :							Optional[Any]									=				GEGLU(_a      ,       _a							)
          elif activation_fn == "geglu-approximate":
                 lowercase       :							List[Any]									=				ApproximateGELU(_a      ,       _a							)

          lowercase       :							Optional[int]									=				nn.ModuleList([]							)
          # project in
          self.net.append(_a							)
          # project dropout
          self.net.append(nn.Dropout(_a							)							)
          # project out
          self.net.append(nn.Linear(_a      ,       _a							)							)
          # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
          if final_dropout:
                 self.net.append(nn.Dropout(_a							)							)

   def      __magic_name__     (     self      ,       _a							):
          for module in self.net:
                 lowercase       :							Union[str, Any]									=				module(_a							)
          return hidden_states


class 						a__		( nn.Module		):
   def __init__(     self      ,       _a      ,       _a      ,       _a = "none"							):
          super().__init__()
          lowercase       :							Dict									=				nn.Linear(_a      ,       _a							)
          lowercase       :							List[Any]									=				approximate
   def      __magic_name__     (     self      ,       _a							):
          if gate.device.type != "mps":
                 return F.gelu(_a      ,       approximate=self.approximate							)
          # mps: gelu is not implemented for float16
          return F.gelu(gate.to(dtype=torch.floataa							)      ,       approximate=self.approximate							).to(dtype=gate.dtype							)

   def      __magic_name__     (     self      ,       _a							):
          lowercase       :							int									=				self.proj(_a							)
          lowercase       :							str									=				self.gelu(_a							)
          return hidden_states


class 						a__		( nn.Module		):
   def __init__(     self      ,       _a      ,       _a							):
          super().__init__()
          lowercase       :							Union[str, Any]									=				nn.Linear(_a      ,       dim_out * 2							)
   def      __magic_name__     (     self      ,       _a							):
          if gate.device.type != "mps":
                 return F.gelu(_a							)
          # mps: gelu is not implemented for float16
          return F.gelu(gate.to(dtype=torch.floataa							)							).to(dtype=gate.dtype							)

   def      __magic_name__     (     self      ,       _a							):
          lowercase						,       lowercase       :							Optional[Any]									=				self.proj(_a							).chunk(2      ,       dim=-1							)
          return hidden_states * self.gelu(_a							)


class 						a__		( nn.Module		):
   def __init__(     self      ,       _a      ,       _a							):
          super().__init__()
          lowercase       :							List[str]									=				nn.Linear(_a      ,       _a							)

   def      __magic_name__     (     self      ,       _a							):
          lowercase       :							List[Any]									=				self.proj(_a							)
          return x * torch.sigmoid(1.7_0_2 * x							)


class 						a__		( nn.Module		):
   def __init__(     self      ,       _a      ,       _a							):
          super().__init__()
          lowercase       :							str									=				nn.Embedding(_a      ,       _a							)
          lowercase       :							Tuple									=				nn.SiLU()
          lowercase       :							Dict									=				nn.Linear(_a      ,       embedding_dim * 2							)
          lowercase       :							List[str]									=				nn.LayerNorm(_a      ,       elementwise_affine=_a							)

   def      __magic_name__     (     self      ,       _a      ,       _a							):
          lowercase       :							Any									=				self.linear(self.silu(self.emb(_a							)							)							)
          lowercase						,       lowercase       :							Union[str, Any]									=				torch.chunk(_a      ,       2							)
          lowercase       :							List[Any]									=				self.norm(_a							) * (1 + scale) + shift
          return x


class 						a__		( nn.Module		):
   def __init__(     self      ,       _a      ,       _a							):
          super().__init__()

          lowercase       :							Optional[int]									=				CombinedTimestepLabelEmbeddings(_a      ,       _a							)

          lowercase       :							Any									=				nn.SiLU()
          lowercase       :							Dict									=				nn.Linear(_a      ,       6 * embedding_dim      ,       bias=_a							)
          lowercase       :							Union[str, Any]									=				nn.LayerNorm(_a      ,       elementwise_affine=_a      ,       eps=1E-6							)

   def      __magic_name__     (     self      ,       _a      ,       _a      ,       _a      ,       _a=None							):
          lowercase       :							Optional[int]									=				self.linear(self.silu(self.emb(_a      ,       _a      ,       hidden_dtype=_a							)							)							)
          lowercase						,       lowercase						,       lowercase						,       lowercase						,       lowercase						,       lowercase       :							List[str]									=				emb.chunk(6      ,       dim=1							)
          lowercase       :							Any									=				self.norm(_a							) * (1 + scale_msa[:, None]) + shift_msa[:, None]
          return x, gate_msa, shift_mlp, scale_mlp, gate_mlp


class 						a__		( nn.Module		):
   def __init__(     self      ,       _a      ,       _a      ,       _a      ,       _a = None      ,       _a = 1E-5							):
          super().__init__()
          lowercase       :							int									=				num_groups
          lowercase       :							Optional[Any]									=				eps

          if act_fn is None:
                 lowercase       :							List[str]									=				None
          else:
                 lowercase       :							List[str]									=				get_activation(_a							)

          lowercase       :							Dict									=				nn.Linear(_a      ,       out_dim * 2							)

   def      __magic_name__     (     self      ,       _a      ,       _a							):
          if self.act:
                 lowercase       :							Union[str, Any]									=				self.act(_a							)
          lowercase       :							Union[str, Any]									=				self.linear(_a							)
          lowercase       :							List[Any]									=				emb[:, :, None, None]
          lowercase						,       lowercase       :							Tuple									=				emb.chunk(2      ,       dim=1							)

          lowercase       :							List[str]									=				F.group_norm(_a      ,       self.num_groups      ,       eps=self.eps							)
          lowercase       :							Optional[int]									=				x * (1 + scale) + shift
          return x




"""simple docstring"""




def     __magic_name__      ( __snake_case							:			int	,					__snake_case							:			int    )					->							Any:
       if b == 0:
              return 1
       if (b % 2) == 0:
              return actual_power(__snake_case	,					int(b / 2    )    ) * actual_power(__snake_case	,					int(b / 2    )    )
       else:
              return a * actual_power(__snake_case	,					int(b / 2    )    ) * actual_power(__snake_case	,					int(b / 2    )    )


def     __magic_name__      ( __snake_case							:			int	,					__snake_case							:			int    )					->							float:
       if b < 0:
              return 1 / actual_power(__snake_case	,					__snake_case    )
       return actual_power(__snake_case	,					__snake_case    )


if __name__ == "__main__":
   print(power(-2, -3))


