





"""simple docstring"""





from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings





@maybe_allow_in_graph
class        _lowerCamelCase      (       nn.Module ):


						def __init__(self   ,							__a   ,							__a   ,							__a   ,							__a=0.0   ,							__a = None   ,							__a = "geglu"   ,							__a = None   ,							__a = False   ,							__a = False   ,							__a = False   ,							__a = False   ,							__a = True   ,							__a = "layer_norm"   ,							__a = False   ,							) ->   Dict:
								super().__init__()
								UpperCamelCase   =	only_cross_attention

								UpperCamelCase   =	(num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
								UpperCamelCase   =	(num_embeds_ada_norm is not None) and norm_type == "ada_norm"

								if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
										raise ValueError(
										    F"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
										    F" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."		)

								# Define 3 blocks. Each block has its own normalization layer.
								# 1. Self-Attn
								if self.use_ada_layer_norm:
										UpperCamelCase   =	AdaLayerNorm(__a   ,							__a		)
								elif self.use_ada_layer_norm_zero:
										UpperCamelCase   =	AdaLayerNormZero(__a   ,							__a		)
								else:
										UpperCamelCase   =	nn.LayerNorm(__a   ,							elementwise_affine=__a		)
								UpperCamelCase   =	Attention(
								    query_dim=__a   ,							heads=__a   ,							dim_head=__a   ,							dropout=__a   ,							bias=__a   ,							cross_attention_dim=cross_attention_dim if only_cross_attention else None   ,							upcast_attention=__a   ,							)

								# 2. Cross-Attn
								if cross_attention_dim is not None or double_self_attention:
										# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
										# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
										# the second cross attention block.
										UpperCamelCase   =	(
										    AdaLayerNorm(__a   ,							__a		)
										    if self.use_ada_layer_norm
										    else nn.LayerNorm(__a   ,							elementwise_affine=__a		)
										)
										UpperCamelCase   =	Attention(
										    query_dim=__a   ,							cross_attention_dim=cross_attention_dim if not double_self_attention else None   ,							heads=__a   ,							dim_head=__a   ,							dropout=__a   ,							bias=__a   ,							upcast_attention=__a   ,							)  # is self-attn if encoder_hidden_states is none
								else:
										UpperCamelCase   =	None
										UpperCamelCase   =	None

								# 3. Feed-forward
								UpperCamelCase   =	nn.LayerNorm(__a   ,							elementwise_affine=__a		)
								UpperCamelCase   =	FeedForward(__a   ,							dropout=__a   ,							activation_fn=__a   ,							final_dropout=__a		)

								# let chunk size default to None
								UpperCamelCase   =	None
								UpperCamelCase   =	0


						def     snake_case_   (self   ,							__a   ,							__a		) ->   Optional[Any]:
								# Sets chunk feed-forward
								UpperCamelCase   =	chunk_size
								UpperCamelCase   =	dim




						def     snake_case_   (self   ,							__a   ,							__a = None   ,							__a = None   ,							__a = None   ,							__a = None   ,							__a = None   ,							__a = None   ,							) ->   Tuple:
								# Notice that normalization is always applied before the real computation in the following blocks.
								# 1. Self-Attention
								if self.use_ada_layer_norm:
										UpperCamelCase   =	self.norma(__a   ,							__a		)
								elif self.use_ada_layer_norm_zero:
										UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   =	self.norma(
										    __a   ,							__a   ,							__a   ,							hidden_dtype=hidden_states.dtype		)
								else:
										UpperCamelCase   =	self.norma(__a		)

								UpperCamelCase   =	cross_attention_kwargs if cross_attention_kwargs is not None else {}

								UpperCamelCase   =	self.attna(
								    __a   ,							encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None   ,							attention_mask=__a   ,							**__a   ,							)
								if self.use_ada_layer_norm_zero:
										UpperCamelCase   =	gate_msa.unsqueeze(1		) * attn_output
								UpperCamelCase   =	attn_output + hidden_states

								# 2. Cross-Attention
								if self.attna is not None:
										UpperCamelCase   =	(
										    self.norma(__a   ,							__a		) if self.use_ada_layer_norm else self.norma(__a		)
										)

										UpperCamelCase   =	self.attna(
										    __a   ,							encoder_hidden_states=__a   ,							attention_mask=__a   ,							**__a   ,							)
										UpperCamelCase   =	attn_output + hidden_states

								# 3. Feed-forward
								UpperCamelCase   =	self.norma(__a		)

								if self.use_ada_layer_norm_zero:
										UpperCamelCase   =	norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

								if self._chunk_size is not None:
										# "feed_forward_chunk_size" can be used to save memory
										if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
												raise ValueError(
												    F"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."		)

										UpperCamelCase   =	norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
										UpperCamelCase   =	torch.cat(
										    [self.ff(__a		) for hid_slice in norm_hidden_states.chunk(__a   ,							dim=self._chunk_dim		)]   ,							dim=self._chunk_dim   ,							)
								else:
										UpperCamelCase   =	self.ff(__a		)

								if self.use_ada_layer_norm_zero:
										UpperCamelCase   =	gate_mlp.unsqueeze(1		) * ff_output

								UpperCamelCase   =	ff_output + hidden_states

								return hidden_states





class        _lowerCamelCase      (       nn.Module ):


						def __init__(self   ,							__a   ,							__a = None   ,							__a = 4   ,							__a = 0.0   ,							__a = "geglu"   ,							__a = False   ,							) ->   List[str]:
								super().__init__()
								UpperCamelCase   =	int(dim * mult		)
								UpperCamelCase   =	dim_out if dim_out is not None else dim

								if activation_fn == "gelu":
										UpperCamelCase   =	GELU(__a   ,							__a		)
								if activation_fn == "gelu-approximate":
										UpperCamelCase   =	GELU(__a   ,							__a   ,							approximate="tanh"		)
								elif activation_fn == "geglu":
										UpperCamelCase   =	GEGLU(__a   ,							__a		)
								elif activation_fn == "geglu-approximate":
										UpperCamelCase   =	ApproximateGELU(__a   ,							__a		)

								UpperCamelCase   =	nn.ModuleList([]		)
								# project in
								self.net.append(__a		)
								# project dropout
								self.net.append(nn.Dropout(__a		)		)
								# project out
								self.net.append(nn.Linear(__a   ,							__a		)		)
								# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
								if final_dropout:
										self.net.append(nn.Dropout(__a		)		)




						def     snake_case_   (self   ,							__a		) ->   Any:
								for module in self.net:
										UpperCamelCase   =	module(__a		)
								return hidden_states





class        _lowerCamelCase      (       nn.Module ):


						def __init__(self   ,							__a   ,							__a   ,							__a = "none"		) ->   Dict:
								super().__init__()
								UpperCamelCase   =	nn.Linear(__a   ,							__a		)
								UpperCamelCase   =	approximate


						def     snake_case_   (self   ,							__a		) ->   Optional[Any]:
								if gate.device.type != "mps":
										return F.gelu(__a   ,							approximate=self.approximate		)
								# mps: gelu is not implemented for float16
								return F.gelu(gate.to(dtype=torch.floataa		)   ,							approximate=self.approximate		).to(dtype=gate.dtype		)




						def     snake_case_   (self   ,							__a		) ->   str:
								UpperCamelCase   =	self.proj(__a		)
								UpperCamelCase   =	self.gelu(__a		)
								return hidden_states





class        _lowerCamelCase      (       nn.Module ):


						def __init__(self   ,							__a   ,							__a		) ->   Any:
								super().__init__()
								UpperCamelCase   =	nn.Linear(__a   ,							dim_out * 2		)


						def     snake_case_   (self   ,							__a		) ->   List[Any]:
								if gate.device.type != "mps":
										return F.gelu(__a		)
								# mps: gelu is not implemented for float16
								return F.gelu(gate.to(dtype=torch.floataa		)		).to(dtype=gate.dtype		)




						def     snake_case_   (self   ,							__a		) ->   Dict:
								UpperCamelCase   ,				UpperCamelCase   =	self.proj(__a		).chunk(2   ,							dim=-1		)
								return hidden_states * self.gelu(__a		)





class        _lowerCamelCase      (       nn.Module ):


						def __init__(self   ,							__a   ,							__a		) ->   str:
								super().__init__()
								UpperCamelCase   =	nn.Linear(__a   ,							__a		)




						def     snake_case_   (self   ,							__a		) ->   str:
								UpperCamelCase   =	self.proj(__a		)
								return x * torch.sigmoid(1.702 * x		)





class        _lowerCamelCase      (       nn.Module ):


						def __init__(self   ,							__a   ,							__a		) ->   int:
								super().__init__()
								UpperCamelCase   =	nn.Embedding(__a   ,							__a		)
								UpperCamelCase   =	nn.SiLU()
								UpperCamelCase   =	nn.Linear(__a   ,							embedding_dim * 2		)
								UpperCamelCase   =	nn.LayerNorm(__a   ,							elementwise_affine=__a		)




						def     snake_case_   (self   ,							__a   ,							__a		) ->   Union[str, Any]:
								UpperCamelCase   =	self.linear(self.silu(self.emb(__a		)		)		)
								UpperCamelCase   ,				UpperCamelCase   =	torch.chunk(__a   ,							2		)
								UpperCamelCase   =	self.norm(__a		) * (1 + scale) + shift
								return x





class        _lowerCamelCase      (       nn.Module ):


						def __init__(self   ,							__a   ,							__a		) ->   int:
								super().__init__()

								UpperCamelCase   =	CombinedTimestepLabelEmbeddings(__a   ,							__a		)

								UpperCamelCase   =	nn.SiLU()
								UpperCamelCase   =	nn.Linear(__a   ,							6 * embedding_dim   ,							bias=__a		)
								UpperCamelCase   =	nn.LayerNorm(__a   ,							elementwise_affine=__a   ,							eps=1e-6		)




						def     snake_case_   (self   ,							__a   ,							__a   ,							__a   ,							__a=None		) ->   List[str]:
								UpperCamelCase   =	self.linear(self.silu(self.emb(__a   ,							__a   ,							hidden_dtype=__a		)		)		)
								UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   ,				UpperCamelCase   =	emb.chunk(6   ,							dim=1		)
								UpperCamelCase   =	self.norm(__a		) * (1 + scale_msa[:, None]) + shift_msa[:, None]
								return x, gate_msa, shift_mlp, scale_mlp, gate_mlp





class        _lowerCamelCase      (       nn.Module ):


						def __init__(self   ,							__a   ,							__a   ,							__a   ,							__a = None   ,							__a = 1e-5		) ->   Tuple:
								super().__init__()
								UpperCamelCase   =	num_groups
								UpperCamelCase   =	eps

								if act_fn is None:
										UpperCamelCase   =	None
								else:
										UpperCamelCase   =	get_activation(__a		)

								UpperCamelCase   =	nn.Linear(__a   ,							out_dim * 2		)




						def     snake_case_   (self   ,							__a   ,							__a		) ->   Optional[int]:
								if self.act:
										UpperCamelCase   =	self.act(__a		)
								UpperCamelCase   =	self.linear(__a		)
								UpperCamelCase   =	emb[:, :, None, None]
								UpperCamelCase   ,				UpperCamelCase   =	emb.chunk(2   ,							dim=1		)

								UpperCamelCase   =	F.group_norm(__a   ,							self.num_groups   ,							eps=self.eps		)
								UpperCamelCase   =	x * (1 + scale) + shift
								return x













"""simple docstring"""





def       a__					(					_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		return number & 1 == 0


if __name__ == "__main__":
						import doctest

						doctest.testmod()






