

from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings

@maybe_allow_in_graph
class 				__UpperCamelCase				(					nn.Module			):







	def __init__(   self: Optional[Any]  , __UpperCamelCase: int  , __UpperCamelCase: int  , __UpperCamelCase: int  , __UpperCamelCase: Dict=0.0  , __UpperCamelCase: Optional[int] = None  , __UpperCamelCase: str = "geglu"  , __UpperCamelCase: Optional[int] = None  , __UpperCamelCase: bool = False  , __UpperCamelCase: bool = False  , __UpperCamelCase: bool = False  , __UpperCamelCase: bool = False  , __UpperCamelCase: bool = True  , __UpperCamelCase: str = "layer_norm"  , __UpperCamelCase: bool = False  , ):

						'''simple docstring'''





						super().__init__()
						__magic_name__							=    only_cross_attention

						__magic_name__							=    (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
						__magic_name__							=    (num_embeds_ada_norm is not None) and norm_type == "ada_norm"

						if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
											raise ValueError(
											    F'`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'
											    F' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'      )

						# Define 3 blocks. Each block has its own normalization layer.
						# 1. Self-Attn
						if self.use_ada_layer_norm:
											__magic_name__							=    AdaLayerNorm(__a  , __a      )
						elif self.use_ada_layer_norm_zero:
											__magic_name__							=    AdaLayerNormZero(__a  , __a      )
						else:
											__magic_name__							=    nn.LayerNorm(__a  , elementwise_affine=__a      )
						__magic_name__							=    Attention(
						    query_dim=__a  , heads=__a  , dim_head=__a  , dropout=__a  , bias=__a  , cross_attention_dim=cross_attention_dim if only_cross_attention else None  , upcast_attention=__a  , )

						# 2. Cross-Attn
						if cross_attention_dim is not None or double_self_attention:
											# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
											# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
											# the second cross attention block.
											__magic_name__							=    (
											    AdaLayerNorm(__a  , __a      )
											    if self.use_ada_layer_norm
											    else nn.LayerNorm(__a  , elementwise_affine=__a      )
											)
											__magic_name__							=    Attention(
											    query_dim=__a  , cross_attention_dim=cross_attention_dim if not double_self_attention else None  , heads=__a  , dim_head=__a  , dropout=__a  , bias=__a  , upcast_attention=__a  , )  # is self-attn if encoder_hidden_states is none
						else:
											__magic_name__							=    None
											__magic_name__							=    None

						# 3. Feed-forward
						__magic_name__							=    nn.LayerNorm(__a  , elementwise_affine=__a      )
						__magic_name__							=    FeedForward(__a  , dropout=__a  , activation_fn=__a  , final_dropout=__a      )

						# let chunk size default to None
						__magic_name__							=    None
						__magic_name__							=    0







	def 	_SCREAMING_SNAKE_CASE							(   self: List[str]  , __UpperCamelCase: Optional[int]  , __UpperCamelCase: int      ):

						'''simple docstring'''





						__magic_name__							=    chunk_size
						__magic_name__							=    dim







	def 	_SCREAMING_SNAKE_CASE							(   self: List[Any]  , __UpperCamelCase: torch.FloatTensor  , __UpperCamelCase: Optional[torch.FloatTensor] = None  , __UpperCamelCase: Optional[torch.FloatTensor] = None  , __UpperCamelCase: Optional[torch.FloatTensor] = None  , __UpperCamelCase: Optional[torch.LongTensor] = None  , __UpperCamelCase: Dict[str, Any] = None  , __UpperCamelCase: Optional[torch.LongTensor] = None  , ):

						'''simple docstring'''





						if self.use_ada_layer_norm:
											__magic_name__							=    self.norma(__a  , __a      )
						elif self.use_ada_layer_norm_zero:
											__magic_name__							=    self.norma(
											    __a  , __a  , __a  , hidden_dtype=hidden_states.dtype      )
						else:
											__magic_name__							=    self.norma(__a      )

						__magic_name__							=    cross_attention_kwargs if cross_attention_kwargs is not None else {}

						__magic_name__							=    self.attna(
						    __a  , encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None  , attention_mask=__a  , **__a  , )
						if self.use_ada_layer_norm_zero:
											__magic_name__							=    gate_msa.unsqueeze(1      ) * attn_output
						__magic_name__							=    attn_output + hidden_states

						# 2. Cross-Attention
						if self.attna is not None:
											__magic_name__							=    (
											    self.norma(__a  , __a      ) if self.use_ada_layer_norm else self.norma(__a      )
											)

											__magic_name__							=    self.attna(
											    __a  , encoder_hidden_states=__a  , attention_mask=__a  , **__a  , )
											__magic_name__							=    attn_output + hidden_states

						# 3. Feed-forward
						__magic_name__							=    self.norma(__a      )

						if self.use_ada_layer_norm_zero:
											__magic_name__							=    norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

						if self._chunk_size is not None:
											# "feed_forward_chunk_size" can be used to save memory
											if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
																raise ValueError(
																    F'`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'      )

											__magic_name__							=    norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
											__magic_name__							=    torch.cat(
											    [self.ff(__a      ) for hid_slice in norm_hidden_states.chunk(__a  , dim=self._chunk_dim      )]  , dim=self._chunk_dim  , )
						else:
											__magic_name__							=    self.ff(__a      )

						if self.use_ada_layer_norm_zero:
											__magic_name__							=    gate_mlp.unsqueeze(1      ) * ff_output

						__magic_name__							=    ff_output + hidden_states

						return hidden_states




class 				__UpperCamelCase				(					nn.Module			):







	def __init__(   self: Optional[int]  , __UpperCamelCase: int  , __UpperCamelCase: Optional[int] = None  , __UpperCamelCase: int = 4  , __UpperCamelCase: float = 0.0  , __UpperCamelCase: str = "geglu"  , __UpperCamelCase: bool = False  , ):

						'''simple docstring'''





						super().__init__()
						__magic_name__							=    int(dim * mult      )
						__magic_name__							=    dim_out if dim_out is not None else dim

						if activation_fn == "gelu":
											__magic_name__							=    GELU(__a  , __a      )
						if activation_fn == "gelu-approximate":
											__magic_name__							=    GELU(__a  , __a  , approximate='tanh'      )
						elif activation_fn == "geglu":
											__magic_name__							=    GEGLU(__a  , __a      )
						elif activation_fn == "geglu-approximate":
											__magic_name__							=    ApproximateGELU(__a  , __a      )

						__magic_name__							=    nn.ModuleList([]      )
						# project in
						self.net.append(__a      )
						# project dropout
						self.net.append(nn.Dropout(__a      )      )
						# project out
						self.net.append(nn.Linear(__a  , __a      )      )
						# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
						if final_dropout:
											self.net.append(nn.Dropout(__a      )      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Tuple  , __UpperCamelCase: Tuple      ):

						'''simple docstring'''





						for module in self.net:
											__magic_name__							=    module(__a      )
						return hidden_states




class 				__UpperCamelCase				(					nn.Module			):







	def __init__(   self: int  , __UpperCamelCase: int  , __UpperCamelCase: int  , __UpperCamelCase: str = "none"      ):

						'''simple docstring'''





						super().__init__()
						__magic_name__							=    nn.Linear(__a  , __a      )
						__magic_name__							=    approximate







	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[int]  , __UpperCamelCase: int      ):

						'''simple docstring'''





						if gate.device.type != "mps":
											return F.gelu(__a  , approximate=self.approximate      )
						# mps: gelu is not implemented for float16
						return F.gelu(gate.to(dtype=torch.floataa      )  , approximate=self.approximate      ).to(dtype=gate.dtype      )







	def 	_SCREAMING_SNAKE_CASE							(   self: List[str]  , __UpperCamelCase: int      ):

						'''simple docstring'''





						__magic_name__							=    self.proj(__a      )
						__magic_name__							=    self.gelu(__a      )
						return hidden_states




class 				__UpperCamelCase				(					nn.Module			):







	def __init__(   self: List[str]  , __UpperCamelCase: int  , __UpperCamelCase: int      ):

						'''simple docstring'''





						super().__init__()
						__magic_name__							=    nn.Linear(__a  , dim_out * 2      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Any  , __UpperCamelCase: Optional[Any]      ):

						'''simple docstring'''





						if gate.device.type != "mps":
											return F.gelu(__a      )
						# mps: gelu is not implemented for float16
						return F.gelu(gate.to(dtype=torch.floataa      )      ).to(dtype=gate.dtype      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Tuple  , __UpperCamelCase: Tuple      ):

						'''simple docstring'''





						__magic_name__							=    self.proj(__a      ).chunk(2  , dim=-1      )
						return hidden_states * self.gelu(__a      )




class 				__UpperCamelCase				(					nn.Module			):







	def __init__(   self: int  , __UpperCamelCase: int  , __UpperCamelCase: int      ):

						'''simple docstring'''





						super().__init__()
						__magic_name__							=    nn.Linear(__a  , __a      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Union[str, Any]  , __UpperCamelCase: Any      ):

						'''simple docstring'''





						__magic_name__							=    self.proj(__a      )
						return x * torch.sigmoid(1.702 * x      )




class 				__UpperCamelCase				(					nn.Module			):







	def __init__(   self: Dict  , __UpperCamelCase: List[str]  , __UpperCamelCase: Tuple      ):

						'''simple docstring'''





						super().__init__()
						__magic_name__							=    nn.Embedding(__a  , __a      )
						__magic_name__							=    nn.SiLU()
						__magic_name__							=    nn.Linear(__a  , embedding_dim * 2      )
						__magic_name__							=    nn.LayerNorm(__a  , elementwise_affine=__a      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Any  , __UpperCamelCase: str  , __UpperCamelCase: List[Any]      ):

						'''simple docstring'''





						__magic_name__							=    self.linear(self.silu(self.emb(__a      )      )      )
						__magic_name__							=    torch.chunk(__a  , 2      )
						__magic_name__							=    self.norm(__a      ) * (1 + scale) + shift
						return x




class 				__UpperCamelCase				(					nn.Module			):







	def __init__(   self: Tuple  , __UpperCamelCase: List[Any]  , __UpperCamelCase: Tuple      ):

						'''simple docstring'''





						super().__init__()

						__magic_name__							=    CombinedTimestepLabelEmbeddings(__a  , __a      )

						__magic_name__							=    nn.SiLU()
						__magic_name__							=    nn.Linear(__a  , 6 * embedding_dim  , bias=__a      )
						__magic_name__							=    nn.LayerNorm(__a  , elementwise_affine=__a  , eps=1E-6      )







	def 	_SCREAMING_SNAKE_CASE							(   self: Optional[int]  , __UpperCamelCase: str  , __UpperCamelCase: Optional[Any]  , __UpperCamelCase: Dict  , __UpperCamelCase: Tuple=None      ):

						'''simple docstring'''





						__magic_name__							=    self.linear(self.silu(self.emb(__a  , __a  , hidden_dtype=__a      )      )      )
						__magic_name__							=    emb.chunk(6  , dim=1      )
						__magic_name__							=    self.norm(__a      ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
						return x, gate_msa, shift_mlp, scale_mlp, gate_mlp




class 				__UpperCamelCase				(					nn.Module			):







	def __init__(   self: str  , __UpperCamelCase: int  , __UpperCamelCase: int  , __UpperCamelCase: int  , __UpperCamelCase: Optional[str] = None  , __UpperCamelCase: float = 1E-5      ):

						'''simple docstring'''





						super().__init__()
						__magic_name__							=    num_groups
						__magic_name__							=    eps

						if act_fn is None:
											__magic_name__							=    None
						else:
											__magic_name__							=    get_activation(__a      )

						__magic_name__							=    nn.Linear(__a  , out_dim * 2      )







	def 	_SCREAMING_SNAKE_CASE							(   self: List[str]  , __UpperCamelCase: Union[str, Any]  , __UpperCamelCase: Any      ):

						'''simple docstring'''





						if self.act:
											__magic_name__							=    self.act(__a      )
						__magic_name__							=    self.linear(__a      )
						__magic_name__							=    emb[:, :, None, None]
						__magic_name__							=    emb.chunk(2  , dim=1      )

						__magic_name__							=    F.group_norm(__a  , self.num_groups  , eps=self.eps      )
						__magic_name__							=    x * (1 + scale) + shift
						return x







import os

A__						=							{"I": 1, "V": 5, "X": 10, "L": 50, "C": 100, "D": 500, "M": 1000}




def 					_lowercase     (				a_				:					str				)				->	int:




					'''simple docstring'''







					__magic_name__							=    0

					__magic_name__							=    0
					while index < len(a_				) - 1:
										__magic_name__							=    SYMBOLS[numerals[index]]
										__magic_name__							=    SYMBOLS[numerals[index + 1]]
										if current_value < next_value:
															total_value -= current_value
										else:
															total_value += current_value
										index += 1
					total_value += SYMBOLS[numerals[index]]

					return total_value




def 					_lowercase     (				a_				:					int				)				->	str:




					'''simple docstring'''







					__magic_name__							=    ''

					__magic_name__							=    num // 1_0_0_0
					numerals += m_count * "M"
					num %= 1_0_0_0

					__magic_name__							=    num // 1_0_0
					if c_count == 9:
										numerals += "CM"
										c_count -= 9
					elif c_count == 4:
										numerals += "CD"
										c_count -= 4
					if c_count >= 5:
										numerals += "D"
										c_count -= 5
					numerals += c_count * "C"
					num %= 1_0_0

					__magic_name__							=    num // 1_0
					if x_count == 9:
										numerals += "XC"
										x_count -= 9
					elif x_count == 4:
										numerals += "XL"
										x_count -= 4
					if x_count >= 5:
										numerals += "L"
										x_count -= 5
					numerals += x_count * "X"
					num %= 1_0

					if num == 9:
										numerals += "IX"
										num -= 9
					elif num == 4:
										numerals += "IV"
										num -= 4
					if num >= 5:
										numerals += "V"
										num -= 5
					numerals += num * "I"

					return numerals




def 					_lowercase     (				a_				:					str = "/p089_roman.txt"				)				->	int:




					'''simple docstring'''







					__magic_name__							=    0

					with open(os.path.dirname(a_				) + roman_numerals_filename				) as filea:
										__magic_name__							=    filea.readlines()

					for line in lines:
										__magic_name__							=    line.strip()
										__magic_name__							=    parse_roman_numerals(a_				)
										__magic_name__							=    generate_roman_numerals(a_				)
										savings += len(a_				) - len(a_				)

					return savings


if __name__ == "__main__":
						print(f'''{solution() = }''')




