

from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings

@maybe_allow_in_graph
class 						__UpperCamelCase    (					nn.Module						):




	def __init__(		self	:						Dict		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						Optional[int]=0.0		,	lowerCAmelCase	:						Optional[int] = None		,	lowerCAmelCase	:						str = "geglu"		,	lowerCAmelCase	:						Optional[int] = None		,	lowerCAmelCase	:						bool = False		,	lowerCAmelCase	:						bool = False		,	lowerCAmelCase	:						bool = False		,	lowerCAmelCase	:						bool = False		,	lowerCAmelCase	:						bool = True		,	lowerCAmelCase	:						str = "layer_norm"		,	lowerCAmelCase	:						bool = False		,	):


		'''simple docstring'''







		super().__init__()
		UpperCAmelCase_       				=			only_cross_attention

		UpperCAmelCase_       				=			(num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero"
		UpperCAmelCase_       				=			(num_embeds_ada_norm is not None) and norm_type == "ada_norm"

		if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
			raise ValueError(
			    F"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to"
			    F" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}."							)

		# Define 3 blocks. Each block has its own normalization layer.
		# 1. Self-Attn
		if self.use_ada_layer_norm:
			UpperCAmelCase_       				=			AdaLayerNorm(lowerCAmelCase		,	lowerCAmelCase							)
		elif self.use_ada_layer_norm_zero:
			UpperCAmelCase_       				=			AdaLayerNormZero(lowerCAmelCase		,	lowerCAmelCase							)
		else:
			UpperCAmelCase_       				=			nn.LayerNorm(lowerCAmelCase		,	elementwise_affine=lowerCAmelCase							)
		UpperCAmelCase_       				=			Attention(
		    query_dim=lowerCAmelCase		,	heads=lowerCAmelCase		,	dim_head=lowerCAmelCase		,	dropout=lowerCAmelCase		,	bias=lowerCAmelCase		,	cross_attention_dim=cross_attention_dim if only_cross_attention else None		,	upcast_attention=lowerCAmelCase		,	)

		# 2. Cross-Attn
		if cross_attention_dim is not None or double_self_attention:
			# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
			# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
			# the second cross attention block.
			UpperCAmelCase_       				=			(
			    AdaLayerNorm(lowerCAmelCase		,	lowerCAmelCase							)
			    if self.use_ada_layer_norm
			    else nn.LayerNorm(lowerCAmelCase		,	elementwise_affine=lowerCAmelCase							)
			)
			UpperCAmelCase_       				=			Attention(
			    query_dim=lowerCAmelCase		,	cross_attention_dim=cross_attention_dim if not double_self_attention else None		,	heads=lowerCAmelCase		,	dim_head=lowerCAmelCase		,	dropout=lowerCAmelCase		,	bias=lowerCAmelCase		,	upcast_attention=lowerCAmelCase		,	)  # is self-attn if encoder_hidden_states is none
		else:
			UpperCAmelCase_       				=			None
			UpperCAmelCase_       				=			None

		# 3. Feed-forward
		UpperCAmelCase_       				=			nn.LayerNorm(lowerCAmelCase		,	elementwise_affine=lowerCAmelCase							)
		UpperCAmelCase_       				=			FeedForward(lowerCAmelCase		,	dropout=lowerCAmelCase		,	activation_fn=lowerCAmelCase		,	final_dropout=lowerCAmelCase							)

		# let chunk size default to None
		UpperCAmelCase_       				=			None
		UpperCAmelCase_       				=			0




	def       __A					(		self	:						str		,	lowerCAmelCase	:						Optional[int]		,	lowerCAmelCase	:						int							):


		'''simple docstring'''







		UpperCAmelCase_       				=			chunk_size
		UpperCAmelCase_       				=			dim




	def       __A					(		self	:						Dict		,	lowerCAmelCase	:						torch.FloatTensor		,	lowerCAmelCase	:						Optional[torch.FloatTensor] = None		,	lowerCAmelCase	:						Optional[torch.FloatTensor] = None		,	lowerCAmelCase	:						Optional[torch.FloatTensor] = None		,	lowerCAmelCase	:						Optional[torch.LongTensor] = None		,	lowerCAmelCase	:						Dict[str, Any] = None		,	lowerCAmelCase	:						Optional[torch.LongTensor] = None		,	):


		'''simple docstring'''







		if self.use_ada_layer_norm:
			UpperCAmelCase_       				=			self.norma(lowerCAmelCase		,	lowerCAmelCase							)
		elif self.use_ada_layer_norm_zero:
			UpperCAmelCase_				,   UpperCAmelCase_				,   UpperCAmelCase_				,   UpperCAmelCase_				,   UpperCAmelCase_       				=			self.norma(
			    lowerCAmelCase		,	lowerCAmelCase		,	lowerCAmelCase		,	hidden_dtype=hidden_states.dtype							)
		else:
			UpperCAmelCase_       				=			self.norma(lowerCAmelCase							)

		UpperCAmelCase_       				=			cross_attention_kwargs if cross_attention_kwargs is not None else {}

		UpperCAmelCase_       				=			self.attna(
		    lowerCAmelCase		,	encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None		,	attention_mask=lowerCAmelCase		,	**lowerCAmelCase		,	)
		if self.use_ada_layer_norm_zero:
			UpperCAmelCase_       				=			gate_msa.unsqueeze(1							) * attn_output
		UpperCAmelCase_       				=			attn_output + hidden_states

		# 2. Cross-Attention
		if self.attna is not None:
			UpperCAmelCase_       				=			(
			    self.norma(lowerCAmelCase		,	lowerCAmelCase							) if self.use_ada_layer_norm else self.norma(lowerCAmelCase							)
			)

			UpperCAmelCase_       				=			self.attna(
			    lowerCAmelCase		,	encoder_hidden_states=lowerCAmelCase		,	attention_mask=lowerCAmelCase		,	**lowerCAmelCase		,	)
			UpperCAmelCase_       				=			attn_output + hidden_states

		# 3. Feed-forward
		UpperCAmelCase_       				=			self.norma(lowerCAmelCase							)

		if self.use_ada_layer_norm_zero:
			UpperCAmelCase_       				=			norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

		if self._chunk_size is not None:
			# "feed_forward_chunk_size" can be used to save memory
			if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
				raise ValueError(
				    F"`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`."							)

			UpperCAmelCase_       				=			norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
			UpperCAmelCase_       				=			torch.cat(
			    [self.ff(lowerCAmelCase							) for hid_slice in norm_hidden_states.chunk(lowerCAmelCase		,	dim=self._chunk_dim							)]		,	dim=self._chunk_dim		,	)
		else:
			UpperCAmelCase_       				=			self.ff(lowerCAmelCase							)

		if self.use_ada_layer_norm_zero:
			UpperCAmelCase_       				=			gate_mlp.unsqueeze(1							) * ff_output

		UpperCAmelCase_       				=			ff_output + hidden_states

		return hidden_states


class 						__UpperCamelCase    (					nn.Module						):




	def __init__(		self	:						Optional[int]		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						Optional[int] = None		,	lowerCAmelCase	:						int = 4		,	lowerCAmelCase	:						float = 0.0		,	lowerCAmelCase	:						str = "geglu"		,	lowerCAmelCase	:						bool = False		,	):


		'''simple docstring'''







		super().__init__()
		UpperCAmelCase_       				=			int(dim * mult							)
		UpperCAmelCase_       				=			dim_out if dim_out is not None else dim

		if activation_fn == "gelu":
			UpperCAmelCase_       				=			GELU(lowerCAmelCase		,	lowerCAmelCase							)
		if activation_fn == "gelu-approximate":
			UpperCAmelCase_       				=			GELU(lowerCAmelCase		,	lowerCAmelCase		,	approximate="tanh"							)
		elif activation_fn == "geglu":
			UpperCAmelCase_       				=			GEGLU(lowerCAmelCase		,	lowerCAmelCase							)
		elif activation_fn == "geglu-approximate":
			UpperCAmelCase_       				=			ApproximateGELU(lowerCAmelCase		,	lowerCAmelCase							)

		UpperCAmelCase_       				=			nn.ModuleList([]							)
		# project in
		self.net.append(lowerCAmelCase							)
		# project dropout
		self.net.append(nn.Dropout(lowerCAmelCase							)							)
		# project out
		self.net.append(nn.Linear(lowerCAmelCase		,	lowerCAmelCase							)							)
		# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
		if final_dropout:
			self.net.append(nn.Dropout(lowerCAmelCase							)							)




	def       __A					(		self	:						Tuple		,	lowerCAmelCase	:						Optional[Any]							):


		'''simple docstring'''







		for module in self.net:
			UpperCAmelCase_       				=			module(lowerCAmelCase							)
		return hidden_states


class 						__UpperCamelCase    (					nn.Module						):




	def __init__(		self	:						Any		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						str = "none"							):


		'''simple docstring'''







		super().__init__()
		UpperCAmelCase_       				=			nn.Linear(lowerCAmelCase		,	lowerCAmelCase							)
		UpperCAmelCase_       				=			approximate




	def       __A					(		self	:						List[str]		,	lowerCAmelCase	:						Optional[int]							):


		'''simple docstring'''







		if gate.device.type != "mps":
			return F.gelu(lowerCAmelCase		,	approximate=self.approximate							)
		# mps: gelu is not implemented for float16
		return F.gelu(gate.to(dtype=torch.floataa							)		,	approximate=self.approximate							).to(dtype=gate.dtype							)




	def       __A					(		self	:						int		,	lowerCAmelCase	:						List[Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.proj(lowerCAmelCase							)
		UpperCAmelCase_       				=			self.gelu(lowerCAmelCase							)
		return hidden_states


class 						__UpperCamelCase    (					nn.Module						):




	def __init__(		self	:						Dict		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						int							):


		'''simple docstring'''







		super().__init__()
		UpperCAmelCase_       				=			nn.Linear(lowerCAmelCase		,	dim_out * 2							)




	def       __A					(		self	:						int		,	lowerCAmelCase	:						Optional[Any]							):


		'''simple docstring'''







		if gate.device.type != "mps":
			return F.gelu(lowerCAmelCase							)
		# mps: gelu is not implemented for float16
		return F.gelu(gate.to(dtype=torch.floataa							)							).to(dtype=gate.dtype							)




	def       __A					(		self	:						int		,	lowerCAmelCase	:						str							):


		'''simple docstring'''







		UpperCAmelCase_				,   UpperCAmelCase_       				=			self.proj(lowerCAmelCase							).chunk(2		,	dim=-1							)
		return hidden_states * self.gelu(lowerCAmelCase							)


class 						__UpperCamelCase    (					nn.Module						):




	def __init__(		self	:						int		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						int							):


		'''simple docstring'''







		super().__init__()
		UpperCAmelCase_       				=			nn.Linear(lowerCAmelCase		,	lowerCAmelCase							)




	def       __A					(		self	:						Any		,	lowerCAmelCase	:						Dict							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.proj(lowerCAmelCase							)
		return x * torch.sigmoid(1.702 * x							)


class 						__UpperCamelCase    (					nn.Module						):




	def __init__(		self	:						Union[str, Any]		,	lowerCAmelCase	:						Optional[int]		,	lowerCAmelCase	:						Optional[int]							):


		'''simple docstring'''







		super().__init__()
		UpperCAmelCase_       				=			nn.Embedding(lowerCAmelCase		,	lowerCAmelCase							)
		UpperCAmelCase_       				=			nn.SiLU()
		UpperCAmelCase_       				=			nn.Linear(lowerCAmelCase		,	embedding_dim * 2							)
		UpperCAmelCase_       				=			nn.LayerNorm(lowerCAmelCase		,	elementwise_affine=lowerCAmelCase							)




	def       __A					(		self	:						List[Any]		,	lowerCAmelCase	:						Union[str, Any]		,	lowerCAmelCase	:						List[str]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.linear(self.silu(self.emb(lowerCAmelCase							)							)							)
		UpperCAmelCase_				,   UpperCAmelCase_       				=			torch.chunk(lowerCAmelCase		,	2							)
		UpperCAmelCase_       				=			self.norm(lowerCAmelCase							) * (1 + scale) + shift
		return x


class 						__UpperCamelCase    (					nn.Module						):




	def __init__(		self	:						Tuple		,	lowerCAmelCase	:						str		,	lowerCAmelCase	:						Optional[int]							):


		'''simple docstring'''







		super().__init__()

		UpperCAmelCase_       				=			CombinedTimestepLabelEmbeddings(lowerCAmelCase		,	lowerCAmelCase							)

		UpperCAmelCase_       				=			nn.SiLU()
		UpperCAmelCase_       				=			nn.Linear(lowerCAmelCase		,	6 * embedding_dim		,	bias=lowerCAmelCase							)
		UpperCAmelCase_       				=			nn.LayerNorm(lowerCAmelCase		,	elementwise_affine=lowerCAmelCase		,	eps=1e-6							)




	def       __A					(		self	:						int		,	lowerCAmelCase	:						str		,	lowerCAmelCase	:						Tuple		,	lowerCAmelCase	:						Union[str, Any]		,	lowerCAmelCase	:						Union[str, Any]=None							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.linear(self.silu(self.emb(lowerCAmelCase		,	lowerCAmelCase		,	hidden_dtype=lowerCAmelCase							)							)							)
		UpperCAmelCase_				,   UpperCAmelCase_				,   UpperCAmelCase_				,   UpperCAmelCase_				,   UpperCAmelCase_				,   UpperCAmelCase_       				=			emb.chunk(6		,	dim=1							)
		UpperCAmelCase_       				=			self.norm(lowerCAmelCase							) * (1 + scale_msa[:, None]) + shift_msa[:, None]
		return x, gate_msa, shift_mlp, scale_mlp, gate_mlp


class 						__UpperCamelCase    (					nn.Module						):




	def __init__(		self	:						Optional[Any]		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						int		,	lowerCAmelCase	:						Optional[str] = None		,	lowerCAmelCase	:						float = 1e-5							):


		'''simple docstring'''







		super().__init__()
		UpperCAmelCase_       				=			num_groups
		UpperCAmelCase_       				=			eps

		if act_fn is None:
			UpperCAmelCase_       				=			None
		else:
			UpperCAmelCase_       				=			get_activation(lowerCAmelCase							)

		UpperCAmelCase_       				=			nn.Linear(lowerCAmelCase		,	out_dim * 2							)




	def       __A					(		self	:						Tuple		,	lowerCAmelCase	:						Dict		,	lowerCAmelCase	:						Tuple							):


		'''simple docstring'''







		if self.act:
			UpperCAmelCase_       				=			self.act(lowerCAmelCase							)
		UpperCAmelCase_       				=			self.linear(lowerCAmelCase							)
		UpperCAmelCase_       				=			emb[:, :, None, None]
		UpperCAmelCase_				,   UpperCAmelCase_       				=			emb.chunk(2		,	dim=1							)

		UpperCAmelCase_       				=			F.group_norm(lowerCAmelCase		,	self.num_groups		,	eps=self.eps							)
		UpperCAmelCase_       				=			x * (1 + scale) + shift
		return x


def       __lowerCAmelCase      (						A    ):
	UpperCAmelCase_       				=			generate_pascal_triangle(A    )
	for row_idx in range(A    ):
		# Print left spaces
		for _ in range(num_rows - row_idx - 1    ):
			print(end=" "    )
		# Print row values
		for col_idx in range(row_idx + 1    ):
			if col_idx != row_idx:
				print(triangle[row_idx][col_idx]       ,     end=" "    )
			else:
				print(triangle[row_idx][col_idx]       ,     end=""    )
		print()



def       __lowerCAmelCase      (						A    ):

	if not isinstance(A       ,     A    ):
		raise TypeError("The input value of 'num_rows' should be 'int'"    )

	if num_rows == 0:
		return []
	elif num_rows < 0:
		raise ValueError(
		    "The input value of 'num_rows' should be greater than or equal to 0"    )

	UpperCAmelCase_       				=			[]
	for current_row_idx in range(A    ):
		UpperCAmelCase_       				=			populate_current_row(A       ,     A    )
		triangle.append(A    )
	return triangle



def       __lowerCAmelCase      (						A       ,     A    ):
	UpperCAmelCase_       				=			[-1] * (current_row_idx + 1)
	# first and last elements of current row are equal to 1
	UpperCAmelCase_				,   UpperCAmelCase_       				=			1, 1
	for current_col_idx in range(1       ,     A    ):
		calculate_current_element(
		    A       ,     A       ,     A       ,     A    )
	return current_row



def       __lowerCAmelCase      (						A       ,     A       ,     A       ,     A       ,     ):
	UpperCAmelCase_       				=			triangle[current_row_idx - 1][current_col_idx - 1]
	UpperCAmelCase_       				=			triangle[current_row_idx - 1][current_col_idx]
	UpperCAmelCase_       				=			above_to_left_elt + above_to_right_elt



def       __lowerCAmelCase      (						A    ):

	if not isinstance(A       ,     A    ):
		raise TypeError("The input value of 'num_rows' should be 'int'"    )

	if num_rows == 0:
		return []
	elif num_rows < 0:
		raise ValueError(
		    "The input value of 'num_rows' should be greater than or equal to 0"    )

	UpperCAmelCase_       				=			[[1]]

	for row_index in range(1       ,     A    ):
		UpperCAmelCase_       				=			[0] + result[-1] + [0]
		UpperCAmelCase_       				=			row_index + 1
		# Calculate the number of distinct elements in a row
		UpperCAmelCase_       				=			sum(divmod(A       ,     2    )    )
		UpperCAmelCase_       				=			[
		    temp_row[i - 1] + temp_row[i] for i in range(1       ,     distinct_elements + 1    )
		]
		UpperCAmelCase_       				=			row_first_half[: (row_index + 1) // 2]
		row_second_half.reverse()
		UpperCAmelCase_       				=			row_first_half + row_second_half
		result.append(A    )

	return result



def       __lowerCAmelCase      (						):
	from collections.abc import Callable
	from timeit import timeit

	def benchmark_a_function(A       ,     A    ) -> None:
		UpperCAmelCase_       				=			F"{func.__name__}({value})"
		UpperCAmelCase_       				=			timeit(F"__main__.{call}"       ,     setup="import __main__"    )
		# print(f"{call:38} = {func(value)} -- {timing:.4f} seconds")
		print(F"{call:38} -- {timing:.4f} seconds"    )

	for value in range(15    ):  # (1, 7, 14):
		for func in (generate_pascal_triangle, generate_pascal_triangle_optimized):
			benchmark_a_function(A       ,     A    )
		print()


if __name__ == "__main__":
				import doctest

				doctest.testmod()
				benchmark()