



import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin







class       __lowerCAmelCase   (      A    ,						A						):



			@register_to_config
			def __init__(						self       :  Any			,		A       :  int = 1_28			,		A       :  int = 2_56			,		A       :  float = 2_0_0_0.0			,		A       :  int = 7_68			,		A       :  int = 12			,		A       :  int = 12			,		A       :  int = 64			,		A       :  int = 20_48			,		A       :  float = 0.1			,		)       ->   Optional[int]:



						"""simple docstring"""






						super().__init__()

						_UpperCAmelCase			=       nn.Sequential(
						    nn.Linear(A			,		d_model * 4			,		bias=A)			,		nn.SiLU()			,		nn.Linear(d_model * 4			,		d_model * 4			,		bias=A)			,		nn.SiLU()			,		)

						_UpperCAmelCase			=       nn.Embedding(A			,		A)
						_UpperCAmelCase			=       False

						_UpperCAmelCase			=       nn.Linear(A			,		A			,		bias=A)

						_UpperCAmelCase			=       nn.Dropout(p=A)

						_UpperCAmelCase			=       nn.ModuleList()
						for lyr_num in range(A):
									# FiLM conditional T5 decoder
									_UpperCAmelCase			=       DecoderLayer(d_model=A			,		d_kv=A			,		num_heads=A			,		d_ff=A			,		dropout_rate=A)
									self.decoders.append(A)

						_UpperCAmelCase			=       TaLayerNorm(A)

						_UpperCAmelCase			=       nn.Dropout(p=A)
						_UpperCAmelCase			=       nn.Linear(A			,		A			,		bias=A)



			def    _lowerCamelCase	(						self       :  List[Any]			,		A       :  Tuple			,		A       :  List[Any])       ->   Any:



						"""simple docstring"""






						_UpperCAmelCase			=       torch.mul(query_input.unsqueeze(-1)			,		key_input.unsqueeze(-2))
						return mask.unsqueeze(-3)



			def    _lowerCamelCase	(						self       :  Any			,		A       :  Any			,		A       :  int			,		A       :  Tuple)       ->   Optional[Any]:



						"""simple docstring"""






						_UpperCAmelCase	,					_UpperCAmelCase	,					_UpperCAmelCase			=       decoder_input_tokens.shape
						assert decoder_noise_time.shape == (batch,)

						# decoder_noise_time is in [0, 1), so rescale to expected timing range.
						_UpperCAmelCase			=       get_timestep_embedding(
						    decoder_noise_time * self.config.max_decoder_noise_time			,		embedding_dim=self.config.d_model			,		max_period=self.config.max_decoder_noise_time			,		).to(dtype=self.dtype)

						_UpperCAmelCase			=       self.conditioning_emb(A).unsqueeze(1)

						assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

						_UpperCAmelCase			=       decoder_input_tokens.shape[1]

						# If we want to use relative positions for audio context, we can just offset
						# this sequence by the length of encodings_and_masks.
						_UpperCAmelCase			=       torch.broadcast_to(
						    torch.arange(A			,		device=decoder_input_tokens.device)			,		(batch, seq_length)			,		)

						_UpperCAmelCase			=       self.position_encoding(A)

						_UpperCAmelCase			=       self.continuous_inputs_projection(A)
						inputs += position_encodings
						_UpperCAmelCase			=       self.dropout(A)

						# decoder: No padding present.
						_UpperCAmelCase			=       torch.ones(
						    decoder_input_tokens.shape[:2]			,		device=decoder_input_tokens.device			,		dtype=inputs.dtype)

						# Translate encoding masks to encoder-decoder masks.
						_UpperCAmelCase			=       [(x, self.encoder_decoder_mask(A			,		A)) for x, y in encodings_and_masks]

						# cross attend style: concat encodings
						_UpperCAmelCase			=       torch.cat([x[0] for x in encodings_and_encdec_masks]			,		dim=1)
						_UpperCAmelCase			=       torch.cat([x[1] for x in encodings_and_encdec_masks]			,		dim=-1)

						for lyr in self.decoders:
									_UpperCAmelCase			=       lyr(
									    A			,		conditioning_emb=A			,		encoder_hidden_states=A			,		encoder_attention_mask=A			,		)[0]

						_UpperCAmelCase			=       self.decoder_norm(A)
						_UpperCAmelCase			=       self.post_dropout(A)

						_UpperCAmelCase			=       self.spec_out(A)
						return spec_out







class       __lowerCAmelCase   (      nn.Module						):



			def __init__(						self       :  List[Any]			,		A       :  Any			,		A       :  Any			,		A       :  Dict			,		A       :  Any			,		A       :  str			,		A       :  List[Any]=1E-6)       ->   List[str]:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase			=       nn.ModuleList()

						# cond self attention: layer 0
						self.layer.append(
						    TaLayerSelfAttentionCond(d_model=A			,		d_kv=A			,		num_heads=A			,		dropout_rate=A))

						# cross attention: layer 1
						self.layer.append(
						    TaLayerCrossAttention(
						        d_model=A			,		d_kv=A			,		num_heads=A			,		dropout_rate=A			,		layer_norm_epsilon=A			,		))

						# Film Cond MLP + dropout: last layer
						self.layer.append(
						    TaLayerFFCond(d_model=A			,		d_ff=A			,		dropout_rate=A			,		layer_norm_epsilon=A))



			def    _lowerCamelCase	(						self       :  Dict			,		A       :  Optional[Any]			,		A       :  int=None			,		A       :  List[Any]=None			,		A       :  List[str]=None			,		A       :  Optional[int]=None			,		A       :  Union[str, Any]=None			,		)       ->   Union[str, Any]:



						"""simple docstring"""






						_UpperCAmelCase			=       self.layer[0](
						    A			,		conditioning_emb=A			,		attention_mask=A			,		)

						if encoder_hidden_states is not None:
									_UpperCAmelCase			=       torch.where(encoder_attention_mask > 0			,		0			,		-1E10).to(
									    encoder_hidden_states.dtype)

									_UpperCAmelCase			=       self.layer[1](
									    A			,		key_value_states=A			,		attention_mask=A			,		)

						# Apply Film Conditional Feed Forward layer
						_UpperCAmelCase			=       self.layer[-1](A			,		A)

						return (hidden_states,)







class       __lowerCAmelCase   (      nn.Module						):



			def __init__(						self       :  Tuple			,		A       :  Dict			,		A       :  List[str]			,		A       :  Tuple			,		A       :  str)       ->   List[Any]:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase			=       TaLayerNorm(A)
						_UpperCAmelCase			=       TaFiLMLayer(in_features=d_model * 4			,		out_features=A)
						_UpperCAmelCase			=       Attention(query_dim=A			,		heads=A			,		dim_head=A			,		out_bias=A			,		scale_qk=A)
						_UpperCAmelCase			=       nn.Dropout(A)



			def    _lowerCamelCase	(						self       :  Tuple			,		A       :  List[Any]			,		A       :  str=None			,		A       :  Union[str, Any]=None			,		)       ->   Union[str, Any]:



						"""simple docstring"""






						_UpperCAmelCase			=       self.layer_norm(A)

						if conditioning_emb is not None:
									_UpperCAmelCase			=       self.FiLMLayer(A			,		A)

						# Self-attention block
						_UpperCAmelCase			=       self.attention(A)

						_UpperCAmelCase			=       hidden_states + self.dropout(A)

						return hidden_states







class       __lowerCAmelCase   (      nn.Module						):



			def __init__(						self       :  str			,		A       :  int			,		A       :  Dict			,		A       :  str			,		A       :  List[str]			,		A       :  Optional[Any])       ->   List[str]:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase			=       Attention(query_dim=A			,		heads=A			,		dim_head=A			,		out_bias=A			,		scale_qk=A)
						_UpperCAmelCase			=       TaLayerNorm(A			,		eps=A)
						_UpperCAmelCase			=       nn.Dropout(A)



			def    _lowerCamelCase	(						self       :  List[Any]			,		A       :  Union[str, Any]			,		A       :  Tuple=None			,		A       :  Optional[int]=None			,		)       ->   int:



						"""simple docstring"""






						_UpperCAmelCase			=       self.layer_norm(A)
						_UpperCAmelCase			=       self.attention(
						    A			,		encoder_hidden_states=A			,		attention_mask=attention_mask.squeeze(1)			,		)
						_UpperCAmelCase			=       hidden_states + self.dropout(A)
						return layer_output







class       __lowerCAmelCase   (      nn.Module						):



			def __init__(						self       :  List[str]			,		A       :  List[str]			,		A       :  List[Any]			,		A       :  List[Any]			,		A       :  Any)       ->   Any:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase			=       TaDenseGatedActDense(d_model=A			,		d_ff=A			,		dropout_rate=A)
						_UpperCAmelCase			=       TaFiLMLayer(in_features=d_model * 4			,		out_features=A)
						_UpperCAmelCase			=       TaLayerNorm(A			,		eps=A)
						_UpperCAmelCase			=       nn.Dropout(A)



			def    _lowerCamelCase	(						self       :  Optional[int]			,		A       :  Union[str, Any]			,		A       :  int=None)       ->   str:



						"""simple docstring"""






						_UpperCAmelCase			=       self.layer_norm(A)
						if conditioning_emb is not None:
									_UpperCAmelCase			=       self.film(A			,		A)

						_UpperCAmelCase			=       self.DenseReluDense(A)
						_UpperCAmelCase			=       hidden_states + self.dropout(A)
						return hidden_states







class       __lowerCAmelCase   (      nn.Module						):



			def __init__(						self       :  str			,		A       :  Union[str, Any]			,		A       :  str			,		A       :  Dict)       ->   Tuple:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase			=       nn.Linear(A			,		A			,		bias=A)
						_UpperCAmelCase			=       nn.Linear(A			,		A			,		bias=A)
						_UpperCAmelCase			=       nn.Linear(A			,		A			,		bias=A)
						_UpperCAmelCase			=       nn.Dropout(A)
						_UpperCAmelCase			=       NewGELUActivation()



			def    _lowerCamelCase	(						self       :  Union[str, Any]			,		A       :  Optional[int])       ->   List[str]:



						"""simple docstring"""






						_UpperCAmelCase			=       self.act(self.wi_a(A))
						_UpperCAmelCase			=       self.wi_a(A)
						_UpperCAmelCase			=       hidden_gelu * hidden_linear
						_UpperCAmelCase			=       self.dropout(A)

						_UpperCAmelCase			=       self.wo(A)
						return hidden_states







class       __lowerCAmelCase   (      nn.Module						):



			def __init__(						self       :  List[str]			,		A       :  List[Any]			,		A       :  Any=1E-6)       ->   Optional[int]:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase			=       nn.Parameter(torch.ones(A))
						_UpperCAmelCase			=       eps



			def    _lowerCamelCase	(						self       :  str			,		A       :  Dict)       ->   Tuple:



						"""simple docstring"""






						_UpperCAmelCase			=       hidden_states.to(torch.floataa).pow(2).mean(-1			,		keepdim=A)
						_UpperCAmelCase			=       hidden_states * torch.rsqrt(variance + self.variance_epsilon)

						# convert into half-precision if necessary
						if self.weight.dtype in [torch.floataa, torch.bfloataa]:
									_UpperCAmelCase			=       hidden_states.to(self.weight.dtype)

						return self.weight * hidden_states







class       __lowerCAmelCase   (      nn.Module						):



			def    _lowerCamelCase	(						self       :  List[Any]			,		A       :  torch.Tensor)       ->   torch.Tensor:



						"""simple docstring"""






						return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (input + 0.0_4_4_7_1_5 * torch.pow(A			,		3.0))))







class       __lowerCAmelCase   (      nn.Module						):



			def __init__(						self       :  str			,		A       :  Optional[Any]			,		A       :  Union[str, Any])       ->   Tuple:



						"""simple docstring"""






						super().__init__()
						_UpperCAmelCase			=       nn.Linear(A			,		out_features * 2			,		bias=A)



			def    _lowerCamelCase	(						self       :  List[Any]			,		A       :  Optional[Any]			,		A       :  Optional[int])       ->   Any:



						"""simple docstring"""






						_UpperCAmelCase			=       self.scale_bias(A)
						_UpperCAmelCase	,					_UpperCAmelCase			=       torch.chunk(A			,		2			,		-1)
						_UpperCAmelCase			=       x * (1 + scale) + shift
						return x





def 							A     (					_UpperCAmelCase    :       int   )    ->			"list[int]":







			'''simple docstring'''
			if upper_limit < 0:
						raise ValueError('Limit for the Catalan sequence must be ≥ 0'   )

			_UpperCAmelCase			=       [0] * (upper_limit + 1)

			# Base case: C(0) = C(1) = 1
			_UpperCAmelCase			=       1
			if upper_limit > 0:
						_UpperCAmelCase			=       1

			# Recurrence relation: C(i) = sum(C(j).C(i-j-1)), from j = 0 to i
			for i in range(2							,       upper_limit + 1   ):
						for j in range(_UpperCAmelCase   ):
									catalan_list[i] += catalan_list[j] * catalan_list[i - j - 1]

			return catalan_list


if __name__ == "__main__":
							print("\n********* Catalan Numbers Using Dynamic Programming ************\n")
							print("\n*** Enter -1 at any time to quit ***")
							print("\nEnter the upper limit (≥ 0) for the Catalan number sequence: ", end="")
							try:
														while True:
																					UpperCAmelCase__       =	int(input().strip())
																					if N < 0:
																												print("\n********* Goodbye!! ************")
																												break
																					else:
																												print(f"""The Catalan numbers from 0 through {N} are:""")
																												print(catalan_numbers(N))
																												print("Try another upper limit for the sequence: ", end="")
							except (NameError, ValueError):
														print("\n********* Invalid input, goodbye! ************\n")

							import doctest

							doctest.testmod()
