



"""simple docstring"""






from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings







@maybe_allow_in_graph
class 	__SCREAMING_SNAKE_CASE     (				nn.Module       ):


					'''simple docstring'''





					def __init__(	self							:  Tuple,  lowerCamelCase							:  int,  lowerCamelCase							:  int,  lowerCamelCase							:  int,  lowerCamelCase							:  List[str]=0.0,  lowerCamelCase							:  Optional[int] = None,  lowerCamelCase							:  str = "geglu",  lowerCamelCase							:  Optional[int] = None,  lowerCamelCase							:  bool = False,  lowerCamelCase							:  bool = False,  lowerCamelCase							:  bool = False,  lowerCamelCase							:  bool = False,  lowerCamelCase							:  bool = True,  lowerCamelCase							:  str = "layer_norm",  lowerCamelCase							:  bool = False,  )-> Optional[Any]:
								super().__init__()
								lowerCamelCase__		:		Union[str, Any]		     =only_cross_attention

								lowerCamelCase__		:		Dict		     =(num_embeds_ada_norm is not None) and norm_type == '''ada_norm_zero'''
								lowerCamelCase__		:		int		     =(num_embeds_ada_norm is not None) and norm_type == '''ada_norm'''

								if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
											raise ValueError(
											    F'''`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'''
											    F''' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'''					)

								# Define 3 blocks. Each block has its own normalization layer.
								# 1. Self-Attn
								if self.use_ada_layer_norm:
											lowerCamelCase__		:		Tuple		     =AdaLayerNorm(lowerCamelCase,  lowerCamelCase					)
								elif self.use_ada_layer_norm_zero:
											lowerCamelCase__		:		Union[str, Any]		     =AdaLayerNormZero(lowerCamelCase,  lowerCamelCase					)
								else:
											lowerCamelCase__		:		List[str]		     =nn.LayerNorm(lowerCamelCase,  elementwise_affine=lowerCamelCase					)
								lowerCamelCase__		:		List[Any]		     =Attention(
								    query_dim=lowerCamelCase,  heads=lowerCamelCase,  dim_head=lowerCamelCase,  dropout=lowerCamelCase,  bias=lowerCamelCase,  cross_attention_dim=cross_attention_dim if only_cross_attention else None,  upcast_attention=lowerCamelCase,  )

								# 2. Cross-Attn
								if cross_attention_dim is not None or double_self_attention:
											# We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
											# I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
											# the second cross attention block.
											lowerCamelCase__		:		int		     =(
											    AdaLayerNorm(lowerCamelCase,  lowerCamelCase					)
											    if self.use_ada_layer_norm
											    else nn.LayerNorm(lowerCamelCase,  elementwise_affine=lowerCamelCase					)
											)
											lowerCamelCase__		:		Any		     =Attention(
											    query_dim=lowerCamelCase,  cross_attention_dim=cross_attention_dim if not double_self_attention else None,  heads=lowerCamelCase,  dim_head=lowerCamelCase,  dropout=lowerCamelCase,  bias=lowerCamelCase,  upcast_attention=lowerCamelCase,  )  # is self-attn if encoder_hidden_states is none
								else:
											lowerCamelCase__		:		Union[str, Any]		     =None
											lowerCamelCase__		:		Tuple		     =None

								# 3. Feed-forward
								lowerCamelCase__		:		List[Any]		     =nn.LayerNorm(lowerCamelCase,  elementwise_affine=lowerCamelCase					)
								lowerCamelCase__		:		List[Any]		     =FeedForward(lowerCamelCase,  dropout=lowerCamelCase,  activation_fn=lowerCamelCase,  final_dropout=lowerCamelCase					)

								# let chunk size default to None
								lowerCamelCase__		:		Optional[Any]		     =None
								lowerCamelCase__		:		int		     =0



					def 		snake_case	(	self							:  List[Any],  lowerCamelCase							:  Optional[int],  lowerCamelCase							:  int					)-> List[str]:
								# Sets chunk feed-forward
								lowerCamelCase__		:		List[Any]		     =chunk_size
								lowerCamelCase__		:		Optional[int]		     =dim




					def 		snake_case	(	self							:  Union[str, Any],  lowerCamelCase							:  torch.FloatTensor,  lowerCamelCase							:  Optional[torch.FloatTensor] = None,  lowerCamelCase							:  Optional[torch.FloatTensor] = None,  lowerCamelCase							:  Optional[torch.FloatTensor] = None,  lowerCamelCase							:  Optional[torch.LongTensor] = None,  lowerCamelCase							:  Dict[str, Any] = None,  lowerCamelCase							:  Optional[torch.LongTensor] = None,  )-> Dict:
								# Notice that normalization is always applied before the real computation in the following blocks.
								# 1. Self-Attention
								if self.use_ada_layer_norm:
											lowerCamelCase__		:		str		     =self.norma(lowerCamelCase,  lowerCamelCase					)
								elif self.use_ada_layer_norm_zero:
											lowerCamelCase__		:		Any		     =self.norma(
											    lowerCamelCase,  lowerCamelCase,  lowerCamelCase,  hidden_dtype=hidden_states.dtype					)
								else:
											lowerCamelCase__		:		Any		     =self.norma(lowerCamelCase					)

								lowerCamelCase__		:		int		     =cross_attention_kwargs if cross_attention_kwargs is not None else {}

								lowerCamelCase__		:		Optional[int]		     =self.attna(
								    lowerCamelCase,  encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None,  attention_mask=lowerCamelCase,  **lowerCamelCase,  )
								if self.use_ada_layer_norm_zero:
											lowerCamelCase__		:		Any		     =gate_msa.unsqueeze(1					) * attn_output
								lowerCamelCase__		:		Dict		     =attn_output + hidden_states

								# 2. Cross-Attention
								if self.attna is not None:
											lowerCamelCase__		:		str		     =(
											    self.norma(lowerCamelCase,  lowerCamelCase					) if self.use_ada_layer_norm else self.norma(lowerCamelCase					)
											)

											lowerCamelCase__		:		Tuple		     =self.attna(
											    lowerCamelCase,  encoder_hidden_states=lowerCamelCase,  attention_mask=lowerCamelCase,  **lowerCamelCase,  )
											lowerCamelCase__		:		Optional[int]		     =attn_output + hidden_states

								# 3. Feed-forward
								lowerCamelCase__		:		Any		     =self.norma(lowerCamelCase					)

								if self.use_ada_layer_norm_zero:
											lowerCamelCase__		:		int		     =norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

								if self._chunk_size is not None:
											# "feed_forward_chunk_size" can be used to save memory
											if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
														raise ValueError(
														    F'''`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'''					)

											lowerCamelCase__		:		Union[str, Any]		     =norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
											lowerCamelCase__		:		Optional[Any]		     =torch.cat(
											    [self.ff(lowerCamelCase					) for hid_slice in norm_hidden_states.chunk(lowerCamelCase,  dim=self._chunk_dim					)],  dim=self._chunk_dim,  )
								else:
											lowerCamelCase__		:		List[str]		     =self.ff(lowerCamelCase					)

								if self.use_ada_layer_norm_zero:
											lowerCamelCase__		:		Any		     =gate_mlp.unsqueeze(1					) * ff_output

								lowerCamelCase__		:		Dict		     =ff_output + hidden_states

								return hidden_states







class 	__SCREAMING_SNAKE_CASE     (				nn.Module       ):


					'''simple docstring'''





					def __init__(	self							:  Tuple,  lowerCamelCase							:  int,  lowerCamelCase							:  Optional[int] = None,  lowerCamelCase							:  int = 4,  lowerCamelCase							:  float = 0.0,  lowerCamelCase							:  str = "geglu",  lowerCamelCase							:  bool = False,  )-> Dict:
								super().__init__()
								lowerCamelCase__		:		List[str]		     =int(dim * mult					)
								lowerCamelCase__		:		Union[str, Any]		     =dim_out if dim_out is not None else dim

								if activation_fn == "gelu":
											lowerCamelCase__		:		Tuple		     =GELU(lowerCamelCase,  lowerCamelCase					)
								if activation_fn == "gelu-approximate":
											lowerCamelCase__		:		Dict		     =GELU(lowerCamelCase,  lowerCamelCase,  approximate='''tanh'''					)
								elif activation_fn == "geglu":
											lowerCamelCase__		:		str		     =GEGLU(lowerCamelCase,  lowerCamelCase					)
								elif activation_fn == "geglu-approximate":
											lowerCamelCase__		:		Dict		     =ApproximateGELU(lowerCamelCase,  lowerCamelCase					)

								lowerCamelCase__		:		Dict		     =nn.ModuleList([]					)
								# project in
								self.net.append(lowerCamelCase					)
								# project dropout
								self.net.append(nn.Dropout(lowerCamelCase					)					)
								# project out
								self.net.append(nn.Linear(lowerCamelCase,  lowerCamelCase					)					)
								# FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
								if final_dropout:
											self.net.append(nn.Dropout(lowerCamelCase					)					)




					def 		snake_case	(	self							:  str,  lowerCamelCase							:  Tuple					)-> Optional[Any]:
								for module in self.net:
											lowerCamelCase__		:		int		     =module(lowerCamelCase					)
								return hidden_states







class 	__SCREAMING_SNAKE_CASE     (				nn.Module       ):


					'''simple docstring'''





					def __init__(	self							:  Optional[int],  lowerCamelCase							:  int,  lowerCamelCase							:  int,  lowerCamelCase							:  str = "none"					)-> Any:
								super().__init__()
								lowerCamelCase__		:		Optional[int]		     =nn.Linear(lowerCamelCase,  lowerCamelCase					)
								lowerCamelCase__		:		Optional[int]		     =approximate



					def 		snake_case	(	self							:  Union[str, Any],  lowerCamelCase							:  List[str]					)-> Tuple:
								if gate.device.type != "mps":
											return F.gelu(lowerCamelCase,  approximate=self.approximate					)
								# mps: gelu is not implemented for float16
								return F.gelu(gate.to(dtype=torch.floataa					),  approximate=self.approximate					).to(dtype=gate.dtype					)




					def 		snake_case	(	self							:  Dict,  lowerCamelCase							:  Union[str, Any]					)-> Union[str, Any]:
								lowerCamelCase__		:		Any		     =self.proj(lowerCamelCase					)
								lowerCamelCase__		:		Union[str, Any]		     =self.gelu(lowerCamelCase					)
								return hidden_states







class 	__SCREAMING_SNAKE_CASE     (				nn.Module       ):


					'''simple docstring'''





					def __init__(	self							:  List[Any],  lowerCamelCase							:  int,  lowerCamelCase							:  int					)-> List[Any]:
								super().__init__()
								lowerCamelCase__		:		Optional[Any]		     =nn.Linear(lowerCamelCase,  dim_out * 2					)



					def 		snake_case	(	self							:  List[Any],  lowerCamelCase							:  List[Any]					)-> str:
								if gate.device.type != "mps":
											return F.gelu(lowerCamelCase					)
								# mps: gelu is not implemented for float16
								return F.gelu(gate.to(dtype=torch.floataa					)					).to(dtype=gate.dtype					)




					def 		snake_case	(	self							:  List[Any],  lowerCamelCase							:  List[Any]					)-> Any:
								lowerCamelCase__		:		Optional[int]		     =self.proj(lowerCamelCase					).chunk(2,  dim=-1					)
								return hidden_states * self.gelu(lowerCamelCase					)







class 	__SCREAMING_SNAKE_CASE     (				nn.Module       ):


					'''simple docstring'''





					def __init__(	self							:  List[Any],  lowerCamelCase							:  int,  lowerCamelCase							:  int					)-> List[Any]:
								super().__init__()
								lowerCamelCase__		:		Any		     =nn.Linear(lowerCamelCase,  lowerCamelCase					)




					def 		snake_case	(	self							:  str,  lowerCamelCase							:  str					)-> Optional[int]:
								lowerCamelCase__		:		List[str]		     =self.proj(lowerCamelCase					)
								return x * torch.sigmoid(1.702 * x					)







class 	__SCREAMING_SNAKE_CASE     (				nn.Module       ):


					'''simple docstring'''





					def __init__(	self							:  Tuple,  lowerCamelCase							:  int,  lowerCamelCase							:  List[Any]					)-> Optional[Any]:
								super().__init__()
								lowerCamelCase__		:		int		     =nn.Embedding(lowerCamelCase,  lowerCamelCase					)
								lowerCamelCase__		:		int		     =nn.SiLU()
								lowerCamelCase__		:		str		     =nn.Linear(lowerCamelCase,  embedding_dim * 2					)
								lowerCamelCase__		:		int		     =nn.LayerNorm(lowerCamelCase,  elementwise_affine=lowerCamelCase					)




					def 		snake_case	(	self							:  Dict,  lowerCamelCase							:  str,  lowerCamelCase							:  List[str]					)-> Optional[Any]:
								lowerCamelCase__		:		List[Any]		     =self.linear(self.silu(self.emb(lowerCamelCase					)					)					)
								lowerCamelCase__		:		Optional[Any]		     =torch.chunk(lowerCamelCase,  2					)
								lowerCamelCase__		:		Optional[Any]		     =self.norm(lowerCamelCase					) * (1 + scale) + shift
								return x







class 	__SCREAMING_SNAKE_CASE     (				nn.Module       ):


					'''simple docstring'''





					def __init__(	self							:  Tuple,  lowerCamelCase							:  Any,  lowerCamelCase							:  Optional[Any]					)-> int:
								super().__init__()

								lowerCamelCase__		:		str		     =CombinedTimestepLabelEmbeddings(lowerCamelCase,  lowerCamelCase					)

								lowerCamelCase__		:		List[str]		     =nn.SiLU()
								lowerCamelCase__		:		Tuple		     =nn.Linear(lowerCamelCase,  6 * embedding_dim,  bias=lowerCamelCase					)
								lowerCamelCase__		:		Tuple		     =nn.LayerNorm(lowerCamelCase,  elementwise_affine=lowerCamelCase,  eps=1E-6					)




					def 		snake_case	(	self							:  List[Any],  lowerCamelCase							:  int,  lowerCamelCase							:  Tuple,  lowerCamelCase							:  str,  lowerCamelCase							:  int=None					)-> List[Any]:
								lowerCamelCase__		:		str		     =self.linear(self.silu(self.emb(lowerCamelCase,  lowerCamelCase,  hidden_dtype=lowerCamelCase					)					)					)
								lowerCamelCase__		:		int		     =emb.chunk(6,  dim=1					)
								lowerCamelCase__		:		int		     =self.norm(lowerCamelCase					) * (1 + scale_msa[:, None]) + shift_msa[:, None]
								return x, gate_msa, shift_mlp, scale_mlp, gate_mlp







class 	__SCREAMING_SNAKE_CASE     (				nn.Module       ):


					'''simple docstring'''





					def __init__(	self							:  Tuple,  lowerCamelCase							:  int,  lowerCamelCase							:  int,  lowerCamelCase							:  int,  lowerCamelCase							:  Optional[str] = None,  lowerCamelCase							:  float = 1E-5					)-> List[str]:
								super().__init__()
								lowerCamelCase__		:		List[Any]		     =num_groups
								lowerCamelCase__		:		List[Any]		     =eps

								if act_fn is None:
											lowerCamelCase__		:		Tuple		     =None
								else:
											lowerCamelCase__		:		List[Any]		     =get_activation(lowerCamelCase					)

								lowerCamelCase__		:		int		     =nn.Linear(lowerCamelCase,  out_dim * 2					)




					def 		snake_case	(	self							:  str,  lowerCamelCase							:  Any,  lowerCamelCase							:  Optional[Any]					)-> Tuple:
								if self.act:
											lowerCamelCase__		:		List[Any]		     =self.act(lowerCamelCase					)
								lowerCamelCase__		:		List[str]		     =self.linear(lowerCamelCase					)
								lowerCamelCase__		:		Tuple		     =emb[:, :, None, None]
								lowerCamelCase__		:		str		     =emb.chunk(2,  dim=1					)

								lowerCamelCase__		:		Tuple		     =F.group_norm(lowerCamelCase,  self.num_groups,  eps=self.eps					)
								lowerCamelCase__		:		Optional[int]		     =x * (1 + scale) + shift
								return x









"""simple docstring"""






from ...processing_utils import ProcessorMixin

class 	__SCREAMING_SNAKE_CASE     (				lowerCAmelCase_       ):


					'''simple docstring'''





					_a             =							'SpeechT5FeatureExtractor'
					_a             =							'SpeechT5Tokenizer'



					def __init__(	self							:  Dict,  lowerCamelCase							:  Optional[int],  lowerCamelCase							:  str					)-> Any:
								super().__init__(lowerCamelCase,  lowerCamelCase					)



					def __call__(	self							:  Tuple,  *lowerCamelCase							:  List[str],  **lowerCamelCase							:  Optional[int]					)-> List[str]:
								lowerCamelCase__		:		List[Any]		     =kwargs.pop('''audio''',  lowerCamelCase					)
								lowerCamelCase__		:		List[str]		     =kwargs.pop('''text''',  lowerCamelCase					)
								lowerCamelCase__		:		int		     =kwargs.pop('''text_target''',  lowerCamelCase					)
								lowerCamelCase__		:		Dict		     =kwargs.pop('''audio_target''',  lowerCamelCase					)
								lowerCamelCase__		:		Any		     =kwargs.pop('''sampling_rate''',  lowerCamelCase					)

								if audio is not None and text is not None:
											raise ValueError(
											    '''Cannot process both `audio` and `text` inputs. Did you mean `audio_target` or `text_target`?'''					)
								if audio_target is not None and text_target is not None:
											raise ValueError(
											    '''Cannot process both `audio_target` and `text_target` inputs. Did you mean `audio` or `text`?'''					)
								if audio is None and audio_target is None and text is None and text_target is None:
											raise ValueError(
											    '''You need to specify either an `audio`, `audio_target`, `text`, or `text_target` input to process.'''					)

								if audio is not None:
											lowerCamelCase__		:		Union[str, Any]		     =self.feature_extractor(lowerCamelCase,  *lowerCamelCase,  sampling_rate=lowerCamelCase,  **lowerCamelCase					)
								elif text is not None:
											lowerCamelCase__		:		List[Any]		     =self.tokenizer(lowerCamelCase,  **lowerCamelCase					)
								else:
											lowerCamelCase__		:		Any		     =None

								if audio_target is not None:
											lowerCamelCase__		:		List[str]		     =self.feature_extractor(audio_target=lowerCamelCase,  *lowerCamelCase,  sampling_rate=lowerCamelCase,  **lowerCamelCase					)
											lowerCamelCase__		:		Tuple		     =targets['''input_values''']
								elif text_target is not None:
											lowerCamelCase__		:		Dict		     =self.tokenizer(lowerCamelCase,  **lowerCamelCase					)
											lowerCamelCase__		:		int		     =targets['''input_ids''']
								else:
											lowerCamelCase__		:		List[str]		     =None

								if inputs is None:
											return targets

								if targets is not None:
											lowerCamelCase__		:		Dict		     =labels

											lowerCamelCase__		:		Any		     =targets.get('''attention_mask'''					)
											if decoder_attention_mask is not None:
														lowerCamelCase__		:		Dict		     =decoder_attention_mask

								return inputs



					def 		snake_case	(	self							:  int,  *lowerCamelCase							:  Optional[Any],  **lowerCamelCase							:  Optional[int]					)-> Optional[Any]:
								lowerCamelCase__		:		List[Any]		     =kwargs.pop('''input_values''',  lowerCamelCase					)
								lowerCamelCase__		:		Union[str, Any]		     =kwargs.pop('''input_ids''',  lowerCamelCase					)
								lowerCamelCase__		:		Optional[Any]		     =kwargs.pop('''labels''',  lowerCamelCase					)

								if input_values is not None and input_ids is not None:
											raise ValueError('''Cannot process both `input_values` and `input_ids` inputs.'''					)
								if input_values is None and input_ids is None and labels is None:
											raise ValueError(
											    '''You need to specify either an `input_values`, `input_ids`, or `labels` input to be padded.'''					)

								if input_values is not None:
											lowerCamelCase__		:		List[str]		     =self.feature_extractor.pad(lowerCamelCase,  *lowerCamelCase,  **lowerCamelCase					)
								elif input_ids is not None:
											lowerCamelCase__		:		Tuple		     =self.tokenizer.pad(lowerCamelCase,  **lowerCamelCase					)
								else:
											lowerCamelCase__		:		Any		     =None

								if labels is not None:
											if "input_ids" in labels or (isinstance(lowerCamelCase,  lowerCamelCase					) and "input_ids" in labels[0]):
														lowerCamelCase__		:		str		     =self.tokenizer.pad(lowerCamelCase,  **lowerCamelCase					)
														lowerCamelCase__		:		List[Any]		     =targets['''input_ids''']
											else:
														lowerCamelCase__		:		Any		     =self.feature_extractor.feature_size
														lowerCamelCase__		:		Optional[Any]		     =self.feature_extractor.num_mel_bins
														lowerCamelCase__		:		Optional[int]		     =self.feature_extractor.pad(lowerCamelCase,  *lowerCamelCase,  **lowerCamelCase					)
														lowerCamelCase__		:		List[Any]		     =feature_size_hack
														lowerCamelCase__		:		Tuple		     =targets['''input_values''']
								else:
											lowerCamelCase__		:		Optional[Any]		     =None

								if inputs is None:
											return targets

								if targets is not None:
											lowerCamelCase__		:		Tuple		     =labels

											lowerCamelCase__		:		Optional[int]		     =targets.get('''attention_mask'''					)
											if decoder_attention_mask is not None:
														lowerCamelCase__		:		Optional[Any]		     =decoder_attention_mask

								return inputs



					def 		snake_case	(	self							:  List[str],  *lowerCamelCase							:  Union[str, Any],  **lowerCamelCase							:  List[Any]					)-> List[Any]:
								return self.tokenizer.batch_decode(*lowerCamelCase,  **lowerCamelCase					)




					def 		snake_case	(	self							:  List[str],  *lowerCamelCase							:  List[Any],  **lowerCamelCase							:  Tuple					)-> int:
								return self.tokenizer.decode(*lowerCamelCase,  **lowerCamelCase					)




