



from typing import Any, Dict, Optional

import torch
import torch.nn.functional as F
from torch import nn

from ..utils import maybe_allow_in_graph
from .activations import get_activation
from .attention_processor import Attention
from .embeddings import CombinedTimestepLabelEmbeddings


@maybe_allow_in_graph
class 	lowerCamelCase__	(       nn.Module):



 """simple docstring"""

 def __init__(    self					:					int			,						__lowerCAmelCase					:					int			,						__lowerCAmelCase					:					int			,						__lowerCAmelCase					:					int			,						__lowerCAmelCase					:					Dict=0.0			,						__lowerCAmelCase					:					Optional[int] = None			,						__lowerCAmelCase					:					str = "geglu"			,						__lowerCAmelCase					:					Optional[int] = None			,						__lowerCAmelCase					:					bool = False			,						__lowerCAmelCase					:					bool = False			,						__lowerCAmelCase					:					bool = False			,						__lowerCAmelCase					:					bool = False			,						__lowerCAmelCase					:					bool = True			,						__lowerCAmelCase					:					str = "layer_norm"			,						__lowerCAmelCase					:					bool = False			,						)   ->	Tuple:
    super().__init__()
    _A      =       only_cross_attention

    _A      =       (num_embeds_ada_norm is not None) and norm_type == '''ada_norm_zero'''
    _A      =       (num_embeds_ada_norm is not None) and norm_type == '''ada_norm'''

    if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None:
       raise ValueError(
           f'''`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to'''
           f''' define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}.'''      )

    # Define 3 blocks. Each block has its own normalization layer.
    # 1. Self-Attn
    if self.use_ada_layer_norm:
       _A      =       AdaLayerNorm(__lowerCAmelCase			,						__lowerCAmelCase      )
    elif self.use_ada_layer_norm_zero:
       _A      =       AdaLayerNormZero(__lowerCAmelCase			,						__lowerCAmelCase      )
    else:
       _A      =       nn.LayerNorm(__lowerCAmelCase			,						elementwise_affine=__lowerCAmelCase      )
    _A      =       Attention(
        query_dim=__lowerCAmelCase			,						heads=__lowerCAmelCase			,						dim_head=__lowerCAmelCase			,						dropout=__lowerCAmelCase			,						bias=__lowerCAmelCase			,						cross_attention_dim=cross_attention_dim if only_cross_attention else None			,						upcast_attention=__lowerCAmelCase			,						)

    # 2. Cross-Attn
    if cross_attention_dim is not None or double_self_attention:
       # We currently only use AdaLayerNormZero for self attention where there will only be one attention block.
       # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during
       # the second cross attention block.
       _A      =       (
           AdaLayerNorm(__lowerCAmelCase			,						__lowerCAmelCase      )
           if self.use_ada_layer_norm
           else nn.LayerNorm(__lowerCAmelCase			,						elementwise_affine=__lowerCAmelCase      )
       )
       _A      =       Attention(
           query_dim=__lowerCAmelCase			,						cross_attention_dim=cross_attention_dim if not double_self_attention else None			,						heads=__lowerCAmelCase			,						dim_head=__lowerCAmelCase			,						dropout=__lowerCAmelCase			,						bias=__lowerCAmelCase			,						upcast_attention=__lowerCAmelCase			,						)  # is self-attn if encoder_hidden_states is none
    else:
       _A      =       None
       _A      =       None

    # 3. Feed-forward
    _A      =       nn.LayerNorm(__lowerCAmelCase			,						elementwise_affine=__lowerCAmelCase      )
    _A      =       FeedForward(__lowerCAmelCase			,						dropout=__lowerCAmelCase			,						activation_fn=__lowerCAmelCase			,						final_dropout=__lowerCAmelCase      )

    # let chunk size default to None
    _A      =       None
    _A      =       0

 def     snake_case_      (    self					:					str			,						__lowerCAmelCase					:					Optional[int]			,						__lowerCAmelCase					:					int      )   ->	Union[str, Any]:
    # Sets chunk feed-forward
    _A      =       chunk_size
    _A      =       dim

 def     snake_case_      (    self					:					int			,						__lowerCAmelCase					:					torch.FloatTensor			,						__lowerCAmelCase					:					Optional[torch.FloatTensor] = None			,						__lowerCAmelCase					:					Optional[torch.FloatTensor] = None			,						__lowerCAmelCase					:					Optional[torch.FloatTensor] = None			,						__lowerCAmelCase					:					Optional[torch.LongTensor] = None			,						__lowerCAmelCase					:					Dict[str, Any] = None			,						__lowerCAmelCase					:					Optional[torch.LongTensor] = None			,						)   ->	Dict:
    # Notice that normalization is always applied before the real computation in the following blocks.
    # 1. Self-Attention
    if self.use_ada_layer_norm:
       _A      =       self.norma(__lowerCAmelCase			,						__lowerCAmelCase      )
    elif self.use_ada_layer_norm_zero:
       _A  ,    _A  ,    _A  ,    _A  ,    _A      =       self.norma(
           __lowerCAmelCase			,						__lowerCAmelCase			,						__lowerCAmelCase			,						hidden_dtype=hidden_states.dtype      )
    else:
       _A      =       self.norma(__lowerCAmelCase      )

    _A      =       cross_attention_kwargs if cross_attention_kwargs is not None else {}

    _A      =       self.attna(
        __lowerCAmelCase			,						encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None			,						attention_mask=__lowerCAmelCase			,						**__lowerCAmelCase			,						)
    if self.use_ada_layer_norm_zero:
       _A      =       gate_msa.unsqueeze(1      ) * attn_output
    _A      =       attn_output + hidden_states

    # 2. Cross-Attention
    if self.attna is not None:
       _A      =       (
           self.norma(__lowerCAmelCase			,						__lowerCAmelCase      ) if self.use_ada_layer_norm else self.norma(__lowerCAmelCase      )
       )

       _A      =       self.attna(
           __lowerCAmelCase			,						encoder_hidden_states=__lowerCAmelCase			,						attention_mask=__lowerCAmelCase			,						**__lowerCAmelCase			,						)
       _A      =       attn_output + hidden_states

    # 3. Feed-forward
    _A      =       self.norma(__lowerCAmelCase      )

    if self.use_ada_layer_norm_zero:
       _A      =       norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None]

    if self._chunk_size is not None:
       # "feed_forward_chunk_size" can be used to save memory
       if norm_hidden_states.shape[self._chunk_dim] % self._chunk_size != 0:
          raise ValueError(
              f'''`hidden_states` dimension to be chunked: {norm_hidden_states.shape[self._chunk_dim]} has to be divisible by chunk size: {self._chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`.'''      )

       _A      =       norm_hidden_states.shape[self._chunk_dim] // self._chunk_size
       _A      =       torch.cat(
           [self.ff(__lowerCAmelCase      ) for hid_slice in norm_hidden_states.chunk(__lowerCAmelCase			,						dim=self._chunk_dim      )]			,						dim=self._chunk_dim			,						)
    else:
       _A      =       self.ff(__lowerCAmelCase      )

    if self.use_ada_layer_norm_zero:
       _A      =       gate_mlp.unsqueeze(1      ) * ff_output

    _A      =       ff_output + hidden_states

    return hidden_states


class 	lowerCamelCase__	(       nn.Module):



 """simple docstring"""

 def __init__(    self					:					Optional[Any]			,						__lowerCAmelCase					:					int			,						__lowerCAmelCase					:					Optional[int] = None			,						__lowerCAmelCase					:					int = 4			,						__lowerCAmelCase					:					float = 0.0			,						__lowerCAmelCase					:					str = "geglu"			,						__lowerCAmelCase					:					bool = False			,						)   ->	int:
    super().__init__()
    _A      =       int(dim * mult      )
    _A      =       dim_out if dim_out is not None else dim

    if activation_fn == "gelu":
       _A      =       GELU(__lowerCAmelCase			,						__lowerCAmelCase      )
    if activation_fn == "gelu-approximate":
       _A      =       GELU(__lowerCAmelCase			,						__lowerCAmelCase			,						approximate='''tanh'''      )
    elif activation_fn == "geglu":
       _A      =       GEGLU(__lowerCAmelCase			,						__lowerCAmelCase      )
    elif activation_fn == "geglu-approximate":
       _A      =       ApproximateGELU(__lowerCAmelCase			,						__lowerCAmelCase      )

    _A      =       nn.ModuleList([]      )
    # project in
    self.net.append(__lowerCAmelCase      )
    # project dropout
    self.net.append(nn.Dropout(__lowerCAmelCase      )      )
    # project out
    self.net.append(nn.Linear(__lowerCAmelCase			,						__lowerCAmelCase      )      )
    # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout
    if final_dropout:
       self.net.append(nn.Dropout(__lowerCAmelCase      )      )

 def     snake_case_      (    self					:					Tuple			,						__lowerCAmelCase					:					List[Any]      )   ->	Dict:
    for module in self.net:
       _A      =       module(__lowerCAmelCase      )
    return hidden_states


class 	lowerCamelCase__	(       nn.Module):



 """simple docstring"""

 def __init__(    self					:					Tuple			,						__lowerCAmelCase					:					int			,						__lowerCAmelCase					:					int			,						__lowerCAmelCase					:					str = "none"      )   ->	Optional[Any]:
    super().__init__()
    _A      =       nn.Linear(__lowerCAmelCase			,						__lowerCAmelCase      )
    _A      =       approximate

 def     snake_case_      (    self					:					Dict			,						__lowerCAmelCase					:					Union[str, Any]      )   ->	Optional[int]:
    if gate.device.type != "mps":
       return F.gelu(__lowerCAmelCase			,						approximate=self.approximate      )
    # mps: gelu is not implemented for float16
    return F.gelu(gate.to(dtype=torch.floataa      )			,						approximate=self.approximate      ).to(dtype=gate.dtype      )

 def     snake_case_      (    self					:					int			,						__lowerCAmelCase					:					str      )   ->	List[Any]:
    _A      =       self.proj(__lowerCAmelCase      )
    _A      =       self.gelu(__lowerCAmelCase      )
    return hidden_states


class 	lowerCamelCase__	(       nn.Module):



 """simple docstring"""

 def __init__(    self					:					int			,						__lowerCAmelCase					:					int			,						__lowerCAmelCase					:					int      )   ->	Any:
    super().__init__()
    _A      =       nn.Linear(__lowerCAmelCase			,						dim_out * 2      )

 def     snake_case_      (    self					:					Optional[Any]			,						__lowerCAmelCase					:					Optional[Any]      )   ->	Optional[int]:
    if gate.device.type != "mps":
       return F.gelu(__lowerCAmelCase      )
    # mps: gelu is not implemented for float16
    return F.gelu(gate.to(dtype=torch.floataa      )      ).to(dtype=gate.dtype      )

 def     snake_case_      (    self					:					Optional[Any]			,						__lowerCAmelCase					:					Any      )   ->	Optional[int]:
    _A  ,    _A      =       self.proj(__lowerCAmelCase      ).chunk(2			,						dim=-1      )
    return hidden_states * self.gelu(__lowerCAmelCase      )


class 	lowerCamelCase__	(       nn.Module):



 """simple docstring"""

 def __init__(    self					:					Tuple			,						__lowerCAmelCase					:					int			,						__lowerCAmelCase					:					int      )   ->	Optional[int]:
    super().__init__()
    _A      =       nn.Linear(__lowerCAmelCase			,						__lowerCAmelCase      )

 def     snake_case_      (    self					:					int			,						__lowerCAmelCase					:					Optional[Any]      )   ->	Union[str, Any]:
    _A      =       self.proj(__lowerCAmelCase      )
    return x * torch.sigmoid(1.702 * x      )


class 	lowerCamelCase__	(       nn.Module):



 """simple docstring"""

 def __init__(    self					:					List[Any]			,						__lowerCAmelCase					:					List[Any]			,						__lowerCAmelCase					:					int      )   ->	str:
    super().__init__()
    _A      =       nn.Embedding(__lowerCAmelCase			,						__lowerCAmelCase      )
    _A      =       nn.SiLU()
    _A      =       nn.Linear(__lowerCAmelCase			,						embedding_dim * 2      )
    _A      =       nn.LayerNorm(__lowerCAmelCase			,						elementwise_affine=__lowerCAmelCase      )

 def     snake_case_      (    self					:					Optional[int]			,						__lowerCAmelCase					:					Optional[int]			,						__lowerCAmelCase					:					Optional[Any]      )   ->	int:
    _A      =       self.linear(self.silu(self.emb(__lowerCAmelCase      )      )      )
    _A  ,    _A      =       torch.chunk(__lowerCAmelCase			,						2      )
    _A      =       self.norm(__lowerCAmelCase      ) * (1 + scale) + shift
    return x


class 	lowerCamelCase__	(       nn.Module):



 """simple docstring"""

 def __init__(    self					:					str			,						__lowerCAmelCase					:					Optional[Any]			,						__lowerCAmelCase					:					Tuple      )   ->	str:
    super().__init__()

    _A      =       CombinedTimestepLabelEmbeddings(__lowerCAmelCase			,						__lowerCAmelCase      )

    _A      =       nn.SiLU()
    _A      =       nn.Linear(__lowerCAmelCase			,						6 * embedding_dim			,						bias=__lowerCAmelCase      )
    _A      =       nn.LayerNorm(__lowerCAmelCase			,						elementwise_affine=__lowerCAmelCase			,						eps=1E-6      )

 def     snake_case_      (    self					:					Optional[Any]			,						__lowerCAmelCase					:					Tuple			,						__lowerCAmelCase					:					int			,						__lowerCAmelCase					:					Any			,						__lowerCAmelCase					:					str=None      )   ->	Optional[Any]:
    _A      =       self.linear(self.silu(self.emb(__lowerCAmelCase			,						__lowerCAmelCase			,						hidden_dtype=__lowerCAmelCase      )      )      )
    _A  ,    _A  ,    _A  ,    _A  ,    _A  ,    _A      =       emb.chunk(6			,						dim=1      )
    _A      =       self.norm(__lowerCAmelCase      ) * (1 + scale_msa[:, None]) + shift_msa[:, None]
    return x, gate_msa, shift_mlp, scale_mlp, gate_mlp


class 	lowerCamelCase__	(       nn.Module):



 """simple docstring"""

 def __init__(    self					:					Tuple			,						__lowerCAmelCase					:					int			,						__lowerCAmelCase					:					int			,						__lowerCAmelCase					:					int			,						__lowerCAmelCase					:					Optional[str] = None			,						__lowerCAmelCase					:					float = 1E-5      )   ->	Optional[Any]:
    super().__init__()
    _A      =       num_groups
    _A      =       eps

    if act_fn is None:
       _A      =       None
    else:
       _A      =       get_activation(__lowerCAmelCase      )

    _A      =       nn.Linear(__lowerCAmelCase			,						out_dim * 2      )

 def     snake_case_      (    self					:					List[Any]			,						__lowerCAmelCase					:					Union[str, Any]			,						__lowerCAmelCase					:					Any      )   ->	Union[str, Any]:
    if self.act:
       _A      =       self.act(__lowerCAmelCase      )
    _A      =       self.linear(__lowerCAmelCase      )
    _A      =       emb[:, :, None, None]
    _A  ,    _A      =       emb.chunk(2			,						dim=1      )

    _A      =       F.group_norm(__lowerCAmelCase			,						self.num_groups			,						eps=self.eps      )
    _A      =       x * (1 + scale) + shift
    return x









'''simple docstring'''







import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase      :   Optional[int]      =       logging.get_logger(__name__)

lowercase      :   Union[str, Any]      =       {
    """microsoft/unispeech-sat-base-100h-libri-ft""": (
        """https://huggingface.co/microsoft/unispeech-sat-base-100h-libri-ft/resolve/main/config.json"""
    ),
    # See all UniSpeechSat models at https://huggingface.co/models?filter=unispeech_sat
}







class 					_a (a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     List[Any]          =					"""unispeech-sat"""







     def __init__(				self							,__a=32							,__a=768							,__a=12							,__a=12							,__a=3_072							,__a="gelu"							,__a=0.1							,__a=0.1							,__a=0.1							,__a=0.0							,__a=0.0							,__a=0.1							,__a=0.1							,__a=0.02							,__a=1E-5							,__a="group"							,__a="gelu"							,__a=(512, 512, 512, 512, 512, 512, 512)							,__a=(5, 2, 2, 2, 2, 2, 2)							,__a=(10, 3, 3, 3, 3, 2, 2)							,__a=False							,__a=128							,__a=16							,__a=False							,__a=True							,__a=0.05							,__a=10							,__a=2							,__a=0.0							,__a=10							,__a=0							,__a=320							,__a=2							,__a=0.1							,__a=100							,__a=256							,__a=256							,__a=0.1							,__a="mean"							,__a=False							,__a=False							,__a=256							,__a=(512, 512, 512, 512, 1_500)							,__a=(5, 3, 3, 1, 1)							,__a=(1, 2, 3, 1, 1)							,__a=512							,__a=0							,__a=1							,__a=2							,__a=504							,**__a							,)      ->				str:
          super().__init__(**__a							,pad_token_id=__a							,bos_token_id=__a							,eos_token_id=__a							)
          snake_case			:      List[Any]        =			hidden_size
          snake_case			:      Any        =			feat_extract_norm
          snake_case			:      Dict        =			feat_extract_activation
          snake_case			:      List[Any]        =			list(__a							)
          snake_case			:      int        =			list(__a							)
          snake_case			:      Dict        =			list(__a							)
          snake_case			:      int        =			conv_bias
          snake_case			:      Optional[Any]        =			num_conv_pos_embeddings
          snake_case			:      Any        =			num_conv_pos_embedding_groups
          snake_case			:      Dict        =			len(self.conv_dim							)
          snake_case			:      str        =			num_hidden_layers
          snake_case			:      Optional[Any]        =			intermediate_size
          snake_case			:      Optional[Any]        =			hidden_act
          snake_case			:      Union[str, Any]        =			num_attention_heads
          snake_case			:      str        =			hidden_dropout
          snake_case			:      Tuple        =			attention_dropout
          snake_case			:      Tuple        =			activation_dropout
          snake_case			:      List[Any]        =			feat_proj_dropout
          snake_case			:      List[str]        =			final_dropout
          snake_case			:      Union[str, Any]        =			layerdrop
          snake_case			:      Optional[int]        =			layer_norm_eps
          snake_case			:      int        =			initializer_range
          snake_case			:      List[Any]        =			vocab_size
          snake_case			:      Any        =			num_clusters
          snake_case			:      Dict        =			do_stable_layer_norm
          snake_case			:      int        =			use_weighted_layer_sum

          if (
              (len(self.conv_stride							) != self.num_feat_extract_layers)
              or (len(self.conv_kernel							) != self.num_feat_extract_layers)
              or (len(self.conv_dim							) != self.num_feat_extract_layers)
          ):
               raise ValueError(
                   """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
                   """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
                   F''' {len(self.conv_dim							)}`, `len(config.conv_stride) = {len(self.conv_stride							)}`,'''
                   F''' `len(config.conv_kernel) = {len(self.conv_kernel							)}`.'''							)

          # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
          snake_case			:      Dict        =			apply_spec_augment
          snake_case			:      Optional[int]        =			mask_time_prob
          snake_case			:      int        =			mask_time_length
          snake_case			:      int        =			mask_time_min_masks
          snake_case			:      int        =			mask_feature_prob
          snake_case			:      Optional[Any]        =			mask_feature_length
          snake_case			:      List[Any]        =			mask_feature_min_masks

          # parameters for pretraining with codevector quantized representations
          snake_case			:      Optional[Any]        =			num_codevectors_per_group
          snake_case			:      Optional[int]        =			num_codevector_groups
          snake_case			:      Any        =			contrastive_logits_temperature
          snake_case			:      List[str]        =			feat_quantizer_dropout
          snake_case			:      Dict        =			num_negatives
          snake_case			:      int        =			codevector_dim
          snake_case			:      Optional[int]        =			proj_codevector_dim
          snake_case			:      Union[str, Any]        =			diversity_loss_weight

          # ctc loss
          snake_case			:      Union[str, Any]        =			ctc_loss_reduction
          snake_case			:      Optional[int]        =			ctc_zero_infinity

          # SequenceClassification-specific parameter. Feel free to ignore for other classes.
          snake_case			:      Dict        =			classifier_proj_size

          # XVector-specific parameters. Feel free to ignore for other classes.
          snake_case			:      int        =			list(__a							)
          snake_case			:      Tuple        =			list(__a							)
          snake_case			:      List[Any]        =			list(__a							)
          snake_case			:      Dict        =			xvector_output_dim







     @property
     def 			snake_case_      (				self							)      ->				List[str]:
          return functools.reduce(operator.mul							,self.conv_stride							,1							)



