
"""simple docstring"""
import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_       :						List[str]     					=	logging.get_logger(__name__)

lowercase_       :						int     					=	{
    '''microsoft/unispeech-large-1500h-cv''': (
        '''https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json'''
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}
class 			UpperCamelCase      (							__SCREAMING_SNAKE_CASE  ):
     A__      =			"""unispeech"""
     def __init__(     self  ,     snake_case__=32  ,     snake_case__=768  ,     snake_case__=12  ,     snake_case__=12  ,     snake_case__=3072  ,     snake_case__="gelu"  ,     snake_case__=0.1  ,     snake_case__=0.1  ,     snake_case__=0.1  ,     snake_case__=0.0  ,     snake_case__=0.0  ,     snake_case__=0.1  ,     snake_case__=0.1  ,     snake_case__=0.02  ,     snake_case__=1E-5  ,     snake_case__="group"  ,     snake_case__="gelu"  ,     snake_case__=(512, 512, 512, 512, 512, 512, 512)  ,     snake_case__=(5, 2, 2, 2, 2, 2, 2)  ,     snake_case__=(10, 3, 3, 3, 3, 2, 2)  ,     snake_case__=False  ,     snake_case__=128  ,     snake_case__=16  ,     snake_case__=False  ,     snake_case__=True  ,     snake_case__=0.05  ,     snake_case__=10  ,     snake_case__=2  ,     snake_case__=0.0  ,     snake_case__=10  ,     snake_case__=0  ,     snake_case__=320  ,     snake_case__=2  ,     snake_case__=0.1  ,     snake_case__=100  ,     snake_case__=256  ,     snake_case__=256  ,     snake_case__=0.1  ,     snake_case__="mean"  ,     snake_case__=False  ,     snake_case__=False  ,     snake_case__=256  ,     snake_case__=80  ,     snake_case__=0  ,     snake_case__=1  ,     snake_case__=2  ,     snake_case__=0.5  ,     **snake_case__  ,     ):




       """simple docstring"""


       super().__init__(**snake_case__  ,     pad_token_id=snake_case__  ,     bos_token_id=snake_case__  ,     eos_token_id=snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Dict							    = hidden_size
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = feat_extract_norm
       _SCREAMING_SNAKE_CASE   :				Tuple							    = feat_extract_activation
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = list(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Tuple							    = list(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				List[str]							    = list(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = conv_bias
       _SCREAMING_SNAKE_CASE   :				Dict							    = num_conv_pos_embeddings
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = num_conv_pos_embedding_groups
       _SCREAMING_SNAKE_CASE   :				Tuple							    = len(self.conv_dim				)
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = num_hidden_layers
       _SCREAMING_SNAKE_CASE   :				Any							    = intermediate_size
       _SCREAMING_SNAKE_CASE   :				Dict							    = hidden_act
       _SCREAMING_SNAKE_CASE   :				int							    = num_attention_heads
       _SCREAMING_SNAKE_CASE   :				Any							    = hidden_dropout
       _SCREAMING_SNAKE_CASE   :				List[str]							    = attention_dropout
       _SCREAMING_SNAKE_CASE   :				int							    = activation_dropout
       _SCREAMING_SNAKE_CASE   :				Tuple							    = feat_proj_dropout
       _SCREAMING_SNAKE_CASE   :				List[str]							    = final_dropout
       _SCREAMING_SNAKE_CASE   :				str							    = layerdrop
       _SCREAMING_SNAKE_CASE   :				Tuple							    = layer_norm_eps
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = initializer_range
       _SCREAMING_SNAKE_CASE   :				int							    = num_ctc_classes
       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = vocab_size
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = do_stable_layer_norm
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = use_weighted_layer_sum
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = classifier_proj_size

       if (
           (len(self.conv_stride				) != self.num_feat_extract_layers)
           or (len(self.conv_kernel				) != self.num_feat_extract_layers)
           or (len(self.conv_dim				) != self.num_feat_extract_layers)
       ):
         raise ValueError(
             "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
             " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
             F''' {len(self.conv_dim				)}`, `len(config.conv_stride) = {len(self.conv_stride				)}`,'''
             F''' `len(config.conv_kernel) = {len(self.conv_kernel				)}`.'''				)

       # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
       _SCREAMING_SNAKE_CASE   :				Any							    = apply_spec_augment
       _SCREAMING_SNAKE_CASE   :				List[str]							    = mask_time_prob
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = mask_time_length
       _SCREAMING_SNAKE_CASE   :				Dict							    = mask_time_min_masks
       _SCREAMING_SNAKE_CASE   :				Any							    = mask_feature_prob
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = mask_feature_length
       _SCREAMING_SNAKE_CASE   :				int							    = mask_feature_min_masks

       # parameters for pretraining with codevector quantized representations
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = num_codevectors_per_group
       _SCREAMING_SNAKE_CASE   :				int							    = num_codevector_groups
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = contrastive_logits_temperature
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = feat_quantizer_dropout
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = num_negatives
       _SCREAMING_SNAKE_CASE   :				int							    = codevector_dim
       _SCREAMING_SNAKE_CASE   :				Tuple							    = proj_codevector_dim
       _SCREAMING_SNAKE_CASE   :				int							    = diversity_loss_weight

       # ctc loss
       _SCREAMING_SNAKE_CASE   :				Tuple							    = ctc_loss_reduction
       _SCREAMING_SNAKE_CASE   :				Tuple							    = ctc_zero_infinity

       # pretraining loss
       _SCREAMING_SNAKE_CASE   :				str							    = replace_prob




     @property
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       return functools.reduce(operator.mul  ,     self.conv_stride  ,     1				)





"""simple docstring"""
from typing import Optional, Tuple, Union

import flax
import flax.linen as nn
import jax
import jax.numpy as jnp
from flax.core.frozen_dict import FrozenDict

from ..configuration_utils import ConfigMixin, flax_register_to_config
from ..utils import BaseOutput
from .embeddings_flax import FlaxTimestepEmbedding, FlaxTimesteps
from .modeling_flax_utils import FlaxModelMixin
from .unet_ad_blocks_flax import (
    FlaxCrossAttnDownBlockaD,
    FlaxCrossAttnUpBlockaD,
    FlaxDownBlockaD,
    FlaxUNetMidBlockaDCrossAttn,
    FlaxUpBlockaD,
)
@flax.struct.dataclass
class 			UpperCamelCase      (							__SCREAMING_SNAKE_CASE  ):
     A__      =			42
@flax_register_to_config
class 			UpperCamelCase      (							nn.Module      ,       __SCREAMING_SNAKE_CASE      ,       __SCREAMING_SNAKE_CASE  ):
     A__      =			32
     A__      =			4
     A__      =			4
     A__      =			(
         "CrossAttnDownBlock2D",
         "CrossAttnDownBlock2D",
         "CrossAttnDownBlock2D",
         "DownBlock2D",
     )
     A__      =			("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")
     A__      =			False
     A__      =			(320, 640, 1280, 1280)
     A__      =			2
     A__      =			8
     A__      =			None
     A__      =			1280
     A__      =			0.0
     A__      =			False
     A__      =			jnp.floataa
     A__      =			True
     A__      =			0
     A__      =			False
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = (1, self.in_channels, self.sample_size, self.sample_size)
       _SCREAMING_SNAKE_CASE   :				int							    = jnp.zeros(snake_case__  ,     dtype=jnp.floataa				)
       _SCREAMING_SNAKE_CASE   :				str							    = jnp.ones((1,)  ,     dtype=jnp.intaa				)
       _SCREAMING_SNAKE_CASE   :				Any							    = jnp.zeros((1, 1, self.cross_attention_dim)  ,     dtype=jnp.floataa				)

       _SCREAMING_SNAKE_CASE    ,		_SCREAMING_SNAKE_CASE   :				Tuple							    = jax.random.split(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = {"params": params_rng, "dropout": dropout_rng}

       return self.init(snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__				)["params"]
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				List[str]							    = self.block_out_channels
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = block_out_channels[0] * 4

       if self.num_attention_heads is not None:
         raise ValueError(
             "At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19."				)

       # If `num_attention_heads` is not defined (which is the case for most models)
       # it will default to `attention_head_dim`. This looks weird upon first reading it and it is.
       # The reason for this behavior is to correct for incorrectly named variables that were introduced
       # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131
       # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking
       # which is why we correct for the naming here.
       _SCREAMING_SNAKE_CASE   :				List[str]							    = self.num_attention_heads or self.attention_head_dim

       # input
       _SCREAMING_SNAKE_CASE   :				int							    = nn.Conv(
           block_out_channels[0]  ,     kernel_size=(3, 3)  ,     strides=(1, 1)  ,     padding=((1, 1), (1, 1))  ,     dtype=self.dtype  ,     )

       # time
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = FlaxTimesteps(
           block_out_channels[0]  ,     flip_sin_to_cos=self.flip_sin_to_cos  ,     freq_shift=self.config.freq_shift				)
       _SCREAMING_SNAKE_CASE   :				Dict							    = FlaxTimestepEmbedding(snake_case__  ,     dtype=self.dtype				)

       _SCREAMING_SNAKE_CASE   :				str							    = self.only_cross_attention
       if isinstance(snake_case__  ,     snake_case__				):
         _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = (only_cross_attention,) * len(self.down_block_types				)

       if isinstance(snake_case__  ,     snake_case__				):
         _SCREAMING_SNAKE_CASE   :				int							    = (num_attention_heads,) * len(self.down_block_types				)

       # down
       _SCREAMING_SNAKE_CASE   :				Tuple							    = []
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = block_out_channels[0]
       for i, down_block_type in enumerate(self.down_block_types				):
         _SCREAMING_SNAKE_CASE   :				str							    = output_channel
         _SCREAMING_SNAKE_CASE   :				List[str]							    = block_out_channels[i]
         _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = i == len(snake_case__				) - 1

         if down_block_type == "CrossAttnDownBlock2D":
           _SCREAMING_SNAKE_CASE   :				Optional[int]							    = FlaxCrossAttnDownBlockaD(
               in_channels=snake_case__  ,     out_channels=snake_case__  ,     dropout=self.dropout  ,     num_layers=self.layers_per_block  ,     num_attention_heads=num_attention_heads[i]  ,     add_downsample=not is_final_block  ,     use_linear_projection=self.use_linear_projection  ,     only_cross_attention=only_cross_attention[i]  ,     use_memory_efficient_attention=self.use_memory_efficient_attention  ,     dtype=self.dtype  ,     )
         else:
           _SCREAMING_SNAKE_CASE   :				str							    = FlaxDownBlockaD(
               in_channels=snake_case__  ,     out_channels=snake_case__  ,     dropout=self.dropout  ,     num_layers=self.layers_per_block  ,     add_downsample=not is_final_block  ,     dtype=self.dtype  ,     )

         down_blocks.append(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Tuple							    = down_blocks

       # mid
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = FlaxUNetMidBlockaDCrossAttn(
           in_channels=block_out_channels[-1]  ,     dropout=self.dropout  ,     num_attention_heads=num_attention_heads[-1]  ,     use_linear_projection=self.use_linear_projection  ,     use_memory_efficient_attention=self.use_memory_efficient_attention  ,     dtype=self.dtype  ,     )

       # up
       _SCREAMING_SNAKE_CASE   :				int							    = []
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = list(reversed(snake_case__				)				)
       _SCREAMING_SNAKE_CASE   :				Any							    = list(reversed(snake_case__				)				)
       _SCREAMING_SNAKE_CASE   :				Any							    = list(reversed(snake_case__				)				)
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = reversed_block_out_channels[0]
       for i, up_block_type in enumerate(self.up_block_types				):
         _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = output_channel
         _SCREAMING_SNAKE_CASE   :				List[str]							    = reversed_block_out_channels[i]
         _SCREAMING_SNAKE_CASE   :				List[Any]							    = reversed_block_out_channels[min(i + 1  ,     len(snake_case__				) - 1				)]

         _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = i == len(snake_case__				) - 1

         if up_block_type == "CrossAttnUpBlock2D":
           _SCREAMING_SNAKE_CASE   :				int							    = FlaxCrossAttnUpBlockaD(
               in_channels=snake_case__  ,     out_channels=snake_case__  ,     prev_output_channel=snake_case__  ,     num_layers=self.layers_per_block + 1  ,     num_attention_heads=reversed_num_attention_heads[i]  ,     add_upsample=not is_final_block  ,     dropout=self.dropout  ,     use_linear_projection=self.use_linear_projection  ,     only_cross_attention=only_cross_attention[i]  ,     use_memory_efficient_attention=self.use_memory_efficient_attention  ,     dtype=self.dtype  ,     )
         else:
           _SCREAMING_SNAKE_CASE   :				str							    = FlaxUpBlockaD(
               in_channels=snake_case__  ,     out_channels=snake_case__  ,     prev_output_channel=snake_case__  ,     num_layers=self.layers_per_block + 1  ,     add_upsample=not is_final_block  ,     dropout=self.dropout  ,     dtype=self.dtype  ,     )

         up_blocks.append(snake_case__				)
         _SCREAMING_SNAKE_CASE   :				Any							    = output_channel
       _SCREAMING_SNAKE_CASE   :				int							    = up_blocks

       # out
       _SCREAMING_SNAKE_CASE   :				str							    = nn.GroupNorm(num_groups=32  ,     epsilon=1E-5				)
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = nn.Conv(
           self.out_channels  ,     kernel_size=(3, 3)  ,     strides=(1, 1)  ,     padding=((1, 1), (1, 1))  ,     dtype=self.dtype  ,     )




     def __call__(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__=None  ,     snake_case__=None  ,     snake_case__ = True  ,     snake_case__ = False  ,     ):




       """simple docstring"""


       if not isinstance(snake_case__  ,     jnp.ndarray				):
         _SCREAMING_SNAKE_CASE   :				Any							    = jnp.array([timesteps]  ,     dtype=jnp.intaa				)
       elif isinstance(snake_case__  ,     jnp.ndarray				) and len(timesteps.shape				) == 0:
         _SCREAMING_SNAKE_CASE   :				Tuple							    = timesteps.astype(dtype=jnp.floataa				)
         _SCREAMING_SNAKE_CASE   :				Tuple							    = jnp.expand_dims(snake_case__  ,     0				)

       _SCREAMING_SNAKE_CASE   :				List[str]							    = self.time_proj(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Dict							    = self.time_embedding(snake_case__				)

       # 2. pre-process
       _SCREAMING_SNAKE_CASE   :				Dict							    = jnp.transpose(snake_case__  ,     (0, 2, 3, 1)				)
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = self.conv_in(snake_case__				)

       # 3. down
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = (sample,)
       for down_block in self.down_blocks:
         if isinstance(snake_case__  ,     snake_case__				):
           _SCREAMING_SNAKE_CASE    ,		_SCREAMING_SNAKE_CASE   :				str							    = down_block(snake_case__  ,     snake_case__  ,     snake_case__  ,     deterministic=not train				)
         else:
           _SCREAMING_SNAKE_CASE    ,		_SCREAMING_SNAKE_CASE   :				int							    = down_block(snake_case__  ,     snake_case__  ,     deterministic=not train				)
         down_block_res_samples += res_samples

       if down_block_additional_residuals is not None:
         _SCREAMING_SNAKE_CASE   :				str							    = ()

         for down_block_res_sample, down_block_additional_residual in zip(
             snake_case__  ,     snake_case__				):
           down_block_res_sample += down_block_additional_residual
           new_down_block_res_samples += (down_block_res_sample,)

         _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = new_down_block_res_samples

       # 4. mid
       _SCREAMING_SNAKE_CASE   :				Dict							    = self.mid_block(snake_case__  ,     snake_case__  ,     snake_case__  ,     deterministic=not train				)

       if mid_block_additional_residual is not None:
         sample += mid_block_additional_residual

       # 5. up
       for up_block in self.up_blocks:
         _SCREAMING_SNAKE_CASE   :				List[str]							    = down_block_res_samples[-(self.layers_per_block + 1) :]
         _SCREAMING_SNAKE_CASE   :				int							    = down_block_res_samples[: -(self.layers_per_block + 1)]
         if isinstance(snake_case__  ,     snake_case__				):
           _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = up_block(
               snake_case__  ,     temb=snake_case__  ,     encoder_hidden_states=snake_case__  ,     res_hidden_states_tuple=snake_case__  ,     deterministic=not train  ,     )
         else:
           _SCREAMING_SNAKE_CASE   :				List[str]							    = up_block(snake_case__  ,     temb=snake_case__  ,     res_hidden_states_tuple=snake_case__  ,     deterministic=not train				)

        # 6. post-process
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = self.conv_norm_out(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				Optional[int]							    = nn.silu(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				List[str]							    = self.conv_out(snake_case__				)
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = jnp.transpose(snake_case__  ,     (0, 3, 1, 2)				)

       if not return_dict:
         return (sample,)

       return FlaxUNetaDConditionOutput(sample=snake_case__				)



