






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_							=    logging.get_logger(__name__)

a_							=    {
    '''microsoft/unispeech-large-1500h-cv''': (
        '''https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json'''
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}


class      lowercase__     (			_UpperCAmelCase	):
	a_         ="""unispeech"""

	def __init__(							self		,    __UpperCAmelCase=32		,    __UpperCAmelCase=768		,    __UpperCAmelCase=12		,    __UpperCAmelCase=12		,    __UpperCAmelCase=3072		,    __UpperCAmelCase="gelu"		,    __UpperCAmelCase=0.1		,    __UpperCAmelCase=0.1		,    __UpperCAmelCase=0.1		,    __UpperCAmelCase=0.0		,    __UpperCAmelCase=0.0		,    __UpperCAmelCase=0.1		,    __UpperCAmelCase=0.1		,    __UpperCAmelCase=0.02		,    __UpperCAmelCase=1E-5		,    __UpperCAmelCase="group"		,    __UpperCAmelCase="gelu"		,    __UpperCAmelCase=(512, 512, 512, 512, 512, 512, 512)		,    __UpperCAmelCase=(5, 2, 2, 2, 2, 2, 2)		,    __UpperCAmelCase=(10, 3, 3, 3, 3, 2, 2)		,    __UpperCAmelCase=False		,    __UpperCAmelCase=128		,    __UpperCAmelCase=16		,    __UpperCAmelCase=False		,    __UpperCAmelCase=True		,    __UpperCAmelCase=0.05		,    __UpperCAmelCase=10		,    __UpperCAmelCase=2		,    __UpperCAmelCase=0.0		,    __UpperCAmelCase=10		,    __UpperCAmelCase=0		,    __UpperCAmelCase=320		,    __UpperCAmelCase=2		,    __UpperCAmelCase=0.1		,    __UpperCAmelCase=100		,    __UpperCAmelCase=256		,    __UpperCAmelCase=256		,    __UpperCAmelCase=0.1		,    __UpperCAmelCase="mean"		,    __UpperCAmelCase=False		,    __UpperCAmelCase=False		,    __UpperCAmelCase=256		,    __UpperCAmelCase=80		,    __UpperCAmelCase=0		,    __UpperCAmelCase=1		,    __UpperCAmelCase=2		,    __UpperCAmelCase=0.5		,    **__UpperCAmelCase		,    )->					Union[str, Any]:







							'''simple docstring'''





							super().__init__(**__UpperCAmelCase		,    pad_token_id=__UpperCAmelCase		,    bos_token_id=__UpperCAmelCase		,    eos_token_id=__UpperCAmelCase       )
							lowerCAmelCase__       =						hidden_size
							lowerCAmelCase__       =						feat_extract_norm
							lowerCAmelCase__       =						feat_extract_activation
							lowerCAmelCase__       =						list(__UpperCAmelCase       )
							lowerCAmelCase__       =						list(__UpperCAmelCase       )
							lowerCAmelCase__       =						list(__UpperCAmelCase       )
							lowerCAmelCase__       =						conv_bias
							lowerCAmelCase__       =						num_conv_pos_embeddings
							lowerCAmelCase__       =						num_conv_pos_embedding_groups
							lowerCAmelCase__       =						len(self.conv_dim       )
							lowerCAmelCase__       =						num_hidden_layers
							lowerCAmelCase__       =						intermediate_size
							lowerCAmelCase__       =						hidden_act
							lowerCAmelCase__       =						num_attention_heads
							lowerCAmelCase__       =						hidden_dropout
							lowerCAmelCase__       =						attention_dropout
							lowerCAmelCase__       =						activation_dropout
							lowerCAmelCase__       =						feat_proj_dropout
							lowerCAmelCase__       =						final_dropout
							lowerCAmelCase__       =						layerdrop
							lowerCAmelCase__       =						layer_norm_eps
							lowerCAmelCase__       =						initializer_range
							lowerCAmelCase__       =						num_ctc_classes
							lowerCAmelCase__       =						vocab_size
							lowerCAmelCase__       =						do_stable_layer_norm
							lowerCAmelCase__       =						use_weighted_layer_sum
							lowerCAmelCase__       =						classifier_proj_size

							if (
							    (len(self.conv_stride       ) != self.num_feat_extract_layers)
							    or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
							    or (len(self.conv_dim       ) != self.num_feat_extract_layers)
							):
													raise ValueError(
													    "Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="
													    " `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="
													    F" {len(self.conv_dim       )}`, `len(config.conv_stride) = {len(self.conv_stride       )}`,"
													    F" `len(config.conv_kernel) = {len(self.conv_kernel       )}`."       )

							# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
							lowerCAmelCase__       =						apply_spec_augment
							lowerCAmelCase__       =						mask_time_prob
							lowerCAmelCase__       =						mask_time_length
							lowerCAmelCase__       =						mask_time_min_masks
							lowerCAmelCase__       =						mask_feature_prob
							lowerCAmelCase__       =						mask_feature_length
							lowerCAmelCase__       =						mask_feature_min_masks

							# parameters for pretraining with codevector quantized representations
							lowerCAmelCase__       =						num_codevectors_per_group
							lowerCAmelCase__       =						num_codevector_groups
							lowerCAmelCase__       =						contrastive_logits_temperature
							lowerCAmelCase__       =						feat_quantizer_dropout
							lowerCAmelCase__       =						num_negatives
							lowerCAmelCase__       =						codevector_dim
							lowerCAmelCase__       =						proj_codevector_dim
							lowerCAmelCase__       =						diversity_loss_weight

							# ctc loss
							lowerCAmelCase__       =						ctc_loss_reduction
							lowerCAmelCase__       =						ctc_zero_infinity

							# pretraining loss
							lowerCAmelCase__       =						replace_prob


	@property
	def 		UpperCAmelCase  (							self       )->					Any:







							'''simple docstring'''





							return functools.reduce(operator.mul		,    self.conv_stride		,    1       )














import flax.linen as nn
import jax.numpy as jnp

from .attention_flax import FlaxTransformeraDModel
from .resnet_flax import FlaxDownsampleaD, FlaxResnetBlockaD, FlaxUpsampleaD


class      lowercase__     (			nn.Module	):
	a_         =42
	a_         =42
	a_         =0.0
	a_         =1
	a_         =1
	a_         =True
	a_         =False
	a_         =False
	a_         =False
	a_         =jnp.floataa

	def 		UpperCAmelCase  (							self       )->					Optional[Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						[]
							lowerCAmelCase__       =						[]

							for i in range(self.num_layers       ):
													lowerCAmelCase__       =						self.in_channels if i == 0 else self.out_channels

													lowerCAmelCase__       =						FlaxResnetBlockaD(
													    in_channels=__UpperCAmelCase		,    out_channels=self.out_channels		,    dropout_prob=self.dropout		,    dtype=self.dtype		,    )
													resnets.append(__UpperCAmelCase       )

													lowerCAmelCase__       =						FlaxTransformeraDModel(
													    in_channels=self.out_channels		,    n_heads=self.num_attention_heads		,    d_head=self.out_channels // self.num_attention_heads		,    depth=1		,    use_linear_projection=self.use_linear_projection		,    only_cross_attention=self.only_cross_attention		,    use_memory_efficient_attention=self.use_memory_efficient_attention		,    dtype=self.dtype		,    )
													attentions.append(__UpperCAmelCase       )

							lowerCAmelCase__       =						resnets
							lowerCAmelCase__       =						attentions

							if self.add_downsample:
													lowerCAmelCase__       =						FlaxDownsampleaD(self.out_channels		,    dtype=self.dtype       )


	def __call__(							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase=True       )->					str:







							'''simple docstring'''





							lowerCAmelCase__       =						()

							for resnet, attn in zip(self.resnets		,    self.attentions       ):
													lowerCAmelCase__       =						resnet(__UpperCAmelCase		,    __UpperCAmelCase		,    deterministic=__UpperCAmelCase       )
													lowerCAmelCase__       =						attn(__UpperCAmelCase		,    __UpperCAmelCase		,    deterministic=__UpperCAmelCase       )
													output_states += (hidden_states,)

							if self.add_downsample:
													lowerCAmelCase__       =						self.downsamplers_a(__UpperCAmelCase       )
													output_states += (hidden_states,)

							return hidden_states, output_states





class      lowercase__     (			nn.Module	):
	a_         =42
	a_         =42
	a_         =0.0
	a_         =1
	a_         =True
	a_         =jnp.floataa

	def 		UpperCAmelCase  (							self       )->					Dict:







							'''simple docstring'''





							lowerCAmelCase__       =						[]

							for i in range(self.num_layers       ):
													lowerCAmelCase__       =						self.in_channels if i == 0 else self.out_channels

													lowerCAmelCase__       =						FlaxResnetBlockaD(
													    in_channels=__UpperCAmelCase		,    out_channels=self.out_channels		,    dropout_prob=self.dropout		,    dtype=self.dtype		,    )
													resnets.append(__UpperCAmelCase       )
							lowerCAmelCase__       =						resnets

							if self.add_downsample:
													lowerCAmelCase__       =						FlaxDownsampleaD(self.out_channels		,    dtype=self.dtype       )


	def __call__(							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase=True       )->					int:







							'''simple docstring'''





							lowerCAmelCase__       =						()

							for resnet in self.resnets:
													lowerCAmelCase__       =						resnet(__UpperCAmelCase		,    __UpperCAmelCase		,    deterministic=__UpperCAmelCase       )
													output_states += (hidden_states,)

							if self.add_downsample:
													lowerCAmelCase__       =						self.downsamplers_a(__UpperCAmelCase       )
													output_states += (hidden_states,)

							return hidden_states, output_states





class      lowercase__     (			nn.Module	):
	a_         =42
	a_         =42
	a_         =42
	a_         =0.0
	a_         =1
	a_         =1
	a_         =True
	a_         =False
	a_         =False
	a_         =False
	a_         =jnp.floataa

	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							lowerCAmelCase__       =						[]
							lowerCAmelCase__       =						[]

							for i in range(self.num_layers       ):
													lowerCAmelCase__       =						self.in_channels if (i == self.num_layers - 1) else self.out_channels
													lowerCAmelCase__       =						self.prev_output_channel if i == 0 else self.out_channels

													lowerCAmelCase__       =						FlaxResnetBlockaD(
													    in_channels=resnet_in_channels + res_skip_channels		,    out_channels=self.out_channels		,    dropout_prob=self.dropout		,    dtype=self.dtype		,    )
													resnets.append(__UpperCAmelCase       )

													lowerCAmelCase__       =						FlaxTransformeraDModel(
													    in_channels=self.out_channels		,    n_heads=self.num_attention_heads		,    d_head=self.out_channels // self.num_attention_heads		,    depth=1		,    use_linear_projection=self.use_linear_projection		,    only_cross_attention=self.only_cross_attention		,    use_memory_efficient_attention=self.use_memory_efficient_attention		,    dtype=self.dtype		,    )
													attentions.append(__UpperCAmelCase       )

							lowerCAmelCase__       =						resnets
							lowerCAmelCase__       =						attentions

							if self.add_upsample:
													lowerCAmelCase__       =						FlaxUpsampleaD(self.out_channels		,    dtype=self.dtype       )


	def __call__(							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase=True       )->					Dict:







							'''simple docstring'''





							for resnet, attn in zip(self.resnets		,    self.attentions       ):
													# pop res hidden states
													lowerCAmelCase__       =						res_hidden_states_tuple[-1]
													lowerCAmelCase__       =						res_hidden_states_tuple[:-1]
													lowerCAmelCase__       =						jnp.concatenate((hidden_states, res_hidden_states)		,    axis=-1       )

													lowerCAmelCase__       =						resnet(__UpperCAmelCase		,    __UpperCAmelCase		,    deterministic=__UpperCAmelCase       )
													lowerCAmelCase__       =						attn(__UpperCAmelCase		,    __UpperCAmelCase		,    deterministic=__UpperCAmelCase       )

							if self.add_upsample:
													lowerCAmelCase__       =						self.upsamplers_a(__UpperCAmelCase       )

							return hidden_states





class      lowercase__     (			nn.Module	):
	a_         =42
	a_         =42
	a_         =42
	a_         =0.0
	a_         =1
	a_         =True
	a_         =jnp.floataa

	def 		UpperCAmelCase  (							self       )->					int:







							'''simple docstring'''





							lowerCAmelCase__       =						[]

							for i in range(self.num_layers       ):
													lowerCAmelCase__       =						self.in_channels if (i == self.num_layers - 1) else self.out_channels
													lowerCAmelCase__       =						self.prev_output_channel if i == 0 else self.out_channels

													lowerCAmelCase__       =						FlaxResnetBlockaD(
													    in_channels=resnet_in_channels + res_skip_channels		,    out_channels=self.out_channels		,    dropout_prob=self.dropout		,    dtype=self.dtype		,    )
													resnets.append(__UpperCAmelCase       )

							lowerCAmelCase__       =						resnets

							if self.add_upsample:
													lowerCAmelCase__       =						FlaxUpsampleaD(self.out_channels		,    dtype=self.dtype       )


	def __call__(							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase=True       )->					int:







							'''simple docstring'''





							for resnet in self.resnets:
													# pop res hidden states
													lowerCAmelCase__       =						res_hidden_states_tuple[-1]
													lowerCAmelCase__       =						res_hidden_states_tuple[:-1]
													lowerCAmelCase__       =						jnp.concatenate((hidden_states, res_hidden_states)		,    axis=-1       )

													lowerCAmelCase__       =						resnet(__UpperCAmelCase		,    __UpperCAmelCase		,    deterministic=__UpperCAmelCase       )

							if self.add_upsample:
													lowerCAmelCase__       =						self.upsamplers_a(__UpperCAmelCase       )

							return hidden_states





class      lowercase__     (			nn.Module	):
	a_         =42
	a_         =0.0
	a_         =1
	a_         =1
	a_         =False
	a_         =False
	a_         =jnp.floataa

	def 		UpperCAmelCase  (							self       )->					Union[str, Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						[
							    FlaxResnetBlockaD(
							        in_channels=self.in_channels		,    out_channels=self.in_channels		,    dropout_prob=self.dropout		,    dtype=self.dtype		,    )
							]

							lowerCAmelCase__       =						[]

							for _ in range(self.num_layers       ):
													lowerCAmelCase__       =						FlaxTransformeraDModel(
													    in_channels=self.in_channels		,    n_heads=self.num_attention_heads		,    d_head=self.in_channels // self.num_attention_heads		,    depth=1		,    use_linear_projection=self.use_linear_projection		,    use_memory_efficient_attention=self.use_memory_efficient_attention		,    dtype=self.dtype		,    )
													attentions.append(__UpperCAmelCase       )

													lowerCAmelCase__       =						FlaxResnetBlockaD(
													    in_channels=self.in_channels		,    out_channels=self.in_channels		,    dropout_prob=self.dropout		,    dtype=self.dtype		,    )
													resnets.append(__UpperCAmelCase       )

							lowerCAmelCase__       =						resnets
							lowerCAmelCase__       =						attentions


	def __call__(							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase=True       )->					Union[str, Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						self.resnets[0](__UpperCAmelCase		,    __UpperCAmelCase       )
							for attn, resnet in zip(self.attentions		,    self.resnets[1:]       ):
													lowerCAmelCase__       =						attn(__UpperCAmelCase		,    __UpperCAmelCase		,    deterministic=__UpperCAmelCase       )
													lowerCAmelCase__       =						resnet(__UpperCAmelCase		,    __UpperCAmelCase		,    deterministic=__UpperCAmelCase       )

							return hidden_states






