





from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxSeqaSeqConfigWithPast
from ...utils import logging


if TYPE_CHECKING:
      from ...feature_extraction_utils import FeatureExtractionMixin
      from ...tokenization_utils_base import PreTrainedTokenizerBase
      from ...utils import TensorType

lowercase_          =		logging.get_logger(__name__)

lowercase_          =		{
    """openai/whisper-base""": """https://huggingface.co/openai/whisper-base/resolve/main/config.json""",
}

# fmt: off
lowercase_          =		[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 357, 366, 438, 532, 685,
    705, 796, 930, 1_058, 1_220, 1_267, 1_279, 1_303, 1_343, 1_377,
    1_391, 1_635, 1_782, 1_875, 2_162, 2_361, 2_488, 3_467, 4_008, 4_211,
    4_600, 4_808, 5_299, 5_855, 6_329, 7_203, 9_609, 9_959, 10_563, 10_786,
    11_420, 11_709, 11_907, 13_163, 13_697, 13_700, 14_808, 15_306, 16_410, 16_791,
    17_992, 19_203, 19_510, 20_724, 22_305, 22_935, 27_007, 30_109, 30_420, 33_409,
    34_949, 40_283, 40_493, 40_549, 47_282, 49_146, 50_257, 50_359, 50_360, 50_361
]
lowercase_          =		[
    1, 2, 7, 8, 9, 10, 14, 25,
    26, 27, 28, 29, 31, 58, 59, 60, 61, 62,
    63, 90, 91, 92, 93, 359, 503, 522, 542, 873,
    893, 902, 918, 922, 931, 1_350, 1_853, 1_982, 2_460, 2_627,
    3_246, 3_253, 3_268, 3_536, 3_846, 3_961, 4_183, 4_667, 6_585, 6_647,
    7_273, 9_061, 9_383, 10_428, 10_929, 11_938, 12_033, 12_331, 12_562, 13_793,
    14_157, 14_635, 15_265, 15_618, 16_553, 16_604, 18_362, 18_956, 20_075, 21_675,
    22_520, 26_130, 26_161, 26_435, 28_279, 29_464, 31_650, 32_302, 32_470, 36_865,
    42_863, 47_425, 49_870, 50_254, 50_258, 50_360, 50_361, 50_362
]
class     SCREAMING_SNAKE_CASE       (UpperCAmelCase				):
   _UpperCamelCase   :		Any			 =	'whisper'
   _UpperCamelCase   :		List[str]			 =	['past_key_values']
   _UpperCamelCase   :		List[str]			 =	{'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}


   def __init__(   self      :    Optional[int]      ,					a      :    Optional[int]=51_865      ,					a      :    List[str]=80      ,					a      :    Dict=6      ,					a      :    Dict=4      ,					a      :    Any=6      ,					a      :    Any=4      ,					a      :    Union[str, Any]=1_536      ,					a      :    int=1_536      ,					a      :    List[str]=0.0      ,					a      :    Optional[Any]=0.0      ,					a      :    List[str]=50_257      ,					a      :    Optional[int]=True      ,					a      :    List[Any]=True      ,					a      :    Any="gelu"      ,					a      :    str=256      ,					a      :    List[Any]=0.0      ,					a      :    Any=0.0      ,					a      :    Optional[Any]=0.0      ,					a      :    Tuple=0.02      ,					a      :    Any=False      ,					a      :    Union[str, Any]=1_500      ,					a      :    int=448      ,					a      :    Optional[int]=50_256      ,					a      :    List[str]=50_256      ,					a      :    int=50_256      ,					a      :    Any=None      ,					a      :    List[Any]=[220, 50_256]      ,					a      :    Any=False      ,					a      :    Tuple=256      ,					a      :    str=False      ,					a      :    Optional[Any]=0.05      ,					a      :    Optional[Any]=10      ,					a      :    List[str]=2      ,					a      :    Any=0.0      ,					a      :    Any=10      ,					a      :    List[str]=0      ,					a      :    List[str]=7      ,					**a      :    Optional[int]      ,					)->       Dict:





     """simple docstring"""







     lowercase__           =  vocab_size
     lowercase__           =  num_mel_bins
     lowercase__           =  d_model
     lowercase__           =  encoder_layers
     lowercase__           =  encoder_attention_heads
     lowercase__           =  decoder_layers
     lowercase__           =  decoder_attention_heads
     lowercase__           =  decoder_ffn_dim
     lowercase__           =  encoder_ffn_dim
     lowercase__           =  dropout
     lowercase__           =  attention_dropout
     lowercase__           =  activation_dropout
     lowercase__           =  activation_function
     lowercase__           =  init_std
     lowercase__           =  encoder_layerdrop
     lowercase__           =  decoder_layerdrop
     lowercase__           =  use_cache
     lowercase__           =  encoder_layers
     lowercase__           =  scale_embedding  # scale factor will be sqrt(d_model) if True
     lowercase__           =  max_source_positions
     lowercase__           =  max_target_positions

     # Audio Classification-specific parameters. Feel free to ignore for other classes.
     lowercase__           =  classifier_proj_size
     lowercase__           =  use_weighted_layer_sum

     # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
     lowercase__           =  apply_spec_augment
     lowercase__           =  mask_time_prob
     lowercase__           =  mask_time_length
     lowercase__           =  mask_time_min_masks
     lowercase__           =  mask_feature_prob
     lowercase__           =  mask_feature_length
     lowercase__           =  mask_feature_min_masks

     lowercase__           =  median_filter_width

     super().__init__(
         pad_token_id=a      ,					bos_token_id=a      ,					eos_token_id=a      ,					is_encoder_decoder=a      ,					decoder_start_token_id=a      ,					suppress_tokens=a      ,					begin_suppress_tokens=a      ,					**a      ,					)






class     SCREAMING_SNAKE_CASE       (UpperCAmelCase				):


   @property
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict	)->       Mapping[str, Mapping[int, str]]:





     """simple docstring"""







     lowercase__           =  OrderedDict(
         [
             ('input_features', {0: 'batch', 1: 'feature_size', 2: 'encoder_sequence'}),
         ]	)
     if self.use_past:
       lowercase__           =  {0: 'batch'}
     else:
       lowercase__           =  {0: 'batch', 1: 'decoder_sequence'}

     if self.use_past:
       self.fill_with_past_key_values_(a      ,					direction='inputs'	)

     return common_inputs


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    Union["PreTrainedTokenizerBase", "FeatureExtractionMixin"]      ,					a      :    int = -1      ,					a      :    int = -1      ,					a      :    bool = False      ,					a      :    Optional["TensorType"] = None      ,					a      :    int = 22_050      ,					a      :    float = 5.0      ,					a      :    int = 220      ,					)->       Mapping[str, Any]:





     """simple docstring"""







     lowercase__           =  OrderedDict()
     lowercase__           =  OnnxConfig.generate_dummy_inputs(
         self      ,					preprocessor=preprocessor.feature_extractor      ,					batch_size=a      ,					framework=a      ,					sampling_rate=a      ,					time_duration=a      ,					frequency=a      ,					)
     lowercase__           =  encoder_inputs['input_features'].shape[2]
     lowercase__           =  encoder_sequence_length // 2 if self.use_past else seq_length

     lowercase__           =  super().generate_dummy_inputs(
         preprocessor.tokenizer      ,					a      ,					a      ,					a      ,					a	)

     lowercase__           =  encoder_inputs.pop('input_features'	)
     lowercase__           =  decoder_inputs.pop('decoder_input_ids'	)

     if "past_key_values" in decoder_inputs:
       lowercase__           =  decoder_inputs.pop('past_key_values'	)

     return dummy_inputs


   @property
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any	)->       float:





     """simple docstring"""







     return 1E-3









import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin
class     SCREAMING_SNAKE_CASE       (UpperCAmelCase      ,     UpperCAmelCase				):


   @register_to_config
   def __init__(   self      :    str      ,					a      :    int = 128      ,					a      :    int = 256      ,					a      :    float = 2000.0      ,					a      :    int = 768      ,					a      :    int = 12      ,					a      :    int = 12      ,					a      :    int = 64      ,					a      :    int = 2_048      ,					a      :    float = 0.1      ,					)->       Any:





     """simple docstring"""







     super().__init__()

     lowercase__           =  nn.Sequential(
         nn.Linear(a      ,					d_model * 4      ,					bias=a	)      ,					nn.SiLU()      ,					nn.Linear(d_model * 4      ,					d_model * 4      ,					bias=a	)      ,					nn.SiLU()      ,					)

     lowercase__           =  nn.Embedding(a      ,					a	)
     lowercase__           =  False

     lowercase__           =  nn.Linear(a      ,					a      ,					bias=a	)

     lowercase__           =  nn.Dropout(p=a	)

     lowercase__           =  nn.ModuleList()
     for lyr_num in range(a	):
       # FiLM conditional T5 decoder
       lowercase__           =  DecoderLayer(d_model=a      ,					d_kv=a      ,					num_heads=a      ,					d_ff=a      ,					dropout_rate=a	)
       self.decoders.append(a	)

     lowercase__           =  TaLayerNorm(a	)

     lowercase__           =  nn.Dropout(p=a	)
     lowercase__           =  nn.Linear(a      ,					a      ,					bias=a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    str      ,					a      :    int      ,					a      :    List[Any]	)->       str:





     """simple docstring"""







     lowercase__           =  torch.mul(query_input.unsqueeze(-1	)      ,					key_input.unsqueeze(-2	)	)
     return mask.unsqueeze(-3	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[str]      ,					a      :    Dict      ,					a      :    int      ,					a      :    int	)->       int:





     """simple docstring"""







     lowercase__      ,   lowercase__      ,   lowercase__           =  decoder_input_tokens.shape
     assert decoder_noise_time.shape == (batch,)

     # decoder_noise_time is in [0, 1), so rescale to expected timing range.
     lowercase__           =  get_timestep_embedding(
         decoder_noise_time * self.config.max_decoder_noise_time      ,					embedding_dim=self.config.d_model      ,					max_period=self.config.max_decoder_noise_time      ,					).to(dtype=self.dtype	)

     lowercase__           =  self.conditioning_emb(a	).unsqueeze(1	)

     assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

     lowercase__           =  decoder_input_tokens.shape[1]

     # If we want to use relative positions for audio context, we can just offset
     # this sequence by the length of encodings_and_masks.
     lowercase__           =  torch.broadcast_to(
         torch.arange(a      ,					device=decoder_input_tokens.device	)      ,					(batch, seq_length)      ,					)

     lowercase__           =  self.position_encoding(a	)

     lowercase__           =  self.continuous_inputs_projection(a	)
     inputs += position_encodings
     lowercase__           =  self.dropout(a	)

     # decoder: No padding present.
     lowercase__           =  torch.ones(
         decoder_input_tokens.shape[:2]      ,					device=decoder_input_tokens.device      ,					dtype=inputs.dtype	)

     # Translate encoding masks to encoder-decoder masks.
     lowercase__           =  [(x, self.encoder_decoder_mask(a      ,					a	)) for x, y in encodings_and_masks]

     # cross attend style: concat encodings
     lowercase__           =  torch.cat([x[0] for x in encodings_and_encdec_masks]      ,					dim=1	)
     lowercase__           =  torch.cat([x[1] for x in encodings_and_encdec_masks]      ,					dim=-1	)

     for lyr in self.decoders:
       lowercase__           =  lyr(
           a      ,					conditioning_emb=a      ,					encoder_hidden_states=a      ,					encoder_attention_mask=a      ,					)[0]

     lowercase__           =  self.decoder_norm(a	)
     lowercase__           =  self.post_dropout(a	)

     lowercase__           =  self.spec_out(a	)
     return spec_out






class     SCREAMING_SNAKE_CASE       (nn.Module				):


   def __init__(   self      :    Dict      ,					a      :    List[Any]      ,					a      :    str      ,					a      :    Union[str, Any]      ,					a      :    Optional[int]      ,					a      :    Optional[int]      ,					a      :    Tuple=1E-6	)->       Union[str, Any]:





     """simple docstring"""







     super().__init__()
     lowercase__           =  nn.ModuleList()

     # cond self attention: layer 0
     self.layer.append(
         TaLayerSelfAttentionCond(d_model=a      ,					d_kv=a      ,					num_heads=a      ,					dropout_rate=a	)	)

     # cross attention: layer 1
     self.layer.append(
         TaLayerCrossAttention(
             d_model=a      ,					d_kv=a      ,					num_heads=a      ,					dropout_rate=a      ,					layer_norm_epsilon=a      ,					)	)

     # Film Cond MLP + dropout: last layer
     self.layer.append(
         TaLayerFFCond(d_model=a      ,					d_ff=a      ,					dropout_rate=a      ,					layer_norm_epsilon=a	)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    str      ,					a      :    Union[str, Any]=None      ,					a      :    Dict=None      ,					a      :    Union[str, Any]=None      ,					a      :    List[Any]=None      ,					a      :    List[str]=None      ,					)->       List[Any]:





     """simple docstring"""







     lowercase__           =  self.layer[0](
         a      ,					conditioning_emb=a      ,					attention_mask=a      ,					)

     if encoder_hidden_states is not None:
       lowercase__           =  torch.where(encoder_attention_mask > 0      ,					0      ,					-1E1_0	).to(
           encoder_hidden_states.dtype	)

       lowercase__           =  self.layer[1](
           a      ,					key_value_states=a      ,					attention_mask=a      ,					)

     # Apply Film Conditional Feed Forward layer
     lowercase__           =  self.layer[-1](a      ,					a	)

     return (hidden_states,)






class     SCREAMING_SNAKE_CASE       (nn.Module				):


   def __init__(   self      :    List[str]      ,					a      :    List[Any]      ,					a      :    Tuple      ,					a      :    Union[str, Any]      ,					a      :    int	)->       List[str]:





     """simple docstring"""







     super().__init__()
     lowercase__           =  TaLayerNorm(a	)
     lowercase__           =  TaFiLMLayer(in_features=d_model * 4      ,					out_features=a	)
     lowercase__           =  Attention(query_dim=a      ,					heads=a      ,					dim_head=a      ,					out_bias=a      ,					scale_qk=a	)
     lowercase__           =  nn.Dropout(a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    List[Any]      ,					a      :    Union[str, Any]=None      ,					a      :    Tuple=None      ,					)->       int:





     """simple docstring"""







     lowercase__           =  self.layer_norm(a	)

     if conditioning_emb is not None:
       lowercase__           =  self.FiLMLayer(a      ,					a	)

     # Self-attention block
     lowercase__           =  self.attention(a	)

     lowercase__           =  hidden_states + self.dropout(a	)

     return hidden_states






class     SCREAMING_SNAKE_CASE       (nn.Module				):


   def __init__(   self      :    Optional[int]      ,					a      :    Optional[int]      ,					a      :    int      ,					a      :    Optional[int]      ,					a      :    Dict      ,					a      :    Any	)->       Optional[int]:





     """simple docstring"""







     super().__init__()
     lowercase__           =  Attention(query_dim=a      ,					heads=a      ,					dim_head=a      ,					out_bias=a      ,					scale_qk=a	)
     lowercase__           =  TaLayerNorm(a      ,					eps=a	)
     lowercase__           =  nn.Dropout(a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int      ,					a      :    List[Any]      ,					a      :    str=None      ,					a      :    Dict=None      ,					)->       int:





     """simple docstring"""







     lowercase__           =  self.layer_norm(a	)
     lowercase__           =  self.attention(
         a      ,					encoder_hidden_states=a      ,					attention_mask=attention_mask.squeeze(1	)      ,					)
     lowercase__           =  hidden_states + self.dropout(a	)
     return layer_output






class     SCREAMING_SNAKE_CASE       (nn.Module				):


   def __init__(   self      :    Dict      ,					a      :    str      ,					a      :    int      ,					a      :    List[Any]      ,					a      :    List[str]	)->       Dict:





     """simple docstring"""







     super().__init__()
     lowercase__           =  TaDenseGatedActDense(d_model=a      ,					d_ff=a      ,					dropout_rate=a	)
     lowercase__           =  TaFiLMLayer(in_features=d_model * 4      ,					out_features=a	)
     lowercase__           =  TaLayerNorm(a      ,					eps=a	)
     lowercase__           =  nn.Dropout(a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]      ,					a      :    List[Any]      ,					a      :    List[Any]=None	)->       List[Any]:





     """simple docstring"""







     lowercase__           =  self.layer_norm(a	)
     if conditioning_emb is not None:
       lowercase__           =  self.film(a      ,					a	)

     lowercase__           =  self.DenseReluDense(a	)
     lowercase__           =  hidden_states + self.dropout(a	)
     return hidden_states






class     SCREAMING_SNAKE_CASE       (nn.Module				):


   def __init__(   self      :    Optional[Any]      ,					a      :    Tuple      ,					a      :    Tuple      ,					a      :    Dict	)->       Union[str, Any]:





     """simple docstring"""







     super().__init__()
     lowercase__           =  nn.Linear(a      ,					a      ,					bias=a	)
     lowercase__           =  nn.Linear(a      ,					a      ,					bias=a	)
     lowercase__           =  nn.Linear(a      ,					a      ,					bias=a	)
     lowercase__           =  nn.Dropout(a	)
     lowercase__           =  NewGELUActivation()


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    str	)->       str:





     """simple docstring"""







     lowercase__           =  self.act(self.wi_a(a	)	)
     lowercase__           =  self.wi_a(a	)
     lowercase__           =  hidden_gelu * hidden_linear
     lowercase__           =  self.dropout(a	)

     lowercase__           =  self.wo(a	)
     return hidden_states






class     SCREAMING_SNAKE_CASE       (nn.Module				):


   def __init__(   self      :    Dict      ,					a      :    Dict      ,					a      :    Dict=1E-6	)->       Optional[int]:





     """simple docstring"""







     super().__init__()
     lowercase__           =  nn.Parameter(torch.ones(a	)	)
     lowercase__           =  eps


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    Optional[Any]	)->       List[str]:





     """simple docstring"""







     lowercase__           =  hidden_states.to(torch.floataa	).pow(2	).mean(-1      ,					keepdim=a	)
     lowercase__           =  hidden_states * torch.rsqrt(variance + self.variance_epsilon	)

     # convert into half-precision if necessary
     if self.weight.dtype in [torch.floataa, torch.bfloataa]:
       lowercase__           =  hidden_states.to(self.weight.dtype	)

     return self.weight * hidden_states






class     SCREAMING_SNAKE_CASE       (nn.Module				):


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    torch.Tensor	)->       torch.Tensor:





     """simple docstring"""







     return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi	) * (input + 0.044715 * torch.pow(a      ,					3.0	))	))






class     SCREAMING_SNAKE_CASE       (nn.Module				):


   def __init__(   self      :    int      ,					a      :    Optional[Any]      ,					a      :    Any	)->       Optional[Any]:





     """simple docstring"""







     super().__init__()
     lowercase__           =  nn.Linear(a      ,					out_features * 2      ,					bias=a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[str]      ,					a      :    Tuple      ,					a      :    Dict	)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  self.scale_bias(a	)
     lowercase__      ,   lowercase__           =  torch.chunk(a      ,					2      ,					-1	)
     lowercase__           =  x * (1 + scale) + shift
     return x


