


from typing import Optional

import numpy as np
import torch
from torch import nn
from transformers import GPTaConfig, GPTaLMHeadModel
from transformers.modeling_utils import ModuleUtilsMixin

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin







class 						_snake_case    ( _snake_case		,       _snake_case		,       _snake_case				):
 SCREAMING_SNAKE_CASE__ =			[r'h\.\d+\.attn\.bias', r'h\.\d+\.attn\.masked_bias']




 @register_to_config
 def __init__(						self   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase = None   ,    _lowerCamelCase = 5_0257   ,    _lowerCamelCase = 1024   ,    _lowerCamelCase = 768   ,    _lowerCamelCase = 12   ,    _lowerCamelCase = 12   ,    _lowerCamelCase = None   ,    _lowerCamelCase = "gelu_new"   ,    _lowerCamelCase = 0.1   ,    _lowerCamelCase = 0.1   ,    _lowerCamelCase = 0.1   ,    _lowerCamelCase = 1e-5   ,    _lowerCamelCase = 0.02   ,    _lowerCamelCase = True   ,    _lowerCamelCase = True   ,    _lowerCamelCase = False   ,    _lowerCamelCase = False   ,    ):
       super().__init__()

       a		:str									=			prefix_length

       if prefix_inner_dim != n_embd and prefix_hidden_dim is None:
             raise ValueError(
                 F'''`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and'''
                 F''' `n_embd`: {n_embd} are not equal.'''       )

       a		:Optional[int]									=			prefix_inner_dim
       a		:Dict									=			prefix_hidden_dim

       a		:int									=			(
           nn.Linear(self.prefix_inner_dim   ,    self.prefix_hidden_dim       )
           if self.prefix_hidden_dim is not None
           else nn.Identity()
       )
       a		:List[Any]									=			(
           nn.Linear(self.prefix_hidden_dim   ,    _lowerCamelCase       ) if self.prefix_hidden_dim is not None else nn.Identity()
       )

       a		:Optional[Any]									=			GPTaConfig(
           vocab_size=_lowerCamelCase   ,    n_positions=_lowerCamelCase   ,    n_embd=_lowerCamelCase   ,    n_layer=_lowerCamelCase   ,    n_head=_lowerCamelCase   ,    n_inner=_lowerCamelCase   ,    activation_function=_lowerCamelCase   ,    resid_pdrop=_lowerCamelCase   ,    embd_pdrop=_lowerCamelCase   ,    attn_pdrop=_lowerCamelCase   ,    layer_norm_epsilon=_lowerCamelCase   ,    initializer_range=_lowerCamelCase   ,    scale_attn_weights=_lowerCamelCase   ,    use_cache=_lowerCamelCase   ,    scale_attn_by_inverse_layer_idx=_lowerCamelCase   ,    reorder_and_upcast_attn=_lowerCamelCase   ,    )
       a		:Union[str, Any]									=			GPTaLMHeadModel(_lowerCamelCase       )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase = None   ,    _lowerCamelCase = None   ,    ):
       a		:Dict									=			self.transformer.transformer.wte(_lowerCamelCase       )
       a		:str									=			self.encode_prefix(_lowerCamelCase       )
       a		:Tuple									=			self.decode_prefix(_lowerCamelCase       )
       a		:int									=			torch.cat((prefix_embeds, embedding_text)   ,    dim=1       )

       if labels is not None:
             a		:Tuple									=			self.get_dummy_token(input_ids.shape[0]   ,    input_ids.device       )
             a		:str									=			torch.cat((dummy_token, input_ids)   ,    dim=1       )
       a		:str									=			self.transformer(inputs_embeds=_lowerCamelCase   ,    labels=_lowerCamelCase   ,    attention_mask=_lowerCamelCase       )
       if self.prefix_hidden_dim is not None:
             return out, hidden
       else:
             return out




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase       ):
       return torch.zeros(_lowerCamelCase   ,    self.prefix_length   ,    dtype=torch.intaa   ,    device=_lowerCamelCase       )




 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase       ):
       return self.encode_prefix(_lowerCamelCase       )




 @torch.no_grad()
 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase   ,    _lowerCamelCase   ,    _lowerCamelCase       ):
       a		:Tuple									=			torch.split(_lowerCamelCase   ,    1   ,    dim=0       )
       a		:Optional[Any]									=			[]
       a		:List[Any]									=			[]
       for feature in features:
             a		:Tuple									=			self.decode_prefix(feature.to(_lowerCamelCase       )       )  # back to the clip feature
             # Only support beam search for now
             a				,							a		:Dict									=			self.generate_beam(
                 input_embeds=_lowerCamelCase   ,    device=_lowerCamelCase   ,    eos_token_id=_lowerCamelCase       )
             generated_tokens.append(output_tokens[0]       )
             generated_seq_lengths.append(seq_lengths[0]       )
       a		:Optional[int]									=			torch.stack(_lowerCamelCase       )
       a		:Optional[int]									=			torch.stack(_lowerCamelCase       )
       return generated_tokens, generated_seq_lengths




 @torch.no_grad()
 def 			SCREAMING_SNAKE_CASE__							(						self   ,    _lowerCamelCase=None   ,    _lowerCamelCase=None   ,    _lowerCamelCase=None   ,    _lowerCamelCase = 5   ,    _lowerCamelCase = 67   ,    _lowerCamelCase = 1.0   ,    _lowerCamelCase = None   ,    ):
       a		:Any									=			eos_token_id
       a		:Optional[Any]									=			None
       a		:Union[str, Any]									=			None
       a		:Union[str, Any]									=			torch.ones(_lowerCamelCase   ,    device=_lowerCamelCase   ,    dtype=torch.int       )
       a		:List[str]									=			torch.zeros(_lowerCamelCase   ,    device=_lowerCamelCase   ,    dtype=torch.bool       )

       if input_embeds is not None:
             a		:Tuple									=			input_embeds
       else:
             a		:Optional[Any]									=			self.transformer.transformer.wte(_lowerCamelCase       )

       for i in range(_lowerCamelCase       ):
             a		:Optional[Any]									=			self.transformer(inputs_embeds=_lowerCamelCase       )
             a		:List[Any]									=			outputs.logits
             a		:Optional[Any]									=			logits[:, -1, :] / (temperature if temperature > 0 else 1.0)
             a		:Tuple									=			logits.softmax(-1       ).log()

             if scores is None:
                   a				,							a		:List[str]									=			logits.topk(_lowerCamelCase   ,    -1       )
                   a		:str									=			generated.expand(_lowerCamelCase   ,    *generated.shape[1:]       )
                   a				,							a		:Optional[int]									=			next_tokens.permute(1   ,    0       ), scores.squeeze(0       )
                   if tokens is None:
                         a		:Optional[int]									=			next_tokens
                   else:
                         a		:Tuple									=			tokens.expand(_lowerCamelCase   ,    *tokens.shape[1:]       )
                         a		:Tuple									=			torch.cat((tokens, next_tokens)   ,    dim=1       )
             else:
                   a		:List[Any]									=			-float(np.inf       )
                   a		:str									=			0
                   a		:Union[str, Any]									=			scores[:, None] + logits
                   seq_lengths[~is_stopped] += 1
                   a		:Dict									=			scores_sum / seq_lengths[:, None]
                   a				,							a		:Dict									=			scores_sum_average.view(-1       ).topk(_lowerCamelCase   ,    -1       )
                   a		:Tuple									=			next_tokens // scores_sum.shape[1]
                   a		:Any									=			seq_lengths[next_tokens_source]
                   a		:Any									=			next_tokens % scores_sum.shape[1]
                   a		:Optional[int]									=			next_tokens.unsqueeze(1       )
                   a		:Dict									=			tokens[next_tokens_source]
                   a		:Tuple									=			torch.cat((tokens, next_tokens)   ,    dim=1       )
                   a		:Optional[int]									=			generated[next_tokens_source]
                   a		:Dict									=			scores_sum_average * seq_lengths
                   a		:List[Any]									=			is_stopped[next_tokens_source]

             a		:Dict									=			self.transformer.transformer.wte(next_tokens.squeeze()       ).view(generated.shape[0]   ,    1   ,    -1       )
             a		:List[Any]									=			torch.cat((generated, next_token_embed)   ,    dim=1       )
             a		:int									=			is_stopped + next_tokens.eq(_lowerCamelCase       ).squeeze()
             if is_stopped.all():
                   break

       a		:int									=			scores / seq_lengths
       a		:Optional[int]									=			scores.argsort(descending=_lowerCamelCase       )
       # tokens tensors are already padded to max_seq_length
       a		:Optional[int]									=			[tokens[i] for i in order]
       a		:Tuple									=			torch.stack(_lowerCamelCase   ,    dim=0       )
       a		:Optional[Any]									=			torch.tensor([seq_lengths[i] for i in order]   ,    dtype=seq_lengths.dtype       )
       return output_texts, seq_lengths









import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


snake_case							:       Optional[int]										=	logging.get_logger(__name__)

snake_case							:       Optional[int]										=	{
    '''microsoft/unispeech-large-1500h-cv''': (
        '''https://huggingface.co/microsoft/unispeech-large-1500h-cv/resolve/main/config.json'''
    ),
    # See all UniSpeech models at https://huggingface.co/models?filter=unispeech
}







class 						_snake_case    ( _snake_case				):
 SCREAMING_SNAKE_CASE__ =			'unispeech'




 def __init__(						self   ,    _lowerCamelCase=32   ,    _lowerCamelCase=768   ,    _lowerCamelCase=12   ,    _lowerCamelCase=12   ,    _lowerCamelCase=3072   ,    _lowerCamelCase="gelu"   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.0   ,    _lowerCamelCase=0.0   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=0.02   ,    _lowerCamelCase=1e-5   ,    _lowerCamelCase="group"   ,    _lowerCamelCase="gelu"   ,    _lowerCamelCase=(512, 512, 512, 512, 512, 512, 512)   ,    _lowerCamelCase=(5, 2, 2, 2, 2, 2, 2)   ,    _lowerCamelCase=(10, 3, 3, 3, 3, 2, 2)   ,    _lowerCamelCase=False   ,    _lowerCamelCase=128   ,    _lowerCamelCase=16   ,    _lowerCamelCase=False   ,    _lowerCamelCase=True   ,    _lowerCamelCase=0.05   ,    _lowerCamelCase=10   ,    _lowerCamelCase=2   ,    _lowerCamelCase=0.0   ,    _lowerCamelCase=10   ,    _lowerCamelCase=0   ,    _lowerCamelCase=320   ,    _lowerCamelCase=2   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase=100   ,    _lowerCamelCase=256   ,    _lowerCamelCase=256   ,    _lowerCamelCase=0.1   ,    _lowerCamelCase="mean"   ,    _lowerCamelCase=False   ,    _lowerCamelCase=False   ,    _lowerCamelCase=256   ,    _lowerCamelCase=80   ,    _lowerCamelCase=0   ,    _lowerCamelCase=1   ,    _lowerCamelCase=2   ,    _lowerCamelCase=0.5   ,    **_lowerCamelCase   ,    ):
       super().__init__(**_lowerCamelCase   ,    pad_token_id=_lowerCamelCase   ,    bos_token_id=_lowerCamelCase   ,    eos_token_id=_lowerCamelCase       )
       a		:Any									=			hidden_size
       a		:str									=			feat_extract_norm
       a		:List[Any]									=			feat_extract_activation
       a		:Tuple									=			list(_lowerCamelCase       )
       a		:Any									=			list(_lowerCamelCase       )
       a		:List[Any]									=			list(_lowerCamelCase       )
       a		:Union[str, Any]									=			conv_bias
       a		:str									=			num_conv_pos_embeddings
       a		:str									=			num_conv_pos_embedding_groups
       a		:Tuple									=			len(self.conv_dim       )
       a		:int									=			num_hidden_layers
       a		:Any									=			intermediate_size
       a		:Optional[Any]									=			hidden_act
       a		:Tuple									=			num_attention_heads
       a		:Any									=			hidden_dropout
       a		:Any									=			attention_dropout
       a		:Optional[Any]									=			activation_dropout
       a		:Optional[Any]									=			feat_proj_dropout
       a		:Any									=			final_dropout
       a		:int									=			layerdrop
       a		:int									=			layer_norm_eps
       a		:Dict									=			initializer_range
       a		:Dict									=			num_ctc_classes
       a		:Optional[Any]									=			vocab_size
       a		:str									=			do_stable_layer_norm
       a		:Tuple									=			use_weighted_layer_sum
       a		:Any									=			classifier_proj_size

       if (
           (len(self.conv_stride       ) != self.num_feat_extract_layers)
           or (len(self.conv_kernel       ) != self.num_feat_extract_layers)
           or (len(self.conv_dim       ) != self.num_feat_extract_layers)
       ):
             raise ValueError(
                 '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
                 ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
                 F''' {len(self.conv_dim       )}`, `len(config.conv_stride) = {len(self.conv_stride       )}`,'''
                 F''' `len(config.conv_kernel) = {len(self.conv_kernel       )}`.'''       )

       # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
       a		:List[Any]									=			apply_spec_augment
       a		:Any									=			mask_time_prob
       a		:Union[str, Any]									=			mask_time_length
       a		:str									=			mask_time_min_masks
       a		:Tuple									=			mask_feature_prob
       a		:Dict									=			mask_feature_length
       a		:int									=			mask_feature_min_masks

       # parameters for pretraining with codevector quantized representations
       a		:Union[str, Any]									=			num_codevectors_per_group
       a		:Dict									=			num_codevector_groups
       a		:List[Any]									=			contrastive_logits_temperature
       a		:Union[str, Any]									=			feat_quantizer_dropout
       a		:Optional[Any]									=			num_negatives
       a		:Tuple									=			codevector_dim
       a		:Optional[Any]									=			proj_codevector_dim
       a		:Union[str, Any]									=			diversity_loss_weight

       # ctc loss
       a		:List[Any]									=			ctc_loss_reduction
       a		:Union[str, Any]									=			ctc_zero_infinity

       # pretraining loss
       a		:int									=			replace_prob




 @property
 def 			SCREAMING_SNAKE_CASE__							(						self       ):
       return functools.reduce(operator.mul   ,    self.conv_stride   ,    1       )





