





import math
from dataclasses import dataclass
from typing import Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, randn_tensor
from .scheduling_utils import SchedulerMixin


@dataclass
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->UnCLIP
class        __magic_name__	(				lowerCamelCase__				):





     """simple docstring"""



     __UpperCamelCase            =  42
     __UpperCamelCase            =  None




def   __snake_case    (       _lowerCAmelCase			:			Optional[int]					,					_lowerCAmelCase			:			str=0.9_99					,					_lowerCAmelCase			:			List[Any]="cosine"					,					)			->    Union[str, Any]:
   if alpha_transform_type == "cosine":

      def alpha_bar_fn(_lowerCAmelCase			:			List[Any]						):
         return math.cos((t + 0.0_08) / 1.0_08 * math.pi / 2						) ** 2

   elif alpha_transform_type == "exp":

      def alpha_bar_fn(_lowerCAmelCase			:			Any						):
         return math.exp(t * -12.0						)

   else:
      raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}"						)

   A_       :					int        =      []
   for i in range(_lowerCAmelCase						):
      A_       :					Optional[Any]        =      i / num_diffusion_timesteps
      A_       :					Optional[Any]        =      (i + 1) / num_diffusion_timesteps
      betas.append(min(1 - alpha_bar_fn(_lowerCAmelCase						) / alpha_bar_fn(_lowerCAmelCase						)					,					_lowerCAmelCase						)						)
   return torch.tensor(_lowerCAmelCase					,					dtype=torch.floataa						)





class        __magic_name__	(				lowerCamelCase__     ,					lowerCamelCase__				):





     """simple docstring"""



     @register_to_config
     def __init__(				self							:Optional[int]      ,   snake_case							:int = 1_000      ,   snake_case							:str = "fixed_small_log"      ,   snake_case							:bool = True      ,   snake_case							:Optional[float] = 1.0      ,   snake_case							:str = "epsilon"      ,   snake_case							:str = "squaredcos_cap_v2"      ,   ):





        '''simple docstring'''




        if beta_schedule != "squaredcos_cap_v2":
           raise ValueError("UnCLIPScheduler only supports `beta_schedule`: 'squaredcos_cap_v2'"	)

        A_       :					int        =      betas_for_alpha_bar(snake_case	)

        A_       :					List[str]        =      1.0 - self.betas
        A_       :					str        =      torch.cumprod(self.alphas      ,   dim=0	)
        A_       :					str        =      torch.tensor(1.0	)

        # standard deviation of the initial noise distribution
        A_       :					List[Any]        =      1.0

        # setable values
        A_       :					Tuple        =      None
        A_       :					Optional[int]        =      torch.from_numpy(np.arange(0      ,   snake_case	)[::-1].copy()	)

        A_       :					Optional[int]        =      variance_type



     def     SCREAMING_SNAKE_CASE		(				self							:List[str]      ,   snake_case							:torch.FloatTensor      ,   snake_case							:Optional[int] = None	):





        '''simple docstring'''




        return sample



     def     SCREAMING_SNAKE_CASE		(				self							:Union[str, Any]      ,   snake_case							:int      ,   snake_case							:Union[str, torch.device] = None	):





        '''simple docstring'''




        A_       :					Any        =      num_inference_steps
        A_       :					str        =      (self.config.num_train_timesteps - 1) / (self.num_inference_steps - 1)
        A_       :					Dict        =      (np.arange(0      ,   snake_case	) * step_ratio).round()[::-1].copy().astype(np.intaa	)
        A_       :					Dict        =      torch.from_numpy(snake_case	).to(snake_case	)



     def     SCREAMING_SNAKE_CASE		(				self							:int      ,   snake_case							:List[str]      ,   snake_case							:Any=None      ,   snake_case							:Optional[Any]=None      ,   snake_case							:List[str]=None	):





        '''simple docstring'''




        if prev_timestep is None:
           A_       :					str        =      t - 1

        A_       :					Optional[Any]        =      self.alphas_cumprod[t]
        A_       :					Dict        =      self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.one
        A_       :					Optional[int]        =      1 - alpha_prod_t
        A_       :					Optional[Any]        =      1 - alpha_prod_t_prev

        if prev_timestep == t - 1:
           A_       :					int        =      self.betas[t]
        else:
           A_       :					Dict        =      1 - alpha_prod_t / alpha_prod_t_prev

        # For t > 0, compute predicted variance βt (see formula (6) and (7) from https://arxiv.org/pdf/2006.11239.pdf)
        # and sample from it to get previous sample
        # x_{t-1} ~ N(pred_prev_sample, variance) == add variance to pred_sample
        A_       :					Optional[Any]        =      beta_prod_t_prev / beta_prod_t * beta

        if variance_type is None:
           A_       :					Tuple        =      self.config.variance_type

        # hacks - were probably added for training stability
        if variance_type == "fixed_small_log":
           A_       :					Tuple        =      torch.log(torch.clamp(snake_case      ,   min=1e-20	)	)
           A_       :					Tuple        =      torch.exp(0.5 * variance	)
        elif variance_type == "learned_range":
           # NOTE difference with DDPM scheduler
           A_       :					int        =      variance.log()
           A_       :					Optional[Any]        =      beta.log()

           A_       :					Any        =      (predicted_variance + 1) / 2
           A_       :					str        =      frac * max_log + (1 - frac) * min_log

        return variance



     def     SCREAMING_SNAKE_CASE		(				self							:Tuple      ,   snake_case							:torch.FloatTensor      ,   snake_case							:int      ,   snake_case							:torch.FloatTensor      ,   snake_case							:Optional[int] = None      ,   snake_case							:Dict=None      ,   snake_case							:bool = True      ,   ):





        '''simple docstring'''




        A_       :					Dict        =      timestep

        if model_output.shape[1] == sample.shape[1] * 2 and self.variance_type == "learned_range":
           A_   ,    A_       :					Dict        =      torch.split(snake_case      ,   sample.shape[1]      ,   dim=1	)
        else:
           A_       :					List[Any]        =      None

        # 1. compute alphas, betas
        if prev_timestep is None:
           A_       :					Any        =      t - 1

        A_       :					List[str]        =      self.alphas_cumprod[t]
        A_       :					Tuple        =      self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.one
        A_       :					Dict        =      1 - alpha_prod_t
        A_       :					Optional[int]        =      1 - alpha_prod_t_prev

        if prev_timestep == t - 1:
           A_       :					Dict        =      self.betas[t]
           A_       :					Union[str, Any]        =      self.alphas[t]
        else:
           A_       :					Optional[int]        =      1 - alpha_prod_t / alpha_prod_t_prev
           A_       :					Dict        =      1 - beta

        # 2. compute predicted original sample from predicted noise also called
        # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf
        if self.config.prediction_type == "epsilon":
           A_       :					Dict        =      (sample - beta_prod_t ** 0.5 * model_output) / alpha_prod_t ** 0.5
        elif self.config.prediction_type == "sample":
           A_       :					str        =      model_output
        else:
           raise ValueError(
               f"prediction_type given as {self.config.prediction_type} must be one of `epsilon` or `sample`"
               " for the UnCLIPScheduler."	)

        # 3. Clip "predicted x_0"
        if self.config.clip_sample:
           A_       :					Tuple        =      torch.clamp(
               snake_case      ,   -self.config.clip_sample_range      ,   self.config.clip_sample_range	)

        # 4. Compute coefficients for pred_original_sample x_0 and current sample x_t
        # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf
        A_       :					int        =      (alpha_prod_t_prev ** 0.5 * beta) / beta_prod_t
        A_       :					Optional[Any]        =      alpha ** 0.5 * beta_prod_t_prev / beta_prod_t

        # 5. Compute predicted previous sample µ_t
        # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf
        A_       :					str        =      pred_original_sample_coeff * pred_original_sample + current_sample_coeff * sample

        # 6. Add noise
        A_       :					Tuple        =      0
        if t > 0:
           A_       :					List[Any]        =      randn_tensor(
               model_output.shape      ,   dtype=model_output.dtype      ,   generator=snake_case      ,   device=model_output.device	)

           A_       :					Optional[Any]        =      self._get_variance(
               snake_case      ,   predicted_variance=snake_case      ,   prev_timestep=snake_case      ,   )

           if self.variance_type == "fixed_small_log":
              A_       :					Union[str, Any]        =      variance
           elif self.variance_type == "learned_range":
              A_       :					int        =      (0.5 * variance).exp()
           else:
              raise ValueError(
                  f"variance_type given as {self.variance_type} must be one of `fixed_small_log` or `learned_range`"
                  " for the UnCLIPScheduler."	)

           A_       :					List[str]        =      variance * variance_noise

        A_       :					Dict        =      pred_prev_sample + variance

        if not return_dict:
           return (pred_prev_sample,)

        return UnCLIPSchedulerOutput(prev_sample=snake_case      ,   pred_original_sample=snake_case	)





     def     SCREAMING_SNAKE_CASE		(				self							:Optional[Any]      ,   snake_case							:torch.FloatTensor      ,   snake_case							:torch.FloatTensor      ,   snake_case							:torch.IntTensor      ,   ):





        '''simple docstring'''




        A_       :					List[Any]        =      self.alphas_cumprod.to(device=original_samples.device      ,   dtype=original_samples.dtype	)
        A_       :					int        =      timesteps.to(original_samples.device	)

        A_       :					Dict        =      alphas_cumprod[timesteps] ** 0.5
        A_       :					Any        =      sqrt_alpha_prod.flatten()
        while len(sqrt_alpha_prod.shape	) < len(original_samples.shape	):
           A_       :					List[str]        =      sqrt_alpha_prod.unsqueeze(-1	)

        A_       :					List[str]        =      (1 - alphas_cumprod[timesteps]) ** 0.5
        A_       :					Dict        =      sqrt_one_minus_alpha_prod.flatten()
        while len(sqrt_one_minus_alpha_prod.shape	) < len(original_samples.shape	):
           A_       :					Tuple        =      sqrt_one_minus_alpha_prod.unsqueeze(-1	)

        A_       :					Dict        =      sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise
        return noisy_samples







import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
 from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
 _lowerCAmelCase				:   Optional[Any]						       =						True
 from torch.cuda.amp import autocast


_lowerCAmelCase				:   Dict						       =						logging.getLogger(__name__)


@dataclass
class        __magic_name__	:





     """simple docstring"""



     __UpperCamelCase            =  field(
         metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}				)
     __UpperCamelCase            =  field(
         default=lowerCamelCase__     ,					metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}     ,					)
     __UpperCamelCase            =  field(
         default=lowerCamelCase__     ,					metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}				)
     __UpperCamelCase            =  field(
         default=lowerCamelCase__     ,					metadata={'''help''': '''Whether to log verbose messages or not.'''}     ,					)
     __UpperCamelCase            =  field(
         default=2.0     ,					metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}				)
     __UpperCamelCase            =  field(
         default=0.5     ,					metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}				)
     __UpperCamelCase            =  field(
         default=0.99_9995     ,					metadata={'''help''': '''Decay of gumbel temperature during training.'''}				)




def   __snake_case    (       _lowerCAmelCase			:			ModelArguments					,					_lowerCAmelCase			:			TrainingArguments						)			->    str:
   logging.basicConfig(
       format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"					,					datefmt="%m/%d/%Y %H:%M:%S"					,					handlers=[logging.StreamHandler(sys.stdout						)]					,					)
   A_       :					Optional[int]        =      logging.WARNING
   if model_args.verbose_logging:
      A_       :					Dict        =      logging.DEBUG
   elif trainer_utils.is_main_process(training_args.local_rank						):
      A_       :					int        =      logging.INFO
   logger.setLevel(_lowerCAmelCase						)


@dataclass
class        __magic_name__	:





     """simple docstring"""



     __UpperCamelCase            =  field(
         default=lowerCamelCase__     ,					metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}				)
     __UpperCamelCase            =  field(
         default=lowerCamelCase__     ,					metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}				)
     __UpperCamelCase            =  field(
         default='''train'''     ,					metadata={
             '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
         }     ,					)
     __UpperCamelCase            =  field(
         default='''validation'''     ,					metadata={
             '''help''': (
                 '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
             )
         }     ,					)
     __UpperCamelCase            =  field(
         default='''file'''     ,					metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}     ,					)
     __UpperCamelCase            =  field(
         default=lowerCamelCase__     ,					metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}				)
     __UpperCamelCase            =  field(
         default=1     ,					metadata={
             '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
         }     ,					)
     __UpperCamelCase            =  field(
         default=lowerCamelCase__     ,					metadata={'''help''': '''The number of processes to use for the preprocessing.'''}     ,					)
     __UpperCamelCase            =  field(
         default=20.0     ,					metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}				)




@dataclass
class        __magic_name__	:





     """simple docstring"""



     __UpperCamelCase            =  42
     __UpperCamelCase            =  42
     __UpperCamelCase            =  "longest"
     __UpperCamelCase            =  None
     __UpperCamelCase            =  None



     def __call__(				self							:Optional[Any]      ,   snake_case							:List[Dict[str, Union[List[int], torch.Tensor]]]	):





        '''simple docstring'''




        A_       :					Optional[Any]        =      self.feature_extractor.pad(
            snake_case      ,   max_length=self.max_length      ,   padding=self.padding      ,   pad_to_multiple_of=self.pad_to_multiple_of      ,   return_tensors="pt"      ,   )
        A_       :					int        =      self.model._get_feat_extract_output_lengths(batch["input_values"].shape[-1]	)

        A_       :					List[str]        =      batch["input_values"].shape[0]

        # make sure that no loss is computed on padded inputs
        if batch["attention_mask"] is not None:
           # compute real output lengths according to convolution formula
           A_       :					List[str]        =      self.model._get_feat_extract_output_lengths(batch["attention_mask"].sum(-1	)	).to(
               torch.long	)

           A_       :					Union[str, Any]        =      torch.zeros(
               (batch_size, mask_indices_seq_length)      ,   dtype=torch.long      ,   device=batch["input_values"].device	)

           # these two operations makes sure that all values
           # before the output lengths indices are attended to
           A_       :					str        =      1
           A_       :					Union[str, Any]        =      attention_mask.flip([-1]	).cumsum(-1	).flip([-1]	).bool()

        # sample randomly masked indices
        A_       :					Optional[int]        =      _compute_mask_indices(
            (batch_size, mask_indices_seq_length)      ,   self.model.config.mask_time_prob      ,   self.model.config.mask_time_length      ,   attention_mask=snake_case      ,   min_masks=2      ,   )

        return batch




class        __magic_name__	(				lowerCamelCase__				):





     """simple docstring"""



     def __init__(				self							:int      ,   *snake_case							:Any      ,   snake_case							:Any=1      ,   snake_case							:Dict=0      ,   snake_case							:Dict=1.0      ,   **snake_case							:Any	):





        '''simple docstring'''




        super().__init__(*snake_case      ,   **snake_case	)
        A_       :					Union[str, Any]        =      0
        A_       :					Dict        =      max_gumbel_temp
        A_       :					Optional[int]        =      min_gumbel_temp
        A_       :					List[Any]        =      gumbel_temp_decay





     def     SCREAMING_SNAKE_CASE		(				self							:Optional[int]      ,   snake_case							:nn.Module      ,   snake_case							:Dict[str, Union[torch.Tensor, Any]]	):





        '''simple docstring'''




        model.train()
        A_       :					List[str]        =      self._prepare_inputs(snake_case	)

        if self.use_amp:
           with autocast():
              A_       :					List[str]        =      self.compute_loss(snake_case      ,   snake_case	)
        else:
           A_       :					str        =      self.compute_loss(snake_case      ,   snake_case	)

        if self.args.n_gpu > 1 or self.deepspeed:
           if model.module.config.ctc_loss_reduction == "mean":
              A_       :					Any        =      loss.mean()
           elif model.module.config.ctc_loss_reduction == "sum":
              A_       :					Optional[Any]        =      loss.sum() / (inputs["mask_time_indices"]).sum()
           else:
              raise ValueError(f"{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']"	)

        if self.args.gradient_accumulation_steps > 1:
           A_       :					Any        =      loss / self.args.gradient_accumulation_steps

        if self.use_amp:
           self.scaler.scale(snake_case	).backward()
        elif self.use_apex:
           with amp.scale_loss(snake_case      ,   self.optimizer	) as scaled_loss:
              scaled_loss.backward()
        elif self.deepspeed:
           self.deepspeed.backward(snake_case	)
        else:
           loss.backward()

        self.num_update_step += 1
        # make sure gumbel softmax temperature is decayed
        if self.args.n_gpu > 1 or self.deepspeed:
           model.module.set_gumbel_temperature(
               max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step      ,   self.min_gumbel_temp	)	)
        else:
           model.set_gumbel_temperature(
               max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step      ,   self.min_gumbel_temp	)	)

        return loss.detach()




def   __snake_case    (       )			->    Tuple:
   # See all possible arguments in src/transformers/training_args.py
   # or by passing the --help flag to this script.
   # We now keep distinct sets of args, for a cleaner separation of concerns.

   A_       :					List[str]        =      HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)						)

   A_   ,    A_   ,    A_       :					Dict        =      parser.parse_args_into_dataclasses()
   configure_logger(_lowerCAmelCase					,					_lowerCAmelCase						)

   # Downloading and loading a dataset from the hub.
   A_       :					Optional[Any]        =      load_dataset(data_args.dataset_name					,					data_args.dataset_config_name					,					cache_dir=model_args.cache_dir						)

   if "validation" not in datasets.keys():
      # make sure only "validation" and "train" keys remain"
      A_       :					Tuple        =      DatasetDict()
      A_       :					Any        =      load_dataset(
          data_args.dataset_name					,					data_args.dataset_config_name					,					split=f"{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"					,					cache_dir=model_args.cache_dir					,					)
      A_       :					Union[str, Any]        =      load_dataset(
          data_args.dataset_name					,					data_args.dataset_config_name					,					split=f"{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"					,					cache_dir=model_args.cache_dir					,					)
   else:
      # make sure only "validation" and "train" keys remain"
      A_       :					Any        =      DatasetDict()
      A_       :					Optional[Any]        =      load_dataset(
          data_args.dataset_name					,					data_args.dataset_config_name					,					split="validation"					,					cache_dir=model_args.cache_dir					,					)
      A_       :					List[str]        =      load_dataset(
          data_args.dataset_name					,					data_args.dataset_config_name					,					split=f"{data_args.train_split_name}"					,					cache_dir=model_args.cache_dir					,					)

   # only normalized-inputs-training is supported
   A_       :					Union[str, Any]        =      WavaVecaFeatureExtractor.from_pretrained(
       model_args.model_name_or_path					,					cache_dir=model_args.cache_dir					,					do_normalize=_lowerCAmelCase						)

   def prepare_dataset(_lowerCAmelCase			:			str						):
      # check that all files have the correct sampling rate
      A_   ,    A_       :					Tuple        =      librosa.load(batch[data_args.speech_file_column]					,					sr=feature_extractor.sampling_rate						)
      return batch

   # load audio files into numpy arrays
   A_       :					List[Any]        =      datasets.map(
       _lowerCAmelCase					,					num_proc=data_args.preprocessing_num_workers					,					remove_columns=datasets["train"].column_names						)

   # filter audio files that are too long
   A_       :					str        =      vectorized_datasets.filter(
       lambda _lowerCAmelCase						: len(data["speech"]						) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate						)						)

   def normalize(_lowerCAmelCase			:			List[str]						):
      return feature_extractor(batch["speech"]					,					sampling_rate=feature_extractor.sampling_rate						)

   # normalize and transform to `BatchFeatures`
   A_       :					Any        =      vectorized_datasets.map(
       _lowerCAmelCase					,					batched=_lowerCAmelCase					,					num_proc=data_args.preprocessing_num_workers					,					load_from_cache_file=not data_args.overwrite_cache					,					remove_columns=vectorized_datasets["train"].column_names					,					)

   # pretraining is only supported for "newer" stable layer norm architecture
   # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
   A_       :					Any        =      WavaVecaConfig.from_pretrained(
       model_args.model_name_or_path					,					cache_dir=model_args.cache_dir					,					gradient_checkpointing=training_args.gradient_checkpointing					,					)

   if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
      raise ValueError(
          "PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"
          " ``config.feat_extract_norm='layer'"						)

   A_       :					int        =      WavaVecaForPreTraining(_lowerCAmelCase						)

   A_       :					str        =      DataCollatorForWavaVecaPretraining(model=_lowerCAmelCase					,					feature_extractor=_lowerCAmelCase						)

   A_       :					List[Any]        =      WavaVecaPreTrainer(
       model=_lowerCAmelCase					,					data_collator=_lowerCAmelCase					,					args=_lowerCAmelCase					,					train_dataset=vectorized_datasets["train"]					,					eval_dataset=vectorized_datasets["validation"]					,					tokenizer=_lowerCAmelCase					,					max_gumbel_temp=model_args.max_gumbel_temperature					,					min_gumbel_temp=model_args.min_gumbel_temperature					,					gumbel_temp_decay=model_args.gumbel_temperature_decay					,					)
   trainer.train()


if __name__ == "__main__":
 main()
