





'''simple docstring'''




import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
 from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("1.6"):
 _SCREAMING_SNAKE_CASE							     =			True
 from torch.cuda.amp import autocast


_SCREAMING_SNAKE_CASE							     =			logging.getLogger(__name__)
@dataclass
class 			_lowerCAmelCase		:






     """simple docstring"""






     snake_case_						      =   field(
         metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}	)
     snake_case_						      =   field(
         default=A__							,     metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}							,     )
     snake_case_						      =   field(
         default=A__							,     metadata={"help": "Whether to freeze the feature extractor layers of the model."}	)
     snake_case_						      =   field(
         default=A__							,     metadata={"help": "Whether to log verbose messages or not."}							,     )
     snake_case_						      =   field(
         default=2.0							,     metadata={"help": "Maximum temperature for gumbel softmax."}	)
     snake_case_						      =   field(
         default=0.5							,     metadata={"help": "Minimum temperature for gumbel softmax."}	)
     snake_case_						      =   field(
         default=0.999995							,     metadata={"help": "Decay of gumbel temperature during training."}	)





def 	__lowerCamelCase			(      __lowerCAmelCase       :							ModelArguments      ,    __lowerCAmelCase       :							TrainingArguments		)		->  int:
   logging.basicConfig(
       format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""      ,    datefmt="""%m/%d/%Y %H:%M:%S"""      ,    handlers=[logging.StreamHandler(sys.stdout		)]      ,    )
   snake_case 						=   logging.WARNING
   if model_args.verbose_logging:
      snake_case 						=   logging.DEBUG
   elif trainer_utils.is_main_process(training_args.local_rank		):
      snake_case 						=   logging.INFO
   logger.setLevel(__lowerCAmelCase		)
@dataclass
class 			_lowerCAmelCase		:






     """simple docstring"""






     snake_case_						      =   field(
         default=A__							,     metadata={"help": "The name of the dataset to use (via the datasets library)."}	)
     snake_case_						      =   field(
         default=A__							,     metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}	)
     snake_case_						      =   field(
         default="train"							,     metadata={
             "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'"
         }							,     )
     snake_case_						      =   field(
         default="validation"							,     metadata={
             "help": (
                 "The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"
             )
         }							,     )
     snake_case_						      =   field(
         default="file"							,     metadata={"help": "Column in the dataset that contains speech file path. Defaults to 'file'"}							,     )
     snake_case_						      =   field(
         default=A__							,     metadata={"help": "Overwrite the cached preprocessed datasets or not."}	)
     snake_case_						      =   field(
         default=1							,     metadata={
             "help": "The percentage of the train set used as validation set in case there's no validation split"
         }							,     )
     snake_case_						      =   field(
         default=A__							,     metadata={"help": "The number of processes to use for the preprocessing."}							,     )
     snake_case_						      =   field(
         default=20.0							,     metadata={"help": "Filter audio files that are longer than `max_duration_in_seconds` seconds"}	)




@dataclass
class 			_lowerCAmelCase		:






     """simple docstring"""






     snake_case_						      =   42
     snake_case_						      =   42
     snake_case_						      =   "longest"
     snake_case_						      =   None
     snake_case_						      =   None



     def __call__(   self							:			Any   ,       __snake_case							:			List[Dict[str, Union[List[int], torch.Tensor]]]						)->	Dict[str, torch.Tensor]:
        # reformat list to dict and set to pytorch format
        snake_case 						=   self.feature_extractor.pad(
            __snake_case   ,       max_length=self.max_length   ,       padding=self.padding   ,       pad_to_multiple_of=self.pad_to_multiple_of   ,       return_tensors="""pt"""   ,       )
        snake_case 						=   self.model._get_feat_extract_output_lengths(batch["""input_values"""].shape[-1]						)

        snake_case 						=   batch["""input_values"""].shape[0]

        # make sure that no loss is computed on padded inputs
        if batch["attention_mask"] is not None:
           # compute real output lengths according to convolution formula
           snake_case 						=   self.model._get_feat_extract_output_lengths(batch["""attention_mask"""].sum(-1						)						).to(
               torch.long						)

           snake_case 						=   torch.zeros(
               (batch_size, mask_indices_seq_length)   ,       dtype=torch.long   ,       device=batch["""input_values"""].device						)

           # these two operations makes sure that all values
           # before the output lengths indices are attended to
           snake_case 						=   1
           snake_case 						=   attention_mask.flip([-1]						).cumsum(-1						).flip([-1]						).bool()

        # sample randomly masked indices
        snake_case 						=   _compute_mask_indices(
            (batch_size, mask_indices_seq_length)   ,       self.model.config.mask_time_prob   ,       self.model.config.mask_time_length   ,       attention_mask=__snake_case   ,       min_masks=2   ,       )

        return batch




class 			_lowerCAmelCase		(			A__	):






     """simple docstring"""






     def __init__(   self							:			Dict   ,       *__snake_case							:			Optional[int]   ,       __snake_case							:			int=1   ,       __snake_case							:			Union[str, Any]=0   ,       __snake_case							:			Dict=1.0   ,       **__snake_case							:			Any						)->	Optional[Any]:
        super().__init__(*__snake_case   ,       **__snake_case						)
        snake_case 						=   0
        snake_case 						=   max_gumbel_temp
        snake_case 						=   min_gumbel_temp
        snake_case 						=   gumbel_temp_decay



     def      lowerCAmelCase			(   self							:			Tuple   ,       __snake_case							:			nn.Module   ,       __snake_case							:			Dict[str, Union[torch.Tensor, Any]]						)->	torch.Tensor:
        model.train()
        snake_case 						=   self._prepare_inputs(__snake_case						)

        if self.use_amp:
           with autocast():
              snake_case 						=   self.compute_loss(__snake_case   ,       __snake_case						)
        else:
           snake_case 						=   self.compute_loss(__snake_case   ,       __snake_case						)

        if self.args.n_gpu > 1 or self.deepspeed:
           if model.module.config.ctc_loss_reduction == "mean":
              snake_case 						=   loss.mean()
           elif model.module.config.ctc_loss_reduction == "sum":
              snake_case 						=   loss.sum() / (inputs["""mask_time_indices"""]).sum()
           else:
              raise ValueError(f'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''						)

        if self.args.gradient_accumulation_steps > 1:
           snake_case 						=   loss / self.args.gradient_accumulation_steps

        if self.use_amp:
           self.scaler.scale(__snake_case						).backward()
        elif self.use_apex:
           with amp.scale_loss(__snake_case   ,       self.optimizer						) as scaled_loss:
              scaled_loss.backward()
        elif self.deepspeed:
           self.deepspeed.backward(__snake_case						)
        else:
           loss.backward()

        self.num_update_step += 1
        # make sure gumbel softmax temperature is decayed
        if self.args.n_gpu > 1 or self.deepspeed:
           model.module.set_gumbel_temperature(
               max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step   ,       self.min_gumbel_temp						)						)
        else:
           model.set_gumbel_temperature(
               max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step   ,       self.min_gumbel_temp						)						)

        return loss.detach()





def 	__lowerCamelCase			(      )		->  int:
   # See all possible arguments in src/transformers/training_args.py
   # or by passing the --help flag to this script.
   # We now keep distinct sets of args, for a cleaner separation of concerns.

   snake_case 						=   HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)		)

   snake_case						,    snake_case						,    snake_case 						=   parser.parse_args_into_dataclasses()
   configure_logger(__lowerCAmelCase      ,    __lowerCAmelCase		)

   # Downloading and loading a dataset from the hub.
   snake_case 						=   load_dataset(data_args.dataset_name      ,    data_args.dataset_config_name      ,    cache_dir=model_args.cache_dir		)

   if "validation" not in datasets.keys():
      # make sure only "validation" and "train" keys remain"
      snake_case 						=   DatasetDict()
      snake_case 						=   load_dataset(
          data_args.dataset_name      ,    data_args.dataset_config_name      ,    split=F'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''      ,    cache_dir=model_args.cache_dir      ,    )
      snake_case 						=   load_dataset(
          data_args.dataset_name      ,    data_args.dataset_config_name      ,    split=F'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''      ,    cache_dir=model_args.cache_dir      ,    )
   else:
      # make sure only "validation" and "train" keys remain"
      snake_case 						=   DatasetDict()
      snake_case 						=   load_dataset(
          data_args.dataset_name      ,    data_args.dataset_config_name      ,    split="""validation"""      ,    cache_dir=model_args.cache_dir      ,    )
      snake_case 						=   load_dataset(
          data_args.dataset_name      ,    data_args.dataset_config_name      ,    split=F'''{data_args.train_split_name}'''      ,    cache_dir=model_args.cache_dir      ,    )

   # only normalized-inputs-training is supported
   snake_case 						=   WavaVecaFeatureExtractor.from_pretrained(
       model_args.model_name_or_path      ,    cache_dir=model_args.cache_dir      ,    do_normalize=__lowerCAmelCase		)

   def prepare_dataset(__lowerCAmelCase       :							str		):
      # check that all files have the correct sampling rate
      snake_case						,    snake_case 						=   librosa.load(batch[data_args.speech_file_column]      ,    sr=feature_extractor.sampling_rate		)
      return batch

   # load audio files into numpy arrays
   snake_case 						=   datasets.map(
       __lowerCAmelCase      ,    num_proc=data_args.preprocessing_num_workers      ,    remove_columns=datasets["""train"""].column_names		)

   # filter audio files that are too long
   snake_case 						=   vectorized_datasets.filter(
       lambda __lowerCAmelCase		: len(data["""speech"""]		) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate		)		)

   def normalize(__lowerCAmelCase       :							Optional[int]		):
      return feature_extractor(batch["""speech"""]      ,    sampling_rate=feature_extractor.sampling_rate		)

   # normalize and transform to `BatchFeatures`
   snake_case 						=   vectorized_datasets.map(
       __lowerCAmelCase      ,    batched=__lowerCAmelCase      ,    num_proc=data_args.preprocessing_num_workers      ,    load_from_cache_file=not data_args.overwrite_cache      ,    remove_columns=vectorized_datasets["""train"""].column_names      ,    )

   # pretraining is only supported for "newer" stable layer norm architecture
   # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
   snake_case 						=   WavaVecaConfig.from_pretrained(
       model_args.model_name_or_path      ,    cache_dir=model_args.cache_dir      ,    gradient_checkpointing=training_args.gradient_checkpointing      ,    )

   if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
      raise ValueError(
          """PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"""
          """ ``config.feat_extract_norm='layer'"""		)

   snake_case 						=   WavaVecaForPreTraining(__lowerCAmelCase		)

   snake_case 						=   DataCollatorForWavaVecaPretraining(model=__lowerCAmelCase      ,    feature_extractor=__lowerCAmelCase		)

   snake_case 						=   WavaVecaPreTrainer(
       model=__lowerCAmelCase      ,    data_collator=__lowerCAmelCase      ,    args=__lowerCAmelCase      ,    train_dataset=vectorized_datasets["""train"""]      ,    eval_dataset=vectorized_datasets["""validation"""]      ,    tokenizer=__lowerCAmelCase      ,    max_gumbel_temp=model_args.max_gumbel_temperature      ,    min_gumbel_temp=model_args.min_gumbel_temperature      ,    gumbel_temp_decay=model_args.gumbel_temperature_decay      ,    )
   trainer.train()


if __name__ == "__main__":
 main()








'''simple docstring'''




from __future__ import annotations





def 	__lowerCamelCase			(      __lowerCAmelCase       :							Any      ,    __lowerCAmelCase       :							str      ,    __lowerCAmelCase       :							Any      ,    __lowerCAmelCase       :							int		)		->  Tuple:  # noqa: E741
   while r - l > 1:
      snake_case 						=   (l + r) // 2
      if v[m] >= key:
         snake_case 						=   m
      else:
         snake_case 						=   m  # noqa: E741
   return r





def 	__lowerCamelCase			(      __lowerCAmelCase       :							list[int]		)		->  int:
   if len(__lowerCAmelCase		) == 0:
      return 0

   snake_case 						=   [0] * len(__lowerCAmelCase		)
   snake_case 						=   1

   snake_case 						=   v[0]

   for i in range(1      ,    len(__lowerCAmelCase		)		):
      if v[i] < tail[0]:
         snake_case 						=   v[i]
      elif v[i] > tail[length - 1]:
         snake_case 						=   v[i]
         length += 1
      else:
         snake_case 						=   v[i]

   return length


if __name__ == "__main__":
 import doctest

 doctest.testmod()

