





'''simple docstring'''




import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
  from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
  lowercase__       : str         =       True
  from torch.cuda.amp import autocast


lowercase__       : Optional[int]         =       logging.getLogger(__name__)


@dataclass
class 					SCREAMING_SNAKE_CASE       :
     lowerCAmelCase						     =			field(
         metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}				)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}   ,						)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}				)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''Whether to log verbose messages or not.'''}   ,						)
     lowerCAmelCase						     =			field(
         default=2.0   ,						metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}				)
     lowerCAmelCase						     =			field(
         default=0.5   ,						metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}				)
     lowerCAmelCase						     =			field(
         default=0.999_995   ,						metadata={'''help''': '''Decay of gumbel temperature during training.'''}				)






def 			_lowerCAmelCase     (    __snake_case    :					ModelArguments					,				__snake_case    :					TrainingArguments     )	->	Optional[int]:
     logging.basicConfig(
         format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'					,				datefmt='%m/%d/%Y %H:%M:%S'					,				handlers=[logging.StreamHandler(sys.stdout     )]					,				)
     __A		:					Union[str, Any]						=			logging.WARNING
     if model_args.verbose_logging:
          __A		:					Optional[int]						=			logging.DEBUG
     elif trainer_utils.is_main_process(training_args.local_rank     ):
          __A		:					str						=			logging.INFO
     logger.setLevel(__snake_case     )


@dataclass
class 					SCREAMING_SNAKE_CASE       :
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}				)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}				)
     lowerCAmelCase						     =			field(
         default='''train'''   ,						metadata={
             '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
         }   ,						)
     lowerCAmelCase						     =			field(
         default='''validation'''   ,						metadata={
             '''help''': (
                 '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
             )
         }   ,						)
     lowerCAmelCase						     =			field(
         default='''file'''   ,						metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}   ,						)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}				)
     lowerCAmelCase						     =			field(
         default=1   ,						metadata={
             '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
         }   ,						)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''The number of processes to use for the preprocessing.'''}   ,						)
     lowerCAmelCase						     =			field(
         default=20.0   ,						metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}				)




@dataclass
class 					SCREAMING_SNAKE_CASE       :
     lowerCAmelCase						     =			42
     lowerCAmelCase						     =			42
     lowerCAmelCase						     =			"longest"
     lowerCAmelCase						     =			None
     lowerCAmelCase						     =			None


     def __call__(		self		,      _UpperCAmelCase):

          '''simple docstring'''






          __A		:					Optional[int]						=			self.feature_extractor.pad(
              _UpperCAmelCase		,      max_length=self.max_length		,      padding=self.padding		,      pad_to_multiple_of=self.pad_to_multiple_of		,      return_tensors='pt'		,      )
          __A		:					Tuple						=			self.model._get_feat_extract_output_lengths(batch['input_values'].shape[-1])

          __A		:					Any						=			batch['input_values'].shape[0]

          # make sure that no loss is computed on padded inputs
          if batch["attention_mask"] is not None:
               # compute real output lengths according to convolution formula
               __A		:					Union[str, Any]						=			self.model._get_feat_extract_output_lengths(batch['attention_mask'].sum(-1)).to(
                   torch.long)

               __A		:					Dict						=			torch.zeros(
                   (batch_size, mask_indices_seq_length)		,      dtype=torch.long		,      device=batch['input_values'].device)

               # these two operations makes sure that all values
               # before the output lengths indices are attended to
               __A		:					List[str]						=			1
               __A		:					Optional[Any]						=			attention_mask.flip([-1]).cumsum(-1).flip([-1]).bool()

          # sample randomly masked indices
          __A		:					Optional[Any]						=			_compute_mask_indices(
              (batch_size, mask_indices_seq_length)		,      self.model.config.mask_time_prob		,      self.model.config.mask_time_length		,      attention_mask=_UpperCAmelCase		,      min_masks=2		,      )

          return batch




class 					SCREAMING_SNAKE_CASE       (a__				):


     def __init__(		self		,      *_UpperCAmelCase		,      _UpperCAmelCase=1		,      _UpperCAmelCase=0		,      _UpperCAmelCase=1.0		,      **_UpperCAmelCase):

          '''simple docstring'''






          super().__init__(*_UpperCAmelCase		,      **_UpperCAmelCase)
          __A		:					Tuple						=			0
          __A		:					List[Any]						=			max_gumbel_temp
          __A		:					int						=			min_gumbel_temp
          __A		:					Union[str, Any]						=			gumbel_temp_decay


     def 				SCREAMING_SNAKE_CASE    (		self		,      _UpperCAmelCase		,      _UpperCAmelCase):

          '''simple docstring'''






          model.train()
          __A		:					Any						=			self._prepare_inputs(_UpperCAmelCase)

          if self.use_amp:
               with autocast():
                    __A		:					Any						=			self.compute_loss(_UpperCAmelCase		,      _UpperCAmelCase)
          else:
               __A		:					Tuple						=			self.compute_loss(_UpperCAmelCase		,      _UpperCAmelCase)

          if self.args.n_gpu > 1 or self.deepspeed:
               if model.module.config.ctc_loss_reduction == "mean":
                    __A		:					Optional[Any]						=			loss.mean()
               elif model.module.config.ctc_loss_reduction == "sum":
                    __A		:					List[Any]						=			loss.sum() / (inputs['mask_time_indices']).sum()
               else:
                    raise ValueError(F'{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']')

          if self.args.gradient_accumulation_steps > 1:
               __A		:					Dict						=			loss / self.args.gradient_accumulation_steps

          if self.use_amp:
               self.scaler.scale(_UpperCAmelCase).backward()
          elif self.use_apex:
               with amp.scale_loss(_UpperCAmelCase		,      self.optimizer) as scaled_loss:
                    scaled_loss.backward()
          elif self.deepspeed:
               self.deepspeed.backward(_UpperCAmelCase)
          else:
               loss.backward()

          self.num_update_step += 1
          # make sure gumbel softmax temperature is decayed
          if self.args.n_gpu > 1 or self.deepspeed:
               model.module.set_gumbel_temperature(
                   max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step		,      self.min_gumbel_temp))
          else:
               model.set_gumbel_temperature(
                   max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step		,      self.min_gumbel_temp))

          return loss.detach()






def 			_lowerCAmelCase     (    )	->	List[Any]:
     # See all possible arguments in src/transformers/training_args.py
     # or by passing the --help flag to this script.
     # We now keep distinct sets of args, for a cleaner separation of concerns.

     __A		:					Optional[int]						=			HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )

     __A					,__A					,__A		:					Any						=			parser.parse_args_into_dataclasses()
     configure_logger(__snake_case					,				__snake_case     )

     # Downloading and loading a dataset from the hub.
     __A		:					Optional[int]						=			load_dataset(data_args.dataset_name					,				data_args.dataset_config_name					,				cache_dir=model_args.cache_dir     )

     if "validation" not in datasets.keys():
          # make sure only "validation" and "train" keys remain"
          __A		:					Optional[int]						=			DatasetDict()
          __A		:					Optional[int]						=			load_dataset(
              data_args.dataset_name					,				data_args.dataset_config_name					,				split=f'{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'					,				cache_dir=model_args.cache_dir					,				)
          __A		:					str						=			load_dataset(
              data_args.dataset_name					,				data_args.dataset_config_name					,				split=f'{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'					,				cache_dir=model_args.cache_dir					,				)
     else:
          # make sure only "validation" and "train" keys remain"
          __A		:					Dict						=			DatasetDict()
          __A		:					List[Any]						=			load_dataset(
              data_args.dataset_name					,				data_args.dataset_config_name					,				split='validation'					,				cache_dir=model_args.cache_dir					,				)
          __A		:					Any						=			load_dataset(
              data_args.dataset_name					,				data_args.dataset_config_name					,				split=f'{data_args.train_split_name}'					,				cache_dir=model_args.cache_dir					,				)

     # only normalized-inputs-training is supported
     __A		:					List[Any]						=			WavaVecaFeatureExtractor.from_pretrained(
         model_args.model_name_or_path					,				cache_dir=model_args.cache_dir					,				do_normalize=__snake_case     )

     def prepare_dataset(__snake_case    :					Optional[Any]     ):
          # check that all files have the correct sampling rate
          __A					,__A		:					Tuple						=			librosa.load(batch[data_args.speech_file_column]					,				sr=feature_extractor.sampling_rate     )
          return batch

     # load audio files into numpy arrays
     __A		:					List[str]						=			datasets.map(
         __snake_case					,				num_proc=data_args.preprocessing_num_workers					,				remove_columns=datasets['train'].column_names     )

     # filter audio files that are too long
     __A		:					List[str]						=			vectorized_datasets.filter(
         lambda __snake_case     : len(data['speech']     ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate     )     )

     def normalize(__snake_case    :					int     ):
          return feature_extractor(batch['speech']					,				sampling_rate=feature_extractor.sampling_rate     )

     # normalize and transform to `BatchFeatures`
     __A		:					str						=			vectorized_datasets.map(
         __snake_case					,				batched=__snake_case					,				num_proc=data_args.preprocessing_num_workers					,				load_from_cache_file=not data_args.overwrite_cache					,				remove_columns=vectorized_datasets['train'].column_names					,				)

     # pretraining is only supported for "newer" stable layer norm architecture
     # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
     __A		:					int						=			WavaVecaConfig.from_pretrained(
         model_args.model_name_or_path					,				cache_dir=model_args.cache_dir					,				gradient_checkpointing=training_args.gradient_checkpointing					,				)

     if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
          raise ValueError(
              'PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'
              ' ``config.feat_extract_norm=\'layer\''     )

     __A		:					int						=			WavaVecaForPreTraining(__snake_case     )

     __A		:					Optional[int]						=			DataCollatorForWavaVecaPretraining(model=__snake_case					,				feature_extractor=__snake_case     )

     __A		:					Dict						=			WavaVecaPreTrainer(
         model=__snake_case					,				data_collator=__snake_case					,				args=__snake_case					,				train_dataset=vectorized_datasets['train']					,				eval_dataset=vectorized_datasets['validation']					,				tokenizer=__snake_case					,				max_gumbel_temp=model_args.max_gumbel_temperature					,				min_gumbel_temp=model_args.min_gumbel_temperature					,				gumbel_temp_decay=model_args.gumbel_temperature_decay					,				)
     trainer.train()


if __name__ == "__main__":
  main()






'''simple docstring'''




import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Union

from .generation.configuration_utils import GenerationConfig
from .training_args import TrainingArguments
from .utils import add_start_docstrings


lowercase__       : Dict         =       logging.getLogger(__name__)


@dataclass
@add_start_docstrings(TrainingArguments.__doc__				)
class 					SCREAMING_SNAKE_CASE       (a__				):
     lowerCAmelCase						     =			field(default=a__   ,						metadata={'''help''': '''Whether to use SortishSampler or not.'''}				)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={'''help''': '''Whether to use generate to calculate generative metrics (ROUGE, BLEU).'''}				)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={
             '''help''': (
                 '''The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
                 '''to the `max_length` value of the model configuration.'''
             )
         }   ,						)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={
             '''help''': (
                 '''The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
                 '''to the `num_beams` value of the model configuration.'''
             )
         }   ,						)
     lowerCAmelCase						     =			field(
         default=a__   ,						metadata={
             '''help''': '''Model id, file path or url pointing to a GenerationConfig json file, to use during prediction.'''
         }   ,						)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					List[Any]						=			super().to_dict()
          for k, v in d.items():
               if isinstance(_UpperCAmelCase		,      _UpperCAmelCase):
                    __A		:					List[Any]						=			v.to_dict()
          return d