




"""simple docstring"""




import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
					from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("""1.6"""):
					lowerCAmelCase							:    Optional[Any]  							=							True
					from torch.cuda.amp import autocast


lowerCAmelCase							:    str  							=							logging.getLogger(__name__)






@dataclass
class 					__magic_name__							:






				'''simple docstring'''
				__UpperCamelCase    						=		field(
				    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}       )
				__UpperCamelCase    						=		field(
				    default=UpperCamelCase_		,      metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}		,      )
				__UpperCamelCase    						=		field(
				    default=UpperCamelCase_		,      metadata={"help": "Whether to freeze the feature extractor layers of the model."}       )
				__UpperCamelCase    						=		field(
				    default=UpperCamelCase_		,      metadata={"help": "Whether to log verbose messages or not."}		,      )
				__UpperCamelCase    						=		field(
				    default=2.0		,      metadata={"help": "Maximum temperature for gumbel softmax."}       )
				__UpperCamelCase    						=		field(
				    default=0.5		,      metadata={"help": "Minimum temperature for gumbel softmax."}       )
				__UpperCamelCase    						=		field(
				    default=0.9_9_9_9_9_5		,      metadata={"help": "Decay of gumbel temperature during training."}       )




def 					a__				(       snake_case__ ,				snake_case__       )		->      str:
			logging.basicConfig(
			    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""" ,				datefmt="""%m/%d/%Y %H:%M:%S""" ,				handlers=[logging.StreamHandler(sys.stdout       )] ,				)
			lowerCamelCase       =      logging.WARNING
			if model_args.verbose_logging:
						lowerCamelCase       =      logging.DEBUG
			elif trainer_utils.is_main_process(training_args.local_rank       ):
						lowerCamelCase       =      logging.INFO
			logger.setLevel(snake_case_       )






@dataclass
class 					__magic_name__							:






				'''simple docstring'''
				__UpperCamelCase    						=		field(
				    default=UpperCamelCase_		,      metadata={"help": "The name of the dataset to use (via the datasets library)."}       )
				__UpperCamelCase    						=		field(
				    default=UpperCamelCase_		,      metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}       )
				__UpperCamelCase    						=		field(
				    default="train"		,      metadata={
				        "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'"
				    }		,      )
				__UpperCamelCase    						=		field(
				    default="validation"		,      metadata={
				        "help": (
				            "The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"
				        )
				    }		,      )
				__UpperCamelCase    						=		field(
				    default="file"		,      metadata={"help": "Column in the dataset that contains speech file path. Defaults to 'file'"}		,      )
				__UpperCamelCase    						=		field(
				    default=UpperCamelCase_		,      metadata={"help": "Overwrite the cached preprocessed datasets or not."}       )
				__UpperCamelCase    						=		field(
				    default=1		,      metadata={
				        "help": "The percentage of the train set used as validation set in case there's no validation split"
				    }		,      )
				__UpperCamelCase    						=		field(
				    default=UpperCamelCase_		,      metadata={"help": "The number of processes to use for the preprocessing."}		,      )
				__UpperCamelCase    						=		field(
				    default=2_0.0		,      metadata={"help": "Filter audio files that are longer than `max_duration_in_seconds` seconds"}       )






@dataclass
class 					__magic_name__							:






				'''simple docstring'''
				__UpperCamelCase    						=		42
				__UpperCamelCase    						=		42
				__UpperCamelCase    						=		"longest"
				__UpperCamelCase    						=		None
				__UpperCamelCase    						=		None





				def __call__( self	,				_a				):





							"""simple docstring"""
							# reformat list to dict and set to pytorch format
							lowerCamelCase       =      self.feature_extractor.pad(
							    _a	,				max_length=self.max_length	,				padding=self.padding	,				pad_to_multiple_of=self.pad_to_multiple_of	,				return_tensors="""pt"""	,				)
							lowerCamelCase       =      self.model._get_feat_extract_output_lengths(batch["""input_values"""].shape[-1]				)

							lowerCamelCase       =      batch['''input_values'''].shape[0]

							# make sure that no loss is computed on padded inputs
							if batch["attention_mask"] is not None:
										# compute real output lengths according to convolution formula
										lowerCamelCase       =      self.model._get_feat_extract_output_lengths(batch["""attention_mask"""].sum(-1				)				).to(
										    torch.long				)

										lowerCamelCase       =      torch.zeros(
										    (batch_size, mask_indices_seq_length)	,				dtype=torch.long	,				device=batch["""input_values"""].device				)

										# these two operations makes sure that all values
										# before the output lengths indices are attended to
										lowerCamelCase       =      1
										lowerCamelCase       =      attention_mask.flip([-1]				).cumsum(-1				).flip([-1]				).bool()

							# sample randomly masked indices
							lowerCamelCase       =      _compute_mask_indices(
							    (batch_size, mask_indices_seq_length)	,				self.model.config.mask_time_prob	,				self.model.config.mask_time_length	,				attention_mask=_a	,				min_masks=2	,				)

							return batch






class 					__magic_name__							( UpperCamelCase_       ):






				'''simple docstring'''





				def __init__( self	,				*_a	,				_a=1	,				_a=0	,				_a=1.0	,				**_a				):





							"""simple docstring"""
							super().__init__(*_a	,				**_a				)
							lowerCamelCase       =      0
							lowerCamelCase       =      max_gumbel_temp
							lowerCamelCase       =      min_gumbel_temp
							lowerCamelCase       =      gumbel_temp_decay





				def      _lowerCAmelCase       ( self	,				_a	,				_a				):





							"""simple docstring"""
							model.train()
							lowerCamelCase       =      self._prepare_inputs(_a				)

							if self.use_amp:
										with autocast():
													lowerCamelCase       =      self.compute_loss(_a	,				_a				)
							else:
										lowerCamelCase       =      self.compute_loss(_a	,				_a				)

							if self.args.n_gpu > 1 or self.deepspeed:
										if model.module.config.ctc_loss_reduction == "mean":
													lowerCamelCase       =      loss.mean()
										elif model.module.config.ctc_loss_reduction == "sum":
													lowerCamelCase       =      loss.sum() / (inputs['''mask_time_indices''']).sum()
										else:
													raise ValueError(f'{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'				)

							if self.args.gradient_accumulation_steps > 1:
										lowerCamelCase       =      loss / self.args.gradient_accumulation_steps

							if self.use_amp:
										self.scaler.scale(_a				).backward()
							elif self.use_apex:
										with amp.scale_loss(_a	,				self.optimizer				) as scaled_loss:
													scaled_loss.backward()
							elif self.deepspeed:
										self.deepspeed.backward(_a				)
							else:
										loss.backward()

							self.num_update_step += 1
							# make sure gumbel softmax temperature is decayed
							if self.args.n_gpu > 1 or self.deepspeed:
										model.module.set_gumbel_temperature(
										    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step	,				self.min_gumbel_temp				)				)
							else:
										model.set_gumbel_temperature(
										    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step	,				self.min_gumbel_temp				)				)

							return loss.detach()




def 					a__				(       )		->      Optional[Any]:
			# See all possible arguments in src/transformers/training_args.py
			# or by passing the --help flag to this script.
			# We now keep distinct sets of args, for a cleaner separation of concerns.

			lowerCamelCase       =      HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)       )

			lowerCamelCase       =      parser.parse_args_into_dataclasses()
			configure_logger(snake_case_ ,				snake_case_       )

			# Downloading and loading a dataset from the hub.
			lowerCamelCase       =      load_dataset(data_args.dataset_name ,				data_args.dataset_config_name ,				cache_dir=model_args.cache_dir       )

			if "validation" not in datasets.keys():
						# make sure only "validation" and "train" keys remain"
						lowerCamelCase       =      DatasetDict()
						lowerCamelCase       =      load_dataset(
						    data_args.dataset_name ,				data_args.dataset_config_name ,				split=F'{data_args.train_split_name}[:{data_args.validation_split_percentage}%]' ,				cache_dir=model_args.cache_dir ,				)
						lowerCamelCase       =      load_dataset(
						    data_args.dataset_name ,				data_args.dataset_config_name ,				split=F'{data_args.train_split_name}[{data_args.validation_split_percentage}%:]' ,				cache_dir=model_args.cache_dir ,				)
			else:
						# make sure only "validation" and "train" keys remain"
						lowerCamelCase       =      DatasetDict()
						lowerCamelCase       =      load_dataset(
						    data_args.dataset_name ,				data_args.dataset_config_name ,				split="""validation""" ,				cache_dir=model_args.cache_dir ,				)
						lowerCamelCase       =      load_dataset(
						    data_args.dataset_name ,				data_args.dataset_config_name ,				split=F'{data_args.train_split_name}' ,				cache_dir=model_args.cache_dir ,				)

			# only normalized-inputs-training is supported
			lowerCamelCase       =      WavaVecaFeatureExtractor.from_pretrained(
			    model_args.model_name_or_path ,				cache_dir=model_args.cache_dir ,				do_normalize=snake_case_       )

			def prepare_dataset(snake_case__       ):
						# check that all files have the correct sampling rate
						lowerCamelCase       =      librosa.load(batch[data_args.speech_file_column] ,				sr=feature_extractor.sampling_rate       )
						return batch

			# load audio files into numpy arrays
			lowerCamelCase       =      datasets.map(
			    snake_case_ ,				num_proc=data_args.preprocessing_num_workers ,				remove_columns=datasets["""train"""].column_names       )

			# filter audio files that are too long
			lowerCamelCase       =      vectorized_datasets.filter(
			    lambda snake_case__       : len(data["""speech"""]       ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate       )       )

			def normalize(snake_case__       ):
						return feature_extractor(batch["""speech"""] ,				sampling_rate=feature_extractor.sampling_rate       )

			# normalize and transform to `BatchFeatures`
			lowerCamelCase       =      vectorized_datasets.map(
			    snake_case_ ,				batched=snake_case_ ,				num_proc=data_args.preprocessing_num_workers ,				load_from_cache_file=not data_args.overwrite_cache ,				remove_columns=vectorized_datasets["""train"""].column_names ,				)

			# pretraining is only supported for "newer" stable layer norm architecture
			# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
			lowerCamelCase       =      WavaVecaConfig.from_pretrained(
			    model_args.model_name_or_path ,				cache_dir=model_args.cache_dir ,				gradient_checkpointing=training_args.gradient_checkpointing ,				)

			if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
						raise ValueError(
						    """PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"""
						    """ ``config.feat_extract_norm=\'layer\'"""       )

			lowerCamelCase       =      WavaVecaForPreTraining(snake_case_       )

			lowerCamelCase       =      DataCollatorForWavaVecaPretraining(model=snake_case_ ,				feature_extractor=snake_case_       )

			lowerCamelCase       =      WavaVecaPreTrainer(
			    model=snake_case_ ,				data_collator=snake_case_ ,				args=snake_case_ ,				train_dataset=vectorized_datasets["""train"""] ,				eval_dataset=vectorized_datasets["""validation"""] ,				tokenizer=snake_case_ ,				max_gumbel_temp=model_args.max_gumbel_temperature ,				min_gumbel_temp=model_args.min_gumbel_temperature ,				gumbel_temp_decay=model_args.gumbel_temperature_decay ,				)
			trainer.train()


if __name__ == "__main__":
					main()









"""simple docstring"""




from ..utils import DummyObject, requires_backends






class 					__magic_name__							( metaclass=UpperCAmelCase__       ):






				'''simple docstring'''
				__UpperCamelCase    						=		["note_seq"]





				def __init__( self	,				*_a	,				**_a				):





							"""simple docstring"""
							requires_backends(self	,				["""note_seq"""]				)





				@classmethod
				def      _lowerCAmelCase       ( cls	,				*_a	,				**_a				):





							"""simple docstring"""
							requires_backends(cls	,				["""note_seq"""]				)





				@classmethod
				def      _lowerCAmelCase       ( cls	,				*_a	,				**_a				):





							"""simple docstring"""
							requires_backends(cls	,				["""note_seq"""]				)



