
import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
		from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
		lowercase   =	True
		from torch.cuda.amp import autocast


lowercase   =	logging.getLogger(__name__)





@dataclass
class    __A:
					SCREAMING_SNAKE_CASE           =     field(
					    metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''Whether to log verbose messages or not.'''}  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=2.0  ,		metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default=0.5  ,		metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default=0.99_99_95  ,		metadata={'''help''': '''Decay of gumbel temperature during training.'''}							)



def   __lowerCAmelCase    (				UpperCAmelCase__					:   ModelArguments      ,					UpperCAmelCase__					:   TrainingArguments							)	->     int:
							logging.basicConfig(
							    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""      ,					datefmt="""%m/%d/%Y %H:%M:%S"""      ,					handlers=[logging.StreamHandler(sys.stdout							)]      ,					)
							lowerCamelCase_					=       logging.WARNING
							if model_args.verbose_logging:
														lowerCamelCase_					=       logging.DEBUG
							elif trainer_utils.is_main_process(training_args.local_rank							):
														lowerCamelCase_					=       logging.INFO
							logger.setLevel(UpperCAmelCase__							)





@dataclass
class    __A:
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default='''train'''  ,		metadata={
					        '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
					    }  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default='''validation'''  ,		metadata={
					        '''help''': (
					            '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
					        )
					    }  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default='''file'''  ,		metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default=1  ,		metadata={
					        '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
					    }  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''The number of processes to use for the preprocessing.'''}  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=20.0  ,		metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}							)





@dataclass
class    __A:
					SCREAMING_SNAKE_CASE           =     42
					SCREAMING_SNAKE_CASE           =     42
					SCREAMING_SNAKE_CASE           =     "longest"
					SCREAMING_SNAKE_CASE           =     None
					SCREAMING_SNAKE_CASE           =     None

					def __call__(						self       :  Union[str, Any] ,       __UpperCamelCase       :  List[Dict[str, Union[List[int], torch.Tensor]]] ):
												# reformat list to dict and set to pytorch format
												lowerCamelCase_					=       self.feature_extractor.pad(
												    __UpperCamelCase ,       max_length=self.max_length ,       padding=self.padding ,       pad_to_multiple_of=self.pad_to_multiple_of ,       return_tensors="""pt""" ,       )
												lowerCamelCase_					=       self.model._get_feat_extract_output_lengths(batch["""input_values"""].shape[-1] )

												lowerCamelCase_					=       batch["""input_values"""].shape[0]

												# make sure that no loss is computed on padded inputs
												if batch["attention_mask"] is not None:
																			# compute real output lengths according to convolution formula
																			lowerCamelCase_					=       self.model._get_feat_extract_output_lengths(batch["""attention_mask"""].sum(-1 ) ).to(
																			    torch.long )

																			lowerCamelCase_					=       torch.zeros(
																			    (batch_size, mask_indices_seq_length) ,       dtype=torch.long ,       device=batch["""input_values"""].device )

																			# these two operations makes sure that all values
																			# before the output lengths indices are attended to
																			lowerCamelCase_					=       1
																			lowerCamelCase_					=       attention_mask.flip([-1] ).cumsum(-1 ).flip([-1] ).bool()

												# sample randomly masked indices
												lowerCamelCase_					=       _compute_mask_indices(
												    (batch_size, mask_indices_seq_length) ,       self.model.config.mask_time_prob ,       self.model.config.mask_time_length ,       attention_mask=__UpperCamelCase ,       min_masks=2 ,       )

												return batch





class    __A(					UpperCAmelCase							):

					def __init__(						self       :  Optional[Any] ,       *__UpperCamelCase       :  Dict ,       __UpperCamelCase       :  List[Any]=1 ,       __UpperCamelCase       :  Tuple=0 ,       __UpperCamelCase       :  Union[str, Any]=1.0 ,       **__UpperCamelCase       :  List[Any] ):
												super().__init__(*__UpperCamelCase ,       **__UpperCamelCase )
												lowerCamelCase_					=       0
												lowerCamelCase_					=       max_gumbel_temp
												lowerCamelCase_					=       min_gumbel_temp
												lowerCamelCase_					=       gumbel_temp_decay



					def  lowercase__   (						self       :  List[Any] ,       __UpperCamelCase       :  nn.Module ,       __UpperCamelCase       :  Dict[str, Union[torch.Tensor, Any]] ):
												model.train()
												lowerCamelCase_					=       self._prepare_inputs(__UpperCamelCase )

												if self.use_amp:
																			with autocast():
																										lowerCamelCase_					=       self.compute_loss(__UpperCamelCase ,       __UpperCamelCase )
												else:
																			lowerCamelCase_					=       self.compute_loss(__UpperCamelCase ,       __UpperCamelCase )

												if self.args.n_gpu > 1 or self.deepspeed:
																			if model.module.config.ctc_loss_reduction == "mean":
																										lowerCamelCase_					=       loss.mean()
																			elif model.module.config.ctc_loss_reduction == "sum":
																										lowerCamelCase_					=       loss.sum() / (inputs["""mask_time_indices"""]).sum()
																			else:
																										raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']''' )

												if self.args.gradient_accumulation_steps > 1:
																			lowerCamelCase_					=       loss / self.args.gradient_accumulation_steps

												if self.use_amp:
																			self.scaler.scale(__UpperCamelCase ).backward()
												elif self.use_apex:
																			with amp.scale_loss(__UpperCamelCase ,       self.optimizer ) as scaled_loss:
																										scaled_loss.backward()
												elif self.deepspeed:
																			self.deepspeed.backward(__UpperCamelCase )
												else:
																			loss.backward()

												self.num_update_step += 1
												# make sure gumbel softmax temperature is decayed
												if self.args.n_gpu > 1 or self.deepspeed:
																			model.module.set_gumbel_temperature(
																			    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step ,       self.min_gumbel_temp ) )
												else:
																			model.set_gumbel_temperature(
																			    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step ,       self.min_gumbel_temp ) )

												return loss.detach()



def   __lowerCAmelCase    (				)	->     Any:
							# See all possible arguments in src/transformers/training_args.py
							# or by passing the --help flag to this script.
							# We now keep distinct sets of args, for a cleaner separation of concerns.

							lowerCamelCase_					=       HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)							)

							lowerCamelCase_	, lowerCamelCase_	, lowerCamelCase_					=       parser.parse_args_into_dataclasses()
							configure_logger(UpperCAmelCase__      ,					UpperCAmelCase__							)

							# Downloading and loading a dataset from the hub.
							lowerCamelCase_					=       load_dataset(data_args.dataset_name      ,					data_args.dataset_config_name      ,					cache_dir=model_args.cache_dir							)

							if "validation" not in datasets.keys():
														# make sure only "validation" and "train" keys remain"
														lowerCamelCase_					=       DatasetDict()
														lowerCamelCase_					=       load_dataset(
														    data_args.dataset_name      ,					data_args.dataset_config_name      ,					split=F'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''      ,					cache_dir=model_args.cache_dir      ,					)
														lowerCamelCase_					=       load_dataset(
														    data_args.dataset_name      ,					data_args.dataset_config_name      ,					split=F'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''      ,					cache_dir=model_args.cache_dir      ,					)
							else:
														# make sure only "validation" and "train" keys remain"
														lowerCamelCase_					=       DatasetDict()
														lowerCamelCase_					=       load_dataset(
														    data_args.dataset_name      ,					data_args.dataset_config_name      ,					split="""validation"""      ,					cache_dir=model_args.cache_dir      ,					)
														lowerCamelCase_					=       load_dataset(
														    data_args.dataset_name      ,					data_args.dataset_config_name      ,					split=F'''{data_args.train_split_name}'''      ,					cache_dir=model_args.cache_dir      ,					)

							# only normalized-inputs-training is supported
							lowerCamelCase_					=       WavaVecaFeatureExtractor.from_pretrained(
							    model_args.model_name_or_path      ,					cache_dir=model_args.cache_dir      ,					do_normalize=UpperCAmelCase__							)

							def prepare_dataset(UpperCAmelCase__					:   Dict							):
														# check that all files have the correct sampling rate
														lowerCamelCase_	, lowerCamelCase_					=       librosa.load(batch[data_args.speech_file_column]      ,					sr=feature_extractor.sampling_rate							)
														return batch

							# load audio files into numpy arrays
							lowerCamelCase_					=       datasets.map(
							    UpperCAmelCase__      ,					num_proc=data_args.preprocessing_num_workers      ,					remove_columns=datasets["""train"""].column_names							)

							# filter audio files that are too long
							lowerCamelCase_					=       vectorized_datasets.filter(
							    lambda UpperCAmelCase__							: len(data["""speech"""]							) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate							)							)

							def normalize(UpperCAmelCase__					:   List[str]							):
														return feature_extractor(batch["""speech"""]      ,					sampling_rate=feature_extractor.sampling_rate							)

							# normalize and transform to `BatchFeatures`
							lowerCamelCase_					=       vectorized_datasets.map(
							    UpperCAmelCase__      ,					batched=UpperCAmelCase__      ,					num_proc=data_args.preprocessing_num_workers      ,					load_from_cache_file=not data_args.overwrite_cache      ,					remove_columns=vectorized_datasets["""train"""].column_names      ,					)

							# pretraining is only supported for "newer" stable layer norm architecture
							# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
							lowerCamelCase_					=       WavaVecaConfig.from_pretrained(
							    model_args.model_name_or_path      ,					cache_dir=model_args.cache_dir      ,					gradient_checkpointing=training_args.gradient_checkpointing      ,					)

							if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
														raise ValueError(
														    """PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"""
														    """ ``config.feat_extract_norm='layer'"""							)

							lowerCamelCase_					=       WavaVecaForPreTraining(UpperCAmelCase__							)

							lowerCamelCase_					=       DataCollatorForWavaVecaPretraining(model=UpperCAmelCase__      ,					feature_extractor=UpperCAmelCase__							)

							lowerCamelCase_					=       WavaVecaPreTrainer(
							    model=UpperCAmelCase__      ,					data_collator=UpperCAmelCase__      ,					args=UpperCAmelCase__      ,					train_dataset=vectorized_datasets["""train"""]      ,					eval_dataset=vectorized_datasets["""validation"""]      ,					tokenizer=UpperCAmelCase__      ,					max_gumbel_temp=model_args.max_gumbel_temperature      ,					min_gumbel_temp=model_args.min_gumbel_temperature      ,					gumbel_temp_decay=model_args.gumbel_temperature_decay      ,					)
							trainer.train()


if __name__ == "__main__":
		main()







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available


lowercase   =	{}


try:
		if not is_sentencepiece_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase   =	['''MLukeTokenizer''']

if TYPE_CHECKING:
		try:
				if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_mluke import MLukeTokenizer


else:
		import sys

		lowercase   =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





