





"""simple docstring"""
import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
       from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
       __UpperCamelCase     :						Optional[Any]						       =  True
       from torch.cuda.amp import autocast


__UpperCamelCase     :						Any						       =  logging.getLogger(__name__)







@dataclass
class       a							:
 snake_case__             =       field(
     metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models'''}  )
 snake_case__             =       field(
     default=lowercase__   , metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}   , )
 snake_case__             =       field(
     default=lowercase__   , metadata={'''help''': '''Whether to freeze the feature extractor layers of the model.'''}  )
 snake_case__             =       field(
     default=lowercase__   , metadata={'''help''': '''Whether to log verbose messages or not.'''}   , )
 snake_case__             =       field(
     default=2.0   , metadata={'''help''': '''Maximum temperature for gumbel softmax.'''}  )
 snake_case__             =       field(
     default=0.5   , metadata={'''help''': '''Minimum temperature for gumbel softmax.'''}  )
 snake_case__             =       field(
     default=0.99_99_95   , metadata={'''help''': '''Decay of gumbel temperature during training.'''}  )


def 		_SCREAMING_SNAKE_CASE			(_UpperCAmelCase					:    ModelArguments  ,     _UpperCAmelCase					:    TrainingArguments			):
 logging.basicConfig(
     format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'  ,     datefmt='%m/%d/%Y %H:%M:%S'  ,     handlers=[logging.StreamHandler(sys.stdout			)]  ,     )
 lowerCAmelCase    				= logging.WARNING
 if model_args.verbose_logging:
  lowerCAmelCase    				= logging.DEBUG
 elif trainer_utils.is_main_process(training_args.local_rank			):
  lowerCAmelCase    				= logging.INFO
 logger.setLevel(_UpperCAmelCase			)







@dataclass
class       a							:
 snake_case__             =       field(
     default=lowercase__   , metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}  )
 snake_case__             =       field(
     default=lowercase__   , metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}  )
 snake_case__             =       field(
     default='''train'''   , metadata={
         '''help''': '''The name of the training data set split to use (via the datasets library). Defaults to \'train\''''
     }   , )
 snake_case__             =       field(
     default='''validation'''   , metadata={
         '''help''': (
             '''The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''''
         )
     }   , )
 snake_case__             =       field(
     default='''file'''   , metadata={'''help''': '''Column in the dataset that contains speech file path. Defaults to \'file\''''}   , )
 snake_case__             =       field(
     default=lowercase__   , metadata={'''help''': '''Overwrite the cached preprocessed datasets or not.'''}  )
 snake_case__             =       field(
     default=1   , metadata={
         '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
     }   , )
 snake_case__             =       field(
     default=lowercase__   , metadata={'''help''': '''The number of processes to use for the preprocessing.'''}   , )
 snake_case__             =       field(
     default=2_0.0   , metadata={'''help''': '''Filter audio files that are longer than `max_duration_in_seconds` seconds'''}  )







@dataclass
class       a							:
 snake_case__             =       4_2
 snake_case__             =       4_2
 snake_case__             =       '''longest'''
 snake_case__             =       None
 snake_case__             =       None



 def __call__(     self				,       _snake_case     ):







  """simple docstring"""






  lowerCAmelCase    				= self.feature_extractor.pad(
      UpperCAmelCase__				,       max_length=self.max_length				,       padding=self.padding				,       pad_to_multiple_of=self.pad_to_multiple_of				,       return_tensors='pt'				,       )
  lowerCAmelCase    				= self.model._get_feat_extract_output_lengths(batch['input_values'].shape[-1]     )

  lowerCAmelCase    				= batch['''input_values'''].shape[0]

  # make sure that no loss is computed on padded inputs
  if batch["attention_mask"] is not None:
   # compute real output lengths according to convolution formula
   lowerCAmelCase    				= self.model._get_feat_extract_output_lengths(batch['attention_mask'].sum(-1     )     ).to(
       torch.long     )

   lowerCAmelCase    				= torch.zeros(
       (batch_size, mask_indices_seq_length)				,       dtype=torch.long				,       device=batch['input_values'].device     )

   # these two operations makes sure that all values
   # before the output lengths indices are attended to
   lowerCAmelCase    				= 1
   lowerCAmelCase    				= attention_mask.flip([-1]     ).cumsum(-1     ).flip([-1]     ).bool()

  # sample randomly masked indices
  lowerCAmelCase    				= _compute_mask_indices(
      (batch_size, mask_indices_seq_length)				,       self.model.config.mask_time_prob				,       self.model.config.mask_time_length				,       attention_mask=UpperCAmelCase__				,       min_masks=2				,       )

  return batch







class       a							(   lowercase__  ):



 def __init__(     self				,       *_snake_case				,       _snake_case=1				,       _snake_case=0				,       _snake_case=1.0				,       **_snake_case     ):







  """simple docstring"""






  super().__init__(*UpperCAmelCase__				,       **UpperCAmelCase__     )
  lowerCAmelCase    				= 0
  lowerCAmelCase    				= max_gumbel_temp
  lowerCAmelCase    				= min_gumbel_temp
  lowerCAmelCase    				= gumbel_temp_decay



 def    UpperCamelCase__  (     self				,       _snake_case				,       _snake_case     ):







  """simple docstring"""






  model.train()
  lowerCAmelCase    				= self._prepare_inputs(UpperCAmelCase__     )

  if self.use_amp:
   with autocast():
    lowerCAmelCase    				= self.compute_loss(UpperCAmelCase__				,       UpperCAmelCase__     )
  else:
   lowerCAmelCase    				= self.compute_loss(UpperCAmelCase__				,       UpperCAmelCase__     )

  if self.args.n_gpu > 1 or self.deepspeed:
   if model.module.config.ctc_loss_reduction == "mean":
    lowerCAmelCase    				= loss.mean()
   elif model.module.config.ctc_loss_reduction == "sum":
    lowerCAmelCase    				= loss.sum() / (inputs['''mask_time_indices''']).sum()
   else:
    raise ValueError(F'{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'     )

  if self.args.gradient_accumulation_steps > 1:
   lowerCAmelCase    				= loss / self.args.gradient_accumulation_steps

  if self.use_amp:
   self.scaler.scale(UpperCAmelCase__     ).backward()
  elif self.use_apex:
   with amp.scale_loss(UpperCAmelCase__				,       self.optimizer     ) as scaled_loss:
    scaled_loss.backward()
  elif self.deepspeed:
   self.deepspeed.backward(UpperCAmelCase__     )
  else:
   loss.backward()

  self.num_update_step += 1
  # make sure gumbel softmax temperature is decayed
  if self.args.n_gpu > 1 or self.deepspeed:
   model.module.set_gumbel_temperature(
       max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step				,       self.min_gumbel_temp     )     )
  else:
   model.set_gumbel_temperature(
       max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step				,       self.min_gumbel_temp     )     )

  return loss.detach()


def 		_SCREAMING_SNAKE_CASE			():
 # See all possible arguments in src/transformers/training_args.py
 # or by passing the --help flag to this script.
 # We now keep distinct sets of args, for a cleaner separation of concerns.

 lowerCAmelCase    				= HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)			)

 lowerCAmelCase    				= parser.parse_args_into_dataclasses()
 configure_logger(_UpperCAmelCase  ,     _UpperCAmelCase			)

 # Downloading and loading a dataset from the hub.
 lowerCAmelCase    				= load_dataset(data_args.dataset_name  ,     data_args.dataset_config_name  ,     cache_dir=model_args.cache_dir			)

 if "validation" not in datasets.keys():
  # make sure only "validation" and "train" keys remain"
  lowerCAmelCase    				= DatasetDict()
  lowerCAmelCase    				= load_dataset(
      data_args.dataset_name  ,     data_args.dataset_config_name  ,     split=F'{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'  ,     cache_dir=model_args.cache_dir  ,     )
  lowerCAmelCase    				= load_dataset(
      data_args.dataset_name  ,     data_args.dataset_config_name  ,     split=F'{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'  ,     cache_dir=model_args.cache_dir  ,     )
 else:
  # make sure only "validation" and "train" keys remain"
  lowerCAmelCase    				= DatasetDict()
  lowerCAmelCase    				= load_dataset(
      data_args.dataset_name  ,     data_args.dataset_config_name  ,     split='validation'  ,     cache_dir=model_args.cache_dir  ,     )
  lowerCAmelCase    				= load_dataset(
      data_args.dataset_name  ,     data_args.dataset_config_name  ,     split=F'{data_args.train_split_name}'  ,     cache_dir=model_args.cache_dir  ,     )

 # only normalized-inputs-training is supported
 lowerCAmelCase    				= WavaVecaFeatureExtractor.from_pretrained(
     model_args.model_name_or_path  ,     cache_dir=model_args.cache_dir  ,     do_normalize=_UpperCAmelCase			)

 def prepare_dataset(_UpperCAmelCase					:    Optional[Any]			):
  # check that all files have the correct sampling rate
  lowerCAmelCase    				= librosa.load(batch[data_args.speech_file_column]  ,     sr=feature_extractor.sampling_rate			)
  return batch

 # load audio files into numpy arrays
 lowerCAmelCase    				= datasets.map(
     _UpperCAmelCase  ,     num_proc=data_args.preprocessing_num_workers  ,     remove_columns=datasets['train'].column_names			)

 # filter audio files that are too long
 lowerCAmelCase    				= vectorized_datasets.filter(
     lambda _UpperCAmelCase			: len(data['speech']			) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate			)			)

 def normalize(_UpperCAmelCase					:    Tuple			):
  return feature_extractor(batch['speech']  ,     sampling_rate=feature_extractor.sampling_rate			)

 # normalize and transform to `BatchFeatures`
 lowerCAmelCase    				= vectorized_datasets.map(
     _UpperCAmelCase  ,     batched=_UpperCAmelCase  ,     num_proc=data_args.preprocessing_num_workers  ,     load_from_cache_file=not data_args.overwrite_cache  ,     remove_columns=vectorized_datasets['train'].column_names  ,     )

 # pretraining is only supported for "newer" stable layer norm architecture
 # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
 lowerCAmelCase    				= WavaVecaConfig.from_pretrained(
     model_args.model_name_or_path  ,     cache_dir=model_args.cache_dir  ,     gradient_checkpointing=training_args.gradient_checkpointing  ,     )

 if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
  raise ValueError(
      'PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'
      ' ``config.feat_extract_norm=\'layer\''			)

 lowerCAmelCase    				= WavaVecaForPreTraining(_UpperCAmelCase			)

 lowerCAmelCase    				= DataCollatorForWavaVecaPretraining(model=_UpperCAmelCase  ,     feature_extractor=_UpperCAmelCase			)

 lowerCAmelCase    				= WavaVecaPreTrainer(
     model=_UpperCAmelCase  ,     data_collator=_UpperCAmelCase  ,     args=_UpperCAmelCase  ,     train_dataset=vectorized_datasets['train']  ,     eval_dataset=vectorized_datasets['validation']  ,     tokenizer=_UpperCAmelCase  ,     max_gumbel_temp=model_args.max_gumbel_temperature  ,     min_gumbel_temp=model_args.min_gumbel_temperature  ,     gumbel_temp_decay=model_args.gumbel_temperature_decay  ,     )
 trainer.train()


if __name__ == "__main__":
       main()








'''simple docstring'''





import logging
import os
from dataclasses import dataclass, field
from typing import Dict, Optional

import numpy as np
from utils_multiple_choice import MultipleChoiceDataset, Split, processors

import transformers
from transformers import (
    AutoConfig,
    AutoModelForMultipleChoice,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import is_main_process


UpperCamelCase_    						=       logging.getLogger(__name__)






def 				_lowerCAmelCase			(   __magic_name__						:     List[Any]   , __magic_name__						:     str						)				->  Union[str, Any]:
						return (preds == labels).mean()





@dataclass
class 		__SCREAMING_SNAKE_CASE							:
				lowerCamelCase_		       =    field(
				    metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}      )
				lowerCamelCase_		       =    field(
				    default=lowercase__						,							metadata={'help': 'Pretrained config name or path if not the same as model_name'}      )
				lowerCamelCase_		       =    field(
				    default=lowercase__						,							metadata={'help': 'Pretrained tokenizer name or path if not the same as model_name'}      )
				lowerCamelCase_		       =    field(
				    default=lowercase__						,							metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}						,							)





@dataclass
class 		__SCREAMING_SNAKE_CASE							:
				lowerCamelCase_		       =    field(metadata={'help': 'The name of the task to train on: ' + ', '.join(processors.keys()      )}      )
				lowerCamelCase_		       =    field(metadata={'help': 'Should contain the data files for the task.'}      )
				lowerCamelCase_		       =    field(
				    default=1_28						,							metadata={
				        'help': (
				            'The maximum total input sequence length after tokenization. Sequences longer '
				            'than this will be truncated, sequences shorter will be padded.'
				        )
				    }						,							)
				lowerCamelCase_		       =    field(
				    default=lowercase__						,							metadata={'help': 'Overwrite the cached training and evaluation sets'}      )






def 				_lowerCAmelCase			(   )				->  int:
						# See all possible arguments in src/transformers/training_args.py
						# or by passing the --help flag to this script.
						# We now keep distinct sets of args, for a cleaner separation of concerns.

						lowercase :							Dict    		=HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)						)
						lowercase		,	lowercase		,	lowercase :							List[Any]    		=parser.parse_args_into_dataclasses()

						if (
						    os.path.exists(training_args.output_dir						)
						    and os.listdir(training_args.output_dir						)
						    and training_args.do_train
						    and not training_args.overwrite_output_dir
						):
												raise ValueError(
												    f'''Output directory ({training_args.output_dir}) already exists and is not empty. Use'''
												    ''' --overwrite_output_dir to overcome.'''						)

						# Setup logging
						logging.basicConfig(
						    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''   , datefmt='''%m/%d/%Y %H:%M:%S'''   , level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN   , )
						logger.warning(
						    '''Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'''   , training_args.local_rank   , training_args.device   , training_args.n_gpu   , bool(training_args.local_rank != -1						)   , training_args.fpaa   , )
						# Set the verbosity to info of the Transformers logger (on main process only):
						if is_main_process(training_args.local_rank						):
												transformers.utils.logging.set_verbosity_info()
												transformers.utils.logging.enable_default_handler()
												transformers.utils.logging.enable_explicit_format()
						logger.info('''Training/evaluation parameters %s'''   , __magic_name__						)

						# Set seed
						set_seed(training_args.seed						)

						try:
												lowercase :							Any    		=processors[data_args.task_name]()
												lowercase :							Optional[int]    		=processor.get_labels()
												lowercase :							str    		=len(__magic_name__						)
						except KeyError:
												raise ValueError('''Task not found: %s''' % (data_args.task_name)						)

						# Load pretrained model and tokenizer
						#
						# Distributed training:
						# The .from_pretrained methods guarantee that only one local process can concurrently
						# download model & vocab.

						lowercase :							List[str]    		=AutoConfig.from_pretrained(
						    model_args.config_name if model_args.config_name else model_args.model_name_or_path   , num_labels=__magic_name__   , finetuning_task=data_args.task_name   , cache_dir=model_args.cache_dir   , )
						lowercase :							int    		=AutoTokenizer.from_pretrained(
						    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path   , cache_dir=model_args.cache_dir   , )
						lowercase :							Any    		=AutoModelForMultipleChoice.from_pretrained(
						    model_args.model_name_or_path   , from_tf=bool('''.ckpt''' in model_args.model_name_or_path						)   , config=__magic_name__   , cache_dir=model_args.cache_dir   , )

						# Get datasets
						lowercase :							int    		=(
						    MultipleChoiceDataset(
						        data_dir=data_args.data_dir   , tokenizer=__magic_name__   , task=data_args.task_name   , max_seq_length=data_args.max_seq_length   , overwrite_cache=data_args.overwrite_cache   , mode=Split.train   , )
						    if training_args.do_train
						    else None
						)
						lowercase :							Union[str, Any]    		=(
						    MultipleChoiceDataset(
						        data_dir=data_args.data_dir   , tokenizer=__magic_name__   , task=data_args.task_name   , max_seq_length=data_args.max_seq_length   , overwrite_cache=data_args.overwrite_cache   , mode=Split.dev   , )
						    if training_args.do_eval
						    else None
						)

						def compute_metrics(__magic_name__						:     EvalPrediction						) -> Dict:
												lowercase :							Dict    		=np.argmax(p.predictions   , axis=1						)
												return {"acc": simple_accuracy(__magic_name__   , p.label_ids						)}

						# Data collator
						lowercase :							List[str]    		=DataCollatorWithPadding(__magic_name__   , pad_to_multiple_of=8						) if training_args.fpaa else None

						# Initialize our Trainer
						lowercase :							Dict    		=Trainer(
						    model=__magic_name__   , args=__magic_name__   , train_dataset=__magic_name__   , eval_dataset=__magic_name__   , compute_metrics=__magic_name__   , data_collator=__magic_name__   , )

						# Training
						if training_args.do_train:
												trainer.train(
												    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path						) else None						)
												trainer.save_model()
												# For convenience, we also re-save the tokenizer to the same directory,
												# so that you can share your model easily on huggingface.co/models =)
												if trainer.is_world_master():
																		tokenizer.save_pretrained(training_args.output_dir						)

    # Evaluation
						lowercase :							Optional[Any]    		={}
						if training_args.do_eval:
												logger.info('''*** Evaluate ***'''						)

												lowercase :							List[Any]    		=trainer.evaluate()

												lowercase :							Any    		=os.path.join(training_args.output_dir   , '''eval_results.txt'''						)
												if trainer.is_world_master():
																		with open(__magic_name__   , '''w'''						) as writer:
																								logger.info('''***** Eval results *****'''						)
																								for key, value in result.items():
																														logger.info('''  %s = %s'''   , __magic_name__   , __magic_name__						)
																														writer.write('''%s = %s\n''' % (key, value)						)

																								results.update(__magic_name__						)

						return results






def 				_lowerCAmelCase			(   __magic_name__						:     Any						)				->  Optional[Any]:
						# For xla_spawn (TPUs)
						main()


if __name__ == "__main__":
							main()





