


import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
	from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("1.6"):
	SCREAMING_SNAKE_CASE :   Dict							      =				True
	from torch.cuda.amp import autocast


SCREAMING_SNAKE_CASE :   int							      =				logging.getLogger(__name__)






@dataclass
class _lowerCamelCase:
							lowercase_				:			str									=    field(
							    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}     )
							lowercase_				:			Optional[str]									=    field(
							    default=_a,				metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""},				)
							lowercase_				:			Optional[bool]									=    field(
							    default=_a,				metadata={"""help""": """Whether to freeze the feature extractor layers of the model."""}     )
							lowercase_				:			Optional[bool]									=    field(
							    default=_a,				metadata={"""help""": """Whether to log verbose messages or not."""},				)
							lowercase_				:			Optional[float]									=    field(
							    default=2.0,				metadata={"""help""": """Maximum temperature for gumbel softmax."""}     )
							lowercase_				:			Optional[float]									=    field(
							    default=0.5,				metadata={"""help""": """Minimum temperature for gumbel softmax."""}     )
							lowercase_				:			Optional[float]									=    field(
							    default=0.999995,				metadata={"""help""": """Decay of gumbel temperature during training."""}     )



def        UpperCamelCase_(   lowerCamelCase_							,				lowerCamelCase_				)					->		Tuple:
				logging.basicConfig(
				    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'							,				datefmt='%m/%d/%Y %H:%M:%S'							,				handlers=[logging.StreamHandler(sys.stdout				)]							,				)
				_lowercase :							Any   	=	logging.WARNING
				if model_args.verbose_logging:
								_lowercase :							Tuple   	=	logging.DEBUG
				elif trainer_utils.is_main_process(training_args.local_rank				):
								_lowercase :							List[str]   	=	logging.INFO
				logger.setLevel(lowerCamelCase_				)






@dataclass
class _lowerCamelCase:
							lowercase_				:			str									=    field(
							    default=_a,				metadata={"""help""": """The name of the dataset to use (via the datasets library)."""}     )
							lowercase_				:			Optional[str]									=    field(
							    default=_a,				metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""}     )
							lowercase_				:			Optional[str]									=    field(
							    default="""train""",				metadata={
							        """help""": """The name of the training data set split to use (via the datasets library). Defaults to 'train'"""
							    },				)
							lowercase_				:			Optional[str]									=    field(
							    default="""validation""",				metadata={
							        """help""": (
							            """The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"""
							        )
							    },				)
							lowercase_				:			Optional[str]									=    field(
							    default="""file""",				metadata={"""help""": """Column in the dataset that contains speech file path. Defaults to 'file'"""},				)
							lowercase_				:			bool									=    field(
							    default=_a,				metadata={"""help""": """Overwrite the cached preprocessed datasets or not."""}     )
							lowercase_				:			Optional[int]									=    field(
							    default=1,				metadata={
							        """help""": """The percentage of the train set used as validation set in case there's no validation split"""
							    },				)
							lowercase_				:			Optional[int]									=    field(
							    default=_a,				metadata={"""help""": """The number of processes to use for the preprocessing."""},				)
							lowercase_				:			Optional[float]									=    field(
							    default=20.0,				metadata={"""help""": """Filter audio files that are longer than `max_duration_in_seconds` seconds"""}     )






@dataclass
class _lowerCamelCase:
							lowercase_				:			WavaVecaForPreTraining
							lowercase_				:			WavaVecaFeatureExtractor
							lowercase_				:			Union[bool, str]									=    "longest"
							lowercase_				:			Optional[int]									=    None
							lowercase_				:			Optional[int]									=    None
							def __call__(      self,   lowerCamelCase) ->     Dict[str, torch.Tensor]:




											"""simple docstring"""

											_lowercase :							int   	=	self.feature_extractor.pad(
											    lowerCamelCase,   max_length=self.max_length,   padding=self.padding,   pad_to_multiple_of=self.pad_to_multiple_of,   return_tensors='pt',   )
											_lowercase :							Any   	=	self.model._get_feat_extract_output_lengths(batch['input_values'].shape[-1])

											_lowercase :							Tuple   	=	batch['input_values'].shape[0]

											# make sure that no loss is computed on padded inputs
											if batch["attention_mask"] is not None:
															# compute real output lengths according to convolution formula
															_lowercase :							Tuple   	=	self.model._get_feat_extract_output_lengths(batch['attention_mask'].sum(-1)).to(
															    torch.long)

															_lowercase :							Dict   	=	torch.zeros(
															    (batch_size, mask_indices_seq_length),   dtype=torch.long,   device=batch['input_values'].device)

															# these two operations makes sure that all values
															# before the output lengths indices are attended to
															_lowercase :							Union[str, Any]   	=	1
															_lowercase :							Any   	=	attention_mask.flip([-1]).cumsum(-1).flip([-1]).bool()

											# sample randomly masked indices
											_lowercase :							List[Any]   	=	_compute_mask_indices(
											    (batch_size, mask_indices_seq_length),   self.model.config.mask_time_prob,   self.model.config.mask_time_length,   attention_mask=lowerCamelCase,   min_masks=2,   )

											return batch






class _lowerCamelCase(		_a     ):
							def __init__(      self,   *lowerCamelCase,   lowerCamelCase=1,   lowerCamelCase=0,   lowerCamelCase=1.0,   **lowerCamelCase) ->     Any:




											"""simple docstring"""

											super().__init__(*lowerCamelCase,   **lowerCamelCase)
											_lowercase :							List[str]   	=	0
											_lowercase :							str   	=	max_gumbel_temp
											_lowercase :							Optional[Any]   	=	min_gumbel_temp
											_lowercase :							Dict   	=	gumbel_temp_decay




							def     UpperCamelCase  (      self,   lowerCamelCase,   lowerCamelCase) ->     torch.Tensor:




											"""simple docstring"""

											model.train()
											_lowercase :							Tuple   	=	self._prepare_inputs(lowerCamelCase)

											if self.use_amp:
															with autocast():
																			_lowercase :							Union[str, Any]   	=	self.compute_loss(lowerCamelCase,   lowerCamelCase)
											else:
															_lowercase :							int   	=	self.compute_loss(lowerCamelCase,   lowerCamelCase)

											if self.args.n_gpu > 1 or self.deepspeed:
															if model.module.config.ctc_loss_reduction == "mean":
																			_lowercase :							Optional[int]   	=	loss.mean()
															elif model.module.config.ctc_loss_reduction == "sum":
																			_lowercase :							Optional[Any]   	=	loss.sum() / (inputs['mask_time_indices']).sum()
															else:
																			raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']''')

											if self.args.gradient_accumulation_steps > 1:
															_lowercase :							Union[str, Any]   	=	loss / self.args.gradient_accumulation_steps

											if self.use_amp:
															self.scaler.scale(lowerCamelCase).backward()
											elif self.use_apex:
															with amp.scale_loss(lowerCamelCase,   self.optimizer) as scaled_loss:
																			scaled_loss.backward()
											elif self.deepspeed:
															self.deepspeed.backward(lowerCamelCase)
											else:
															loss.backward()

											self.num_update_step += 1
											# make sure gumbel softmax temperature is decayed
											if self.args.n_gpu > 1 or self.deepspeed:
															model.module.set_gumbel_temperature(
															    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step,   self.min_gumbel_temp))
											else:
															model.set_gumbel_temperature(
															    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step,   self.min_gumbel_temp))

											return loss.detach()



def        UpperCamelCase_(   )					->		int:
				# See all possible arguments in src/transformers/training_args.py
				# or by passing the --help flag to this script.
				# We now keep distinct sets of args, for a cleaner separation of concerns.

				_lowercase :							Any   	=	HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)				)

				_lowercase :							Optional[int]   	=	parser.parse_args_into_dataclasses()
				configure_logger(lowerCamelCase_							,				lowerCamelCase_				)

				# Downloading and loading a dataset from the hub.
				_lowercase :							List[str]   	=	load_dataset(data_args.dataset_name							,				data_args.dataset_config_name							,				cache_dir=model_args.cache_dir				)

				if "validation" not in datasets.keys():
								# make sure only "validation" and "train" keys remain"
								_lowercase :							Dict   	=	DatasetDict()
								_lowercase :							Optional[int]   	=	load_dataset(
								    data_args.dataset_name							,				data_args.dataset_config_name							,				split=F'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''							,				cache_dir=model_args.cache_dir							,				)
								_lowercase :							str   	=	load_dataset(
								    data_args.dataset_name							,				data_args.dataset_config_name							,				split=F'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''							,				cache_dir=model_args.cache_dir							,				)
				else:
								# make sure only "validation" and "train" keys remain"
								_lowercase :							str   	=	DatasetDict()
								_lowercase :							Any   	=	load_dataset(
								    data_args.dataset_name							,				data_args.dataset_config_name							,				split='validation'							,				cache_dir=model_args.cache_dir							,				)
								_lowercase :							Union[str, Any]   	=	load_dataset(
								    data_args.dataset_name							,				data_args.dataset_config_name							,				split=F'''{data_args.train_split_name}'''							,				cache_dir=model_args.cache_dir							,				)

				# only normalized-inputs-training is supported
				_lowercase :							Optional[int]   	=	WavaVecaFeatureExtractor.from_pretrained(
				    model_args.model_name_or_path							,				cache_dir=model_args.cache_dir							,				do_normalize=lowerCamelCase_				)

				def prepare_dataset(lowerCamelCase_				):
								# check that all files have the correct sampling rate
								_lowercase :							str   	=	librosa.load(batch[data_args.speech_file_column]							,				sr=feature_extractor.sampling_rate				)
								return batch

				# load audio files into numpy arrays
				_lowercase :							Optional[Any]   	=	datasets.map(
				    lowerCamelCase_							,				num_proc=data_args.preprocessing_num_workers							,				remove_columns=datasets['train'].column_names				)

				# filter audio files that are too long
				_lowercase :							Any   	=	vectorized_datasets.filter(
				    lambda lowerCamelCase_				: len(data['speech']				) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate				)				)

				def normalize(lowerCamelCase_				):
								return feature_extractor(batch['speech']							,				sampling_rate=feature_extractor.sampling_rate				)

				# normalize and transform to `BatchFeatures`
				_lowercase :							List[str]   	=	vectorized_datasets.map(
				    lowerCamelCase_							,				batched=lowerCamelCase_							,				num_proc=data_args.preprocessing_num_workers							,				load_from_cache_file=not data_args.overwrite_cache							,				remove_columns=vectorized_datasets['train'].column_names							,				)

				# pretraining is only supported for "newer" stable layer norm architecture
				# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
				_lowercase :							int   	=	WavaVecaConfig.from_pretrained(
				    model_args.model_name_or_path							,				cache_dir=model_args.cache_dir							,				gradient_checkpointing=training_args.gradient_checkpointing							,				)

				if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
								raise ValueError(
								    'PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'
								    ' ``config.feat_extract_norm=\'layer\''				)

				_lowercase :							Any   	=	WavaVecaForPreTraining(lowerCamelCase_				)

				_lowercase :							List[str]   	=	DataCollatorForWavaVecaPretraining(model=lowerCamelCase_							,				feature_extractor=lowerCamelCase_				)

				_lowercase :							Optional[int]   	=	WavaVecaPreTrainer(
				    model=lowerCamelCase_							,				data_collator=lowerCamelCase_							,				args=lowerCamelCase_							,				train_dataset=vectorized_datasets['train']							,				eval_dataset=vectorized_datasets['validation']							,				tokenizer=lowerCamelCase_							,				max_gumbel_temp=model_args.max_gumbel_temperature							,				min_gumbel_temp=model_args.min_gumbel_temperature							,				gumbel_temp_decay=model_args.gumbel_temperature_decay							,				)
				trainer.train()


if __name__ == "__main__":
	main()






import logging
import os
import sys
from pathlib import Path
from unittest.mock import patch

from parameterized import parameterized
from run_eval import run_generate
from run_eval_search import run_search

from transformers.testing_utils import CaptureStdout, TestCasePlus, slow
from utils import ROUGE_KEYS


logging.basicConfig(level=logging.DEBUG)
SCREAMING_SNAKE_CASE :   Optional[Any]							      =				logging.getLogger()



def        UpperCamelCase_(   lowerCamelCase_							,				lowerCamelCase_				)					->		Optional[Any]:
    _lowercase :							List[Any]   	=	'\n'.join(lowerCamelCase_				)
    Path(lowerCamelCase_				).open('w'				).writelines(lowerCamelCase_				)


SCREAMING_SNAKE_CASE :   str							      =				"patrickvonplaten/t5-tiny-random"
SCREAMING_SNAKE_CASE :   List[Any]							      =				"sshleifer/bart-tiny-random"
SCREAMING_SNAKE_CASE :   int							      =				"sshleifer/tiny-mbart"

SCREAMING_SNAKE_CASE :   Optional[int]							      =				logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)
logging.disable(logging.CRITICAL)  # remove noisy download output from tracebacks






class _lowerCamelCase(		_a     ):
       def     UpperCamelCase  (      self,   lowerCamelCase) ->     Dict:




           """simple docstring"""

           _lowercase :							int   	=	Path(self.get_auto_remove_tmp_dir()) / 'utest_input.source'
           _lowercase :							str   	=	input_file_name.parent / 'utest_output.txt'
           assert not output_file_name.exists()
           _lowercase :							str   	=	[' New York (CNN)When Liana Barrientos was 23 years old, she got married in Westchester County.']
           _dump_articles(lowerCamelCase,   lowerCamelCase)

           _lowercase :							List[str]   	=	str(Path(self.get_auto_remove_tmp_dir()) / 'scores.json')
           _lowercase :							Optional[int]   	=	'translation_en_to_de' if model == T5_TINY else 'summarization'
           _lowercase :							Any   	=	F'''
            run_eval_search.py
            {model}
            {input_file_name}
            {output_file_name}
            --score_path {score_path}
            --task {task}
            --num_beams 2
            --length_penalty 2.0
            '''.split()

           with patch.object(lowerCamelCase,   'argv',   lowerCamelCase):
               run_generate()
               assert Path(lowerCamelCase).exists()
               # os.remove(Path(output_file_name))
       def     UpperCamelCase  (      self) ->     Dict:




           """simple docstring"""

           self.run_eval_tester(lowerCamelCase)
       @parameterized.expand([BART_TINY, MBART_TINY])
       @slow
       def     UpperCamelCase  (      self,   lowerCamelCase) ->     int:




           """simple docstring"""

           self.run_eval_tester(lowerCamelCase)




       @parameterized.expand([T5_TINY, MBART_TINY])
       @slow
       def     UpperCamelCase  (      self,   lowerCamelCase) ->     List[str]:




           """simple docstring"""

           _lowercase :							str   	=	Path(self.get_auto_remove_tmp_dir()) / 'utest_input.source'
           _lowercase :							Any   	=	input_file_name.parent / 'utest_output.txt'
           assert not output_file_name.exists()

           _lowercase :							List[str]   	=	{
               'en': ['Machine learning is great, isn\'t it?', 'I like to eat bananas', 'Tomorrow is another great day!'],
               'de': [
                   'Maschinelles Lernen ist großartig, oder?',
                   'Ich esse gerne Bananen',
                   'Morgen ist wieder ein toller Tag!',
               ],
           }

           _lowercase :							Optional[Any]   	=	Path(self.get_auto_remove_tmp_dir())
           _lowercase :							Optional[Any]   	=	str(tmp_dir / 'scores.json')
           _lowercase :							str   	=	str(tmp_dir / 'val.target')
           _dump_articles(lowerCamelCase,   text['en'])
           _dump_articles(lowerCamelCase,   text['de'])
           _lowercase :							Tuple   	=	'translation_en_to_de' if model == T5_TINY else 'summarization'
           _lowercase :							Tuple   	=	F'''
            run_eval_search.py
            {model}
            {str(lowerCamelCase)}
            {str(lowerCamelCase)}
            --score_path {score_path}
            --reference_path {reference_path}
            --task {task}
            '''.split()
           testargs.extend(['--search', 'num_beams=1:2 length_penalty=0.9:1.0'])

           with patch.object(lowerCamelCase,   'argv',   lowerCamelCase):
               with CaptureStdout() as cs:
                   run_search()
               _lowercase :							Dict   	=	[' num_beams | length_penalty', model, 'Best score args']
               _lowercase :							Optional[Any]   	=	['Info']
               if "translation" in task:
                   expected_strings.append('bleu')
               else:
                   expected_strings.extend(lowerCamelCase)
               for w in expected_strings:
                   assert w in cs.out
               for w in un_expected_strings:
                   assert w not in cs.out
               assert Path(lowerCamelCase).exists()
               os.remove(Path(lowerCamelCase))


