


'''simple docstring'''






import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
		from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('''1.6'''):
		__UpperCAmelCase					       =							True
		from torch.cuda.amp import autocast


__UpperCAmelCase					       =							logging.getLogger(__name__)




@dataclass
class 	a__       :





							'''simple docstring'''






							lowercase__    :     str					=   field(
							    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}						)
							lowercase__    :     Optional[str]					=   field(
							    default=a__       ,       metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}       ,       )
							lowercase__    :     Optional[bool]					=   field(
							    default=a__       ,       metadata={"help": "Whether to freeze the feature extractor layers of the model."}						)
							lowercase__    :     Optional[bool]					=   field(
							    default=a__       ,       metadata={"help": "Whether to log verbose messages or not."}       ,       )
							lowercase__    :     Optional[float]					=   field(
							    default=2.0       ,       metadata={"help": "Maximum temperature for gumbel softmax."}						)
							lowercase__    :     Optional[float]					=   field(
							    default=0.5       ,       metadata={"help": "Minimum temperature for gumbel softmax."}						)
							lowercase__    :     Optional[float]					=   field(
							    default=0.9_9_9_9_9_5       ,       metadata={"help": "Decay of gumbel temperature during training."}						)


def       _snake_case     (  A				,					A )		->					Optional[int]:
							logging.basicConfig(
							    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''				,					datefmt='''%m/%d/%Y %H:%M:%S'''				,					handlers=[logging.StreamHandler(sys.stdout )]				,					)
							lowerCAmelCase__      =			logging.WARNING
							if model_args.verbose_logging:
														lowerCAmelCase__      =			logging.DEBUG
							elif trainer_utils.is_main_process(training_args.local_rank ):
														lowerCAmelCase__      =			logging.INFO
							logger.setLevel(A )




@dataclass
class 	a__       :





							'''simple docstring'''






							lowercase__    :     str					=   field(
							    default=a__       ,       metadata={"help": "The name of the dataset to use (via the datasets library)."}						)
							lowercase__    :     Optional[str]					=   field(
							    default=a__       ,       metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}						)
							lowercase__    :     Optional[str]					=   field(
							    default="train"       ,       metadata={
							        "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'"
							    }       ,       )
							lowercase__    :     Optional[str]					=   field(
							    default="validation"       ,       metadata={
							        "help": (
							            "The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"
							        )
							    }       ,       )
							lowercase__    :     Optional[str]					=   field(
							    default="file"       ,       metadata={"help": "Column in the dataset that contains speech file path. Defaults to 'file'"}       ,       )
							lowercase__    :     bool					=   field(
							    default=a__       ,       metadata={"help": "Overwrite the cached preprocessed datasets or not."}						)
							lowercase__    :     Optional[int]					=   field(
							    default=1       ,       metadata={
							        "help": "The percentage of the train set used as validation set in case there's no validation split"
							    }       ,       )
							lowercase__    :     Optional[int]					=   field(
							    default=a__       ,       metadata={"help": "The number of processes to use for the preprocessing."}       ,       )
							lowercase__    :     Optional[float]					=   field(
							    default=2_0.0       ,       metadata={"help": "Filter audio files that are longer than `max_duration_in_seconds` seconds"}						)




@dataclass
class 	a__       :





							'''simple docstring'''






							lowercase__    :     WavaVecaForPreTraining
							lowercase__    :     WavaVecaFeatureExtractor
							lowercase__    :     Union[bool, str]					=   "longest"
							lowercase__    :     Optional[int]					=   None
							lowercase__    :     Optional[int]					=   None







							def __call__(	self      ,					lowerCamelCase_     ) ->							Dict[str, torch.Tensor]:
														# reformat list to dict and set to pytorch format
														lowerCAmelCase__      =			self.feature_extractor.pad(
														    lowerCamelCase_      ,					max_length=self.max_length      ,					padding=self.padding      ,					pad_to_multiple_of=self.pad_to_multiple_of      ,					return_tensors='''pt'''      ,					)
														lowerCAmelCase__      =			self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1]     )

														lowerCAmelCase__      =			batch['''input_values'''].shape[0]

														# make sure that no loss is computed on padded inputs
														if batch["attention_mask"] is not None:
																					# compute real output lengths according to convolution formula
																					lowerCAmelCase__      =			self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1     )     ).to(
																					    torch.long     )

																					lowerCAmelCase__      =			torch.zeros(
																					    (batch_size, mask_indices_seq_length)      ,					dtype=torch.long      ,					device=batch['''input_values'''].device     )

																					# these two operations makes sure that all values
																					# before the output lengths indices are attended to
																					lowerCAmelCase__      =			1
																					lowerCAmelCase__      =			attention_mask.flip([-1]     ).cumsum(-1     ).flip([-1]     ).bool()

														# sample randomly masked indices
														lowerCAmelCase__      =			_compute_mask_indices(
														    (batch_size, mask_indices_seq_length)      ,					self.model.config.mask_time_prob      ,					self.model.config.mask_time_length      ,					attention_mask=lowerCamelCase_      ,					min_masks=2      ,					)

														return batch




class 	a__       (							a__						):





							'''simple docstring'''






							def __init__(	self      ,					*lowerCamelCase_      ,					lowerCamelCase_=1      ,					lowerCamelCase_=0      ,					lowerCamelCase_=1.0      ,					**lowerCamelCase_     ) ->							List[Any]:
														super().__init__(*lowerCamelCase_      ,					**lowerCamelCase_     )
														lowerCAmelCase__      =			0
														lowerCAmelCase__      =			max_gumbel_temp
														lowerCAmelCase__      =			min_gumbel_temp
														lowerCAmelCase__      =			gumbel_temp_decay







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_     ) ->							torch.Tensor:
														model.train()
														lowerCAmelCase__      =			self._prepare_inputs(lowerCamelCase_     )

														if self.use_amp:
																					with autocast():
																												lowerCAmelCase__      =			self.compute_loss(lowerCamelCase_      ,					lowerCamelCase_     )
														else:
																					lowerCAmelCase__      =			self.compute_loss(lowerCamelCase_      ,					lowerCamelCase_     )

														if self.args.n_gpu > 1 or self.deepspeed:
																					if model.module.config.ctc_loss_reduction == "mean":
																												lowerCAmelCase__      =			loss.mean()
																					elif model.module.config.ctc_loss_reduction == "sum":
																												lowerCAmelCase__      =			loss.sum() / (inputs['''mask_time_indices''']).sum()
																					else:
																												raise ValueError(F"""{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']"""     )

														if self.args.gradient_accumulation_steps > 1:
																					lowerCAmelCase__      =			loss / self.args.gradient_accumulation_steps

														if self.use_amp:
																					self.scaler.scale(lowerCamelCase_     ).backward()
														elif self.use_apex:
																					with amp.scale_loss(lowerCamelCase_      ,					self.optimizer     ) as scaled_loss:
																												scaled_loss.backward()
														elif self.deepspeed:
																					self.deepspeed.backward(lowerCamelCase_     )
														else:
																					loss.backward()

														self.num_update_step += 1
														# make sure gumbel softmax temperature is decayed
														if self.args.n_gpu > 1 or self.deepspeed:
																					model.module.set_gumbel_temperature(
																					    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step      ,					self.min_gumbel_temp     )     )
														else:
																					model.set_gumbel_temperature(
																					    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step      ,					self.min_gumbel_temp     )     )

														return loss.detach()


def       _snake_case     (  )		->					List[str]:
							# See all possible arguments in src/transformers/training_args.py
							# or by passing the --help flag to this script.
							# We now keep distinct sets of args, for a cleaner separation of concerns.

							lowerCAmelCase__      =			HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments) )

							lowerCAmelCase__     ,      lowerCAmelCase__     ,      lowerCAmelCase__      =			parser.parse_args_into_dataclasses()
							configure_logger(A				,					A )

							# Downloading and loading a dataset from the hub.
							lowerCAmelCase__      =			load_dataset(data_args.dataset_name				,					data_args.dataset_config_name				,					cache_dir=model_args.cache_dir )

							if "validation" not in datasets.keys():
														# make sure only "validation" and "train" keys remain"
														lowerCAmelCase__      =			DatasetDict()
														lowerCAmelCase__      =			load_dataset(
														    data_args.dataset_name				,					data_args.dataset_config_name				,					split=F"""{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"""				,					cache_dir=model_args.cache_dir				,					)
														lowerCAmelCase__      =			load_dataset(
														    data_args.dataset_name				,					data_args.dataset_config_name				,					split=F"""{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"""				,					cache_dir=model_args.cache_dir				,					)
							else:
														# make sure only "validation" and "train" keys remain"
														lowerCAmelCase__      =			DatasetDict()
														lowerCAmelCase__      =			load_dataset(
														    data_args.dataset_name				,					data_args.dataset_config_name				,					split='''validation'''				,					cache_dir=model_args.cache_dir				,					)
														lowerCAmelCase__      =			load_dataset(
														    data_args.dataset_name				,					data_args.dataset_config_name				,					split=F"""{data_args.train_split_name}"""				,					cache_dir=model_args.cache_dir				,					)

							# only normalized-inputs-training is supported
							lowerCAmelCase__      =			WavaVecaFeatureExtractor.from_pretrained(
							    model_args.model_name_or_path				,					cache_dir=model_args.cache_dir				,					do_normalize=A )

							def prepare_dataset(A ):
														# check that all files have the correct sampling rate
														lowerCAmelCase__     ,      lowerCAmelCase__      =			librosa.load(batch[data_args.speech_file_column]				,					sr=feature_extractor.sampling_rate )
														return batch

							# load audio files into numpy arrays
							lowerCAmelCase__      =			datasets.map(
							    A				,					num_proc=data_args.preprocessing_num_workers				,					remove_columns=datasets['''train'''].column_names )

							# filter audio files that are too long
							lowerCAmelCase__      =			vectorized_datasets.filter(
							    lambda A : len(data['''speech'''] ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate ) )

							def normalize(A ):
														return feature_extractor(batch['''speech''']				,					sampling_rate=feature_extractor.sampling_rate )

							# normalize and transform to `BatchFeatures`
							lowerCAmelCase__      =			vectorized_datasets.map(
							    A				,					batched=A				,					num_proc=data_args.preprocessing_num_workers				,					load_from_cache_file=not data_args.overwrite_cache				,					remove_columns=vectorized_datasets['''train'''].column_names				,					)

							# pretraining is only supported for "newer" stable layer norm architecture
							# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
							lowerCAmelCase__      =			WavaVecaConfig.from_pretrained(
							    model_args.model_name_or_path				,					cache_dir=model_args.cache_dir				,					gradient_checkpointing=training_args.gradient_checkpointing				,					)

							if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
														raise ValueError(
														    '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
														    ''' ``config.feat_extract_norm=\'layer\'''' )

							lowerCAmelCase__      =			WavaVecaForPreTraining(A )

							lowerCAmelCase__      =			DataCollatorForWavaVecaPretraining(model=A				,					feature_extractor=A )

							lowerCAmelCase__      =			WavaVecaPreTrainer(
							    model=A				,					data_collator=A				,					args=A				,					train_dataset=vectorized_datasets['''train''']				,					eval_dataset=vectorized_datasets['''validation''']				,					tokenizer=A				,					max_gumbel_temp=model_args.max_gumbel_temperature				,					min_gumbel_temp=model_args.min_gumbel_temperature				,					gumbel_temp_decay=model_args.gumbel_temperature_decay				,					)
							trainer.train()


if __name__ == "__main__":
		main()



'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__UpperCAmelCase					       =							logging.get_logger(__name__)

__UpperCAmelCase					       =							{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

__UpperCAmelCase					       =							{
    '''vocab_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/vocab.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/vocab.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/vocab.json''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/merges.txt''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/merges.txt''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/merges.txt''',
        '''roberta-base-openai-detector''': '''https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt''',
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''roberta-base''': '''https://huggingface.co/roberta-base/resolve/main/tokenizer.json''',
        '''roberta-large''': '''https://huggingface.co/roberta-large/resolve/main/tokenizer.json''',
        '''roberta-large-mnli''': '''https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json''',
        '''distilroberta-base''': '''https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json''',
        '''roberta-base-openai-detector''': (
            '''https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json'''
        ),
        '''roberta-large-openai-detector''': (
            '''https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json'''
        ),
    },
}

__UpperCAmelCase					       =							{
    '''roberta-base''': 512,
    '''roberta-large''': 512,
    '''roberta-large-mnli''': 512,
    '''distilroberta-base''': 512,
    '''roberta-base-openai-detector''': 512,
    '''roberta-large-openai-detector''': 512,
}




class 	a__       (							a__						):





							'''simple docstring'''






							lowercase__    :     Optional[Any]					=   VOCAB_FILES_NAMES
							lowercase__    :     int					=   PRETRAINED_VOCAB_FILES_MAP
							lowercase__    :     Any					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase__    :     List[Any]					=   ["input_ids", "attention_mask"]
							lowercase__    :     Dict					=   RobertaTokenizer







							def __init__(	self      ,					lowerCamelCase_=None      ,					lowerCamelCase_=None      ,					lowerCamelCase_=None      ,					lowerCamelCase_="replace"      ,					lowerCamelCase_="<s>"      ,					lowerCamelCase_="</s>"      ,					lowerCamelCase_="</s>"      ,					lowerCamelCase_="<s>"      ,					lowerCamelCase_="<unk>"      ,					lowerCamelCase_="<pad>"      ,					lowerCamelCase_="<mask>"      ,					lowerCamelCase_=False      ,					lowerCamelCase_=True      ,					**lowerCamelCase_      ,					) ->							Tuple:
														super().__init__(
														    lowerCamelCase_      ,					lowerCamelCase_      ,					tokenizer_file=lowerCamelCase_      ,					errors=lowerCamelCase_      ,					bos_token=lowerCamelCase_      ,					eos_token=lowerCamelCase_      ,					sep_token=lowerCamelCase_      ,					cls_token=lowerCamelCase_      ,					unk_token=lowerCamelCase_      ,					pad_token=lowerCamelCase_      ,					mask_token=lowerCamelCase_      ,					add_prefix_space=lowerCamelCase_      ,					trim_offsets=lowerCamelCase_      ,					**lowerCamelCase_      ,					)

														lowerCAmelCase__      =			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()     )
														if pre_tok_state.get('''add_prefix_space'''      ,					lowerCamelCase_     ) != add_prefix_space:
																					lowerCAmelCase__      =			getattr(lowerCamelCase_      ,					pre_tok_state.pop('''type'''     )     )
																					lowerCAmelCase__      =			add_prefix_space
																					lowerCAmelCase__      =			pre_tok_class(**lowerCamelCase_     )

														lowerCAmelCase__      =			add_prefix_space

														lowerCAmelCase__      =			'''post_processor'''
														lowerCAmelCase__      =			getattr(self.backend_tokenizer      ,					lowerCamelCase_      ,					lowerCamelCase_     )
														if tokenizer_component_instance:
																					lowerCAmelCase__      =			json.loads(tokenizer_component_instance.__getstate__()     )

																					# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																					if "sep" in state:
																												lowerCAmelCase__      =			tuple(state['''sep''']     )
																					if "cls" in state:
																												lowerCAmelCase__      =			tuple(state['''cls''']     )

																					lowerCAmelCase__      =			False

																					if state.get('''add_prefix_space'''      ,					lowerCamelCase_     ) != add_prefix_space:
																												lowerCAmelCase__      =			add_prefix_space
																												lowerCAmelCase__      =			True

																					if state.get('''trim_offsets'''      ,					lowerCamelCase_     ) != trim_offsets:
																												lowerCAmelCase__      =			trim_offsets
																												lowerCAmelCase__      =			True

																					if changes_to_apply:
																												lowerCAmelCase__      =			getattr(lowerCamelCase_      ,					state.pop('''type'''     )     )
																												lowerCAmelCase__      =			component_class(**lowerCamelCase_     )
																												setattr(self.backend_tokenizer      ,					lowerCamelCase_      ,					lowerCamelCase_     )







							@property
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							str:
														if self._mask_token is None:
																					if self.verbose:
																												logger.error('''Using mask_token, but it is not set yet.'''     )
																					return None
														return str(self._mask_token     )







							@mask_token.setter
							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							Optional[Any]:
														lowerCAmelCase__      =			AddedToken(lowerCamelCase_      ,					lstrip=lowerCamelCase_      ,					rstrip=lowerCamelCase_     ) if isinstance(lowerCamelCase_      ,					lowerCamelCase_     ) else value
														lowerCAmelCase__      =			value







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					*lowerCamelCase_      ,					**lowerCamelCase_     ) ->							BatchEncoding:
														lowerCAmelCase__      =			kwargs.get('''is_split_into_words'''      ,					lowerCamelCase_     )
														assert self.add_prefix_space or not is_split_into_words, (
														    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
														    "to use it with pretokenized inputs."
														)

														return super()._batch_encode_plus(*lowerCamelCase_      ,					**lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					*lowerCamelCase_      ,					**lowerCamelCase_     ) ->							BatchEncoding:
														lowerCAmelCase__      =			kwargs.get('''is_split_into_words'''      ,					lowerCamelCase_     )

														assert self.add_prefix_space or not is_split_into_words, (
														    F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
														    "to use it with pretokenized inputs."
														)

														return super()._encode_plus(*lowerCamelCase_      ,					**lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = None     ) ->							Tuple[str]:
														lowerCAmelCase__      =			self._tokenizer.model.save(lowerCamelCase_      ,					name=lowerCamelCase_     )
														return tuple(lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_=None     ) ->							Tuple:
														lowerCAmelCase__      =			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
														if token_ids_a is None:
																					return output

														return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = None     ) ->							List[int]:
														lowerCAmelCase__      =			[self.sep_token_id]
														lowerCAmelCase__      =			[self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep     ) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep     ) * [0]