




"""simple docstring"""







import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
						from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('1.6'):
						__A   :  Optional[Any]					=      True
						from torch.cuda.amp import autocast


__A   :  str					=      logging.getLogger(__name__)




@dataclass
class       lowerCAmelCase__       :


				"""simple docstring"""





				__UpperCAmelCase :     Optional[Any]       =      field(
				    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}    )
				__UpperCAmelCase :     Any       =      field(
				    default=UpperCamelCase_  ,			metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}  ,			)
				__UpperCAmelCase :     Tuple       =      field(
				    default=UpperCamelCase_  ,			metadata={"help": "Whether to freeze the feature extractor layers of the model."}    )
				__UpperCAmelCase :     Optional[Any]       =      field(
				    default=UpperCamelCase_  ,			metadata={"help": "Whether to log verbose messages or not."}  ,			)
				__UpperCAmelCase :     int       =      field(
				    default=2.0  ,			metadata={"help": "Maximum temperature for gumbel softmax."}    )
				__UpperCAmelCase :     int       =      field(
				    default=0.5  ,			metadata={"help": "Minimum temperature for gumbel softmax."}    )
				__UpperCAmelCase :     Union[str, Any]       =      field(
				    default=0.99_99_95  ,			metadata={"help": "Decay of gumbel temperature during training."}    )





def 						snake_case__       (					_lowerCamelCase,  _lowerCamelCase				)			->str:






		"""simple docstring"""


		logging.basicConfig(
		    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",  datefmt="%m/%d/%Y %H:%M:%S",  handlers=[logging.StreamHandler(sys.stdout				)],  )
		__lowercase				:		Optional[int]												=				logging.WARNING
		if model_args.verbose_logging:
				__lowercase				:		Union[str, Any]												=				logging.DEBUG
		elif trainer_utils.is_main_process(training_args.local_rank				):
				__lowercase				:		Any												=				logging.INFO
		logger.setLevel(snake_case_				)




@dataclass
class       lowerCAmelCase__       :


				"""simple docstring"""





				__UpperCAmelCase :     Optional[int]       =      field(
				    default=UpperCamelCase_  ,			metadata={"help": "The name of the dataset to use (via the datasets library)."}    )
				__UpperCAmelCase :     Any       =      field(
				    default=UpperCamelCase_  ,			metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}    )
				__UpperCAmelCase :     Dict       =      field(
				    default="train"  ,			metadata={
				        "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'"
				    }  ,			)
				__UpperCAmelCase :     Optional[Any]       =      field(
				    default="validation"  ,			metadata={
				        "help": (
				            "The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"
				        )
				    }  ,			)
				__UpperCAmelCase :     int       =      field(
				    default="file"  ,			metadata={"help": "Column in the dataset that contains speech file path. Defaults to 'file'"}  ,			)
				__UpperCAmelCase :     Any       =      field(
				    default=UpperCamelCase_  ,			metadata={"help": "Overwrite the cached preprocessed datasets or not."}    )
				__UpperCAmelCase :     Union[str, Any]       =      field(
				    default=1  ,			metadata={
				        "help": "The percentage of the train set used as validation set in case there's no validation split"
				    }  ,			)
				__UpperCAmelCase :     Any       =      field(
				    default=UpperCamelCase_  ,			metadata={"help": "The number of processes to use for the preprocessing."}  ,			)
				__UpperCAmelCase :     str       =      field(
				    default=20.0  ,			metadata={"help": "Filter audio files that are longer than `max_duration_in_seconds` seconds"}    )





@dataclass
class       lowerCAmelCase__       :


				"""simple docstring"""





				__UpperCAmelCase :     Union[str, Any]       =      42
				__UpperCAmelCase :     Tuple       =      42
				__UpperCAmelCase :     Union[str, Any]       =      "longest"
				__UpperCAmelCase :     Any       =      None
				__UpperCAmelCase :     Optional[int]       =      None



				def __call__(     self					:	Optional[Any]    ,   lowercase__					:	List[str]						):
						# reformat list to dict and set to pytorch format
						__lowercase				:		str												=				self.feature_extractor.pad(
						    lowercase__    ,   max_length=self.max_length    ,   padding=self.padding    ,   pad_to_multiple_of=self.pad_to_multiple_of    ,   return_tensors="pt"    ,   )
						__lowercase				:		List[Any]												=				self.model._get_feat_extract_output_lengths(batch["input_values"].shape[-1]						)

						__lowercase				:		Tuple												=				batch['''input_values'''].shape[0]

						# make sure that no loss is computed on padded inputs
						if batch["attention_mask"] is not None:
								# compute real output lengths according to convolution formula
								__lowercase				:		List[Any]												=				self.model._get_feat_extract_output_lengths(batch["attention_mask"].sum(-1						)						).to(
								    torch.long						)

								__lowercase				:		int												=				torch.zeros(
								    (batch_size, mask_indices_seq_length)    ,   dtype=torch.long    ,   device=batch["input_values"].device						)

								# these two operations makes sure that all values
								# before the output lengths indices are attended to
								__lowercase				:		Tuple												=				1
								__lowercase				:		Any												=				attention_mask.flip([-1]						).cumsum(-1						).flip([-1]						).bool()

						# sample randomly masked indices
						__lowercase				:		Dict												=				_compute_mask_indices(
						    (batch_size, mask_indices_seq_length)    ,   self.model.config.mask_time_prob    ,   self.model.config.mask_time_length    ,   attention_mask=lowercase__    ,   min_masks=2    ,   )

						return batch





class       lowerCAmelCase__       (			UpperCamelCase_    ):


				"""simple docstring"""





				def __init__(     self					:	List[str]    ,   *lowercase__					:	Tuple    ,   lowercase__					:	str=1    ,   lowercase__					:	str=0    ,   lowercase__					:	List[Any]=1.0    ,   **lowercase__					:	List[str]						):
						super().__init__(*lowercase__    ,   **lowercase__						)
						__lowercase				:		Optional[Any]												=				0
						__lowercase				:		Optional[int]												=				max_gumbel_temp
						__lowercase				:		Any												=				min_gumbel_temp
						__lowercase				:		Union[str, Any]												=				gumbel_temp_decay



				def       snake_case    (     self					:	Optional[Any]    ,   lowercase__					:	Dict    ,   lowercase__					:	List[Any]						):
						model.train()
						__lowercase				:		Any												=				self._prepare_inputs(lowercase__						)

						if self.use_amp:
								with autocast():
										__lowercase				:		Dict												=				self.compute_loss(lowercase__    ,   lowercase__						)
						else:
								__lowercase				:		Dict												=				self.compute_loss(lowercase__    ,   lowercase__						)

						if self.args.n_gpu > 1 or self.deepspeed:
								if model.module.config.ctc_loss_reduction == "mean":
										__lowercase				:		Dict												=				loss.mean()
								elif model.module.config.ctc_loss_reduction == "sum":
										__lowercase				:		Optional[int]												=				loss.sum() / (inputs['''mask_time_indices''']).sum()
								else:
										raise ValueError(f'{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'						)

						if self.args.gradient_accumulation_steps > 1:
								__lowercase				:		Any												=				loss / self.args.gradient_accumulation_steps

						if self.use_amp:
								self.scaler.scale(lowercase__						).backward()
						elif self.use_apex:
								with amp.scale_loss(lowercase__    ,   self.optimizer						) as scaled_loss:
										scaled_loss.backward()
						elif self.deepspeed:
								self.deepspeed.backward(lowercase__						)
						else:
								loss.backward()

						self.num_update_step += 1
						# make sure gumbel softmax temperature is decayed
						if self.args.n_gpu > 1 or self.deepspeed:
								model.module.set_gumbel_temperature(
								    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step    ,   self.min_gumbel_temp						)						)
						else:
								model.set_gumbel_temperature(
								    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step    ,   self.min_gumbel_temp						)						)

						return loss.detach()





def 						snake_case__       (					)			->Dict:






		"""simple docstring"""


		__lowercase				:		str												=				HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)				)

		__lowercase				:		List[Any]												=				parser.parse_args_into_dataclasses()
		configure_logger(snake_case_,  snake_case_				)

		# Downloading and loading a dataset from the hub.
		__lowercase				:		List[str]												=				load_dataset(data_args.dataset_name,  data_args.dataset_config_name,  cache_dir=model_args.cache_dir				)

		if "validation" not in datasets.keys():
				# make sure only "validation" and "train" keys remain"
				__lowercase				:		int												=				DatasetDict()
				__lowercase				:		List[Any]												=				load_dataset(
				    data_args.dataset_name,  data_args.dataset_config_name,  split=F'{data_args.train_split_name}[:{data_args.validation_split_percentage}%]',  cache_dir=model_args.cache_dir,  )
				__lowercase				:		Union[str, Any]												=				load_dataset(
				    data_args.dataset_name,  data_args.dataset_config_name,  split=F'{data_args.train_split_name}[{data_args.validation_split_percentage}%:]',  cache_dir=model_args.cache_dir,  )
		else:
				# make sure only "validation" and "train" keys remain"
				__lowercase				:		List[str]												=				DatasetDict()
				__lowercase				:		Dict												=				load_dataset(
				    data_args.dataset_name,  data_args.dataset_config_name,  split="validation",  cache_dir=model_args.cache_dir,  )
				__lowercase				:		int												=				load_dataset(
				    data_args.dataset_name,  data_args.dataset_config_name,  split=F'{data_args.train_split_name}',  cache_dir=model_args.cache_dir,  )

		# only normalized-inputs-training is supported
		__lowercase				:		Optional[int]												=				WavaVecaFeatureExtractor.from_pretrained(
		    model_args.model_name_or_path,  cache_dir=model_args.cache_dir,  do_normalize=snake_case_				)

		def prepare_dataset(_lowerCamelCase				):
				# check that all files have the correct sampling rate
				__lowercase				:		List[str]												=				librosa.load(batch[data_args.speech_file_column],  sr=feature_extractor.sampling_rate				)
				return batch

		# load audio files into numpy arrays
		__lowercase				:		Optional[int]												=				datasets.map(
		    snake_case_,  num_proc=data_args.preprocessing_num_workers,  remove_columns=datasets["train"].column_names				)

		# filter audio files that are too long
		__lowercase				:		str												=				vectorized_datasets.filter(
		    lambda _lowerCamelCase				: len(data["speech"]				) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate				)				)

		def normalize(_lowerCamelCase				):
				return feature_extractor(batch["speech"],  sampling_rate=feature_extractor.sampling_rate				)

		# normalize and transform to `BatchFeatures`
		__lowercase				:		List[str]												=				vectorized_datasets.map(
		    snake_case_,  batched=snake_case_,  num_proc=data_args.preprocessing_num_workers,  load_from_cache_file=not data_args.overwrite_cache,  remove_columns=vectorized_datasets["train"].column_names,  )

		# pretraining is only supported for "newer" stable layer norm architecture
		# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
		__lowercase				:		Optional[int]												=				WavaVecaConfig.from_pretrained(
		    model_args.model_name_or_path,  cache_dir=model_args.cache_dir,  gradient_checkpointing=training_args.gradient_checkpointing,  )

		if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
				raise ValueError(
				    "PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"
				    " ``config.feat_extract_norm=\'layer\'"				)

		__lowercase				:		List[str]												=				WavaVecaForPreTraining(snake_case_				)

		__lowercase				:		str												=				DataCollatorForWavaVecaPretraining(model=snake_case_,  feature_extractor=snake_case_				)

		__lowercase				:		Tuple												=				WavaVecaPreTrainer(
		    model=snake_case_,  data_collator=snake_case_,  args=snake_case_,  train_dataset=vectorized_datasets["train"],  eval_dataset=vectorized_datasets["validation"],  tokenizer=snake_case_,  max_gumbel_temp=model_args.max_gumbel_temperature,  min_gumbel_temp=model_args.min_gumbel_temperature,  gumbel_temp_decay=model_args.gumbel_temperature_decay,  )
		trainer.train()


if __name__ == "__main__":
						main()







"""simple docstring"""







from collections import deque

def 						snake_case__       (					_lowerCamelCase				)			->Dict:






		"""simple docstring"""


		__lowercase				:		Any												=				len(_lowerCamelCase				)
		__lowercase				:		str												=				deque()
		__lowercase				:		Tuple												=				[False for _ in range(_lowerCamelCase				)]
		__lowercase				:		Optional[int]												=				[-1 for _ in range(_lowerCamelCase				)]
		__lowercase				:		Dict												=				index_of[:]

		def strong_connect(_lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase				):
				__lowercase				:		List[str]												=				index  # the number when this node is seen
				__lowercase				:		List[str]												=				index  # lowest rank node reachable from here
				index += 1
				stack.append(_lowerCamelCase				)
				__lowercase				:		List[Any]												=				True

				for w in g[v]:
						if index_of[w] == -1:
								__lowercase				:		Optional[int]												=				strong_connect(_lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase				)
								__lowercase				:		List[Any]												=				(
								    lowlink_of[w] if lowlink_of[w] < lowlink_of[v] else lowlink_of[v]
								)
						elif on_stack[w]:
								__lowercase				:		List[Any]												=				(
								    lowlink_of[w] if lowlink_of[w] < lowlink_of[v] else lowlink_of[v]
								)

				if lowlink_of[v] == index_of[v]:
						__lowercase				:		Any												=				[]
						__lowercase				:		List[Any]												=				stack.pop()
						__lowercase				:		Any												=				False
						component.append(_lowerCamelCase				)
						while w != v:
								__lowercase				:		List[str]												=				stack.pop()
								__lowercase				:		Dict												=				False
								component.append(_lowerCamelCase				)
						components.append(_lowerCamelCase				)
				return index

		__lowercase				:		Tuple												=				[]
		for v in range(_lowerCamelCase				):
				if index_of[v] == -1:
						strong_connect(_lowerCamelCase,  0,  _lowerCamelCase				)

		return components

def 						snake_case__       (					_lowerCamelCase,  _lowerCamelCase				)			->Tuple:






		"""simple docstring"""


		__lowercase				:		int												=				[[] for _ in range(_lowerCamelCase				)]
		for u, v in edges:
				g[u].append(_lowerCamelCase				)
		return g


if __name__ == "__main__":
						# Test
						__A   :  int					=      7
						__A   :  int					=      [0, 0, 1, 2, 3, 3, 4, 4, 6]
						__A   :  List[Any]					=      [1, 3, 2, 0, 1, 4, 5, 6, 5]
						__A   :  List[str]					=      [(u, v) for u, v in zip(source, target)]
						__A   :  List[str]					=      create_graph(n_vertices, edges)

						assert [[5], [6], [4], [3, 2, 1, 0]] == tarjan(g)
