
"""simple docstring"""
import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
    from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("""1.6"""):
    _a      :			Dict					=							True
    from torch.cuda.amp import autocast


_a      :			Union[str, Any]					=							logging.getLogger(__name__)






@dataclass
class      _UpperCAmelCase				:
    __lowercase							: int  =					field(
        metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""})
    __lowercase							: Optional[Any]  =					field(
        default=snake_case__		,					metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}		,					)
    __lowercase							: List[str]  =					field(
        default=snake_case__		,					metadata={"""help""": """Whether to freeze the feature extractor layers of the model."""})
    __lowercase							: Optional[int]  =					field(
        default=snake_case__		,					metadata={"""help""": """Whether to log verbose messages or not."""}		,					)
    __lowercase							: Union[str, Any]  =					field(
        default=2.0		,					metadata={"""help""": """Maximum temperature for gumbel softmax."""})
    __lowercase							: Any  =					field(
        default=0.5		,					metadata={"""help""": """Minimum temperature for gumbel softmax."""})
    __lowercase							: Union[str, Any]  =					field(
        default=0.999_995		,					metadata={"""help""": """Decay of gumbel temperature during training."""})


def 				a__   (   a   :				Optional[Any]       ,     a   :				Tuple		):

  """simple docstring"""







  logging.basicConfig(
      format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"       ,     datefmt="%m/%d/%Y %H:%M:%S"       ,     handlers=[logging.StreamHandler(sys.stdout		)]       ,     )
  _snake_case     :						Any						=   logging.WARNING
  if model_args.verbose_logging:
    _snake_case     :						str						=   logging.DEBUG
  elif trainer_utils.is_main_process(training_args.local_rank		):
    _snake_case     :						List[str]						=   logging.INFO
  logger.setLevel(__SCREAMING_SNAKE_CASE		)






@dataclass
class      _UpperCAmelCase				:
    __lowercase							: List[Any]  =					field(
        default=snake_case__		,					metadata={"""help""": """The name of the dataset to use (via the datasets library)."""})
    __lowercase							: int  =					field(
        default=snake_case__		,					metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""})
    __lowercase							: Dict  =					field(
        default="""train"""		,					metadata={
            """help""": """The name of the training data set split to use (via the datasets library). Defaults to 'train'"""
        }		,					)
    __lowercase							: Dict  =					field(
        default="""validation"""		,					metadata={
            """help""": (
                """The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"""
            )
        }		,					)
    __lowercase							: Optional[Any]  =					field(
        default="""file"""		,					metadata={"""help""": """Column in the dataset that contains speech file path. Defaults to 'file'"""}		,					)
    __lowercase							: Union[str, Any]  =					field(
        default=snake_case__		,					metadata={"""help""": """Overwrite the cached preprocessed datasets or not."""})
    __lowercase							: Optional[int]  =					field(
        default=1		,					metadata={
            """help""": """The percentage of the train set used as validation set in case there's no validation split"""
        }		,					)
    __lowercase							: Dict  =					field(
        default=snake_case__		,					metadata={"""help""": """The number of processes to use for the preprocessing."""}		,					)
    __lowercase							: Tuple  =					field(
        default=2_0.0		,					metadata={"""help""": """Filter audio files that are longer than `max_duration_in_seconds` seconds"""})






@dataclass
class      _UpperCAmelCase				:
    __lowercase							: Optional[Any]  =					4_2
    __lowercase							: Optional[int]  =					4_2
    __lowercase							: Tuple  =					"""longest"""
    __lowercase							: Tuple  =					None
    __lowercase							: List[str]  =					None
    def __call__(    self						,    snake_case_     ):
      # reformat list to dict and set to pytorch format
      _snake_case     :						List[Any]						=   self.feature_extractor.pad(
          lowercase_						,    max_length=self.max_length						,    padding=self.padding						,    pad_to_multiple_of=self.pad_to_multiple_of						,    return_tensors="pt"						,    )
      _snake_case     :						Optional[int]						=   self.model._get_feat_extract_output_lengths(batch["input_values"].shape[-1]     )

      _snake_case     :						Tuple						=   batch["input_values"].shape[0]

      # make sure that no loss is computed on padded inputs
      if batch["attention_mask"] is not None:
        # compute real output lengths according to convolution formula
        _snake_case     :						Tuple						=   self.model._get_feat_extract_output_lengths(batch["attention_mask"].sum(-1     )     ).to(
            torch.long     )

        _snake_case     :						Union[str, Any]						=   torch.zeros(
            (batch_size, mask_indices_seq_length)						,    dtype=torch.long						,    device=batch["input_values"].device     )

        # these two operations makes sure that all values
        # before the output lengths indices are attended to
        _snake_case     :						Optional[Any]						=   1
        _snake_case     :						Dict						=   attention_mask.flip([-1]     ).cumsum(-1     ).flip([-1]     ).bool()

      # sample randomly masked indices
      _snake_case     :						str						=   _compute_mask_indices(
          (batch_size, mask_indices_seq_length)						,    self.model.config.mask_time_prob						,    self.model.config.mask_time_length						,    attention_mask=lowercase_						,    min_masks=2						,    )

      return batch






class      _UpperCAmelCase				(					snake_case__):
    def __init__(    self						,    *snake_case_						,    snake_case_=1						,    snake_case_=0						,    snake_case_=1.0						,    **snake_case_     ):
      super().__init__(*lowercase_						,    **lowercase_     )
      _snake_case     :						Union[str, Any]						=   0
      _snake_case     :						int						=   max_gumbel_temp
      _snake_case     :						Union[str, Any]						=   min_gumbel_temp
      _snake_case     :						List[str]						=   gumbel_temp_decay


    def 					lowerCamelCase__	(    self						,    snake_case_						,    snake_case_     ):
      model.train()
      _snake_case     :						Dict						=   self._prepare_inputs(lowercase_     )

      if self.use_amp:
        with autocast():
          _snake_case     :						Dict						=   self.compute_loss(lowercase_						,    lowercase_     )
      else:
        _snake_case     :						Tuple						=   self.compute_loss(lowercase_						,    lowercase_     )

      if self.args.n_gpu > 1 or self.deepspeed:
        if model.module.config.ctc_loss_reduction == "mean":
          _snake_case     :						Any						=   loss.mean()
        elif model.module.config.ctc_loss_reduction == "sum":
          _snake_case     :						List[Any]						=   loss.sum() / (inputs["mask_time_indices"]).sum()
        else:
          raise ValueError(F'{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'     )

      if self.args.gradient_accumulation_steps > 1:
        _snake_case     :						int						=   loss / self.args.gradient_accumulation_steps

      if self.use_amp:
        self.scaler.scale(lowercase_     ).backward()
      elif self.use_apex:
        with amp.scale_loss(lowercase_						,    self.optimizer     ) as scaled_loss:
          scaled_loss.backward()
      elif self.deepspeed:
        self.deepspeed.backward(lowercase_     )
      else:
        loss.backward()

      self.num_update_step += 1
      # make sure gumbel softmax temperature is decayed
      if self.args.n_gpu > 1 or self.deepspeed:
        model.module.set_gumbel_temperature(
            max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step						,    self.min_gumbel_temp     )     )
      else:
        model.set_gumbel_temperature(
            max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step						,    self.min_gumbel_temp     )     )

      return loss.detach()


def 				a__   (   ):

  """simple docstring"""







  _snake_case     :						str						=   HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)		)

  _snake_case     :						str						=   parser.parse_args_into_dataclasses()
  configure_logger(__SCREAMING_SNAKE_CASE       ,     __SCREAMING_SNAKE_CASE		)

  # Downloading and loading a dataset from the hub.
  _snake_case     :						Optional[Any]						=   load_dataset(data_args.dataset_name       ,     data_args.dataset_config_name       ,     cache_dir=model_args.cache_dir		)

  if "validation" not in datasets.keys():
    # make sure only "validation" and "train" keys remain"
    _snake_case     :						Optional[int]						=   DatasetDict()
    _snake_case     :						Optional[Any]						=   load_dataset(
        data_args.dataset_name       ,     data_args.dataset_config_name       ,     split=f'{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'       ,     cache_dir=model_args.cache_dir       ,     )
    _snake_case     :						Tuple						=   load_dataset(
        data_args.dataset_name       ,     data_args.dataset_config_name       ,     split=f'{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'       ,     cache_dir=model_args.cache_dir       ,     )
  else:
    # make sure only "validation" and "train" keys remain"
    _snake_case     :						Tuple						=   DatasetDict()
    _snake_case     :						Dict						=   load_dataset(
        data_args.dataset_name       ,     data_args.dataset_config_name       ,     split="validation"       ,     cache_dir=model_args.cache_dir       ,     )
    _snake_case     :						Union[str, Any]						=   load_dataset(
        data_args.dataset_name       ,     data_args.dataset_config_name       ,     split=f'{data_args.train_split_name}'       ,     cache_dir=model_args.cache_dir       ,     )

  # only normalized-inputs-training is supported
  _snake_case     :						List[str]						=   WavaVecaFeatureExtractor.from_pretrained(
      model_args.model_name_or_path       ,     cache_dir=model_args.cache_dir       ,     do_normalize=__SCREAMING_SNAKE_CASE		)

  def prepare_dataset(a   :				int		):
    # check that all files have the correct sampling rate
    _snake_case     :						Optional[Any]						=   librosa.load(batch[data_args.speech_file_column]       ,     sr=feature_extractor.sampling_rate		)
    return batch

  # load audio files into numpy arrays
  _snake_case     :						Optional[int]						=   datasets.map(
      __SCREAMING_SNAKE_CASE       ,     num_proc=data_args.preprocessing_num_workers       ,     remove_columns=datasets["train"].column_names		)

  # filter audio files that are too long
  _snake_case     :						Any						=   vectorized_datasets.filter(
      lambda a		: len(data["speech"]		) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate		)		)

  def normalize(a   :				int		):
    return feature_extractor(batch["speech"]       ,     sampling_rate=feature_extractor.sampling_rate		)

  # normalize and transform to `BatchFeatures`
  _snake_case     :						Optional[Any]						=   vectorized_datasets.map(
      __SCREAMING_SNAKE_CASE       ,     batched=__SCREAMING_SNAKE_CASE       ,     num_proc=data_args.preprocessing_num_workers       ,     load_from_cache_file=not data_args.overwrite_cache       ,     remove_columns=vectorized_datasets["train"].column_names       ,     )

  # pretraining is only supported for "newer" stable layer norm architecture
  # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
  _snake_case     :						Optional[int]						=   WavaVecaConfig.from_pretrained(
      model_args.model_name_or_path       ,     cache_dir=model_args.cache_dir       ,     gradient_checkpointing=training_args.gradient_checkpointing       ,     )

  if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
    raise ValueError(
        "PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"
        " ``config.feat_extract_norm='layer'"		)

  _snake_case     :						int						=   WavaVecaForPreTraining(__SCREAMING_SNAKE_CASE		)

  _snake_case     :						Dict						=   DataCollatorForWavaVecaPretraining(model=__SCREAMING_SNAKE_CASE       ,     feature_extractor=__SCREAMING_SNAKE_CASE		)

  _snake_case     :						str						=   WavaVecaPreTrainer(
      model=__SCREAMING_SNAKE_CASE       ,     data_collator=__SCREAMING_SNAKE_CASE       ,     args=__SCREAMING_SNAKE_CASE       ,     train_dataset=vectorized_datasets["train"]       ,     eval_dataset=vectorized_datasets["validation"]       ,     tokenizer=__SCREAMING_SNAKE_CASE       ,     max_gumbel_temp=model_args.max_gumbel_temperature       ,     min_gumbel_temp=model_args.min_gumbel_temperature       ,     gumbel_temp_decay=model_args.gumbel_temperature_decay       ,     )
  trainer.train()


if __name__ == "__main__":
    main()





"""simple docstring"""
from typing import List, Optional

import numpy as np

from ...processing_utils import ProcessorMixin
from ...utils import to_numpy






class      _UpperCAmelCase				(					_snake_case):
    __lowercase							: int  =					"""EncodecFeatureExtractor"""
    __lowercase							: str  =					("""T5Tokenizer""", """T5TokenizerFast""")
    def __init__(    self						,    snake_case_						,    snake_case_     ):
      super().__init__(snake_case_						,    snake_case_     )
      _snake_case     :						Dict						=   self.feature_extractor
      _snake_case     :						Any						=   False
    def 					lowerCamelCase__	(    self						,    snake_case_=None						,    snake_case_=None						,    snake_case_=True     ):
      return self.tokenizer.get_decoder_prompt_ids(task=snake_case_						,    language=snake_case_						,    no_timestamps=snake_case_     )
    def __call__(    self						,    *snake_case_						,    **snake_case_     ):
      # For backward compatibility
      if self._in_target_context_manager:
        return self.current_processor(*snake_case_						,    **snake_case_     )

      _snake_case     :						str						=   kwargs.pop("audio"						,    snake_case_     )
      _snake_case     :						Optional[int]						=   kwargs.pop("sampling_rate"						,    snake_case_     )
      _snake_case     :						Optional[Any]						=   kwargs.pop("text"						,    snake_case_     )
      if len(snake_case_     ) > 0:
        _snake_case     :						Any						=   args[0]
        _snake_case     :						Union[str, Any]						=   args[1:]

      if audio is None and text is None:
        raise ValueError("You need to specify either an `audio` or `text` input to process."     )

      if text is not None:
        _snake_case     :						Any						=   self.tokenizer(snake_case_						,    **snake_case_     )

      if audio is not None:
        _snake_case     :						Any						=   self.feature_extractor(snake_case_						,    *snake_case_						,    sampling_rate=snake_case_						,    **snake_case_     )

      if audio is None:
        return inputs

      elif text is None:
        return audio_inputs

      else:
        _snake_case     :						str						=   audio_inputs["input_values"]
        if "padding_mask" in audio_inputs:
          _snake_case     :						List[str]						=   audio_inputs["padding_mask"]
        return inputs
    def 					lowerCamelCase__	(    self						,    *snake_case_						,    **snake_case_     ):
      _snake_case     :						Tuple						=   kwargs.pop("audio"						,    snake_case_     )
      _snake_case     :						List[str]						=   kwargs.pop("padding_mask"						,    snake_case_     )

      if len(snake_case_     ) > 0:
        _snake_case     :						Tuple						=   args[0]
        _snake_case     :						Dict						=   args[1:]

      if audio_values is not None:
        return self._decode_audio(snake_case_						,    padding_mask=snake_case_     )
      else:
        return self.tokenizer.batch_decode(*snake_case_						,    **snake_case_     )
    def 					lowerCamelCase__	(    self						,    *snake_case_						,    **snake_case_     ):
      return self.tokenizer.decode(*snake_case_						,    **snake_case_     )


    def 					lowerCamelCase__	(    self						,    snake_case_						,    snake_case_ = None     ):
      _snake_case     :						Optional[int]						=   to_numpy(snake_case_     )
      _snake_case			,					_snake_case			,					_snake_case     :						Tuple						=   audio_values.shape

      if padding_mask is None:
        return list(snake_case_     )

      _snake_case     :						Optional[int]						=   to_numpy(snake_case_     )

      # match the sequence length of the padding mask to the generated audio arrays by padding with the **non-padding**
      # token (so that the generated audio values are **not** treated as padded tokens)
      _snake_case     :						Any						=   seq_len - padding_mask.shape[-1]
      _snake_case     :						Optional[Any]						=   1 - self.feature_extractor.padding_value
      _snake_case     :						Optional[int]						=   np.pad(snake_case_						,    ((0, 0), (0, difference))						,    "constant"						,    constant_values=snake_case_     )

      _snake_case     :						Any						=   audio_values.tolist()
      for i in range(snake_case_     ):
        _snake_case     :						Tuple						=   np.asarray(audio_values[i]     )[
            padding_mask[i][None, :] != self.feature_extractor.padding_value
        ]
        _snake_case     :						Tuple						=   sliced_audio.reshape(snake_case_						,    -1     )

      return audio_values

