"""simple docstring"""


import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
    from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("""1.6"""):
    UpperCAmelCase__			=    True
    from torch.cuda.amp import autocast


UpperCAmelCase__			=    logging.getLogger(__name__)


@dataclass
class 							a					:
    _snake_case  :	str  				=				field(
        metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'} )
    _snake_case  :	Optional[str]  				=				field(
        default=lowerCAmelCase_						,				metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}						,				)
    _snake_case  :	Optional[bool]  				=				field(
        default=lowerCAmelCase_						,				metadata={'help': 'Whether to freeze the feature extractor layers of the model.'} )
    _snake_case  :	Optional[bool]  				=				field(
        default=lowerCAmelCase_						,				metadata={'help': 'Whether to log verbose messages or not.'}						,				)
    _snake_case  :	Optional[float]  				=				field(
        default=2.0						,				metadata={'help': 'Maximum temperature for gumbel softmax.'} )
    _snake_case  :	Optional[float]  				=				field(
        default=0.5						,				metadata={'help': 'Minimum temperature for gumbel softmax.'} )
    _snake_case  :	Optional[float]  				=				field(
        default=0.99_99_95						,				metadata={'help': 'Decay of gumbel temperature during training.'} )






def      __UpperCAmelCase       (      lowercase ,lowercase		):







  """simple docstring"""
  logging.basicConfig(
      format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""" ,datefmt="""%m/%d/%Y %H:%M:%S""" ,handlers=[logging.StreamHandler(sys.stdout		)] ,)
  _UpperCAmelCase    			=   logging.WARNING
  if model_args.verbose_logging:
    _UpperCAmelCase    			=   logging.DEBUG
  elif trainer_utils.is_main_process(training_args.local_rank		):
    _UpperCAmelCase    			=   logging.INFO
  logger.setLevel(lowercase		)


@dataclass
class 							a					:
    _snake_case  :	str  				=				field(
        default=lowerCAmelCase_						,				metadata={'help': 'The name of the dataset to use (via the datasets library).'} )
    _snake_case  :	Optional[str]  				=				field(
        default=lowerCAmelCase_						,				metadata={'help': 'The configuration name of the dataset to use (via the datasets library).'} )
    _snake_case  :	Optional[str]  				=				field(
        default='train'						,				metadata={
            'help': 'The name of the training data set split to use (via the datasets library). Defaults to \'train\''
        }						,				)
    _snake_case  :	Optional[str]  				=				field(
        default='validation'						,				metadata={
            'help': (
                'The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''
            )
        }						,				)
    _snake_case  :	Optional[str]  				=				field(
        default='file'						,				metadata={'help': 'Column in the dataset that contains speech file path. Defaults to \'file\''}						,				)
    _snake_case  :	bool  				=				field(
        default=lowerCAmelCase_						,				metadata={'help': 'Overwrite the cached preprocessed datasets or not.'} )
    _snake_case  :	Optional[int]  				=				field(
        default=1						,				metadata={
            'help': 'The percentage of the train set used as validation set in case there\'s no validation split'
        }						,				)
    _snake_case  :	Optional[int]  				=				field(
        default=lowerCAmelCase_						,				metadata={'help': 'The number of processes to use for the preprocessing.'}						,				)
    _snake_case  :	Optional[float]  				=				field(
        default=20.0						,				metadata={'help': 'Filter audio files that are longer than `max_duration_in_seconds` seconds'} )



@dataclass
class 							a					:
    _snake_case  :	WavaVecaForPreTraining
    _snake_case  :	WavaVecaFeatureExtractor
    _snake_case  :	Union[bool, str]  				=				"longest"
    _snake_case  :	Optional[int]  				=				None
    _snake_case  :	Optional[int]  				=				None


    def __call__(       self						:	Dict ,					__lowerCAmelCase						:	List[Dict[str, Union[List[int], torch.Tensor]]]       ):
      # reformat list to dict and set to pytorch format
      _UpperCAmelCase    			=   self.feature_extractor.pad(
          __lowerCAmelCase ,					max_length=self.max_length ,					padding=self.padding ,					pad_to_multiple_of=self.pad_to_multiple_of ,					return_tensors="""pt""" ,					)
      _UpperCAmelCase    			=   self.model._get_feat_extract_output_lengths(batch["""input_values"""].shape[-1]       )

      _UpperCAmelCase    			=   batch["""input_values"""].shape[0]

      # make sure that no loss is computed on padded inputs
      if batch["attention_mask"] is not None:
        # compute real output lengths according to convolution formula
        _UpperCAmelCase    			=   self.model._get_feat_extract_output_lengths(batch["""attention_mask"""].sum(-1       )       ).to(
            torch.long       )

        _UpperCAmelCase    			=   torch.zeros(
            (batch_size, mask_indices_seq_length) ,					dtype=torch.long ,					device=batch["""input_values"""].device       )

        # these two operations makes sure that all values
        # before the output lengths indices are attended to
        _UpperCAmelCase    			=   1
        _UpperCAmelCase    			=   attention_mask.flip([-1]       ).cumsum(-1       ).flip([-1]       ).bool()

      # sample randomly masked indices
      _UpperCAmelCase    			=   _compute_mask_indices(
          (batch_size, mask_indices_seq_length) ,					self.model.config.mask_time_prob ,					self.model.config.mask_time_length ,					attention_mask=__lowerCAmelCase ,					min_masks=2 ,					)

      return batch



class 							a					(		lowerCAmelCase_ ):


    def __init__(       self						:	List[Any] ,					*__lowerCAmelCase						:	Union[str, Any] ,					__lowerCAmelCase						:	Optional[int]=1 ,					__lowerCAmelCase						:	Dict=0 ,					__lowerCAmelCase						:	str=1.0 ,					**__lowerCAmelCase						:	Any       ):
      super().__init__(*__lowerCAmelCase ,					**__lowerCAmelCase       )
      _UpperCAmelCase    			=   0
      _UpperCAmelCase    			=   max_gumbel_temp
      _UpperCAmelCase    			=   min_gumbel_temp
      _UpperCAmelCase    			=   gumbel_temp_decay


    def   lowerCAmelCase_	(       self						:	Optional[int] ,					__lowerCAmelCase						:	nn.Module ,					__lowerCAmelCase						:	Dict[str, Union[torch.Tensor, Any]]       ):
      model.train()
      _UpperCAmelCase    			=   self._prepare_inputs(__lowerCAmelCase       )

      if self.use_amp:
        with autocast():
          _UpperCAmelCase    			=   self.compute_loss(__lowerCAmelCase ,					__lowerCAmelCase       )
      else:
        _UpperCAmelCase    			=   self.compute_loss(__lowerCAmelCase ,					__lowerCAmelCase       )

      if self.args.n_gpu > 1 or self.deepspeed:
        if model.module.config.ctc_loss_reduction == "mean":
          _UpperCAmelCase    			=   loss.mean()
        elif model.module.config.ctc_loss_reduction == "sum":
          _UpperCAmelCase    			=   loss.sum() / (inputs["""mask_time_indices"""]).sum()
        else:
          raise ValueError(f'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''       )

      if self.args.gradient_accumulation_steps > 1:
        _UpperCAmelCase    			=   loss / self.args.gradient_accumulation_steps

      if self.use_amp:
        self.scaler.scale(__lowerCAmelCase       ).backward()
      elif self.use_apex:
        with amp.scale_loss(__lowerCAmelCase ,					self.optimizer       ) as scaled_loss:
          scaled_loss.backward()
      elif self.deepspeed:
        self.deepspeed.backward(__lowerCAmelCase       )
      else:
        loss.backward()

      self.num_update_step += 1
      # make sure gumbel softmax temperature is decayed
      if self.args.n_gpu > 1 or self.deepspeed:
        model.module.set_gumbel_temperature(
            max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step ,					self.min_gumbel_temp       )       )
      else:
        model.set_gumbel_temperature(
            max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step ,					self.min_gumbel_temp       )       )

      return loss.detach()






def      __UpperCAmelCase       (      ):







  """simple docstring"""
  # See all possible arguments in src/transformers/training_args.py
  # or by passing the --help flag to this script.
  # We now keep distinct sets of args, for a cleaner separation of concerns.

  _UpperCAmelCase    			=   HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)		)

  _UpperCAmelCase			,					_UpperCAmelCase			,					_UpperCAmelCase    			=   parser.parse_args_into_dataclasses()
  configure_logger(lowercase ,lowercase		)

  # Downloading and loading a dataset from the hub.
  _UpperCAmelCase    			=   load_dataset(data_args.dataset_name ,data_args.dataset_config_name ,cache_dir=model_args.cache_dir		)

  if "validation" not in datasets.keys():
    # make sure only "validation" and "train" keys remain"
    _UpperCAmelCase    			=   DatasetDict()
    _UpperCAmelCase    			=   load_dataset(
        data_args.dataset_name ,data_args.dataset_config_name ,split=f'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]''' ,cache_dir=model_args.cache_dir ,)
    _UpperCAmelCase    			=   load_dataset(
        data_args.dataset_name ,data_args.dataset_config_name ,split=f'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]''' ,cache_dir=model_args.cache_dir ,)
  else:
    # make sure only "validation" and "train" keys remain"
    _UpperCAmelCase    			=   DatasetDict()
    _UpperCAmelCase    			=   load_dataset(
        data_args.dataset_name ,data_args.dataset_config_name ,split="""validation""" ,cache_dir=model_args.cache_dir ,)
    _UpperCAmelCase    			=   load_dataset(
        data_args.dataset_name ,data_args.dataset_config_name ,split=f'''{data_args.train_split_name}''' ,cache_dir=model_args.cache_dir ,)

  # only normalized-inputs-training is supported
  _UpperCAmelCase    			=   WavaVecaFeatureExtractor.from_pretrained(
      model_args.model_name_or_path ,cache_dir=model_args.cache_dir ,do_normalize=lowercase		)

  def prepare_dataset(lowercase		):
    # check that all files have the correct sampling rate
    _UpperCAmelCase			,					_UpperCAmelCase    			=   librosa.load(batch[data_args.speech_file_column] ,sr=feature_extractor.sampling_rate		)
    return batch

  # load audio files into numpy arrays
  _UpperCAmelCase    			=   datasets.map(
      lowercase ,num_proc=data_args.preprocessing_num_workers ,remove_columns=datasets["""train"""].column_names		)

  # filter audio files that are too long
  _UpperCAmelCase    			=   vectorized_datasets.filter(
      lambda lowercase		: len(data["""speech"""]		) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate		)		)

  def normalize(lowercase		):
    return feature_extractor(batch["""speech"""] ,sampling_rate=feature_extractor.sampling_rate		)

  # normalize and transform to `BatchFeatures`
  _UpperCAmelCase    			=   vectorized_datasets.map(
      lowercase ,batched=lowercase ,num_proc=data_args.preprocessing_num_workers ,load_from_cache_file=not data_args.overwrite_cache ,remove_columns=vectorized_datasets["""train"""].column_names ,)

  # pretraining is only supported for "newer" stable layer norm architecture
  # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
  _UpperCAmelCase    			=   WavaVecaConfig.from_pretrained(
      model_args.model_name_or_path ,cache_dir=model_args.cache_dir ,gradient_checkpointing=training_args.gradient_checkpointing ,)

  if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
    raise ValueError(
        """PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"""
        """ ``config.feat_extract_norm='layer'"""		)

  _UpperCAmelCase    			=   WavaVecaForPreTraining(lowercase		)

  _UpperCAmelCase    			=   DataCollatorForWavaVecaPretraining(model=lowercase ,feature_extractor=lowercase		)

  _UpperCAmelCase    			=   WavaVecaPreTrainer(
      model=lowercase ,data_collator=lowercase ,args=lowercase ,train_dataset=vectorized_datasets["""train"""] ,eval_dataset=vectorized_datasets["""validation"""] ,tokenizer=lowercase ,max_gumbel_temp=model_args.max_gumbel_temperature ,min_gumbel_temp=model_args.min_gumbel_temperature ,gumbel_temp_decay=model_args.gumbel_temperature_decay ,)
  trainer.train()


if __name__ == "__main__":
    main()

"""simple docstring"""


import copy
import json
import os
import tempfile

from transformers import is_torch_available

from .test_configuration_utils import config_common_kwargs


class 							a					(		lowerCAmelCase_ ):


    def __init__(       self						:	Union[str, Any] ,					__lowerCAmelCase						:	Union[str, Any] ,					__lowerCAmelCase						:	str=None ,					__lowerCAmelCase						:	List[Any]=True ,					__lowerCAmelCase						:	Union[str, Any]=None ,					**__lowerCAmelCase						:	Dict       ):
      _UpperCAmelCase    			=   parent
      _UpperCAmelCase    			=   config_class
      _UpperCAmelCase    			=   has_text_modality
      _UpperCAmelCase    			=   kwargs
      _UpperCAmelCase    			=   common_properties


    def   lowerCAmelCase_	(       self						:	Dict       ):
      _UpperCAmelCase    			=   self.config_class(**self.inputs_dict       )
      _UpperCAmelCase    			=   (
          ["""hidden_size""", """num_attention_heads""", """num_hidden_layers"""]
          if self.common_properties is None
          else self.common_properties
      )

      # Add common fields for text models
      if self.has_text_modality:
        common_properties.extend(["""vocab_size"""]       )

      # Test that config has the common properties as getters
      for prop in common_properties:
        self.parent.assertTrue(hasattr(__lowerCAmelCase ,					__lowerCAmelCase       ) ,					msg=f'''`{prop}` does not exist'''       )

      # Test that config has the common properties as setter
      for idx, name in enumerate(__lowerCAmelCase       ):
        try:
          setattr(__lowerCAmelCase ,					__lowerCAmelCase ,					__lowerCAmelCase       )
          self.parent.assertEqual(
              getattr(__lowerCAmelCase ,					__lowerCAmelCase       ) ,					__lowerCAmelCase ,					msg=f'''`{name} value {idx} expected, but was {getattr(__lowerCAmelCase ,					__lowerCAmelCase       )}'''       )
        except NotImplementedError:
          # Some models might not be able to implement setters for common_properties
          # In that case, a NotImplementedError is raised
          pass

        # Test if config class can be called with Config(prop_name=..)
      for idx, name in enumerate(__lowerCAmelCase       ):
        try:
          _UpperCAmelCase    			=   self.config_class(**{name: idx}       )
          self.parent.assertEqual(
              getattr(__lowerCAmelCase ,					__lowerCAmelCase       ) ,					__lowerCAmelCase ,					msg=f'''`{name} value {idx} expected, but was {getattr(__lowerCAmelCase ,					__lowerCAmelCase       )}'''       )
        except NotImplementedError:
          # Some models might not be able to implement setters for common_properties
          # In that case, a NotImplementedError is raised
          pass


    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      _UpperCAmelCase    			=   self.config_class(**self.inputs_dict       )
      _UpperCAmelCase    			=   json.loads(config.to_json_string()       )
      for key, value in self.inputs_dict.items():
        self.parent.assertEqual(obj[key] ,					__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	Union[str, Any]       ):
      _UpperCAmelCase    			=   self.config_class(**self.inputs_dict       )

      with tempfile.TemporaryDirectory() as tmpdirname:
        _UpperCAmelCase    			=   os.path.join(__lowerCAmelCase ,					"""config.json"""       )
        config_first.to_json_file(__lowerCAmelCase       )
        _UpperCAmelCase    			=   self.config_class.from_json_file(__lowerCAmelCase       )

      self.parent.assertEqual(config_second.to_dict() ,					config_first.to_dict()       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      _UpperCAmelCase    			=   self.config_class(**self.inputs_dict       )

      with tempfile.TemporaryDirectory() as tmpdirname:
        config_first.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   self.config_class.from_pretrained(__lowerCAmelCase       )

      self.parent.assertEqual(config_second.to_dict() ,					config_first.to_dict()       )


    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      _UpperCAmelCase    			=   self.config_class(**self.inputs_dict       )

      _UpperCAmelCase    			=   """test"""
      with tempfile.TemporaryDirectory() as tmpdirname:
        _UpperCAmelCase    			=   os.path.join(__lowerCAmelCase ,					__lowerCAmelCase       )
        config_first.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   self.config_class.from_pretrained(__lowerCAmelCase ,					subfolder=__lowerCAmelCase       )

      self.parent.assertEqual(config_second.to_dict() ,					config_first.to_dict()       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      _UpperCAmelCase    			=   self.config_class(**self.inputs_dict ,					num_labels=5       )
      self.parent.assertEqual(len(config.idalabel       ) ,					5       )
      self.parent.assertEqual(len(config.labelaid       ) ,					5       )

      _UpperCAmelCase    			=   3
      self.parent.assertEqual(len(config.idalabel       ) ,					3       )
      self.parent.assertEqual(len(config.labelaid       ) ,					3       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      if self.config_class.is_composition:
        return
      _UpperCAmelCase    			=   self.config_class()
      self.parent.assertIsNotNone(__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	Any       ):
      _UpperCAmelCase    			=   copy.deepcopy(__lowerCAmelCase       )
      _UpperCAmelCase    			=   self.config_class(**__lowerCAmelCase       )
      _UpperCAmelCase    			=   []
      for key, value in config_common_kwargs.items():
        if key == "torch_dtype":
          if not is_torch_available():
            continue
          else:
            import torch

            if config.torch_dtype != torch.floataa:
              wrong_values.append(("""torch_dtype""", config.torch_dtype, torch.floataa)       )
        elif getattr(__lowerCAmelCase ,					__lowerCAmelCase       ) != value:
          wrong_values.append((key, getattr(__lowerCAmelCase ,					__lowerCAmelCase       ), value)       )

      if len(__lowerCAmelCase       ) > 0:
        _UpperCAmelCase    			=   """\n""".join([f'''- {v[0]}: got {v[1]} instead of {v[2]}''' for v in wrong_values]       )
        raise ValueError(f'''The following keys were not properly set in the config:\n{errors}'''       )


    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      self.create_and_test_config_common_properties()
      self.create_and_test_config_to_json_string()
      self.create_and_test_config_to_json_file()
      self.create_and_test_config_from_and_save_pretrained()
      self.create_and_test_config_from_and_save_pretrained_subfolder()
      self.create_and_test_config_with_num_labels()
      self.check_config_can_be_init_without_params()
      self.check_config_arguments_init()
