


import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
 from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("1.6"):
 _SCREAMING_SNAKE_CASE     :					Any      =       True
 from torch.cuda.amp import autocast


_SCREAMING_SNAKE_CASE     :					List[str]      =       logging.getLogger(__name__)

@dataclass
class     A__   :







    """simple docstring"""

    __magic_name__         =						field(
        metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}      )
    __magic_name__         =						field(
        default=snake_case__			,   metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}			,   )
    __magic_name__         =						field(
        default=snake_case__			,   metadata={'help': 'Whether to freeze the feature extractor layers of the model.'}      )
    __magic_name__         =						field(
        default=snake_case__			,   metadata={'help': 'Whether to log verbose messages or not.'}			,   )
    __magic_name__         =						field(
        default=2.0			,   metadata={'help': 'Maximum temperature for gumbel softmax.'}      )
    __magic_name__         =						field(
        default=0.5			,   metadata={'help': 'Minimum temperature for gumbel softmax.'}      )
    __magic_name__         =						field(
        default=0.9_9_9_9_9_5			,   metadata={'help': 'Decay of gumbel temperature during training.'}      )






def      UpperCAmelCase__		(UpperCamelCase_				,UpperCamelCase_							):



     """simple docstring"""






     logging.basicConfig(
         format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''				,datefmt='''%m/%d/%Y %H:%M:%S'''				,handlers=[logging.StreamHandler(sys.stdout							)]				,)
     snake_case											=							logging.WARNING
     if model_args.verbose_logging:
          snake_case											=							logging.DEBUG
     elif trainer_utils.is_main_process(training_args.local_rank							):
          snake_case											=							logging.INFO
     logger.setLevel(UpperCamelCase_							)

@dataclass
class     A__   :







    """simple docstring"""

    __magic_name__         =						field(
        default=snake_case__			,   metadata={'help': 'The name of the dataset to use (via the datasets library).'}      )
    __magic_name__         =						field(
        default=snake_case__			,   metadata={'help': 'The configuration name of the dataset to use (via the datasets library).'}      )
    __magic_name__         =						field(
        default='train'			,   metadata={
            'help': 'The name of the training data set split to use (via the datasets library). Defaults to \'train\''
        }			,   )
    __magic_name__         =						field(
        default='validation'			,   metadata={
            'help': (
                'The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''
            )
        }			,   )
    __magic_name__         =						field(
        default='file'			,   metadata={'help': 'Column in the dataset that contains speech file path. Defaults to \'file\''}			,   )
    __magic_name__         =						field(
        default=snake_case__			,   metadata={'help': 'Overwrite the cached preprocessed datasets or not.'}      )
    __magic_name__         =						field(
        default=1			,   metadata={
            'help': 'The percentage of the train set used as validation set in case there\'s no validation split'
        }			,   )
    __magic_name__         =						field(
        default=snake_case__			,   metadata={'help': 'The number of processes to use for the preprocessing.'}			,   )
    __magic_name__         =						field(
        default=2_0.0			,   metadata={'help': 'Filter audio files that are longer than `max_duration_in_seconds` seconds'}      )






@dataclass
class     A__   :







    """simple docstring"""

    __magic_name__         =						42
    __magic_name__         =						42
    __magic_name__         =						"longest"
    __magic_name__         =						None
    __magic_name__         =						None
    def __call__(			self     ,   __snake_case       ):
         snake_case											=							self.feature_extractor.pad(
             lowerCAmelCase__     ,   max_length=self.max_length     ,   padding=self.padding     ,   pad_to_multiple_of=self.pad_to_multiple_of     ,   return_tensors='''pt'''     ,   )
         snake_case											=							self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1]       )

         snake_case											=							batch['''input_values'''].shape[0]

         # make sure that no loss is computed on padded inputs
         if batch["attention_mask"] is not None:
              # compute real output lengths according to convolution formula
              snake_case											=							self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1       )       ).to(
                  torch.long       )

              snake_case											=							torch.zeros(
                  (batch_size, mask_indices_seq_length)     ,   dtype=torch.long     ,   device=batch['''input_values'''].device       )

              # these two operations makes sure that all values
              # before the output lengths indices are attended to
              snake_case											=							1
              snake_case											=							attention_mask.flip([-1]       ).cumsum(-1       ).flip([-1]       ).bool()

         # sample randomly masked indices
         snake_case											=							_compute_mask_indices(
             (batch_size, mask_indices_seq_length)     ,   self.model.config.mask_time_prob     ,   self.model.config.mask_time_length     ,   attention_mask=lowerCAmelCase__     ,   min_masks=2     ,   )

         return batch






class     A__   (       snake_case__      ):







    """simple docstring"""

    def __init__(			self     ,   *__snake_case     ,   __snake_case=1     ,   __snake_case=0     ,   __snake_case=1.0     ,   **__snake_case       ):
         super().__init__(*lowerCAmelCase__     ,   **lowerCAmelCase__       )
         snake_case											=							0
         snake_case											=							max_gumbel_temp
         snake_case											=							min_gumbel_temp
         snake_case											=							gumbel_temp_decay




    def 						a_     (			self     ,   __snake_case     ,   __snake_case       ):
         model.train()
         snake_case											=							self._prepare_inputs(lowerCAmelCase__       )

         if self.use_amp:
              with autocast():
                   snake_case											=							self.compute_loss(lowerCAmelCase__     ,   lowerCAmelCase__       )
         else:
              snake_case											=							self.compute_loss(lowerCAmelCase__     ,   lowerCAmelCase__       )

         if self.args.n_gpu > 1 or self.deepspeed:
              if model.module.config.ctc_loss_reduction == "mean":
                   snake_case											=							loss.mean()
              elif model.module.config.ctc_loss_reduction == "sum":
                   snake_case											=							loss.sum() / (inputs['''mask_time_indices''']).sum()
              else:
                   raise ValueError(F'''{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'''       )

         if self.args.gradient_accumulation_steps > 1:
              snake_case											=							loss / self.args.gradient_accumulation_steps

         if self.use_amp:
              self.scaler.scale(lowerCAmelCase__       ).backward()
         elif self.use_apex:
              with amp.scale_loss(lowerCAmelCase__     ,   self.optimizer       ) as scaled_loss:
                   scaled_loss.backward()
         elif self.deepspeed:
              self.deepspeed.backward(lowerCAmelCase__       )
         else:
              loss.backward()

         self.num_update_step += 1
         # make sure gumbel softmax temperature is decayed
         if self.args.n_gpu > 1 or self.deepspeed:
              model.module.set_gumbel_temperature(
                  max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step     ,   self.min_gumbel_temp       )       )
         else:
              model.set_gumbel_temperature(
                  max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step     ,   self.min_gumbel_temp       )       )

         return loss.detach()






def      UpperCAmelCase__		():



     """simple docstring"""






     snake_case											=							HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)							)

     snake_case					,    snake_case					,    snake_case											=							parser.parse_args_into_dataclasses()
     configure_logger(UpperCamelCase_				,UpperCamelCase_							)

     # Downloading and loading a dataset from the hub.
     snake_case											=							load_dataset(data_args.dataset_name				,data_args.dataset_config_name				,cache_dir=model_args.cache_dir							)

     if "validation" not in datasets.keys():
          # make sure only "validation" and "train" keys remain"
          snake_case											=							DatasetDict()
          snake_case											=							load_dataset(
              data_args.dataset_name				,data_args.dataset_config_name				,split=F'''{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'''				,cache_dir=model_args.cache_dir				,)
          snake_case											=							load_dataset(
              data_args.dataset_name				,data_args.dataset_config_name				,split=F'''{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'''				,cache_dir=model_args.cache_dir				,)
     else:
          # make sure only "validation" and "train" keys remain"
          snake_case											=							DatasetDict()
          snake_case											=							load_dataset(
              data_args.dataset_name				,data_args.dataset_config_name				,split='''validation'''				,cache_dir=model_args.cache_dir				,)
          snake_case											=							load_dataset(
              data_args.dataset_name				,data_args.dataset_config_name				,split=F'''{data_args.train_split_name}'''				,cache_dir=model_args.cache_dir				,)

     # only normalized-inputs-training is supported
     snake_case											=							WavaVecaFeatureExtractor.from_pretrained(
         model_args.model_name_or_path				,cache_dir=model_args.cache_dir				,do_normalize=UpperCamelCase_							)

     def prepare_dataset(UpperCamelCase_							):
          # check that all files have the correct sampling rate
          snake_case					,    snake_case											=							librosa.load(batch[data_args.speech_file_column]				,sr=feature_extractor.sampling_rate							)
          return batch

     # load audio files into numpy arrays
     snake_case											=							datasets.map(
         UpperCamelCase_				,num_proc=data_args.preprocessing_num_workers				,remove_columns=datasets['''train'''].column_names							)

     # filter audio files that are too long
     snake_case											=							vectorized_datasets.filter(
         lambda UpperCamelCase_							: len(data['''speech''']							) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate							)							)

     def normalize(UpperCamelCase_							):
          return feature_extractor(batch['''speech''']				,sampling_rate=feature_extractor.sampling_rate							)

     # normalize and transform to `BatchFeatures`
     snake_case											=							vectorized_datasets.map(
         UpperCamelCase_				,batched=UpperCamelCase_				,num_proc=data_args.preprocessing_num_workers				,load_from_cache_file=not data_args.overwrite_cache				,remove_columns=vectorized_datasets['''train'''].column_names				,)

     # pretraining is only supported for "newer" stable layer norm architecture
     # apply_spec_augment has to be True, mask_feature_prob has to be 0.0
     snake_case											=							WavaVecaConfig.from_pretrained(
         model_args.model_name_or_path				,cache_dir=model_args.cache_dir				,gradient_checkpointing=training_args.gradient_checkpointing				,)

     if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
          raise ValueError(
              '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
              ''' ``config.feat_extract_norm=\'layer\''''							)

     snake_case											=							WavaVecaForPreTraining(UpperCamelCase_							)

     snake_case											=							DataCollatorForWavaVecaPretraining(model=UpperCamelCase_				,feature_extractor=UpperCamelCase_							)

     snake_case											=							WavaVecaPreTrainer(
         model=UpperCamelCase_				,data_collator=UpperCamelCase_				,args=UpperCamelCase_				,train_dataset=vectorized_datasets['''train''']				,eval_dataset=vectorized_datasets['''validation''']				,tokenizer=UpperCamelCase_				,max_gumbel_temp=model_args.max_gumbel_temperature				,min_gumbel_temp=model_args.min_gumbel_temperature				,gumbel_temp_decay=model_args.gumbel_temperature_decay				,)
     trainer.train()


if __name__ == "__main__":
 main()











'''simple docstring'''

import itertools
import json
import linecache
import os
import pickle
import re
import socket
import string
from collections import Counter
from logging import getLogger
from pathlib import Path
from typing import Callable, Dict, Iterable, List

import git
import torch
from torch.utils.data import Dataset

from transformers import BartTokenizer, RagTokenizer, TaTokenizer







def    a__							(							lowercase   :	Tuple,   lowercase   :	List[str],   lowercase   :	Optional[int],   lowercase   :	List[str],   lowercase   :	List[str]=True,   lowercase   :	str="pt" )						->						Union[str, Any]:





     """simple docstring"""

     _UpperCamelCase							       =			{'''add_prefix_space''': True} if isinstance(lowercase,   lowercase ) and not line.startswith(''' ''' ) else {}
     _UpperCamelCase							       =			padding_side
     return tokenizer(
         [line],   max_length=lowercase,   padding='''max_length''' if pad_to_max_length else None,   truncation=lowercase,   return_tensors=lowercase,   add_special_tokens=lowercase,   **lowercase,   )







def    a__							(							lowercase   :	str,   lowercase   :	int,   lowercase   :	Tuple=None,   )						->						Union[str, Any]:





     """simple docstring"""

     _UpperCamelCase							       =			input_ids.ne(lowercase ).any(dim=0 )
     if attention_mask is None:
          return input_ids[:, keep_column_mask]
     else:
          return (input_ids[:, keep_column_mask], attention_mask[:, keep_column_mask])
class 							__lowerCAmelCase	(       __magic_name__					):






       """simple docstring"""





       def __init__(				self  :						Optional[int]			,					lowerCAmelCase__  :						int			,					lowerCAmelCase__  :						int			,					lowerCAmelCase__  :						str			,					lowerCAmelCase__  :						Tuple			,					lowerCAmelCase__  :						str="train"			,					lowerCAmelCase__  :						List[str]=None			,					lowerCAmelCase__  :						Dict=None			,					lowerCAmelCase__  :						Optional[Any]=None			,					lowerCAmelCase__  :						Dict=""			,					)       ->	Any:

            '''simple docstring'''




            super().__init__()
            _UpperCamelCase							       =			Path(lowerCAmelCase__	).joinpath(type_path + '''.source'''	)
            _UpperCamelCase							       =			Path(lowerCAmelCase__	).joinpath(type_path + '''.target'''	)
            _UpperCamelCase							       =			self.get_char_lens(self.src_file	)
            _UpperCamelCase							       =			max_source_length
            _UpperCamelCase							       =			max_target_length
            assert min(self.src_lens	) > 0, f"""found empty line in {self.src_file}"""
            _UpperCamelCase							       =			tokenizer
            _UpperCamelCase							       =			prefix
            if n_obs is not None:
                 _UpperCamelCase							       =			self.src_lens[:n_obs]
            _UpperCamelCase							       =			src_lang
            _UpperCamelCase							       =			tgt_lang





       def __len__(				self  :						Optional[Any]	)       ->	List[Any]:

            '''simple docstring'''




            return len(self.src_lens	)





       def __getitem__(				self  :						Optional[Any]			,					lowerCAmelCase__  :						Any	)       ->	Dict[str, torch.Tensor]:

            '''simple docstring'''




            _UpperCamelCase							       =			index + 1  # linecache starts at 1
            _UpperCamelCase							       =			self.prefix + linecache.getline(str(self.src_file	)			,					lowerCAmelCase__	).rstrip('''\n'''	)
            _UpperCamelCase							       =			linecache.getline(str(self.tgt_file	)			,					lowerCAmelCase__	).rstrip('''\n'''	)
            assert source_line, f"""empty source line for index {index}"""
            assert tgt_line, f"""empty tgt line for index {index}"""

            # Need to add eos token manually for T5
            if isinstance(self.tokenizer			,					lowerCAmelCase__	):
                 source_line += self.tokenizer.eos_token
                 tgt_line += self.tokenizer.eos_token

            # Pad source and target to the right
            _UpperCamelCase							       =			(
                self.tokenizer.question_encoder if isinstance(self.tokenizer			,					lowerCAmelCase__	) else self.tokenizer
            )
            _UpperCamelCase							       =			self.tokenizer.generator if isinstance(self.tokenizer			,					lowerCAmelCase__	) else self.tokenizer

            _UpperCamelCase							       =			encode_line(lowerCAmelCase__			,					lowerCAmelCase__			,					self.max_source_length			,					'''right'''	)
            _UpperCamelCase							       =			encode_line(lowerCAmelCase__			,					lowerCAmelCase__			,					self.max_target_length			,					'''right'''	)

            _UpperCamelCase							       =			source_inputs['''input_ids'''].squeeze()
            _UpperCamelCase							       =			target_inputs['''input_ids'''].squeeze()
            _UpperCamelCase							       =			source_inputs['''attention_mask'''].squeeze()
            return {
                "input_ids": source_ids,
                "attention_mask": src_mask,
                "decoder_input_ids": target_ids,
            }





       @staticmethod
       def snake_case__		(				lowerCAmelCase__  :						List[Any]	)       ->	Optional[int]:

            '''simple docstring'''




            return [len(lowerCAmelCase__	) for x in Path(lowerCAmelCase__	).open().readlines()]







       def snake_case__		(				self  :						Dict			,					lowerCAmelCase__  :						List[str]	)       ->	Dict[str, torch.Tensor]:

            '''simple docstring'''




            _UpperCamelCase							       =			torch.stack([x['''input_ids'''] for x in batch]	)
            _UpperCamelCase							       =			torch.stack([x['''attention_mask'''] for x in batch]	)
            _UpperCamelCase							       =			torch.stack([x['''decoder_input_ids'''] for x in batch]	)
            _UpperCamelCase							       =			(
                self.tokenizer.generator.pad_token_id
                if isinstance(self.tokenizer			,					lowerCAmelCase__	)
                else self.tokenizer.pad_token_id
            )
            _UpperCamelCase							       =			(
                self.tokenizer.question_encoder.pad_token_id
                if isinstance(self.tokenizer			,					lowerCAmelCase__	)
                else self.tokenizer.pad_token_id
            )
            _UpperCamelCase							       =			trim_batch(lowerCAmelCase__			,					lowerCAmelCase__	)
            _UpperCamelCase       ,							_UpperCamelCase							       =			trim_batch(lowerCAmelCase__			,					lowerCAmelCase__			,					attention_mask=lowerCAmelCase__	)
            _UpperCamelCase							       =			{
                '''input_ids''': source_ids,
                '''attention_mask''': source_mask,
                '''decoder_input_ids''': y,
            }
            return batch

lowercase__		:     Union[str, Any]			=							getLogger(__name__)







def    a__							(							lowercase   :	List[List] )						->						Optional[int]:





     """simple docstring"""

     return list(itertools.chain.from_iterable(lowercase ) )







def    a__							(							lowercase   :	str )						->						None:





     """simple docstring"""

     _UpperCamelCase							       =			get_git_info()
     save_json(lowercase,   os.path.join(lowercase,   '''git_log.json''' ) )







def    a__							(							lowercase   :	Optional[int],   lowercase   :	List[str],   lowercase   :	str=4,   **lowercase   :	Optional[Any] )						->						Any:





     """simple docstring"""

     with open(lowercase,   '''w''' ) as f:
          json.dump(lowercase,   lowercase,   indent=lowercase,   **lowercase )







def    a__							(							lowercase   :	Union[str, Any] )						->						Optional[int]:





     """simple docstring"""

     with open(lowercase ) as f:
          return json.load(lowercase )







def    a__							(							)						->						List[Any]:





     """simple docstring"""

     _UpperCamelCase							       =			git.Repo(search_parent_directories=lowercase )
     _UpperCamelCase							       =			{
         '''repo_id''': str(lowercase ),
         '''repo_sha''': str(repo.head.object.hexsha ),
         '''repo_branch''': str(repo.active_branch ),
         '''hostname''': str(socket.gethostname() ),
     }
     return repo_infos







def    a__							(							lowercase   :	Callable,   lowercase   :	Iterable )						->						List:





     """simple docstring"""

     return list(map(lowercase,   lowercase ) )







def    a__							(							lowercase   :	List[Any],   lowercase   :	Dict )						->						Optional[int]:





     """simple docstring"""

     with open(lowercase,   '''wb''' ) as f:
          return pickle.dump(lowercase,   lowercase )







def    a__							(							lowercase   :	Tuple )						->						Tuple:





     """simple docstring"""

     def remove_articles(lowercase   :	Tuple ):
          return re.sub(r'''\b(a|an|the)\b''',   ''' ''',   lowercase )

     def white_space_fix(lowercase   :	Tuple ):
          return " ".join(text.split() )

     def remove_punc(lowercase   :	Optional[int] ):
          _UpperCamelCase							       =			set(string.punctuation )
          return "".join(ch for ch in text if ch not in exclude )

     def lower(lowercase   :	Tuple ):
          return text.lower()

     return white_space_fix(remove_articles(remove_punc(lower(lowercase ) ) ) )







def    a__							(							lowercase   :	Tuple,   lowercase   :	Union[str, Any] )						->						Dict:





     """simple docstring"""

     _UpperCamelCase							       =			normalize_answer(lowercase ).split()
     _UpperCamelCase							       =			normalize_answer(lowercase ).split()
     _UpperCamelCase							       =			Counter(lowercase ) & Counter(lowercase )
     _UpperCamelCase							       =			sum(common.values() )
     if num_same == 0:
          return 0
     _UpperCamelCase							       =			1.0 * num_same / len(lowercase )
     _UpperCamelCase							       =			1.0 * num_same / len(lowercase )
     _UpperCamelCase							       =			(2 * precision * recall) / (precision + recall)
     return fa







def    a__							(							lowercase   :	Tuple,   lowercase   :	Any )						->						List[str]:





     """simple docstring"""

     return normalize_answer(lowercase ) == normalize_answer(lowercase )







def    a__							(							lowercase   :	List[str],   lowercase   :	List[str] )						->						Dict:





     """simple docstring"""

     assert len(lowercase ) == len(lowercase )
     _UpperCamelCase							       =			0
     for hypo, pred in zip(lowercase,   lowercase ):
          em += exact_match_score(lowercase,   lowercase )
     if len(lowercase ) > 0:
          em /= len(lowercase )
     return {"em": em}







def    a__							(							lowercase   :	Tuple )						->						Union[str, Any]:





     """simple docstring"""

     return model_prefix.startswith('''rag''' )







def    a__							(							lowercase   :	int,   lowercase   :	List[Any],   lowercase   :	List[Any] )						->						Optional[int]:





     """simple docstring"""

     _UpperCamelCase							       =			{p: p for p in extra_params}
     # T5 models don't have `dropout` param, they have `dropout_rate` instead
     _UpperCamelCase							       =			'''dropout_rate'''
     for p in extra_params:
          if getattr(lowercase,   lowercase,   lowercase ):
               if not hasattr(lowercase,   lowercase ) and not hasattr(lowercase,   equivalent_param[p] ):
                    logger.info('''config doesn\'t have a `{}` attribute'''.format(lowercase ) )
                    delattr(lowercase,   lowercase )
                    continue
               _UpperCamelCase							       =			p if hasattr(lowercase,   lowercase ) else equivalent_param[p]
               setattr(lowercase,   lowercase,   getattr(lowercase,   lowercase ) )
               delattr(lowercase,   lowercase )
     return hparams, config



