

"""simple docstring"""




import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
						from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("1.6"):
						A_				:      List[Any]           =			True
						from torch.cuda.amp import autocast


A_				:      Optional[Any]           =			logging.getLogger(__name__)





@dataclass
class  lowerCamelCase    :
			lowerCamelCase__ :		str       		=				field(
			    metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}      )
			lowerCamelCase__ :		Optional[str]       		=				field(
			    default=A__      ,metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}      ,)
			lowerCamelCase__ :		Optional[bool]       		=				field(
			    default=A__      ,metadata={'help': 'Whether to freeze the feature extractor layers of the model.'}      )
			lowerCamelCase__ :		Optional[bool]       		=				field(
			    default=A__      ,metadata={'help': 'Whether to log verbose messages or not.'}      ,)
			lowerCamelCase__ :		Optional[float]       		=				field(
			    default=2.0      ,metadata={'help': 'Maximum temperature for gumbel softmax.'}      )
			lowerCamelCase__ :		Optional[float]       		=				field(
			    default=0.5      ,metadata={'help': 'Minimum temperature for gumbel softmax.'}      )
			lowerCamelCase__ :		Optional[float]       		=				field(
			    default=0.9_9_9_9_9_5      ,metadata={'help': 'Decay of gumbel temperature during training.'}      )

def 						A							(  snake_case__		,					snake_case__     ):




			'''simple docstring'''


			logging.basicConfig(
			    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""		,					datefmt="""%m/%d/%Y %H:%M:%S"""		,					handlers=[logging.StreamHandler(sys.stdout     )]		,					)
			SCREAMING_SNAKE_CASE__										=   logging.WARNING
			if model_args.verbose_logging:
						SCREAMING_SNAKE_CASE__										=   logging.DEBUG
			elif trainer_utils.is_main_process(training_args.local_rank     ):
						SCREAMING_SNAKE_CASE__										=   logging.INFO
			logger.setLevel(snake_case__     )





@dataclass
class  lowerCamelCase    :
			lowerCamelCase__ :		str       		=				field(
			    default=A__      ,metadata={'help': 'The name of the dataset to use (via the datasets library).'}      )
			lowerCamelCase__ :		Optional[str]       		=				field(
			    default=A__      ,metadata={'help': 'The configuration name of the dataset to use (via the datasets library).'}      )
			lowerCamelCase__ :		Optional[str]       		=				field(
			    default='train'      ,metadata={
			        'help': 'The name of the training data set split to use (via the datasets library). Defaults to \'train\''
			    }      ,)
			lowerCamelCase__ :		Optional[str]       		=				field(
			    default='validation'      ,metadata={
			        'help': (
			            'The name of the validation data set split to use (via the datasets library). Defaults to \'validation\''
			        )
			    }      ,)
			lowerCamelCase__ :		Optional[str]       		=				field(
			    default='file'      ,metadata={'help': 'Column in the dataset that contains speech file path. Defaults to \'file\''}      ,)
			lowerCamelCase__ :		bool       		=				field(
			    default=A__      ,metadata={'help': 'Overwrite the cached preprocessed datasets or not.'}      )
			lowerCamelCase__ :		Optional[int]       		=				field(
			    default=1      ,metadata={
			        'help': 'The percentage of the train set used as validation set in case there\'s no validation split'
			    }      ,)
			lowerCamelCase__ :		Optional[int]       		=				field(
			    default=A__      ,metadata={'help': 'The number of processes to use for the preprocessing.'}      ,)
			lowerCamelCase__ :		Optional[float]       		=				field(
			    default=2_0.0      ,metadata={'help': 'Filter audio files that are longer than `max_duration_in_seconds` seconds'}      )





@dataclass
class  lowerCamelCase    :
			lowerCamelCase__ :		WavaVecaForPreTraining
			lowerCamelCase__ :		WavaVecaFeatureExtractor
			lowerCamelCase__ :		Union[bool, str]       		=				"longest"
			lowerCamelCase__ :		Optional[int]       		=				None
			lowerCamelCase__ :		Optional[int]       		=				None

			def __call__(       self					:    List[Any]     ,					__UpperCAmelCase					:    List[Dict[str, Union[List[int], torch.Tensor]]]							) ->     Dict[str, torch.Tensor]:
						# reformat list to dict and set to pytorch format
						SCREAMING_SNAKE_CASE__										=   self.feature_extractor.pad(
						    __UpperCAmelCase     ,					max_length=self.max_length     ,					padding=self.padding     ,					pad_to_multiple_of=self.pad_to_multiple_of     ,					return_tensors="""pt"""     ,					)
						SCREAMING_SNAKE_CASE__										=   self.model._get_feat_extract_output_lengths(batch["""input_values"""].shape[-1]							)

						SCREAMING_SNAKE_CASE__										=   batch["""input_values"""].shape[0]

						# make sure that no loss is computed on padded inputs
						if batch["attention_mask"] is not None:
									# compute real output lengths according to convolution formula
									SCREAMING_SNAKE_CASE__										=   self.model._get_feat_extract_output_lengths(batch["""attention_mask"""].sum(-1							)							).to(
									    torch.long							)

									SCREAMING_SNAKE_CASE__										=   torch.zeros(
									    (batch_size, mask_indices_seq_length)     ,					dtype=torch.long     ,					device=batch["""input_values"""].device							)

									# these two operations makes sure that all values
									# before the output lengths indices are attended to
									SCREAMING_SNAKE_CASE__										=   1
									SCREAMING_SNAKE_CASE__										=   attention_mask.flip([-1]							).cumsum(-1							).flip([-1]							).bool()

						# sample randomly masked indices
						SCREAMING_SNAKE_CASE__										=   _compute_mask_indices(
						    (batch_size, mask_indices_seq_length)     ,					self.model.config.mask_time_prob     ,					self.model.config.mask_time_length     ,					attention_mask=__UpperCAmelCase     ,					min_masks=2     ,					)

						return batch





class  lowerCamelCase    (A__      ):

			def __init__(       self					:    Any     ,					*__UpperCAmelCase					:    Optional[Any]     ,					__UpperCAmelCase					:    Tuple=1     ,					__UpperCAmelCase					:    int=0     ,					__UpperCAmelCase					:    str=1.0     ,					**__UpperCAmelCase					:    str							) ->     Optional[Any]:
						super().__init__(*__UpperCAmelCase     ,					**__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   0
						SCREAMING_SNAKE_CASE__										=   max_gumbel_temp
						SCREAMING_SNAKE_CASE__										=   min_gumbel_temp
						SCREAMING_SNAKE_CASE__										=   gumbel_temp_decay





			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]     ,					__UpperCAmelCase					:    nn.Module     ,					__UpperCAmelCase					:    Dict[str, Union[torch.Tensor, Any]]							) ->     torch.Tensor:
						model.train()
						SCREAMING_SNAKE_CASE__										=   self._prepare_inputs(__UpperCAmelCase							)

						if self.use_amp:
									with autocast():
												SCREAMING_SNAKE_CASE__										=   self.compute_loss(__UpperCAmelCase     ,					__UpperCAmelCase							)
						else:
									SCREAMING_SNAKE_CASE__										=   self.compute_loss(__UpperCAmelCase     ,					__UpperCAmelCase							)

						if self.args.n_gpu > 1 or self.deepspeed:
									if model.module.config.ctc_loss_reduction == "mean":
												SCREAMING_SNAKE_CASE__										=   loss.mean()
									elif model.module.config.ctc_loss_reduction == "sum":
												SCREAMING_SNAKE_CASE__										=   loss.sum() / (inputs["""mask_time_indices"""]).sum()
									else:
												raise ValueError(F"""{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']"""							)

						if self.args.gradient_accumulation_steps > 1:
									SCREAMING_SNAKE_CASE__										=   loss / self.args.gradient_accumulation_steps

						if self.use_amp:
									self.scaler.scale(__UpperCAmelCase							).backward()
						elif self.use_apex:
									with amp.scale_loss(__UpperCAmelCase     ,					self.optimizer							) as scaled_loss:
												scaled_loss.backward()
						elif self.deepspeed:
									self.deepspeed.backward(__UpperCAmelCase							)
						else:
									loss.backward()

						self.num_update_step += 1
						# make sure gumbel softmax temperature is decayed
						if self.args.n_gpu > 1 or self.deepspeed:
									model.module.set_gumbel_temperature(
									    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step     ,					self.min_gumbel_temp							)							)
						else:
									model.set_gumbel_temperature(
									    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step     ,					self.min_gumbel_temp							)							)

						return loss.detach()

def 						A							(  ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__										=   HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )

			SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   parser.parse_args_into_dataclasses()
			configure_logger(snake_case__		,					snake_case__     )

			# Downloading and loading a dataset from the hub.
			SCREAMING_SNAKE_CASE__										=   load_dataset(data_args.dataset_name		,					data_args.dataset_config_name		,					cache_dir=model_args.cache_dir     )

			if "validation" not in datasets.keys():
						# make sure only "validation" and "train" keys remain"
						SCREAMING_SNAKE_CASE__										=   DatasetDict()
						SCREAMING_SNAKE_CASE__										=   load_dataset(
						    data_args.dataset_name		,					data_args.dataset_config_name		,					split=f"""{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"""		,					cache_dir=model_args.cache_dir		,					)
						SCREAMING_SNAKE_CASE__										=   load_dataset(
						    data_args.dataset_name		,					data_args.dataset_config_name		,					split=f"""{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"""		,					cache_dir=model_args.cache_dir		,					)
			else:
						# make sure only "validation" and "train" keys remain"
						SCREAMING_SNAKE_CASE__										=   DatasetDict()
						SCREAMING_SNAKE_CASE__										=   load_dataset(
						    data_args.dataset_name		,					data_args.dataset_config_name		,					split="""validation"""		,					cache_dir=model_args.cache_dir		,					)
						SCREAMING_SNAKE_CASE__										=   load_dataset(
						    data_args.dataset_name		,					data_args.dataset_config_name		,					split=f"""{data_args.train_split_name}"""		,					cache_dir=model_args.cache_dir		,					)

			# only normalized-inputs-training is supported
			SCREAMING_SNAKE_CASE__										=   WavaVecaFeatureExtractor.from_pretrained(
			    model_args.model_name_or_path		,					cache_dir=model_args.cache_dir		,					do_normalize=snake_case__     )

			def prepare_dataset(snake_case__     ):
						# check that all files have the correct sampling rate
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   librosa.load(batch[data_args.speech_file_column]		,					sr=feature_extractor.sampling_rate     )
						return batch

			# load audio files into numpy arrays
			SCREAMING_SNAKE_CASE__										=   datasets.map(
			    snake_case__		,					num_proc=data_args.preprocessing_num_workers		,					remove_columns=datasets["""train"""].column_names     )

			# filter audio files that are too long
			SCREAMING_SNAKE_CASE__										=   vectorized_datasets.filter(
			    lambda snake_case__     : len(data["""speech"""]     ) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate     )     )

			def normalize(snake_case__     ):
						return feature_extractor(batch["""speech"""]		,					sampling_rate=feature_extractor.sampling_rate     )

			# normalize and transform to `BatchFeatures`
			SCREAMING_SNAKE_CASE__										=   vectorized_datasets.map(
			    snake_case__		,					batched=snake_case__		,					num_proc=data_args.preprocessing_num_workers		,					load_from_cache_file=not data_args.overwrite_cache		,					remove_columns=vectorized_datasets["""train"""].column_names		,					)

			# pretraining is only supported for "newer" stable layer norm architecture
			# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
			SCREAMING_SNAKE_CASE__										=   WavaVecaConfig.from_pretrained(
			    model_args.model_name_or_path		,					cache_dir=model_args.cache_dir		,					gradient_checkpointing=training_args.gradient_checkpointing		,					)

			if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
						raise ValueError(
						    """PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"""
						    """ ``config.feat_extract_norm='layer'"""     )

			SCREAMING_SNAKE_CASE__										=   WavaVecaForPreTraining(snake_case__     )

			SCREAMING_SNAKE_CASE__										=   DataCollatorForWavaVecaPretraining(model=snake_case__		,					feature_extractor=snake_case__     )

			SCREAMING_SNAKE_CASE__										=   WavaVecaPreTrainer(
			    model=snake_case__		,					data_collator=snake_case__		,					args=snake_case__		,					train_dataset=vectorized_datasets["""train"""]		,					eval_dataset=vectorized_datasets["""validation"""]		,					tokenizer=snake_case__		,					max_gumbel_temp=model_args.max_gumbel_temperature		,					min_gumbel_temp=model_args.min_gumbel_temperature		,					gumbel_temp_decay=model_args.gumbel_temperature_decay		,					)
			trainer.train()


if __name__ == "__main__":
						main()





"""simple docstring"""




import inspect
import unittest

import numpy as np

from transformers import ViTConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor


if is_flax_available():
						import jax

						from transformers.models.vit.modeling_flax_vit import FlaxViTForImageClassification, FlaxViTModel





class  lowerCamelCase    (unittest.TestCase      ):

			def __init__(       self					:    List[Any]     ,					__UpperCAmelCase					:    Dict     ,					__UpperCAmelCase					:    Optional[Any]=1_3     ,					__UpperCAmelCase					:    Tuple=3_0     ,					__UpperCAmelCase					:    str=2     ,					__UpperCAmelCase					:    List[Any]=3     ,					__UpperCAmelCase					:    Any=True     ,					__UpperCAmelCase					:    Dict=True     ,					__UpperCAmelCase					:    Optional[Any]=3_2     ,					__UpperCAmelCase					:    Any=5     ,					__UpperCAmelCase					:    Optional[Any]=4     ,					__UpperCAmelCase					:    Tuple=3_7     ,					__UpperCAmelCase					:    List[str]="gelu"     ,					__UpperCAmelCase					:    Union[str, Any]=0.1     ,					__UpperCAmelCase					:    List[str]=0.1     ,					__UpperCAmelCase					:    int=1_0     ,					__UpperCAmelCase					:    Any=0.02     ,					) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   parent
						SCREAMING_SNAKE_CASE__										=   batch_size
						SCREAMING_SNAKE_CASE__										=   image_size
						SCREAMING_SNAKE_CASE__										=   patch_size
						SCREAMING_SNAKE_CASE__										=   num_channels
						SCREAMING_SNAKE_CASE__										=   is_training
						SCREAMING_SNAKE_CASE__										=   use_labels
						SCREAMING_SNAKE_CASE__										=   hidden_size
						SCREAMING_SNAKE_CASE__										=   num_hidden_layers
						SCREAMING_SNAKE_CASE__										=   num_attention_heads
						SCREAMING_SNAKE_CASE__										=   intermediate_size
						SCREAMING_SNAKE_CASE__										=   hidden_act
						SCREAMING_SNAKE_CASE__										=   hidden_dropout_prob
						SCREAMING_SNAKE_CASE__										=   attention_probs_dropout_prob
						SCREAMING_SNAKE_CASE__										=   type_sequence_label_size
						SCREAMING_SNAKE_CASE__										=   initializer_range

						# in ViT, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
						SCREAMING_SNAKE_CASE__										=   (image_size // patch_size) ** 2
						SCREAMING_SNAKE_CASE__										=   num_patches + 1

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

						SCREAMING_SNAKE_CASE__										=   ViTConfig(
						    image_size=self.image_size     ,					patch_size=self.patch_size     ,					num_channels=self.num_channels     ,					hidden_size=self.hidden_size     ,					num_hidden_layers=self.num_hidden_layers     ,					num_attention_heads=self.num_attention_heads     ,					intermediate_size=self.intermediate_size     ,					hidden_act=self.hidden_act     ,					hidden_dropout_prob=self.hidden_dropout_prob     ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,					is_decoder=__UpperCAmelCase     ,					initializer_range=self.initializer_range     ,					)

						return config, pixel_values

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict     ,					__UpperCAmelCase					:    int     ,					__UpperCAmelCase					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   FlaxViTModel(config=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   model(__UpperCAmelCase							)
						# expected sequence length = num_patches + 1 (we add 1 for the [CLS] token)
						SCREAMING_SNAKE_CASE__										=   (self.image_size, self.image_size)
						SCREAMING_SNAKE_CASE__										=   (self.patch_size, self.patch_size)
						SCREAMING_SNAKE_CASE__										=   (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
						self.parent.assertEqual(result.last_hidden_state.shape     ,					(self.batch_size, num_patches + 1, self.hidden_size)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]     ,					__UpperCAmelCase					:    Optional[int]     ,					__UpperCAmelCase					:    Optional[Any]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   self.type_sequence_label_size
						SCREAMING_SNAKE_CASE__										=   FlaxViTForImageClassification(config=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   model(__UpperCAmelCase							)
						self.parent.assertEqual(result.logits.shape     ,					(self.batch_size, self.type_sequence_label_size)							)

						# test greyscale images
						SCREAMING_SNAKE_CASE__										=   1
						SCREAMING_SNAKE_CASE__										=   FlaxViTForImageClassification(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
						SCREAMING_SNAKE_CASE__										=   model(__UpperCAmelCase							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     str:
						SCREAMING_SNAKE_CASE__										=   self.prepare_config_and_inputs()
						(
						    (
						    SCREAMING_SNAKE_CASE__
						)				,		(
						    SCREAMING_SNAKE_CASE__
						)				,		
						)										=   config_and_inputs
						SCREAMING_SNAKE_CASE__										=   {"""pixel_values""": pixel_values}
						return config, inputs_dict





@require_flax
class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		Tuple       		=				(FlaxViTModel, FlaxViTForImageClassification) if is_flax_available() else ()

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     None:
						SCREAMING_SNAKE_CASE__										=   FlaxViTModelTester(self							)
						SCREAMING_SNAKE_CASE__										=   ConfigTester(self     ,					config_class=__UpperCAmelCase     ,					has_text_modality=__UpperCAmelCase     ,					hidden_size=3_7							)

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Dict:
						self.config_tester.run_common_tests()

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_image_classification(*__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     str:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
									SCREAMING_SNAKE_CASE__										=   model_class(__UpperCAmelCase							)
									SCREAMING_SNAKE_CASE__										=   inspect.signature(model.__call__							)
									# signature.parameters is an OrderedDict => so arg_names order is deterministic
									SCREAMING_SNAKE_CASE__										=   [*signature.parameters.keys()]

									SCREAMING_SNAKE_CASE__										=   ["""pixel_values"""]
									self.assertListEqual(arg_names[:1]     ,					__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
									with self.subTest(model_class.__name__							):
												SCREAMING_SNAKE_CASE__										=   self._prepare_for_class(__UpperCAmelCase     ,					__UpperCAmelCase							)
												SCREAMING_SNAKE_CASE__										=   model_class(__UpperCAmelCase							)

												@jax.jit
												def model_jitted(__UpperCAmelCase					:    int     ,					**__UpperCAmelCase					:    Tuple							):
															return model(pixel_values=__UpperCAmelCase     ,					**__UpperCAmelCase							)

												with self.subTest("""JIT Enabled"""							):
															SCREAMING_SNAKE_CASE__										=   model_jitted(**__UpperCAmelCase							).to_tuple()

												with self.subTest("""JIT Disabled"""							):
															with jax.disable_jit():
																		SCREAMING_SNAKE_CASE__										=   model_jitted(**__UpperCAmelCase							).to_tuple()

												self.assertEqual(len(__UpperCAmelCase							)     ,					len(__UpperCAmelCase							)							)
												for jitted_output, output in zip(__UpperCAmelCase     ,					__UpperCAmelCase							):
															self.assertEqual(jitted_output.shape     ,					output.shape							)





			@slow
			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Dict:
						for model_class_name in self.all_model_classes:
									SCREAMING_SNAKE_CASE__										=   model_class_name.from_pretrained("""google/vit-base-patch16-224"""							)
									SCREAMING_SNAKE_CASE__										=   model(np.ones((1, 3, 2_2_4, 2_2_4)							)							)
									self.assertIsNotNone(__UpperCAmelCase							)


