


import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
			from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse("1.6"):
			_snake_case  :	Optional[Any]			      =       True
			from torch.cuda.amp import autocast


_snake_case  :	str			      =       logging.getLogger(__name__)

@dataclass
class        a	:




							"""simple docstring"""


							__UpperCAmelCase					:							List[str]  						=						field(
							    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}   )
							__UpperCAmelCase					:							Union[str, Any]  						=						field(
							    default=UpperCamelCase_  ,       metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}  ,       )
							__UpperCAmelCase					:							int  						=						field(
							    default=UpperCamelCase_  ,       metadata={"help": "Whether to freeze the feature extractor layers of the model."}   )
							__UpperCAmelCase					:							int  						=						field(
							    default=UpperCamelCase_  ,       metadata={"help": "Whether to log verbose messages or not."}  ,       )
							__UpperCAmelCase					:							Any  						=						field(
							    default=2.0  ,       metadata={"help": "Maximum temperature for gumbel softmax."}   )
							__UpperCAmelCase					:							List[Any]  						=						field(
							    default=0.5  ,       metadata={"help": "Minimum temperature for gumbel softmax."}   )
							__UpperCAmelCase					:							Tuple  						=						field(
							    default=0.999995  ,       metadata={"help": "Decay of gumbel temperature during training."}   )







def        lowerCAmelCase_  (      __lowerCamelCase							,			__lowerCamelCase			):
							logging.basicConfig(
							    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"							,			datefmt="%m/%d/%Y %H:%M:%S"							,			handlers=[logging.StreamHandler(sys.stdout			)]							,			)
							__snake_case	:    Optional[int]											=     logging.WARNING
							if model_args.verbose_logging:
														__snake_case	:    Union[str, Any]											=     logging.DEBUG
							elif trainer_utils.is_main_process(training_args.local_rank			):
														__snake_case	:    Any											=     logging.INFO
							logger.setLevel(snake_case_			)

@dataclass
class        a	:




							"""simple docstring"""


							__UpperCAmelCase					:							List[Any]  						=						field(
							    default=UpperCamelCase_  ,       metadata={"help": "The name of the dataset to use (via the datasets library)."}   )
							__UpperCAmelCase					:							Any  						=						field(
							    default=UpperCamelCase_  ,       metadata={"help": "The configuration name of the dataset to use (via the datasets library)."}   )
							__UpperCAmelCase					:							Tuple  						=						field(
							    default="train"  ,       metadata={
							        "help": "The name of the training data set split to use (via the datasets library). Defaults to 'train'"
							    }  ,       )
							__UpperCAmelCase					:							Dict  						=						field(
							    default="validation"  ,       metadata={
							        "help": (
							            "The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"
							        )
							    }  ,       )
							__UpperCAmelCase					:							List[Any]  						=						field(
							    default="file"  ,       metadata={"help": "Column in the dataset that contains speech file path. Defaults to 'file'"}  ,       )
							__UpperCAmelCase					:							List[str]  						=						field(
							    default=UpperCamelCase_  ,       metadata={"help": "Overwrite the cached preprocessed datasets or not."}   )
							__UpperCAmelCase					:							str  						=						field(
							    default=1  ,       metadata={
							        "help": "The percentage of the train set used as validation set in case there's no validation split"
							    }  ,       )
							__UpperCAmelCase					:							Optional[int]  						=						field(
							    default=UpperCamelCase_  ,       metadata={"help": "The number of processes to use for the preprocessing."}  ,       )
							__UpperCAmelCase					:							Optional[Any]  						=						field(
							    default=20.0  ,       metadata={"help": "Filter audio files that are longer than `max_duration_in_seconds` seconds"}   )

@dataclass
class        a	:




							"""simple docstring"""


							__UpperCAmelCase					:							Optional[Any]  						=						42
							__UpperCAmelCase					:							List[Any]  						=						42
							__UpperCAmelCase					:							Union[str, Any]  						=						"longest"
							__UpperCAmelCase					:							Optional[Any]  						=						None
							__UpperCAmelCase					:							Dict  						=						None

							def __call__(							self :						Any					,			lowerCamelCase :						List[Any]       )			->					Dict[str, torch.Tensor]:
														# reformat list to dict and set to pytorch format
														__snake_case	:    str											=     self.feature_extractor.pad(
														    lowerCamelCase					,			max_length=self.max_length					,			padding=self.padding					,			pad_to_multiple_of=self.pad_to_multiple_of					,			return_tensors="pt"					,			)
														__snake_case	:    List[Any]											=     self.model._get_feat_extract_output_lengths(batch["input_values"].shape[-1]       )

														__snake_case	:    Tuple											=     batch['''input_values'''].shape[0]

														# make sure that no loss is computed on padded inputs
														if batch["attention_mask"] is not None:
																					# compute real output lengths according to convolution formula
																					__snake_case	:    List[Any]											=     self.model._get_feat_extract_output_lengths(batch["attention_mask"].sum(-1       )       ).to(
																					    torch.long       )

																					__snake_case	:    int											=     torch.zeros(
																					    (batch_size, mask_indices_seq_length)					,			dtype=torch.long					,			device=batch["input_values"].device       )

																					# these two operations makes sure that all values
																					# before the output lengths indices are attended to
																					__snake_case	:    Tuple											=     1
																					__snake_case	:    Any											=     attention_mask.flip([-1]       ).cumsum(-1       ).flip([-1]       ).bool()

														# sample randomly masked indices
														__snake_case	:    Dict											=     _compute_mask_indices(
														    (batch_size, mask_indices_seq_length)					,			self.model.config.mask_time_prob					,			self.model.config.mask_time_length					,			attention_mask=lowerCamelCase					,			min_masks=2					,			)

														return batch

class        a	(UpperCamelCase_   ):




							"""simple docstring"""


							def __init__(							self :						int					,			*lowerCamelCase :						Optional[Any]					,			lowerCamelCase :						List[Any]=1					,			lowerCamelCase :						int=0					,			lowerCamelCase :						Optional[Any]=1.0					,			**lowerCamelCase :						Dict       )			->					Optional[int]:
														super().__init__(*lowerCamelCase					,			**lowerCamelCase       )
														__snake_case	:    Optional[Any]											=     0
														__snake_case	:    Optional[int]											=     max_gumbel_temp
														__snake_case	:    Any											=     min_gumbel_temp
														__snake_case	:    Union[str, Any]											=     gumbel_temp_decay






							def 					UpperCamelCase      (							self :						List[Any]					,			lowerCamelCase :						Dict					,			lowerCamelCase :						Tuple       )			->					torch.Tensor:
														model.train()
														__snake_case	:    Any											=     self._prepare_inputs(lowerCamelCase       )

														if self.use_amp:
																					with autocast():
																												__snake_case	:    Dict											=     self.compute_loss(lowerCamelCase					,			lowerCamelCase       )
														else:
																					__snake_case	:    Dict											=     self.compute_loss(lowerCamelCase					,			lowerCamelCase       )

														if self.args.n_gpu > 1 or self.deepspeed:
																					if model.module.config.ctc_loss_reduction == "mean":
																												__snake_case	:    Dict											=     loss.mean()
																					elif model.module.config.ctc_loss_reduction == "sum":
																												__snake_case	:    Optional[int]											=     loss.sum() / (inputs['''mask_time_indices''']).sum()
																					else:
																												raise ValueError(F'{model.config.ctc_loss_reduction} is not valid. Choose one of [\'mean\', \'sum\']'       )

														if self.args.gradient_accumulation_steps > 1:
																					__snake_case	:    Any											=     loss / self.args.gradient_accumulation_steps

														if self.use_amp:
																					self.scaler.scale(lowerCamelCase       ).backward()
														elif self.use_apex:
																					with amp.scale_loss(lowerCamelCase					,			self.optimizer       ) as scaled_loss:
																												scaled_loss.backward()
														elif self.deepspeed:
																					self.deepspeed.backward(lowerCamelCase       )
														else:
																					loss.backward()

														self.num_update_step += 1
														# make sure gumbel softmax temperature is decayed
														if self.args.n_gpu > 1 or self.deepspeed:
																					model.module.set_gumbel_temperature(
																					    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step					,			self.min_gumbel_temp       )       )
														else:
																					model.set_gumbel_temperature(
																					    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step					,			self.min_gumbel_temp       )       )

														return loss.detach()







def        lowerCAmelCase_  (      ):
							# See all possible arguments in src/transformers/training_args.py
							# or by passing the --help flag to this script.
							# We now keep distinct sets of args, for a cleaner separation of concerns.

							__snake_case	:    str											=     HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)			)

							__snake_case	:    List[Any]											=     parser.parse_args_into_dataclasses()
							configure_logger(snake_case_							,			snake_case_			)

							# Downloading and loading a dataset from the hub.
							__snake_case	:    List[str]											=     load_dataset(data_args.dataset_name							,			data_args.dataset_config_name							,			cache_dir=model_args.cache_dir			)

							if "validation" not in datasets.keys():
														# make sure only "validation" and "train" keys remain"
														__snake_case	:    int											=     DatasetDict()
														__snake_case	:    List[Any]											=     load_dataset(
														    data_args.dataset_name							,			data_args.dataset_config_name							,			split=F'{data_args.train_split_name}[:{data_args.validation_split_percentage}%]'							,			cache_dir=model_args.cache_dir							,			)
														__snake_case	:    Union[str, Any]											=     load_dataset(
														    data_args.dataset_name							,			data_args.dataset_config_name							,			split=F'{data_args.train_split_name}[{data_args.validation_split_percentage}%:]'							,			cache_dir=model_args.cache_dir							,			)
							else:
														# make sure only "validation" and "train" keys remain"
														__snake_case	:    List[str]											=     DatasetDict()
														__snake_case	:    Dict											=     load_dataset(
														    data_args.dataset_name							,			data_args.dataset_config_name							,			split="validation"							,			cache_dir=model_args.cache_dir							,			)
														__snake_case	:    int											=     load_dataset(
														    data_args.dataset_name							,			data_args.dataset_config_name							,			split=F'{data_args.train_split_name}'							,			cache_dir=model_args.cache_dir							,			)

							# only normalized-inputs-training is supported
							__snake_case	:    Optional[int]											=     WavaVecaFeatureExtractor.from_pretrained(
							    model_args.model_name_or_path							,			cache_dir=model_args.cache_dir							,			do_normalize=snake_case_			)

							def prepare_dataset(__lowerCamelCase			):
														# check that all files have the correct sampling rate
														__snake_case	:    List[str]											=     librosa.load(batch[data_args.speech_file_column]							,			sr=feature_extractor.sampling_rate			)
														return batch

							# load audio files into numpy arrays
							__snake_case	:    Optional[int]											=     datasets.map(
							    snake_case_							,			num_proc=data_args.preprocessing_num_workers							,			remove_columns=datasets["train"].column_names			)

							# filter audio files that are too long
							__snake_case	:    str											=     vectorized_datasets.filter(
							    lambda __lowerCamelCase			: len(data["speech"]			) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate			)			)

							def normalize(__lowerCamelCase			):
														return feature_extractor(batch["speech"]							,			sampling_rate=feature_extractor.sampling_rate			)

							# normalize and transform to `BatchFeatures`
							__snake_case	:    List[str]											=     vectorized_datasets.map(
							    snake_case_							,			batched=snake_case_							,			num_proc=data_args.preprocessing_num_workers							,			load_from_cache_file=not data_args.overwrite_cache							,			remove_columns=vectorized_datasets["train"].column_names							,			)

							# pretraining is only supported for "newer" stable layer norm architecture
							# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
							__snake_case	:    Optional[int]											=     WavaVecaConfig.from_pretrained(
							    model_args.model_name_or_path							,			cache_dir=model_args.cache_dir							,			gradient_checkpointing=training_args.gradient_checkpointing							,			)

							if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
														raise ValueError(
														    "PreTraining is only supported for ``config.do_stable_layer_norm=True`` and"
														    " ``config.feat_extract_norm=\'layer\'"			)

							__snake_case	:    List[str]											=     WavaVecaForPreTraining(snake_case_			)

							__snake_case	:    str											=     DataCollatorForWavaVecaPretraining(model=snake_case_							,			feature_extractor=snake_case_			)

							__snake_case	:    Tuple											=     WavaVecaPreTrainer(
							    model=snake_case_							,			data_collator=snake_case_							,			args=snake_case_							,			train_dataset=vectorized_datasets["train"]							,			eval_dataset=vectorized_datasets["validation"]							,			tokenizer=snake_case_							,			max_gumbel_temp=model_args.max_gumbel_temperature							,			min_gumbel_temp=model_args.min_gumbel_temperature							,			gumbel_temp_decay=model_args.gumbel_temperature_decay							,			)
							trainer.train()


if __name__ == "__main__":
			main()





import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import AutoencoderKL, DDIMScheduler, LDMTextToImagePipeline, UNetaDConditionModel
from diffusers.utils.testing_utils import (
    enable_full_determinism,
    load_numpy,
    nightly,
    require_torch_gpu,
    slow,
    torch_device,
)

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()

class        a	(_lowerCAmelCase  ,       unittest.TestCase   ):




							"""simple docstring"""


							__UpperCAmelCase					:							Optional[Any]  						=						LDMTextToImagePipeline
							__UpperCAmelCase					:							List[Any]  						=						TEXT_TO_IMAGE_PARAMS - {
							    "negative_prompt",
							    "negative_prompt_embeds",
							    "cross_attention_kwargs",
							    "prompt_embeds",
							}
							__UpperCAmelCase					:							int  						=						PipelineTesterMixin.required_optional_params - {
							    "num_images_per_prompt",
							    "callback",
							    "callback_steps",
							}
							__UpperCAmelCase					:							Any  						=						TEXT_TO_IMAGE_BATCH_PARAMS
							__UpperCAmelCase					:							int  						=						False

							def 					__snake_case      (							self :						Union[str, Any]       )			->					str:
														torch.manual_seed(0       )
														__snake_case	:    List[Any]											=     UNetaDConditionModel(
														    block_out_channels=(32, 64)					,			layers_per_block=2					,			sample_size=32					,			in_channels=4					,			out_channels=4					,			down_block_types=("DownBlock2D", "CrossAttnDownBlock2D")					,			up_block_types=("CrossAttnUpBlock2D", "UpBlock2D")					,			cross_attention_dim=32					,			)
														__snake_case	:    Any											=     DDIMScheduler(
														    beta_start=0.0_00_85					,			beta_end=0.0_12					,			beta_schedule="scaled_linear"					,			clip_sample=lowerCamelCase					,			set_alpha_to_one=lowerCamelCase					,			)
														torch.manual_seed(0       )
														__snake_case	:    Dict											=     AutoencoderKL(
														    block_out_channels=(32, 64)					,			in_channels=3					,			out_channels=3					,			down_block_types=("DownEncoderBlock2D", "DownEncoderBlock2D")					,			up_block_types=("UpDecoderBlock2D", "UpDecoderBlock2D")					,			latent_channels=4					,			)
														torch.manual_seed(0       )
														__snake_case	:    Any											=     CLIPTextConfig(
														    bos_token_id=0					,			eos_token_id=2					,			hidden_size=32					,			intermediate_size=37					,			layer_norm_eps=1E-05					,			num_attention_heads=4					,			num_hidden_layers=5					,			pad_token_id=1					,			vocab_size=1000					,			)
														__snake_case	:    Any											=     CLIPTextModel(lowerCamelCase       )
														__snake_case	:    Dict											=     CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip"       )

														__snake_case	:    List[str]											=     {
														    "unet": unet,
														    "scheduler": scheduler,
														    "vqvae": vae,
														    "bert": text_encoder,
														    "tokenizer": tokenizer,
														}
														return components

							def 					__snake_case      (							self :						List[Any]					,			lowerCamelCase :						str					,			lowerCamelCase :						Any=0       )			->					str:
														if str(lowerCamelCase       ).startswith("mps"       ):
																					__snake_case	:    Union[str, Any]											=     torch.manual_seed(lowerCamelCase       )
														else:
																					__snake_case	:    str											=     torch.Generator(device=lowerCamelCase       ).manual_seed(lowerCamelCase       )
														__snake_case	:    Dict											=     {
														    "prompt": "A painting of a squirrel eating a burger",
														    "generator": generator,
														    "num_inference_steps": 2,
														    "guidance_scale": 6.0,
														    "output_type": "numpy",
														}
														return inputs






							def 					__snake_case      (							self :						Dict       )			->					Optional[Any]:
														__snake_case	:    List[Any]											=     "cpu"  # ensure determinism for the device-dependent torch.Generator

														__snake_case	:    Union[str, Any]											=     self.get_dummy_components()
														__snake_case	:    int											=     LDMTextToImagePipeline(**lowerCamelCase       )
														pipe.to(lowerCamelCase       )
														pipe.set_progress_bar_config(disable=lowerCamelCase       )

														__snake_case	:    Optional[Any]											=     self.get_dummy_inputs(lowerCamelCase       )
														__snake_case	:    Optional[Any]											=     pipe(**lowerCamelCase       ).images
														__snake_case	:    Optional[int]											=     image[0, -3:, -3:, -1]

														assert image.shape == (1, 16, 16, 3)
														__snake_case	:    int											=     np.array([0.61_01, 0.61_56, 0.56_22, 0.48_95, 0.66_61, 0.38_04, 0.57_48, 0.61_36, 0.50_14]       )

														assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1E-3

@slow
@require_torch_gpu
class        a	(unittest.TestCase   ):




							"""simple docstring"""


							def 					__snake_case      (							self :						int       )			->					str:
														super().tearDown()
														gc.collect()
														torch.cuda.empty_cache()

							def 					__snake_case      (							self :						Optional[int]					,			lowerCamelCase :						Dict					,			lowerCamelCase :						Any=torch.floataa					,			lowerCamelCase :						List[Any]=0       )			->					Dict:
														__snake_case	:    List[Any]											=     torch.manual_seed(lowerCamelCase       )
														__snake_case	:    Tuple											=     np.random.RandomState(lowerCamelCase       ).standard_normal((1, 4, 32, 32)       )
														__snake_case	:    int											=     torch.from_numpy(lowerCamelCase       ).to(device=lowerCamelCase					,			dtype=lowerCamelCase       )
														__snake_case	:    List[str]											=     {
														    "prompt": "A painting of a squirrel eating a burger",
														    "latents": latents,
														    "generator": generator,
														    "num_inference_steps": 3,
														    "guidance_scale": 6.0,
														    "output_type": "numpy",
														}
														return inputs






							def 					__snake_case      (							self :						Tuple       )			->					Optional[Any]:
														__snake_case	:    str											=     LDMTextToImagePipeline.from_pretrained("CompVis/ldm-text2im-large-256"       ).to(lowerCamelCase       )
														pipe.set_progress_bar_config(disable=lowerCamelCase       )

														__snake_case	:    List[str]											=     self.get_inputs(lowerCamelCase       )
														__snake_case	:    str											=     pipe(**lowerCamelCase       ).images
														__snake_case	:    Dict											=     image[0, -3:, -3:, -1].flatten()

														assert image.shape == (1, 256, 256, 3)
														__snake_case	:    int											=     np.array([0.5_18_25, 0.5_28_50, 0.5_25_43, 0.5_42_58, 0.5_23_04, 0.5_25_69, 0.5_43_63, 0.5_52_76, 0.5_68_78]       )
														__snake_case	:    List[str]											=     np.abs(expected_slice - image_slice       ).max()
														assert max_diff < 1E-3

@nightly
@require_torch_gpu
class        a	(unittest.TestCase   ):




							"""simple docstring"""


							def 					__snake_case      (							self :						List[Any]       )			->					List[Any]:
														super().tearDown()
														gc.collect()
														torch.cuda.empty_cache()

							def 					__snake_case      (							self :						Any					,			lowerCamelCase :						Dict					,			lowerCamelCase :						Any=torch.floataa					,			lowerCamelCase :						List[Any]=0       )			->					Optional[Any]:
														__snake_case	:    int											=     torch.manual_seed(lowerCamelCase       )
														__snake_case	:    Tuple											=     np.random.RandomState(lowerCamelCase       ).standard_normal((1, 4, 32, 32)       )
														__snake_case	:    Optional[Any]											=     torch.from_numpy(lowerCamelCase       ).to(device=lowerCamelCase					,			dtype=lowerCamelCase       )
														__snake_case	:    List[Any]											=     {
														    "prompt": "A painting of a squirrel eating a burger",
														    "latents": latents,
														    "generator": generator,
														    "num_inference_steps": 50,
														    "guidance_scale": 6.0,
														    "output_type": "numpy",
														}
														return inputs






							def 					__snake_case      (							self :						List[str]       )			->					int:
														__snake_case	:    Union[str, Any]											=     LDMTextToImagePipeline.from_pretrained("CompVis/ldm-text2im-large-256"       ).to(lowerCamelCase       )
														pipe.set_progress_bar_config(disable=lowerCamelCase       )

														__snake_case	:    Optional[Any]											=     self.get_inputs(lowerCamelCase       )
														__snake_case	:    str											=     pipe(**lowerCamelCase       ).images[0]

														__snake_case	:    Tuple											=     load_numpy(
														    "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/ldm_text2img/ldm_large_256_ddim.npy"       )
														__snake_case	:    str											=     np.abs(expected_image - image       ).max()
														assert max_diff < 1E-3

