




"""simple docstring"""

import logging
import sys
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Union

import librosa
import torch
from datasets import DatasetDict, load_dataset
from packaging import version
from torch import nn

from transformers import (
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaForPreTraining,
    is_apex_available,
    trainer_utils,
)
from transformers.models.wavaveca.modeling_wavaveca import _compute_mask_indices


if is_apex_available():
		from apex import amp

if version.parse(version.parse(torch.__version__).base_version) >= version.parse('1.6'):
		_lowerCamelCase  :			int        =       True
		from torch.cuda.amp import autocast


_lowerCamelCase  :			List[str]        =       logging.getLogger(__name__)

@dataclass
class        lowercase      :
				__lowerCAmelCase    :    str         =   field(
				    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""})
				__lowerCAmelCase    :    Optional[str]         =   field(
				    default=__UpperCAmelCase     ,	metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}     ,	)
				__lowerCAmelCase    :    Optional[bool]         =   field(
				    default=__UpperCAmelCase     ,	metadata={"""help""": """Whether to freeze the feature extractor layers of the model."""})
				__lowerCAmelCase    :    Optional[bool]         =   field(
				    default=__UpperCAmelCase     ,	metadata={"""help""": """Whether to log verbose messages or not."""}     ,	)
				__lowerCAmelCase    :    Optional[float]         =   field(
				    default=2.0     ,	metadata={"""help""": """Maximum temperature for gumbel softmax."""})
				__lowerCAmelCase    :    Optional[float]         =   field(
				    default=0.5     ,	metadata={"""help""": """Minimum temperature for gumbel softmax."""})
				__lowerCAmelCase    :    Optional[float]         =   field(
				    default=0.9_9_9_9_9_5     ,	metadata={"""help""": """Decay of gumbel temperature during training."""})

def 		lowercase_      (		_UpperCAmelCase  ,      _UpperCAmelCase	):




	"""simple docstring"""






	logging.basicConfig(
	    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''  ,      datefmt='''%m/%d/%Y %H:%M:%S'''  ,      handlers=[logging.StreamHandler(sys.stdout	)]  ,      )
	A_  :				Optional[int]	   =		logging.WARNING
	if model_args.verbose_logging:
		A_  :				Tuple	   =		logging.DEBUG
	elif trainer_utils.is_main_process(training_args.local_rank	):
		A_  :				List[Any]	   =		logging.INFO
	logger.setLevel(_UpperCAmelCase	)

@dataclass
class        lowercase      :
				__lowerCAmelCase    :    str         =   field(
				    default=__UpperCAmelCase     ,	metadata={"""help""": """The name of the dataset to use (via the datasets library)."""})
				__lowerCAmelCase    :    Optional[str]         =   field(
				    default=__UpperCAmelCase     ,	metadata={"""help""": """The configuration name of the dataset to use (via the datasets library)."""})
				__lowerCAmelCase    :    Optional[str]         =   field(
				    default="""train"""     ,	metadata={
				        """help""": """The name of the training data set split to use (via the datasets library). Defaults to 'train'"""
				    }     ,	)
				__lowerCAmelCase    :    Optional[str]         =   field(
				    default="""validation"""     ,	metadata={
				        """help""": (
				            """The name of the validation data set split to use (via the datasets library). Defaults to 'validation'"""
				        )
				    }     ,	)
				__lowerCAmelCase    :    Optional[str]         =   field(
				    default="""file"""     ,	metadata={"""help""": """Column in the dataset that contains speech file path. Defaults to 'file'"""}     ,	)
				__lowerCAmelCase    :    bool         =   field(
				    default=__UpperCAmelCase     ,	metadata={"""help""": """Overwrite the cached preprocessed datasets or not."""})
				__lowerCAmelCase    :    Optional[int]         =   field(
				    default=1     ,	metadata={
				        """help""": """The percentage of the train set used as validation set in case there's no validation split"""
				    }     ,	)
				__lowerCAmelCase    :    Optional[int]         =   field(
				    default=__UpperCAmelCase     ,	metadata={"""help""": """The number of processes to use for the preprocessing."""}     ,	)
				__lowerCAmelCase    :    Optional[float]         =   field(
				    default=2_0.0     ,	metadata={"""help""": """Filter audio files that are longer than `max_duration_in_seconds` seconds"""})

@dataclass
class        lowercase      :
				__lowerCAmelCase    :    WavaVecaForPreTraining
				__lowerCAmelCase    :    WavaVecaFeatureExtractor
				__lowerCAmelCase    :    Union[bool, str]         =   "longest"
				__lowerCAmelCase    :    Optional[int]         =   None
				__lowerCAmelCase    :    Optional[int]         =   None







				def __call__(							self   :					int      ,     _lowerCamelCase   :					List[Dict[str, Union[List[int], torch.Tensor]]]       ):



					"""simple docstring"""

					A_  :				Dict	   =		self.feature_extractor.pad(
					    _lowerCamelCase      ,     max_length=self.max_length      ,     padding=self.padding      ,     pad_to_multiple_of=self.pad_to_multiple_of      ,     return_tensors='''pt'''      ,     )
					A_  :				List[str]	   =		self.model._get_feat_extract_output_lengths(batch['''input_values'''].shape[-1]       )

					A_  :				Optional[int]	   =		batch['''input_values'''].shape[0]

					# make sure that no loss is computed on padded inputs
					if batch["attention_mask"] is not None:
						# compute real output lengths according to convolution formula
						A_  :				str	   =		self.model._get_feat_extract_output_lengths(batch['''attention_mask'''].sum(-1       )       ).to(
						    torch.long       )

						A_  :				int	   =		torch.zeros(
						    (batch_size, mask_indices_seq_length)      ,     dtype=torch.long      ,     device=batch['''input_values'''].device       )

						# these two operations makes sure that all values
						# before the output lengths indices are attended to
						A_  :				int	   =		1
						A_  :				int	   =		attention_mask.flip([-1]       ).cumsum(-1       ).flip([-1]       ).bool()

					# sample randomly masked indices
					A_  :				List[str]	   =		_compute_mask_indices(
					    (batch_size, mask_indices_seq_length)      ,     self.model.config.mask_time_prob      ,     self.model.config.mask_time_length      ,     attention_mask=_lowerCamelCase      ,     min_masks=2      ,     )

					return batch

class        lowercase      (    __UpperCAmelCase):







				def __init__(							self   :					Tuple      ,     *_lowerCamelCase   :					Optional[Any]      ,     _lowerCamelCase   :					Union[str, Any]=1      ,     _lowerCamelCase   :					Optional[Any]=0      ,     _lowerCamelCase   :					Optional[Any]=1.0      ,     **_lowerCamelCase   :					Union[str, Any]       ):



					"""simple docstring"""

					super().__init__(*_lowerCamelCase      ,     **_lowerCamelCase       )
					A_  :				Any	   =		0
					A_  :				Union[str, Any]	   =		max_gumbel_temp
					A_  :				Any	   =		min_gumbel_temp
					A_  :				Tuple	   =		gumbel_temp_decay







				def 					a_  (							self   :					Dict      ,     _lowerCamelCase   :					nn.Module      ,     _lowerCamelCase   :					Dict[str, Union[torch.Tensor, Any]]       ):



					"""simple docstring"""

					model.train()
					A_  :				str	   =		self._prepare_inputs(_lowerCamelCase       )

					if self.use_amp:
						with autocast():
							A_  :				int	   =		self.compute_loss(_lowerCamelCase      ,     _lowerCamelCase       )
					else:
						A_  :				List[Any]	   =		self.compute_loss(_lowerCamelCase      ,     _lowerCamelCase       )

					if self.args.n_gpu > 1 or self.deepspeed:
						if model.module.config.ctc_loss_reduction == "mean":
							A_  :				Optional[Any]	   =		loss.mean()
						elif model.module.config.ctc_loss_reduction == "sum":
							A_  :				Optional[Any]	   =		loss.sum() / (inputs['''mask_time_indices''']).sum()
						else:
							raise ValueError(F"""{model.config.ctc_loss_reduction} is not valid. Choose one of ['mean', 'sum']"""       )

					if self.args.gradient_accumulation_steps > 1:
						A_  :				int	   =		loss / self.args.gradient_accumulation_steps

					if self.use_amp:
						self.scaler.scale(_lowerCamelCase       ).backward()
					elif self.use_apex:
						with amp.scale_loss(_lowerCamelCase      ,     self.optimizer       ) as scaled_loss:
							scaled_loss.backward()
					elif self.deepspeed:
						self.deepspeed.backward(_lowerCamelCase       )
					else:
						loss.backward()

					self.num_update_step += 1
					# make sure gumbel softmax temperature is decayed
					if self.args.n_gpu > 1 or self.deepspeed:
						model.module.set_gumbel_temperature(
						    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step      ,     self.min_gumbel_temp       )       )
					else:
						model.set_gumbel_temperature(
						    max(self.max_gumbel_temp * self.gumbel_temp_decay**self.num_update_step      ,     self.min_gumbel_temp       )       )

					return loss.detach()

def 		lowercase_      (		):




	"""simple docstring"""






	A_  :				Optional[Any]	   =		HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)	)

	A_   , A_   , A_  :				List[str]	   =		parser.parse_args_into_dataclasses()
	configure_logger(_UpperCAmelCase  ,      _UpperCAmelCase	)

	# Downloading and loading a dataset from the hub.
	A_  :				Optional[int]	   =		load_dataset(data_args.dataset_name  ,      data_args.dataset_config_name  ,      cache_dir=model_args.cache_dir	)

	if "validation" not in datasets.keys():
		# make sure only "validation" and "train" keys remain"
		A_  :				int	   =		DatasetDict()
		A_  :				List[Any]	   =		load_dataset(
		    data_args.dataset_name  ,      data_args.dataset_config_name  ,      split=f"""{data_args.train_split_name}[:{data_args.validation_split_percentage}%]"""  ,      cache_dir=model_args.cache_dir  ,      )
		A_  :				List[Any]	   =		load_dataset(
		    data_args.dataset_name  ,      data_args.dataset_config_name  ,      split=f"""{data_args.train_split_name}[{data_args.validation_split_percentage}%:]"""  ,      cache_dir=model_args.cache_dir  ,      )
	else:
		# make sure only "validation" and "train" keys remain"
		A_  :				Optional[Any]	   =		DatasetDict()
		A_  :				Optional[int]	   =		load_dataset(
		    data_args.dataset_name  ,      data_args.dataset_config_name  ,      split='''validation'''  ,      cache_dir=model_args.cache_dir  ,      )
		A_  :				List[str]	   =		load_dataset(
		    data_args.dataset_name  ,      data_args.dataset_config_name  ,      split=f"""{data_args.train_split_name}"""  ,      cache_dir=model_args.cache_dir  ,      )

	# only normalized-inputs-training is supported
	A_  :				str	   =		WavaVecaFeatureExtractor.from_pretrained(
	    model_args.model_name_or_path  ,      cache_dir=model_args.cache_dir  ,      do_normalize=_UpperCAmelCase	)

	def prepare_dataset(_UpperCAmelCase	):
		# check that all files have the correct sampling rate
		A_   , A_  :				List[str]	   =		librosa.load(batch[data_args.speech_file_column]  ,      sr=feature_extractor.sampling_rate	)
		return batch

	# load audio files into numpy arrays
	A_  :				Optional[int]	   =		datasets.map(
	    _UpperCAmelCase  ,      num_proc=data_args.preprocessing_num_workers  ,      remove_columns=datasets['''train'''].column_names	)

	# filter audio files that are too long
	A_  :				Optional[Any]	   =		vectorized_datasets.filter(
	    lambda _UpperCAmelCase	: len(data['''speech''']	) < int(data_args.max_duration_in_seconds * feature_extractor.sampling_rate	)	)

	def normalize(_UpperCAmelCase	):
		return feature_extractor(batch['''speech''']  ,      sampling_rate=feature_extractor.sampling_rate	)

	# normalize and transform to `BatchFeatures`
	A_  :				List[Any]	   =		vectorized_datasets.map(
	    _UpperCAmelCase  ,      batched=_UpperCAmelCase  ,      num_proc=data_args.preprocessing_num_workers  ,      load_from_cache_file=not data_args.overwrite_cache  ,      remove_columns=vectorized_datasets['''train'''].column_names  ,      )

	# pretraining is only supported for "newer" stable layer norm architecture
	# apply_spec_augment has to be True, mask_feature_prob has to be 0.0
	A_  :				List[Any]	   =		WavaVecaConfig.from_pretrained(
	    model_args.model_name_or_path  ,      cache_dir=model_args.cache_dir  ,      gradient_checkpointing=training_args.gradient_checkpointing  ,      )

	if not config.do_stable_layer_norm or config.feat_extract_norm != "layer":
		raise ValueError(
		    '''PreTraining is only supported for ``config.do_stable_layer_norm=True`` and'''
		    ''' ``config.feat_extract_norm=\'layer\''''	)

	A_  :				Any	   =		WavaVecaForPreTraining(_UpperCAmelCase	)

	A_  :				Optional[int]	   =		DataCollatorForWavaVecaPretraining(model=_UpperCAmelCase  ,      feature_extractor=_UpperCAmelCase	)

	A_  :				Union[str, Any]	   =		WavaVecaPreTrainer(
	    model=_UpperCAmelCase  ,      data_collator=_UpperCAmelCase  ,      args=_UpperCAmelCase  ,      train_dataset=vectorized_datasets['''train''']  ,      eval_dataset=vectorized_datasets['''validation''']  ,      tokenizer=_UpperCAmelCase  ,      max_gumbel_temp=model_args.max_gumbel_temperature  ,      min_gumbel_temp=model_args.min_gumbel_temperature  ,      gumbel_temp_decay=model_args.gumbel_temperature_decay  ,      )
	trainer.train()


if __name__ == "__main__":
		main()











"""simple docstring"""

import argparse

import torch

from diffusers.pipelines.stable_diffusion.convert_from_ckpt import download_from_original_stable_diffusion_ckpt


if __name__ == "__main__":
		_lowerCamelCase  :			Optional[int]        =       argparse.ArgumentParser()

		parser.add_argument(
		    '--checkpoint_path', default=None, type=str, required=True, help='Path to the checkpoint to convert.'
		)
		# !wget https://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yaml
		parser.add_argument(
		    '--original_config_file',
		    default=None,
		    type=str,
		    help='The YAML config file corresponding to the original architecture.',
		)
		parser.add_argument(
		    '--num_in_channels',
		    default=None,
		    type=int,
		    help='The number of input channels. If `None` number of input channels will be automatically inferred.',
		)
		parser.add_argument(
		    '--scheduler_type',
		    default='pndm',
		    type=str,
		    help='Type of scheduler to use. Should be one of [\'pndm\', \'lms\', \'ddim\', \'euler\', \'euler-ancestral\', \'dpm\']',
		)
		parser.add_argument(
		    '--pipeline_type',
		    default=None,
		    type=str,
		    help=(
		        'The pipeline type. One of \'FrozenOpenCLIPEmbedder\', \'FrozenCLIPEmbedder\', \'PaintByExample\''
		        '. If `None` pipeline will be automatically inferred.'
		    ),
		)
		parser.add_argument(
		    '--image_size',
		    default=None,
		    type=int,
		    help=(
		        'The image size that the model was trained on. Use 512 for Stable Diffusion v1.X and Stable Siffusion v2'
		        ' Base. Use 768 for Stable Diffusion v2.'
		    ),
		)
		parser.add_argument(
		    '--prediction_type',
		    default=None,
		    type=str,
		    help=(
		        'The prediction type that the model was trained on. Use \'epsilon\' for Stable Diffusion v1.X and Stable'
		        ' Diffusion v2 Base. Use \'v_prediction\' for Stable Diffusion v2.'
		    ),
		)
		parser.add_argument(
		    '--extract_ema',
		    action='store_true',
		    help=(
		        'Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights'
		        ' or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield'
		        ' higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning.'
		    ),
		)
		parser.add_argument(
		    '--upcast_attention',
		    action='store_true',
		    help=(
		        'Whether the attention computation should always be upcasted. This is necessary when running stable'
		        ' diffusion 2.1.'
		    ),
		)
		parser.add_argument(
		    '--from_safetensors',
		    action='store_true',
		    help='If `--checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch.',
		)
		parser.add_argument(
		    '--to_safetensors',
		    action='store_true',
		    help='Whether to store pipeline in safetensors format or not.',
		)
		parser.add_argument('--dump_path', default=None, type=str, required=True, help='Path to the output model.')
		parser.add_argument('--device', type=str, help='Device to use (e.g. cpu, cuda:0, cuda:1, etc.)')
		parser.add_argument(
		    '--stable_unclip',
		    type=str,
		    default=None,
		    required=False,
		    help='Set if this is a stable unCLIP model. One of \'txt2img\' or \'img2img\'.',
		)
		parser.add_argument(
		    '--stable_unclip_prior',
		    type=str,
		    default=None,
		    required=False,
		    help='Set if this is a stable unCLIP txt2img model. Selects which prior to use. If `--stable_unclip` is set to `txt2img`, the karlo prior (https://huggingface.co/kakaobrain/karlo-v1-alpha/tree/main/prior) is selected by default.',
		)
		parser.add_argument(
		    '--clip_stats_path',
		    type=str,
		    help='Path to the clip stats file. Only required if the stable unclip model\'s config specifies `model.params.noise_aug_config.params.clip_stats_path`.',
		    required=False,
		)
		parser.add_argument(
		    '--controlnet', action='store_true', default=None, help='Set flag if this is a controlnet checkpoint.'
		)
		parser.add_argument('--half', action='store_true', help='Save weights in half precision.')
		parser.add_argument(
		    '--vae_path',
		    type=str,
		    default=None,
		    required=False,
		    help='Set to a path, hub id to an already converted vae to not convert it again.',
		)
		_lowerCamelCase  :			Optional[Any]        =       parser.parse_args()

		_lowerCamelCase  :			Optional[Any]        =       download_from_original_stable_diffusion_ckpt(
		    checkpoint_path=args.checkpoint_path,
		    original_config_file=args.original_config_file,
		    image_size=args.image_size,
		    prediction_type=args.prediction_type,
		    model_type=args.pipeline_type,
		    extract_ema=args.extract_ema,
		    scheduler_type=args.scheduler_type,
		    num_in_channels=args.num_in_channels,
		    upcast_attention=args.upcast_attention,
		    from_safetensors=args.from_safetensors,
		    device=args.device,
		    stable_unclip=args.stable_unclip,
		    stable_unclip_prior=args.stable_unclip_prior,
		    clip_stats_path=args.clip_stats_path,
		    controlnet=args.controlnet,
		    vae_path=args.vae_path,
		)

		if args.half:
				pipe.to(torch_dtype=torch.floataa)

		if args.controlnet:
				# only save the controlnet model
				pipe.controlnet.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors)
		else:
				pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors)





