





'''simple docstring'''



import math
import warnings
from dataclasses import dataclass
from typing import List, Optional, Tuple, Union

import numpy as np
import torch
from scipy import integrate

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput
from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin






@dataclass
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->LMSDiscrete
class 							__lowercase				(					lowerCAmelCase__  ):







	'''simple docstring'''

	a			: torch.FloatTensor
	a			: Optional[torch.FloatTensor]      =				None





def       _lowerCAmelCase  ( lowerCamelCase_	:     str						,				lowerCamelCase_	:     Dict=0.9_99						,				lowerCamelCase_	:     List[str]="cosine"						,				):
			if alpha_transform_type == "cosine":

						def alpha_bar_fn(lowerCamelCase_	:     List[Any] ):
									return math.cos((t + 0.0_08) / 1.0_08 * math.pi / 2 ) ** 2

			elif alpha_transform_type == "exp":

						def alpha_bar_fn(lowerCamelCase_	:     List[Any] ):
									return math.exp(t * -12.0 )

			else:
						raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}" )

			__lowercase   =							[]
			for i in range(lowerCamelCase_ ):
						__lowercase   =							i / num_diffusion_timesteps
						__lowercase   =							(i + 1) / num_diffusion_timesteps
						betas.append(min(1 - alpha_bar_fn(lowerCamelCase_ ) / alpha_bar_fn(lowerCamelCase_ )						,				lowerCamelCase_ ) )
			return torch.tensor(lowerCamelCase_						,				dtype=torch.floataa )






class 							__lowercase				(					lowerCAmelCase__       ,					lowerCAmelCase__  ):







	'''simple docstring'''

	a			: Dict      =				[e.name for e in KarrasDiffusionSchedulers]
	a			: Dict      =				1


	@register_to_config
	def __init__(self   ,_lowerCamelCase = 1000   ,_lowerCamelCase = 0.0_0_0_1   ,_lowerCamelCase = 0.0_2   ,_lowerCamelCase = "linear"   ,_lowerCamelCase = None   ,_lowerCamelCase = False   ,_lowerCamelCase = "epsilon"   ,_lowerCamelCase = "linspace"   ,_lowerCamelCase = 0   ,)							->							Union[str, Any]:







				'''simple docstring'''





				if trained_betas is not None:
							__lowercase   =							torch.tensor(_lowerCamelCase   ,dtype=torch.floataa       )
				elif beta_schedule == "linear":
							__lowercase   =							torch.linspace(_lowerCamelCase   ,_lowerCamelCase   ,_lowerCamelCase   ,dtype=torch.floataa       )
				elif beta_schedule == "scaled_linear":
							# this schedule is very specific to the latent diffusion model.
							__lowercase   =							(
							    torch.linspace(beta_start**0.5   ,beta_end**0.5   ,_lowerCamelCase   ,dtype=torch.floataa       ) ** 2
							)
				elif beta_schedule == "squaredcos_cap_v2":
							# Glide cosine schedule
							__lowercase   =							betas_for_alpha_bar(_lowerCamelCase       )
				else:
							raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}"       )

				__lowercase   =							1.0 - self.betas
				__lowercase   =							torch.cumprod(self.alphas   ,dim=0       )

				__lowercase   =							np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5       )
				__lowercase   =							np.concatenate([sigmas[::-1], [0.0]]       ).astype(np.floataa       )
				__lowercase   =							torch.from_numpy(_lowerCamelCase       )

				# setable values
				__lowercase   =							None
				__lowercase   =							use_karras_sigmas
				self.set_timesteps(_lowerCamelCase   ,_lowerCamelCase       )
				__lowercase   =							[]
				__lowercase   =							False


	@property
	def   _UpperCAmelCase   (self       )							->							Union[str, Any]:







				'''simple docstring'''





				if self.config.timestep_spacing in ["linspace", "trailing"]:
							return self.sigmas.max()

				return (self.sigmas.max() ** 2 + 1) ** 0.5


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase       )							->							torch.FloatTensor:







				'''simple docstring'''





				if isinstance(_lowerCamelCase   ,torch.Tensor       ):
							__lowercase   =							timestep.to(self.timesteps.device       )
				__lowercase   =							(self.timesteps == timestep).nonzero().item()
				__lowercase   =							self.sigmas[step_index]
				__lowercase   =							sample / ((sigma**2 + 1) ** 0.5)
				__lowercase   =							True
				return sample


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase   ,_lowerCamelCase       )							->							Tuple:







				'''simple docstring'''





				def lms_derivative(_lowerCamelCase       ):
							__lowercase   =							1.0
							for k in range(_lowerCamelCase       ):
										if current_order == k:
													continue
										prod *= (tau - self.sigmas[t - k]) / (self.sigmas[t - current_order] - self.sigmas[t - k])
							return prod

				__lowercase   =							integrate.quad(_lowerCamelCase   ,self.sigmas[t]   ,self.sigmas[t + 1]   ,epsrel=1E-4       )[0]

				return integrated_coeff


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase = None       )							->							List[str]:







				'''simple docstring'''





				__lowercase   =							num_inference_steps

				# "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891
				if self.config.timestep_spacing == "linspace":
							__lowercase   =							np.linspace(0   ,self.config.num_train_timesteps - 1   ,_lowerCamelCase   ,dtype=_lowerCamelCase       )[
							    ::-1
							].copy()
				elif self.config.timestep_spacing == "leading":
							__lowercase   =							self.config.num_train_timesteps // self.num_inference_steps
							# creates integer timesteps by multiplying by ratio
							# casting to int to avoid issues when num_inference_step is power of 3
							__lowercase   =							(np.arange(0   ,_lowerCamelCase       ) * step_ratio).round()[::-1].copy().astype(_lowerCamelCase       )
							timesteps += self.config.steps_offset
				elif self.config.timestep_spacing == "trailing":
							__lowercase   =							self.config.num_train_timesteps / self.num_inference_steps
							# creates integer timesteps by multiplying by ratio
							# casting to int to avoid issues when num_inference_step is power of 3
							__lowercase   =							(np.arange(self.config.num_train_timesteps   ,0   ,-step_ratio       )).round().copy().astype(_lowerCamelCase       )
							timesteps -= 1
				else:
							raise ValueError(
							    f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'."       )

				__lowercase   =							np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5       )
				__lowercase   =							np.log(_lowerCamelCase       )
				__lowercase   =							np.interp(_lowerCamelCase   ,np.arange(0   ,len(_lowerCamelCase       )       )   ,_lowerCamelCase       )

				if self.use_karras_sigmas:
							__lowercase   =							self._convert_to_karras(in_sigmas=_lowerCamelCase       )
							__lowercase   =							np.array([self._sigma_to_t(_lowerCamelCase   ,_lowerCamelCase       ) for sigma in sigmas]       )

				__lowercase   =							np.concatenate([sigmas, [0.0]]       ).astype(np.floataa       )

				__lowercase   =							torch.from_numpy(_lowerCamelCase       ).to(device=_lowerCamelCase       )
				if str(_lowerCamelCase       ).startswith('''mps'''       ):
							# mps does not support float64
							__lowercase   =							torch.from_numpy(_lowerCamelCase       ).to(_lowerCamelCase   ,dtype=torch.floataa       )
				else:
							__lowercase   =							torch.from_numpy(_lowerCamelCase       ).to(device=_lowerCamelCase       )

				__lowercase   =							[]


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase       )							->							Any:







				'''simple docstring'''





				__lowercase   =							np.log(_lowerCamelCase       )

				# get distribution
				__lowercase   =							log_sigma - log_sigmas[:, np.newaxis]

				# get sigmas range
				__lowercase   =							np.cumsum((dists >= 0)   ,axis=0       ).argmax(axis=0       ).clip(max=log_sigmas.shape[0] - 2       )
				__lowercase   =							low_idx + 1

				__lowercase   =							log_sigmas[low_idx]
				__lowercase   =							log_sigmas[high_idx]

				# interpolate sigmas
				__lowercase   =							(low - log_sigma) / (low - high)
				__lowercase   =							np.clip(_lowerCamelCase   ,0   ,1       )

				# transform interpolation to time range
				__lowercase   =							(1 - w) * low_idx + w * high_idx
				__lowercase   =							t.reshape(sigma.shape       )
				return t


	def   _UpperCAmelCase   (self   ,_lowerCamelCase       )							->							torch.FloatTensor:







				'''simple docstring'''





				__lowercase   =							in_sigmas[-1].item()
				__lowercase   =							in_sigmas[0].item()

				__lowercase   =							7.0  # 7.0 is the value used in the paper
				__lowercase   =							np.linspace(0   ,1   ,self.num_inference_steps       )
				__lowercase   =							sigma_min ** (1 / rho)
				__lowercase   =							sigma_max ** (1 / rho)
				__lowercase   =							(max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
				return sigmas


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase   ,_lowerCamelCase   ,_lowerCamelCase = 4   ,_lowerCamelCase = True   ,)							->							Union[LMSDiscreteSchedulerOutput, Tuple]:







				'''simple docstring'''





				if not self.is_scale_input_called:
							warnings.warn(
							    '''The `scale_model_input` function should be called before `step` to ensure correct denoising. '''
							    '''See `StableDiffusionPipeline` for a usage example.'''       )

				if isinstance(_lowerCamelCase   ,torch.Tensor       ):
							__lowercase   =							timestep.to(self.timesteps.device       )
				__lowercase   =							(self.timesteps == timestep).nonzero().item()
				__lowercase   =							self.sigmas[step_index]

				# 1. compute predicted original sample (x_0) from sigma-scaled predicted noise
				if self.config.prediction_type == "epsilon":
							__lowercase   =							sample - sigma * model_output
				elif self.config.prediction_type == "v_prediction":
							# * c_out + input * c_skip
							__lowercase   =							model_output * (-sigma / (sigma**2 + 1) ** 0.5) + (sample / (sigma**2 + 1))
				elif self.config.prediction_type == "sample":
							__lowercase   =							model_output
				else:
							raise ValueError(
							    f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`"       )

				# 2. Convert to an ODE derivative
				__lowercase   =							(sample - pred_original_sample) / sigma
				self.derivatives.append(_lowerCamelCase       )
				if len(self.derivatives       ) > order:
							self.derivatives.pop(0       )

				# 3. Compute linear multistep coefficients
				__lowercase   =							min(step_index + 1   ,_lowerCamelCase       )
				__lowercase   =							[self.get_lms_coefficient(_lowerCamelCase   ,_lowerCamelCase   ,_lowerCamelCase       ) for curr_order in range(_lowerCamelCase       )]

				# 4. Compute previous sample based on the derivatives path
				__lowercase   =							sample + sum(
				    coeff * derivative for coeff, derivative in zip(_lowerCamelCase   ,reversed(self.derivatives       )       )       )

				if not return_dict:
							return (prev_sample,)

				return LMSDiscreteSchedulerOutput(prev_sample=_lowerCamelCase   ,pred_original_sample=_lowerCamelCase       )


	def   _UpperCAmelCase   (self   ,_lowerCamelCase   ,_lowerCamelCase   ,_lowerCamelCase   ,)							->							torch.FloatTensor:







				'''simple docstring'''





				__lowercase   =							self.sigmas.to(device=original_samples.device   ,dtype=original_samples.dtype       )
				if original_samples.device.type == "mps" and torch.is_floating_point(_lowerCamelCase       ):
							# mps does not support float64
							__lowercase   =							self.timesteps.to(original_samples.device   ,dtype=torch.floataa       )
							__lowercase   =							timesteps.to(original_samples.device   ,dtype=torch.floataa       )
				else:
							__lowercase   =							self.timesteps.to(original_samples.device       )
							__lowercase   =							timesteps.to(original_samples.device       )

				__lowercase   =							[(schedule_timesteps == t).nonzero().item() for t in timesteps]

				__lowercase   =							sigmas[step_indices].flatten()
				while len(sigma.shape       ) < len(original_samples.shape       ):
							__lowercase   =							sigma.unsqueeze(-1       )

				__lowercase   =							original_samples + noise * sigma
				return noisy_samples


	def __len__(self       )							->							Optional[Any]:







				'''simple docstring'''





				return self.config.num_train_timesteps













import tempfile
import unittest

import torch

from diffusers import UNetaDConditionModel
from diffusers.training_utils import EMAModel
from diffusers.utils.testing_utils import enable_full_determinism, skip_mps, torch_device


enable_full_determinism()





class   SCREAMING_SNAKE_CASE (       unittest.TestCase    ):





							"""simple docstring"""

							A_       =    "hf-internal-testing/tiny-stable-diffusion-pipe"
							A_       =    1
							A_       =    77
							A_       =    32
							A_       =    4
							A_       =    A_       =    64
							A_       =    torch.manual_seed(0    )

							def 					__A       (     self:      List[Any]						,   __A:      List[Any]=0.9_999      )			-> Dict:
												_A									=       UNetaDConditionModel.from_pretrained(self.model_id						,   subfolder='''unet'''      )
												_A									=       unet.to(__A      )
												_A									=       EMAModel(unet.parameters()						,   decay=__A						,   model_cls=__A						,   model_config=unet.config      )
												return unet, ema_unet

							def 					__A       (     self:      Tuple      )			-> Union[str, Any]:
												_A									=       torch.randn(
												    self.batch_size						,   self.num_in_channels						,   self.latent_height						,   self.latent_width						,   generator=self.generator      ).to(__A      )
												_A									=       torch.randint(0						,   10_00						,   size=(self.batch_size,)						,   generator=self.generator      ).to(__A      )
												_A									=       torch.randn(
												    self.batch_size						,   self.prompt_length						,   self.text_encoder_hidden_dim						,   generator=self.generator      ).to(__A      )
												return noisy_latents, timesteps, encoder_hidden_states

							def 					__A       (     self:      Tuple						,   __A:      Optional[int]      )			-> Optional[Any]:
												_A									=       {}
												for k, param in unet.state_dict().items():
																	_A									=       torch.randn_like(__A      ) + (param * torch.randn_like(__A      ))
																	updated_state_dict.update({k: updated_param}      )
												unet.load_state_dict(__A      )
												return unet

							def 					__A       (     self:      Any      )			-> Any:
												_A							,_A									=       self.get_models()
												# Take the first (hypothetical) EMA step.
												ema_unet.step(unet.parameters()      )
												assert ema_unet.optimization_step == 1

												# Take two more.
												for _ in range(2      ):
																	ema_unet.step(unet.parameters()      )
												assert ema_unet.optimization_step == 3

							def 					__A       (     self:      Union[str, Any]      )			-> str:
												_A							,_A									=       self.get_models()
												# Since the `unet` is not being updated (i.e., backprop'd)
												# there won't be any difference between the `params` of `unet`
												# and `ema_unet` even if we call `ema_unet.step(unet.parameters())`.
												ema_unet.step(unet.parameters()      )
												_A									=       list(unet.parameters()      )
												for s_param, param in zip(ema_unet.shadow_params						,   __A      ):
																	assert torch.allclose(__A						,   __A      )

												# The above holds true even if we call `ema.step()` multiple times since
												# `unet` params are still not being updated.
												for _ in range(4      ):
																	ema_unet.step(unet.parameters()      )
												for s_param, param in zip(ema_unet.shadow_params						,   __A      ):
																	assert torch.allclose(__A						,   __A      )

							def 					__A       (     self:      Tuple      )			-> List[str]:
												_A							,_A									=       self.get_models()
												# Here we simulate the parameter updates for `unet`. Since there might
												# be some parameters which are initialized to zero we take extra care to
												# initialize their values to something non-zero before the multiplication.
												_A									=       self.simulate_backprop(__A      )

												# Take the EMA step.
												ema_unet.step(unet_pseudo_updated_step_one.parameters()      )

												# Now the EMA'd parameters won't be equal to the original model parameters.
												_A									=       list(unet_pseudo_updated_step_one.parameters()      )
												for s_param, param in zip(ema_unet.shadow_params						,   __A      ):
																	assert ~torch.allclose(__A						,   __A      )

												# Ensure this is the case when we take multiple EMA steps.
												for _ in range(4      ):
																	ema_unet.step(unet.parameters()      )
												for s_param, param in zip(ema_unet.shadow_params						,   __A      ):
																	assert ~torch.allclose(__A						,   __A      )

							def 					__A       (     self:      Optional[int]      )			-> Optional[int]:
												# If we call EMA step after a backpropagation consecutively for two times,
												# the shadow params from those two steps should be different.
												_A							,_A									=       self.get_models()

												# First backprop + EMA
												_A									=       self.simulate_backprop(__A      )
												ema_unet.step(unet_step_one.parameters()      )
												_A									=       ema_unet.shadow_params

												# Second backprop + EMA
												_A									=       self.simulate_backprop(__A      )
												ema_unet.step(unet_step_two.parameters()      )
												_A									=       ema_unet.shadow_params

												for step_one, step_two in zip(__A						,   __A      ):
																	assert ~torch.allclose(__A						,   __A      )

							def 					__A       (     self:      int      )			-> Any:
												# If there's no decay even if there are backprops, EMA steps
												# won't take any effect i.e., the shadow params would remain the
												# same.
												_A							,_A									=       self.get_models(decay=0.0      )
												_A									=       self.simulate_backprop(__A      )
												ema_unet.step(unet_step_one.parameters()      )
												_A									=       ema_unet.shadow_params

												_A									=       self.simulate_backprop(__A      )
												ema_unet.step(unet_step_two.parameters()      )
												_A									=       ema_unet.shadow_params

												for step_one, step_two in zip(__A						,   __A      ):
																	assert torch.allclose(__A						,   __A      )



							@skip_mps
							def 					__A       (     self:      Optional[Any]      )			-> List[str]:
												_A							,_A									=       self.get_models()
												_A							,_A							,_A									=       self.get_dummy_inputs()

												with tempfile.TemporaryDirectory() as tmpdir:
																	ema_unet.save_pretrained(__A      )
																	_A									=       UNetaDConditionModel.from_pretrained(__A						,   model_cls=__A      )
																	_A									=       loaded_unet.to(unet.device      )

												# Since no EMA step has been performed the outputs should match.
												_A									=       unet(__A						,   __A						,   __A      ).sample
												_A									=       loaded_unet(__A						,   __A						,   __A      ).sample

												assert torch.allclose(__A						,   __A						,   atol=1e-4      )



