





'''simple docstring'''




import tensorflow as tf

from ...tf_utils import shape_list




class      UpperCamelCase__					(					tf.keras.layers.Layer			):




					"""simple docstring"""


					def __init__(						self				:			Dict      ,						lowerCamelCase_				:			Optional[int]      ,						lowerCamelCase_				:			Any      ,						lowerCamelCase_				:			Dict      ,						lowerCamelCase_				:			Optional[Any]      ,						lowerCamelCase_				:			Tuple=1      ,						lowerCamelCase_				:			List[str]=False      ,						**lowerCamelCase_				:			int						):

							'''simple docstring'''



							super().__init__(**lowerCamelCase_						)

							SCREAMING_SNAKE_CASE :				List[str]		=			vocab_size
							SCREAMING_SNAKE_CASE :				List[Any]		=			d_embed
							SCREAMING_SNAKE_CASE :				Optional[Any]		=			d_proj

							SCREAMING_SNAKE_CASE :				List[str]		=			cutoffs + [vocab_size]
							SCREAMING_SNAKE_CASE :				List[str]		=			[0] + self.cutoffs
							SCREAMING_SNAKE_CASE :				List[Any]		=			div_val

							SCREAMING_SNAKE_CASE :				Optional[Any]		=			self.cutoffs[0]
							SCREAMING_SNAKE_CASE :				Tuple		=			len(self.cutoffs						) - 1
							SCREAMING_SNAKE_CASE :				Dict		=			self.shortlist_size + self.n_clusters
							SCREAMING_SNAKE_CASE :				List[Any]		=			keep_order

							SCREAMING_SNAKE_CASE :				Tuple		=			[]
							SCREAMING_SNAKE_CASE :				Any		=			[]


					def lowerCamelCase_			(						self				:			str      ,						lowerCamelCase_				:			Union[str, Any]						):

							'''simple docstring'''



							if self.n_clusters > 0:
									SCREAMING_SNAKE_CASE :				Any		=			self.add_weight(
									    shape=(self.n_clusters, self.d_embed)      ,						initializer="""zeros"""      ,						trainable=lowerCamelCase_      ,						name="""cluster_weight"""						)
									SCREAMING_SNAKE_CASE :				Optional[int]		=			self.add_weight(
									    shape=(self.n_clusters,)      ,						initializer="""zeros"""      ,						trainable=lowerCamelCase_      ,						name="""cluster_bias"""						)

							if self.div_val == 1:
									for i in range(len(self.cutoffs						)						):
											if self.d_proj != self.d_embed:
													SCREAMING_SNAKE_CASE :				Tuple		=			self.add_weight(
													    shape=(self.d_embed, self.d_proj)      ,						initializer="""zeros"""      ,						trainable=lowerCamelCase_      ,						name=f'''out_projs_._{i}'''      ,						)
													self.out_projs.append(lowerCamelCase_						)
											else:
													self.out_projs.append(lowerCamelCase_						)
											SCREAMING_SNAKE_CASE :				Optional[Any]		=			self.add_weight(
											    shape=(self.vocab_size, self.d_embed)      ,						initializer="""zeros"""      ,						trainable=lowerCamelCase_      ,						name=f'''out_layers_._{i}_._weight'''      ,						)
											SCREAMING_SNAKE_CASE :				Union[str, Any]		=			self.add_weight(
											    shape=(self.vocab_size,)      ,						initializer="""zeros"""      ,						trainable=lowerCamelCase_      ,						name=f'''out_layers_._{i}_._bias'''      ,						)
											self.out_layers.append((weight, bias)						)
							else:
									for i in range(len(self.cutoffs						)						):
											SCREAMING_SNAKE_CASE,							SCREAMING_SNAKE_CASE :				Optional[Any]		=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
											SCREAMING_SNAKE_CASE :				List[str]		=			self.d_embed // (self.div_val**i)

											SCREAMING_SNAKE_CASE :				Union[str, Any]		=			self.add_weight(
											    shape=(d_emb_i, self.d_proj)      ,						initializer="""zeros"""      ,						trainable=lowerCamelCase_      ,						name=f'''out_projs_._{i}'''						)
											self.out_projs.append(lowerCamelCase_						)
											SCREAMING_SNAKE_CASE :				Dict		=			self.add_weight(
											    shape=(r_idx - l_idx, d_emb_i)      ,						initializer="""zeros"""      ,						trainable=lowerCamelCase_      ,						name=f'''out_layers_._{i}_._weight'''      ,						)
											SCREAMING_SNAKE_CASE :				Optional[Any]		=			self.add_weight(
											    shape=(r_idx - l_idx,)      ,						initializer="""zeros"""      ,						trainable=lowerCamelCase_      ,						name=f'''out_layers_._{i}_._bias'''      ,						)
											self.out_layers.append((weight, bias)						)
							super().build(lowerCamelCase_						)


					@staticmethod
					def lowerCamelCase_			(						lowerCamelCase_				:			Union[str, Any]      ,						lowerCamelCase_				:			str      ,						lowerCamelCase_				:			Dict      ,						lowerCamelCase_				:			Dict=None						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				int		=			x
							if proj is not None:
									SCREAMING_SNAKE_CASE :				Optional[int]		=			tf.einsum("""ibd,ed->ibe"""      ,						lowerCamelCase_      ,						lowerCamelCase_						)
							return tf.einsum("""ibd,nd->ibn"""      ,						lowerCamelCase_      ,						lowerCamelCase_						) + b


					@staticmethod
					def lowerCamelCase_			(						lowerCamelCase_				:			Dict      ,						lowerCamelCase_				:			Optional[Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Optional[int]		=			shape_list(lowerCamelCase_						)
							SCREAMING_SNAKE_CASE :				Optional[Any]		=			tf.range(lp_size[0]      ,						dtype=target.dtype						)
							SCREAMING_SNAKE_CASE :				Dict		=			tf.stack([r, target]      ,						1						)
							return tf.gather_nd(lowerCamelCase_      ,						lowerCamelCase_						)



					def lowerCamelCase_			(						self				:			Optional[Any]      ,						lowerCamelCase_				:			Tuple      ,						lowerCamelCase_				:			Any      ,						lowerCamelCase_				:			List[Any]=True      ,						lowerCamelCase_				:			List[str]=False						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Optional[int]		=			0
							if self.n_clusters == 0:
									SCREAMING_SNAKE_CASE :				Dict		=			self._logit(lowerCamelCase_      ,						self.out_layers[0][0]      ,						self.out_layers[0][1]      ,						self.out_projs[0]						)
									if target is not None:
											SCREAMING_SNAKE_CASE :				int		=			tf.nn.sparse_softmax_cross_entropy_with_logits(labels=lowerCamelCase_      ,						logits=lowerCamelCase_						)
									SCREAMING_SNAKE_CASE :				Tuple		=			tf.nn.log_softmax(lowerCamelCase_      ,						axis=-1						)
							else:
									SCREAMING_SNAKE_CASE :				Union[str, Any]		=			shape_list(lowerCamelCase_						)
									SCREAMING_SNAKE_CASE :				Optional[int]		=			[]
									SCREAMING_SNAKE_CASE :				Dict		=			tf.zeros(hidden_sizes[:2]						)
									for i in range(len(self.cutoffs						)						):
											SCREAMING_SNAKE_CASE,							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
											if target is not None:
													SCREAMING_SNAKE_CASE :				Dict		=			(target >= l_idx) & (target < r_idx)
													SCREAMING_SNAKE_CASE :				Optional[Any]		=			tf.where(lowerCamelCase_						)
													SCREAMING_SNAKE_CASE :				Dict		=			tf.boolean_mask(lowerCamelCase_      ,						lowerCamelCase_						) - l_idx

											if self.div_val == 1:
													SCREAMING_SNAKE_CASE :				str		=			self.out_layers[0][0][l_idx:r_idx]
													SCREAMING_SNAKE_CASE :				str		=			self.out_layers[0][1][l_idx:r_idx]
											else:
													SCREAMING_SNAKE_CASE :				Tuple		=			self.out_layers[i][0]
													SCREAMING_SNAKE_CASE :				List[Any]		=			self.out_layers[i][1]

											if i == 0:
													SCREAMING_SNAKE_CASE :				Any		=			tf.concat([cur_W, self.cluster_weight]      ,						0						)
													SCREAMING_SNAKE_CASE :				Optional[Any]		=			tf.concat([cur_b, self.cluster_bias]      ,						0						)

													SCREAMING_SNAKE_CASE :				int		=			self._logit(lowerCamelCase_      ,						lowerCamelCase_      ,						lowerCamelCase_      ,						self.out_projs[0]						)
													SCREAMING_SNAKE_CASE :				Tuple		=			tf.nn.log_softmax(lowerCamelCase_						)
													out.append(head_logprob[..., : self.cutoffs[0]]						)
													if target is not None:
															SCREAMING_SNAKE_CASE :				List[str]		=			tf.boolean_mask(lowerCamelCase_      ,						lowerCamelCase_						)
															SCREAMING_SNAKE_CASE :				str		=			self._gather_logprob(lowerCamelCase_      ,						lowerCamelCase_						)
											else:
													SCREAMING_SNAKE_CASE :				Dict		=			self._logit(lowerCamelCase_      ,						lowerCamelCase_      ,						lowerCamelCase_      ,						self.out_projs[i]						)
													SCREAMING_SNAKE_CASE :				str		=			tf.nn.log_softmax(lowerCamelCase_						)
													SCREAMING_SNAKE_CASE :				List[str]		=			self.cutoffs[0] + i - 1  # No probability for the head cluster
													SCREAMING_SNAKE_CASE :				Any		=			head_logprob[..., cluster_prob_idx, None] + tail_logprob
													out.append(lowerCamelCase_						)
													if target is not None:
															SCREAMING_SNAKE_CASE :				Tuple		=			tf.boolean_mask(lowerCamelCase_      ,						lowerCamelCase_						)
															SCREAMING_SNAKE_CASE :				Dict		=			tf.boolean_mask(lowerCamelCase_      ,						lowerCamelCase_						)
															SCREAMING_SNAKE_CASE :				Optional[int]		=			self._gather_logprob(lowerCamelCase_      ,						lowerCamelCase_						)
															cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
											if target is not None:
													loss += tf.scatter_nd(lowerCamelCase_      ,						-cur_logprob      ,						shape_list(lowerCamelCase_						)						)
									SCREAMING_SNAKE_CASE :				Any		=			tf.concat(lowerCamelCase_      ,						axis=-1						)

							if target is not None:
									if return_mean:
											SCREAMING_SNAKE_CASE :				List[Any]		=			tf.reduce_mean(lowerCamelCase_						)
									# Add the training-time loss value to the layer using `self.add_loss()`.
									self.add_loss(lowerCamelCase_						)

									# Log the loss as a metric (we could log arbitrary metrics,
									# including different metrics for training and inference.
									self.add_metric(lowerCamelCase_      ,						name=self.name      ,						aggregation="""mean""" if return_mean else """"""						)

							return out










'''simple docstring'''




import torch

from diffusers import UnCLIPScheduler

from .test_schedulers import SchedulerCommonTest




class      UpperCamelCase__					(					lowercase_			):




					"""simple docstring"""
					SCREAMING_SNAKE_CASE__				=	(UnCLIPScheduler,)


					def lowerCamelCase_			(						self				:			List[str]      ,						**lowerCamelCase_				:			List[Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				List[Any]		=			{
							    """num_train_timesteps""": 10_00,
							    """variance_type""": """fixed_small_log""",
							    """clip_sample""": True,
							    """clip_sample_range""": 1.0,
							    """prediction_type""": """epsilon""",
							}

							config.update(**lowerCamelCase_						)
							return config


					def lowerCamelCase_			(						self				:			List[Any]						):

							'''simple docstring'''



							for timesteps in [1, 5, 1_00, 10_00]:
									self.check_over_configs(num_train_timesteps=lowerCamelCase_						)


					def lowerCamelCase_			(						self				:			str						):

							'''simple docstring'''



							for variance in ["fixed_small_log", "learned_range"]:
									self.check_over_configs(variance_type=lowerCamelCase_						)


					def lowerCamelCase_			(						self				:			List[Any]						):

							'''simple docstring'''



							for clip_sample in [True, False]:
									self.check_over_configs(clip_sample=lowerCamelCase_						)


					def lowerCamelCase_			(						self				:			Union[str, Any]						):

							'''simple docstring'''



							for clip_sample_range in [1, 5, 10, 20]:
									self.check_over_configs(clip_sample_range=lowerCamelCase_						)


					def lowerCamelCase_			(						self				:			List[Any]						):

							'''simple docstring'''



							for prediction_type in ["epsilon", "sample"]:
									self.check_over_configs(prediction_type=lowerCamelCase_						)


					def lowerCamelCase_			(						self				:			List[Any]						):

							'''simple docstring'''



							for time_step in [0, 5_00, 9_99]:
									for prev_timestep in [None, 5, 1_00, 2_50, 5_00, 7_50]:
											if prev_timestep is not None and prev_timestep >= time_step:
													continue

											self.check_over_forward(time_step=lowerCamelCase_      ,						prev_timestep=lowerCamelCase_						)


					def lowerCamelCase_			(						self				:			List[str]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				int		=			self.scheduler_classes[0]
							SCREAMING_SNAKE_CASE :				Dict		=			self.get_scheduler_config(variance_type="""fixed_small_log"""						)
							SCREAMING_SNAKE_CASE :				Optional[Any]		=			scheduler_class(**lowerCamelCase_						)

							assert torch.sum(torch.abs(scheduler._get_variance(0						) - 1.0000e-10						)						) < 1e-5
							assert torch.sum(torch.abs(scheduler._get_variance(4_87						) - 0.0_549_625						)						) < 1e-5
							assert torch.sum(torch.abs(scheduler._get_variance(9_99						) - 0.9_994_987						)						) < 1e-5


					def lowerCamelCase_			(						self				:			Optional[Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Tuple		=			self.scheduler_classes[0]
							SCREAMING_SNAKE_CASE :				Optional[int]		=			self.get_scheduler_config(variance_type="""learned_range"""						)
							SCREAMING_SNAKE_CASE :				List[str]		=			scheduler_class(**lowerCamelCase_						)

							SCREAMING_SNAKE_CASE :				List[str]		=			0.5

							assert scheduler._get_variance(1      ,						predicted_variance=lowerCamelCase_						) - -10.1_712_790 < 1e-5
							assert scheduler._get_variance(4_87      ,						predicted_variance=lowerCamelCase_						) - -5.7_998_052 < 1e-5
							assert scheduler._get_variance(9_99      ,						predicted_variance=lowerCamelCase_						) - -0.0_010_011 < 1e-5


					def lowerCamelCase_			(						self				:			str						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Tuple		=			self.scheduler_classes[0]
							SCREAMING_SNAKE_CASE :				int		=			self.get_scheduler_config()
							SCREAMING_SNAKE_CASE :				List[Any]		=			scheduler_class(**lowerCamelCase_						)

							SCREAMING_SNAKE_CASE :				Optional[Any]		=			scheduler.timesteps

							SCREAMING_SNAKE_CASE :				Optional[int]		=			self.dummy_model()
							SCREAMING_SNAKE_CASE :				Optional[int]		=			self.dummy_sample_deter
							SCREAMING_SNAKE_CASE :				str		=			torch.manual_seed(0						)

							for i, t in enumerate(lowerCamelCase_						):
									# 1. predict noise residual
									SCREAMING_SNAKE_CASE :				List[str]		=			model(lowerCamelCase_      ,						lowerCamelCase_						)

									# 2. predict previous mean of sample x_t-1
									SCREAMING_SNAKE_CASE :				Optional[int]		=			scheduler.step(lowerCamelCase_      ,						lowerCamelCase_      ,						lowerCamelCase_      ,						generator=lowerCamelCase_						).prev_sample

									SCREAMING_SNAKE_CASE :				Union[str, Any]		=			pred_prev_sample

							SCREAMING_SNAKE_CASE :				Optional[int]		=			torch.sum(torch.abs(lowerCamelCase_						)						)
							SCREAMING_SNAKE_CASE :				int		=			torch.mean(torch.abs(lowerCamelCase_						)						)

							assert abs(result_sum.item() - 252.2_682_495						) < 1e-2
							assert abs(result_mean.item() - 0.3_284_743						) < 1e-3


					def lowerCamelCase_			(						self				:			Optional[Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Optional[Any]		=			self.scheduler_classes[0]
							SCREAMING_SNAKE_CASE :				Dict		=			self.get_scheduler_config()
							SCREAMING_SNAKE_CASE :				List[str]		=			scheduler_class(**lowerCamelCase_						)

							scheduler.set_timesteps(25						)

							SCREAMING_SNAKE_CASE :				List[str]		=			scheduler.timesteps

							SCREAMING_SNAKE_CASE :				Optional[int]		=			self.dummy_model()
							SCREAMING_SNAKE_CASE :				Tuple		=			self.dummy_sample_deter
							SCREAMING_SNAKE_CASE :				Any		=			torch.manual_seed(0						)

							for i, t in enumerate(lowerCamelCase_						):
									# 1. predict noise residual
									SCREAMING_SNAKE_CASE :				int		=			model(lowerCamelCase_      ,						lowerCamelCase_						)

									if i + 1 == timesteps.shape[0]:
											SCREAMING_SNAKE_CASE :				Union[str, Any]		=			None
									else:
											SCREAMING_SNAKE_CASE :				int		=			timesteps[i + 1]

									# 2. predict previous mean of sample x_t-1
									SCREAMING_SNAKE_CASE :				int		=			scheduler.step(
									    lowerCamelCase_      ,						lowerCamelCase_      ,						lowerCamelCase_      ,						prev_timestep=lowerCamelCase_      ,						generator=lowerCamelCase_						).prev_sample

									SCREAMING_SNAKE_CASE :				Union[str, Any]		=			pred_prev_sample

							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			torch.sum(torch.abs(lowerCamelCase_						)						)
							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			torch.mean(torch.abs(lowerCamelCase_						)						)

							assert abs(result_sum.item() - 258.2_044_983						) < 1e-2
							assert abs(result_mean.item() - 0.3_362_038						) < 1e-3


					def lowerCamelCase_			(						self				:			Dict						):

							'''simple docstring'''



							pass



					def lowerCamelCase_			(						self				:			int						):

							'''simple docstring'''



							pass



