


import tensorflow as tf

from ...tf_utils import shape_list







class 	__lowercase    (tf.keras.layers.Layer  ):





							"""simple docstring"""







							def __init__(      self			,						lowerCAmelCase__			,						lowerCAmelCase__			,						lowerCAmelCase__			,						lowerCAmelCase__			,						lowerCAmelCase__=1			,						lowerCAmelCase__=False			,						**lowerCAmelCase__       ):





										"""simple docstring"""




										super().__init__(**lowerCAmelCase__       )

										SCREAMING_SNAKE_CASE_		:     int					=					vocab_size
										SCREAMING_SNAKE_CASE_		:     Tuple					=					d_embed
										SCREAMING_SNAKE_CASE_		:     str					=					d_proj

										SCREAMING_SNAKE_CASE_		:     List[Any]					=					cutoffs + [vocab_size]
										SCREAMING_SNAKE_CASE_		:     Any					=					[0] + self.cutoffs
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					div_val

										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self.cutoffs[0]
										SCREAMING_SNAKE_CASE_		:     List[str]					=					len(self.cutoffs       ) - 1
										SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self.shortlist_size + self.n_clusters
										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					keep_order

										SCREAMING_SNAKE_CASE_		:     str					=					[]
										SCREAMING_SNAKE_CASE_		:     int					=					[]






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__       ):





										"""simple docstring"""




										if self.n_clusters > 0:
													SCREAMING_SNAKE_CASE_		:     List[str]					=					self.add_weight(
													    shape=(self.n_clusters, self.d_embed)			,						initializer='zeros'			,						trainable=lowerCAmelCase__			,						name='cluster_weight'       )
													SCREAMING_SNAKE_CASE_		:     Any					=					self.add_weight(
													    shape=(self.n_clusters,)			,						initializer='zeros'			,						trainable=lowerCAmelCase__			,						name='cluster_bias'       )

										if self.div_val == 1:
													for i in range(len(self.cutoffs       )       ):
																if self.d_proj != self.d_embed:
																			SCREAMING_SNAKE_CASE_		:     str					=					self.add_weight(
																			    shape=(self.d_embed, self.d_proj)			,						initializer='zeros'			,						trainable=lowerCAmelCase__			,						name=F'''out_projs_._{i}'''			,						)
																			self.out_projs.append(lowerCAmelCase__       )
																else:
																			self.out_projs.append(lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self.add_weight(
																    shape=(self.vocab_size, self.d_embed)			,						initializer='zeros'			,						trainable=lowerCAmelCase__			,						name=F'''out_layers_._{i}_._weight'''			,						)
																SCREAMING_SNAKE_CASE_		:     Tuple					=					self.add_weight(
																    shape=(self.vocab_size,)			,						initializer='zeros'			,						trainable=lowerCAmelCase__			,						name=F'''out_layers_._{i}_._bias'''			,						)
																self.out_layers.append((weight, bias)       )
										else:
													for i in range(len(self.cutoffs       )       ):
																SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		:     List[Any]					=					self.cutoff_ends[i], self.cutoff_ends[i + 1]
																SCREAMING_SNAKE_CASE_		:     Optional[int]					=					self.d_embed // (self.div_val**i)

																SCREAMING_SNAKE_CASE_		:     int					=					self.add_weight(
																    shape=(d_emb_i, self.d_proj)			,						initializer='zeros'			,						trainable=lowerCAmelCase__			,						name=F'''out_projs_._{i}'''       )
																self.out_projs.append(lowerCAmelCase__       )
																SCREAMING_SNAKE_CASE_		:     List[str]					=					self.add_weight(
																    shape=(r_idx - l_idx, d_emb_i)			,						initializer='zeros'			,						trainable=lowerCAmelCase__			,						name=F'''out_layers_._{i}_._weight'''			,						)
																SCREAMING_SNAKE_CASE_		:     Dict					=					self.add_weight(
																    shape=(r_idx - l_idx,)			,						initializer='zeros'			,						trainable=lowerCAmelCase__			,						name=F'''out_layers_._{i}_._bias'''			,						)
																self.out_layers.append((weight, bias)       )
										super().build(lowerCAmelCase__       )






							@staticmethod
							def 					UpperCamelCase__     (      lowerCAmelCase__			,						lowerCAmelCase__			,						lowerCAmelCase__			,						lowerCAmelCase__=None       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					x
										if proj is not None:
													SCREAMING_SNAKE_CASE_		:     Dict					=					tf.einsum('ibd,ed->ibe'			,						lowerCAmelCase__			,						lowerCAmelCase__       )
										return tf.einsum('ibd,nd->ibn'			,						lowerCAmelCase__			,						lowerCAmelCase__       ) + b






							@staticmethod
							def 					UpperCamelCase__     (      lowerCAmelCase__			,						lowerCAmelCase__       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Any					=					shape_list(lowerCAmelCase__       )
										SCREAMING_SNAKE_CASE_		:     Optional[int]					=					tf.range(lp_size[0]			,						dtype=target.dtype       )
										SCREAMING_SNAKE_CASE_		:     Any					=					tf.stack([r, target]			,						1       )
										return tf.gather_nd(lowerCAmelCase__			,						lowerCAmelCase__       )






							def 					UpperCamelCase__     (      self			,						lowerCAmelCase__			,						lowerCAmelCase__			,						lowerCAmelCase__=True			,						lowerCAmelCase__=False       ):





										"""simple docstring"""




										SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					0
										if self.n_clusters == 0:
													SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					self._logit(lowerCAmelCase__			,						self.out_layers[0][0]			,						self.out_layers[0][1]			,						self.out_projs[0]       )
													if target is not None:
																SCREAMING_SNAKE_CASE_		:     Dict					=					tf.nn.sparse_softmax_cross_entropy_with_logits(labels=lowerCAmelCase__			,						logits=lowerCAmelCase__       )
													SCREAMING_SNAKE_CASE_		:     List[str]					=					tf.nn.log_softmax(lowerCAmelCase__			,						axis=-1       )
										else:
													SCREAMING_SNAKE_CASE_		:     List[Any]					=					shape_list(lowerCAmelCase__       )
													SCREAMING_SNAKE_CASE_		:     str					=					[]
													SCREAMING_SNAKE_CASE_		:     int					=					tf.zeros(hidden_sizes[:2]       )
													for i in range(len(self.cutoffs       )       ):
																SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		:     int					=					self.cutoff_ends[i], self.cutoff_ends[i + 1]
																if target is not None:
																			SCREAMING_SNAKE_CASE_		:     Dict					=					(target >= l_idx) & (target < r_idx)
																			SCREAMING_SNAKE_CASE_		:     Any					=					tf.where(lowerCAmelCase__       )
																			SCREAMING_SNAKE_CASE_		:     List[Any]					=					tf.boolean_mask(lowerCAmelCase__			,						lowerCAmelCase__       ) - l_idx

																if self.div_val == 1:
																			SCREAMING_SNAKE_CASE_		:     str					=					self.out_layers[0][0][l_idx:r_idx]
																			SCREAMING_SNAKE_CASE_		:     List[str]					=					self.out_layers[0][1][l_idx:r_idx]
																else:
																			SCREAMING_SNAKE_CASE_		:     List[str]					=					self.out_layers[i][0]
																			SCREAMING_SNAKE_CASE_		:     List[Any]					=					self.out_layers[i][1]

																if i == 0:
																			SCREAMING_SNAKE_CASE_		:     Any					=					tf.concat([cur_W, self.cluster_weight]			,						0       )
																			SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					tf.concat([cur_b, self.cluster_bias]			,						0       )

																			SCREAMING_SNAKE_CASE_		:     List[str]					=					self._logit(lowerCAmelCase__			,						lowerCAmelCase__			,						lowerCAmelCase__			,						self.out_projs[0]       )
																			SCREAMING_SNAKE_CASE_		:     Any					=					tf.nn.log_softmax(lowerCAmelCase__       )
																			out.append(head_logprob[..., : self.cutoffs[0]]       )
																			if target is not None:
																						SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					tf.boolean_mask(lowerCAmelCase__			,						lowerCAmelCase__       )
																						SCREAMING_SNAKE_CASE_		:     int					=					self._gather_logprob(lowerCAmelCase__			,						lowerCAmelCase__       )
																else:
																			SCREAMING_SNAKE_CASE_		:     Any					=					self._logit(lowerCAmelCase__			,						lowerCAmelCase__			,						lowerCAmelCase__			,						self.out_projs[i]       )
																			SCREAMING_SNAKE_CASE_		:     Tuple					=					tf.nn.log_softmax(lowerCAmelCase__       )
																			SCREAMING_SNAKE_CASE_		:     Dict					=					self.cutoffs[0] + i - 1  # No probability for the head cluster
																			SCREAMING_SNAKE_CASE_		:     Optional[int]					=					head_logprob[..., cluster_prob_idx, None] + tail_logprob
																			out.append(lowerCAmelCase__       )
																			if target is not None:
																						SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					tf.boolean_mask(lowerCAmelCase__			,						lowerCAmelCase__       )
																						SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					tf.boolean_mask(lowerCAmelCase__			,						lowerCAmelCase__       )
																						SCREAMING_SNAKE_CASE_		:     int					=					self._gather_logprob(lowerCAmelCase__			,						lowerCAmelCase__       )
																						cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
																if target is not None:
																			loss += tf.scatter_nd(lowerCAmelCase__			,						-cur_logprob			,						shape_list(lowerCAmelCase__       )       )
													SCREAMING_SNAKE_CASE_		:     Tuple					=					tf.concat(lowerCAmelCase__			,						axis=-1       )

										if target is not None:
													if return_mean:
																SCREAMING_SNAKE_CASE_		:     int					=					tf.reduce_mean(lowerCAmelCase__       )
													# Add the training-time loss value to the layer using `self.add_loss()`.
													self.add_loss(lowerCAmelCase__       )

													# Log the loss as a metric (we could log arbitrary metrics,
													# including different metrics for training and inference.
													self.add_metric(lowerCAmelCase__			,						name=self.name			,						aggregation='mean' if return_mean else ''       )

										return out









import numpy as np
def 							UpperCamelCase_(					_A :np.array	)->		np.array:
			return 1 / (1 + np.exp(-vector	))


if __name__ == "__main__":
						import doctest

						doctest.testmod()


