





'''simple docstring'''






import tensorflow as tf

from ...tf_utils import shape_list
class 		lowerCamelCase  (      tf.keras.layers.Layer       ):



	def __init__(   self		,      a_		,      a_		,      a_		,      a_		,      a_=1		,      a_=False		,      **a_	):
							super().__init__(**a_	)

							lowerCAmelCase     :    str						=   vocab_size
							lowerCAmelCase     :    str						=   d_embed
							lowerCAmelCase     :    Optional[Any]						=   d_proj

							lowerCAmelCase     :    List[str]						=   cutoffs + [vocab_size]
							lowerCAmelCase     :    List[Any]						=   [0] + self.cutoffs
							lowerCAmelCase     :    Any						=   div_val

							lowerCAmelCase     :    Optional[int]						=   self.cutoffs[0]
							lowerCAmelCase     :    Optional[int]						=   len(self.cutoffs	) - 1
							lowerCAmelCase     :    str						=   self.shortlist_size + self.n_clusters
							lowerCAmelCase     :    Tuple						=   keep_order

							lowerCAmelCase     :    str						=   []
							lowerCAmelCase     :    Optional[int]						=   []



	def   _lowerCamelCase    (   self		,      a_	):
							if self.n_clusters > 0:
													lowerCAmelCase     :    Tuple						=   self.add_weight(
													    shape=(self.n_clusters, self.d_embed)		,      initializer="zeros"		,      trainable=a_		,      name="cluster_weight"	)
													lowerCAmelCase     :    Union[str, Any]						=   self.add_weight(
													    shape=(self.n_clusters,)		,      initializer="zeros"		,      trainable=a_		,      name="cluster_bias"	)

							if self.div_val == 1:
													for i in range(len(self.cutoffs	)	):
																			if self.d_proj != self.d_embed:
																									lowerCAmelCase     :    List[str]						=   self.add_weight(
																									    shape=(self.d_embed, self.d_proj)		,      initializer="zeros"		,      trainable=a_		,      name=F'''out_projs_._{i}'''		,      )
																									self.out_projs.append(a_	)
																			else:
																									self.out_projs.append(a_	)
																			lowerCAmelCase     :    List[str]						=   self.add_weight(
																			    shape=(self.vocab_size, self.d_embed)		,      initializer="zeros"		,      trainable=a_		,      name=F'''out_layers_._{i}_._weight'''		,      )
																			lowerCAmelCase     :    Optional[int]						=   self.add_weight(
																			    shape=(self.vocab_size,)		,      initializer="zeros"		,      trainable=a_		,      name=F'''out_layers_._{i}_._bias'''		,      )
																			self.out_layers.append((weight, bias)	)
							else:
													for i in range(len(self.cutoffs	)	):
																			lowerCAmelCase     :    Optional[Any]						=   self.cutoff_ends[i], self.cutoff_ends[i + 1]
																			lowerCAmelCase     :    Tuple						=   self.d_embed // (self.div_val**i)

																			lowerCAmelCase     :    int						=   self.add_weight(
																			    shape=(d_emb_i, self.d_proj)		,      initializer="zeros"		,      trainable=a_		,      name=F'''out_projs_._{i}'''	)
																			self.out_projs.append(a_	)
																			lowerCAmelCase     :    Tuple						=   self.add_weight(
																			    shape=(r_idx - l_idx, d_emb_i)		,      initializer="zeros"		,      trainable=a_		,      name=F'''out_layers_._{i}_._weight'''		,      )
																			lowerCAmelCase     :    Dict						=   self.add_weight(
																			    shape=(r_idx - l_idx,)		,      initializer="zeros"		,      trainable=a_		,      name=F'''out_layers_._{i}_._bias'''		,      )
																			self.out_layers.append((weight, bias)	)
							super().build(a_	)



	@staticmethod
	def   _lowerCamelCase    (   a_		,      a_		,      a_		,      a_=None	):
							lowerCAmelCase     :    str						=   x
							if proj is not None:
													lowerCAmelCase     :    Union[str, Any]						=   tf.einsum("ibd,ed->ibe"		,      a_		,      a_	)
							return tf.einsum("ibd,nd->ibn"		,      a_		,      a_	) + b



	@staticmethod
	def   _lowerCamelCase    (   a_		,      a_	):
							lowerCAmelCase     :    List[Any]						=   shape_list(a_	)
							lowerCAmelCase     :    List[str]						=   tf.range(lp_size[0]		,      dtype=target.dtype	)
							lowerCAmelCase     :    Optional[int]						=   tf.stack([r, target]		,      1	)
							return tf.gather_nd(a_		,      a_	)





	def   _lowerCamelCase    (   self		,      a_		,      a_		,      a_=True		,      a_=False	):
							lowerCAmelCase     :    List[Any]						=   0
							if self.n_clusters == 0:
													lowerCAmelCase     :    Any						=   self._logit(a_		,      self.out_layers[0][0]		,      self.out_layers[0][1]		,      self.out_projs[0]	)
													if target is not None:
																			lowerCAmelCase     :    int						=   tf.nn.sparse_softmax_cross_entropy_with_logits(labels=a_		,      logits=a_	)
													lowerCAmelCase     :    Tuple						=   tf.nn.log_softmax(a_		,      axis=-1	)
							else:
													lowerCAmelCase     :    Dict						=   shape_list(a_	)
													lowerCAmelCase     :    List[str]						=   []
													lowerCAmelCase     :    Union[str, Any]						=   tf.zeros(hidden_sizes[:2]	)
													for i in range(len(self.cutoffs	)	):
																			lowerCAmelCase     :    Any						=   self.cutoff_ends[i], self.cutoff_ends[i + 1]
																			if target is not None:
																									lowerCAmelCase     :    int						=   (target >= l_idx) & (target < r_idx)
																									lowerCAmelCase     :    Any						=   tf.where(a_	)
																									lowerCAmelCase     :    List[Any]						=   tf.boolean_mask(a_		,      a_	) - l_idx

																			if self.div_val == 1:
																									lowerCAmelCase     :    int						=   self.out_layers[0][0][l_idx:r_idx]
																									lowerCAmelCase     :    Dict						=   self.out_layers[0][1][l_idx:r_idx]
																			else:
																									lowerCAmelCase     :    Any						=   self.out_layers[i][0]
																									lowerCAmelCase     :    Optional[Any]						=   self.out_layers[i][1]

																			if i == 0:
																									lowerCAmelCase     :    Any						=   tf.concat([cur_W, self.cluster_weight]		,      0	)
																									lowerCAmelCase     :    Union[str, Any]						=   tf.concat([cur_b, self.cluster_bias]		,      0	)

																									lowerCAmelCase     :    str						=   self._logit(a_		,      a_		,      a_		,      self.out_projs[0]	)
																									lowerCAmelCase     :    List[Any]						=   tf.nn.log_softmax(a_	)
																									out.append(head_logprob[..., : self.cutoffs[0]]	)
																									if target is not None:
																															lowerCAmelCase     :    Optional[int]						=   tf.boolean_mask(a_		,      a_	)
																															lowerCAmelCase     :    str						=   self._gather_logprob(a_		,      a_	)
																			else:
																									lowerCAmelCase     :    List[str]						=   self._logit(a_		,      a_		,      a_		,      self.out_projs[i]	)
																									lowerCAmelCase     :    str						=   tf.nn.log_softmax(a_	)
																									lowerCAmelCase     :    List[str]						=   self.cutoffs[0] + i - 1  # No probability for the head cluster
																									lowerCAmelCase     :    str						=   head_logprob[..., cluster_prob_idx, None] + tail_logprob
																									out.append(a_	)
																									if target is not None:
																															lowerCAmelCase     :    Dict						=   tf.boolean_mask(a_		,      a_	)
																															lowerCAmelCase     :    Union[str, Any]						=   tf.boolean_mask(a_		,      a_	)
																															lowerCAmelCase     :    List[Any]						=   self._gather_logprob(a_		,      a_	)
																															cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
																			if target is not None:
																									loss += tf.scatter_nd(a_		,      -cur_logprob		,      shape_list(a_	)	)
													lowerCAmelCase     :    Union[str, Any]						=   tf.concat(a_		,      axis=-1	)

							if target is not None:
													if return_mean:
																			lowerCAmelCase     :    int						=   tf.reduce_mean(a_	)
													# Add the training-time loss value to the layer using `self.add_loss()`.
													self.add_loss(a_	)

													# Log the loss as a metric (we could log arbitrary metrics,
													# including different metrics for training and inference.
													self.add_metric(a_		,      name=self.name		,      aggregation="mean" if return_mean else ""	)

							return out










'''simple docstring'''






import numpy as np






def      __A						(							a_						:  np.array	):
		return (2 / (1 + np.exp(-2 * vector	))) - 1


if __name__ == "__main__":
					import doctest

					doctest.testmod()



