






import tensorflow as tf

from ...tf_utils import shape_list




class 					UpperCamelCase_ (     tf.keras.layers.Layer							):



	'''simple docstring'''




	def __init__(			self			,     a			,     a			,     a			,     a			,     a=1			,     a=False			,     **a   )			->       List[str]:
								super().__init__(**a   )

								snake_case_           =				vocab_size
								snake_case_           =				d_embed
								snake_case_           =				d_proj

								snake_case_           =				cutoffs + [vocab_size]
								snake_case_           =				[0] + self.cutoffs
								snake_case_           =				div_val

								snake_case_           =				self.cutoffs[0]
								snake_case_           =				len(self.cutoffs   ) - 1
								snake_case_           =				self.shortlist_size + self.n_clusters
								snake_case_           =				keep_order

								snake_case_           =				[]
								snake_case_           =				[]




	def 				_UpperCamelCase     (			self			,     a   )			->       Union[str, Any]:
								if self.n_clusters > 0:
															snake_case_           =				self.add_weight(
															    shape=(self.n_clusters, self.d_embed)			,     initializer='zeros'			,     trainable=a			,     name='cluster_weight'   )
															snake_case_           =				self.add_weight(
															    shape=(self.n_clusters,)			,     initializer='zeros'			,     trainable=a			,     name='cluster_bias'   )

								if self.div_val == 1:
															for i in range(len(self.cutoffs   )   ):
																						if self.d_proj != self.d_embed:
																													snake_case_           =				self.add_weight(
																													    shape=(self.d_embed, self.d_proj)			,     initializer='zeros'			,     trainable=a			,     name=F'''out_projs_._{i}'''			,     )
																													self.out_projs.append(a   )
																						else:
																													self.out_projs.append(a   )
																						snake_case_           =				self.add_weight(
																						    shape=(self.vocab_size, self.d_embed)			,     initializer='zeros'			,     trainable=a			,     name=F'''out_layers_._{i}_._weight'''			,     )
																						snake_case_           =				self.add_weight(
																						    shape=(self.vocab_size,)			,     initializer='zeros'			,     trainable=a			,     name=F'''out_layers_._{i}_._bias'''			,     )
																						self.out_layers.append((weight, bias)   )
								else:
															for i in range(len(self.cutoffs   )   ):
																						snake_case_     ,						snake_case_           =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						snake_case_           =				self.d_embed // (self.div_val**i)

																						snake_case_           =				self.add_weight(
																						    shape=(d_emb_i, self.d_proj)			,     initializer='zeros'			,     trainable=a			,     name=F'''out_projs_._{i}'''   )
																						self.out_projs.append(a   )
																						snake_case_           =				self.add_weight(
																						    shape=(r_idx - l_idx, d_emb_i)			,     initializer='zeros'			,     trainable=a			,     name=F'''out_layers_._{i}_._weight'''			,     )
																						snake_case_           =				self.add_weight(
																						    shape=(r_idx - l_idx,)			,     initializer='zeros'			,     trainable=a			,     name=F'''out_layers_._{i}_._bias'''			,     )
																						self.out_layers.append((weight, bias)   )
								super().build(a   )




	@staticmethod
	def 				_UpperCamelCase     (			a			,     a			,     a			,     a=None   )			->       Optional[int]:
								snake_case_           =				x
								if proj is not None:
															snake_case_           =				tf.einsum('ibd,ed->ibe'			,     a			,     a   )
								return tf.einsum('ibd,nd->ibn'			,     a			,     a   ) + b




	@staticmethod
	def 				_UpperCamelCase     (			a			,     a   )			->       Optional[Any]:
								snake_case_           =				shape_list(a   )
								snake_case_           =				tf.range(lp_size[0]			,     dtype=target.dtype   )
								snake_case_           =				tf.stack([r, target]			,     1   )
								return tf.gather_nd(a			,     a   )






	def 				_UpperCamelCase     (			self			,     a			,     a			,     a=True			,     a=False   )			->       List[str]:
								snake_case_           =				0
								if self.n_clusters == 0:
															snake_case_           =				self._logit(a			,     self.out_layers[0][0]			,     self.out_layers[0][1]			,     self.out_projs[0]   )
															if target is not None:
																						snake_case_           =				tf.nn.sparse_softmax_cross_entropy_with_logits(labels=a			,     logits=a   )
															snake_case_           =				tf.nn.log_softmax(a			,     axis=-1   )
								else:
															snake_case_           =				shape_list(a   )
															snake_case_           =				[]
															snake_case_           =				tf.zeros(hidden_sizes[:2]   )
															for i in range(len(self.cutoffs   )   ):
																						snake_case_     ,						snake_case_           =				self.cutoff_ends[i], self.cutoff_ends[i + 1]
																						if target is not None:
																													snake_case_           =				(target >= l_idx) & (target < r_idx)
																													snake_case_           =				tf.where(a   )
																													snake_case_           =				tf.boolean_mask(a			,     a   ) - l_idx

																						if self.div_val == 1:
																													snake_case_           =				self.out_layers[0][0][l_idx:r_idx]
																													snake_case_           =				self.out_layers[0][1][l_idx:r_idx]
																						else:
																													snake_case_           =				self.out_layers[i][0]
																													snake_case_           =				self.out_layers[i][1]

																						if i == 0:
																													snake_case_           =				tf.concat([cur_W, self.cluster_weight]			,     0   )
																													snake_case_           =				tf.concat([cur_b, self.cluster_bias]			,     0   )

																													snake_case_           =				self._logit(a			,     a			,     a			,     self.out_projs[0]   )
																													snake_case_           =				tf.nn.log_softmax(a   )
																													out.append(head_logprob[..., : self.cutoffs[0]]   )
																													if target is not None:
																																				snake_case_           =				tf.boolean_mask(a			,     a   )
																																				snake_case_           =				self._gather_logprob(a			,     a   )
																						else:
																													snake_case_           =				self._logit(a			,     a			,     a			,     self.out_projs[i]   )
																													snake_case_           =				tf.nn.log_softmax(a   )
																													snake_case_           =				self.cutoffs[0] + i - 1  # No probability for the head cluster
																													snake_case_           =				head_logprob[..., cluster_prob_idx, None] + tail_logprob
																													out.append(a   )
																													if target is not None:
																																				snake_case_           =				tf.boolean_mask(a			,     a   )
																																				snake_case_           =				tf.boolean_mask(a			,     a   )
																																				snake_case_           =				self._gather_logprob(a			,     a   )
																																				cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
																						if target is not None:
																													loss += tf.scatter_nd(a			,     -cur_logprob			,     shape_list(a   )   )
															snake_case_           =				tf.concat(a			,     axis=-1   )

								if target is not None:
															if return_mean:
																						snake_case_           =				tf.reduce_mean(a   )
															# Add the training-time loss value to the layer using `self.add_loss()`.
															self.add_loss(a   )

															# Log the loss as a metric (we could log arbitrary metrics,
															# including different metrics for training and inference.
															self.add_metric(a			,     name=self.name			,     aggregation='mean' if return_mean else ''   )

								return out













from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Features, Value
from .base import TaskTemplate




@dataclass(frozen=snake_case_							)
class 					UpperCamelCase_ (     snake_case_							):



	'''simple docstring'''




	lowerCAmelCase						=					field(default='''language-modeling'''			,   metadata={'''include_in_asdict_even_if_is_default''': True}							)

	lowerCAmelCase						=					Features({'''text''': Value('''string'''							)}							)
	lowerCAmelCase						=					Features({}							)
	lowerCAmelCase						=					"text"




	@property
	def 				_UpperCamelCase     (			self   )			->       Dict[str, str]:
								return {self.text_column: "text"}





