
'''simple docstring'''
import tensorflow as tf

from ...tf_utils import shape_list


class 			SCREAMING_SNAKE_CASE__      (     tf.keras.layers.Layer):







				def __init__(       self						,				A_						,				A_						,				A_						,				A_						,				A_=1						,				A_=False						,				**A_     )->		Any:







											'''simple docstring'''






											super().__init__(**A_     )

											UpperCamelCase   			=     vocab_size
											UpperCamelCase   			=     d_embed
											UpperCamelCase   			=     d_proj

											UpperCamelCase   			=     cutoffs + [vocab_size]
											UpperCamelCase   			=     [0] + self.cutoffs
											UpperCamelCase   			=     div_val

											UpperCamelCase   			=     self.cutoffs[0]
											UpperCamelCase   			=     len(self.cutoffs     ) - 1
											UpperCamelCase   			=     self.shortlist_size + self.n_clusters
											UpperCamelCase   			=     keep_order

											UpperCamelCase   			=     []
											UpperCamelCase   			=     []







				def 						UpperCAmelCase_    (       self						,				A_     )->		Optional[Any]:







											'''simple docstring'''






											if self.n_clusters > 0:
																		UpperCamelCase   			=     self.add_weight(
																		    shape=(self.n_clusters, self.d_embed)						,				initializer='zeros'						,				trainable=A_						,				name='cluster_weight'     )
																		UpperCamelCase   			=     self.add_weight(
																		    shape=(self.n_clusters,)						,				initializer='zeros'						,				trainable=A_						,				name='cluster_bias'     )

											if self.div_val == 1:
																		for i in range(len(self.cutoffs     )     ):
																									if self.d_proj != self.d_embed:
																																UpperCamelCase   			=     self.add_weight(
																																    shape=(self.d_embed, self.d_proj)						,				initializer='zeros'						,				trainable=A_						,				name=F'''out_projs_._{i}'''						,				)
																																self.out_projs.append(A_     )
																									else:
																																self.out_projs.append(A_     )
																									UpperCamelCase   			=     self.add_weight(
																									    shape=(self.vocab_size, self.d_embed)						,				initializer='zeros'						,				trainable=A_						,				name=F'''out_layers_._{i}_._weight'''						,				)
																									UpperCamelCase   			=     self.add_weight(
																									    shape=(self.vocab_size,)						,				initializer='zeros'						,				trainable=A_						,				name=F'''out_layers_._{i}_._bias'''						,				)
																									self.out_layers.append((weight, bias)     )
											else:
																		for i in range(len(self.cutoffs     )     ):
																									UpperCamelCase			,	UpperCamelCase   			=     self.cutoff_ends[i], self.cutoff_ends[i + 1]
																									UpperCamelCase   			=     self.d_embed // (self.div_val**i)

																									UpperCamelCase   			=     self.add_weight(
																									    shape=(d_emb_i, self.d_proj)						,				initializer='zeros'						,				trainable=A_						,				name=F'''out_projs_._{i}'''     )
																									self.out_projs.append(A_     )
																									UpperCamelCase   			=     self.add_weight(
																									    shape=(r_idx - l_idx, d_emb_i)						,				initializer='zeros'						,				trainable=A_						,				name=F'''out_layers_._{i}_._weight'''						,				)
																									UpperCamelCase   			=     self.add_weight(
																									    shape=(r_idx - l_idx,)						,				initializer='zeros'						,				trainable=A_						,				name=F'''out_layers_._{i}_._bias'''						,				)
																									self.out_layers.append((weight, bias)     )
											super().build(A_     )







				@staticmethod
				def 						UpperCAmelCase_    (       A_						,				A_						,				A_						,				A_=None     )->		Optional[int]:







											'''simple docstring'''






											UpperCamelCase   			=     x
											if proj is not None:
																		UpperCamelCase   			=     tf.einsum('ibd,ed->ibe'						,				A_						,				A_     )
											return tf.einsum('ibd,nd->ibn'						,				A_						,				A_     ) + b







				@staticmethod
				def 						UpperCAmelCase_    (       A_						,				A_     )->		Dict:







											'''simple docstring'''






											UpperCamelCase   			=     shape_list(A_     )
											UpperCamelCase   			=     tf.range(lp_size[0]						,				dtype=target.dtype     )
											UpperCamelCase   			=     tf.stack([r, target]						,				1     )
											return tf.gather_nd(A_						,				A_     )







				def 						UpperCAmelCase_    (       self						,				A_						,				A_						,				A_=True						,				A_=False     )->		Dict:







											'''simple docstring'''






											UpperCamelCase   			=     0
											if self.n_clusters == 0:
																		UpperCamelCase   			=     self._logit(A_						,				self.out_layers[0][0]						,				self.out_layers[0][1]						,				self.out_projs[0]     )
																		if target is not None:
																									UpperCamelCase   			=     tf.nn.sparse_softmax_cross_entropy_with_logits(labels=A_						,				logits=A_     )
																		UpperCamelCase   			=     tf.nn.log_softmax(A_						,				axis=-1     )
											else:
																		UpperCamelCase   			=     shape_list(A_     )
																		UpperCamelCase   			=     []
																		UpperCamelCase   			=     tf.zeros(hidden_sizes[:2]     )
																		for i in range(len(self.cutoffs     )     ):
																									UpperCamelCase			,	UpperCamelCase   			=     self.cutoff_ends[i], self.cutoff_ends[i + 1]
																									if target is not None:
																																UpperCamelCase   			=     (target >= l_idx) & (target < r_idx)
																																UpperCamelCase   			=     tf.where(A_     )
																																UpperCamelCase   			=     tf.boolean_mask(A_						,				A_     ) - l_idx

																									if self.div_val == 1:
																																UpperCamelCase   			=     self.out_layers[0][0][l_idx:r_idx]
																																UpperCamelCase   			=     self.out_layers[0][1][l_idx:r_idx]
																									else:
																																UpperCamelCase   			=     self.out_layers[i][0]
																																UpperCamelCase   			=     self.out_layers[i][1]

																									if i == 0:
																																UpperCamelCase   			=     tf.concat([cur_W, self.cluster_weight]						,				0     )
																																UpperCamelCase   			=     tf.concat([cur_b, self.cluster_bias]						,				0     )

																																UpperCamelCase   			=     self._logit(A_						,				A_						,				A_						,				self.out_projs[0]     )
																																UpperCamelCase   			=     tf.nn.log_softmax(A_     )
																																out.append(head_logprob[..., : self.cutoffs[0]]     )
																																if target is not None:
																																							UpperCamelCase   			=     tf.boolean_mask(A_						,				A_     )
																																							UpperCamelCase   			=     self._gather_logprob(A_						,				A_     )
																									else:
																																UpperCamelCase   			=     self._logit(A_						,				A_						,				A_						,				self.out_projs[i]     )
																																UpperCamelCase   			=     tf.nn.log_softmax(A_     )
																																UpperCamelCase   			=     self.cutoffs[0] + i - 1  # No probability for the head cluster
																																UpperCamelCase   			=     head_logprob[..., cluster_prob_idx, None] + tail_logprob
																																out.append(A_     )
																																if target is not None:
																																							UpperCamelCase   			=     tf.boolean_mask(A_						,				A_     )
																																							UpperCamelCase   			=     tf.boolean_mask(A_						,				A_     )
																																							UpperCamelCase   			=     self._gather_logprob(A_						,				A_     )
																																							cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
																									if target is not None:
																																loss += tf.scatter_nd(A_						,				-cur_logprob						,				shape_list(A_     )     )
																		UpperCamelCase   			=     tf.concat(A_						,				axis=-1     )

											if target is not None:
																		if return_mean:
																									UpperCamelCase   			=     tf.reduce_mean(A_     )
																		# Add the training-time loss value to the layer using `self.add_loss()`.
																		self.add_loss(A_     )

																		# Log the loss as a metric (we could log arbitrary metrics,
																		# including different metrics for training and inference.
																		self.add_metric(A_						,				name=self.name						,				aggregation='mean' if return_mean else ''     )

											return out






'''simple docstring'''
from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Features, Value
from .base import TaskTemplate


@dataclass(frozen=snake_case_)
class 			SCREAMING_SNAKE_CASE__      (     snake_case_):
				lowerCAmelCase_     			=							field(default="""language-modeling"""							,       metadata={"""include_in_asdict_even_if_is_default""": True})

				lowerCAmelCase_     			=							Features({"""text""": Value("""string""")})
				lowerCAmelCase_     			=							Features({})
				lowerCAmelCase_     			=							"text"







				@property
				def 						UpperCAmelCase_    (       self     )->		Dict[str, str]:







											'''simple docstring'''






											return {self.text_column: "text"}




