






import tensorflow as tf

from ...tf_utils import shape_list


class      lowercase__     (			tf.keras.layers.Layer	):

	def __init__(							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase=1		,    __UpperCAmelCase=False		,    **__UpperCAmelCase       )->					List[Any]:







							'''simple docstring'''





							super().__init__(**__UpperCAmelCase       )

							lowerCAmelCase__       =						vocab_size
							lowerCAmelCase__       =						d_embed
							lowerCAmelCase__       =						d_proj

							lowerCAmelCase__       =						cutoffs + [vocab_size]
							lowerCAmelCase__       =						[0] + self.cutoffs
							lowerCAmelCase__       =						div_val

							lowerCAmelCase__       =						self.cutoffs[0]
							lowerCAmelCase__       =						len(self.cutoffs       ) - 1
							lowerCAmelCase__       =						self.shortlist_size + self.n_clusters
							lowerCAmelCase__       =						keep_order

							lowerCAmelCase__       =						[]
							lowerCAmelCase__       =						[]

	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase       )->					Optional[int]:







							'''simple docstring'''





							if self.n_clusters > 0:
													lowerCAmelCase__       =						self.add_weight(
													    shape=(self.n_clusters, self.d_embed)		,    initializer="zeros"		,    trainable=__UpperCAmelCase		,    name="cluster_weight"       )
													lowerCAmelCase__       =						self.add_weight(
													    shape=(self.n_clusters,)		,    initializer="zeros"		,    trainable=__UpperCAmelCase		,    name="cluster_bias"       )

							if self.div_val == 1:
													for i in range(len(self.cutoffs       )       ):
																			if self.d_proj != self.d_embed:
																									lowerCAmelCase__       =						self.add_weight(
																									    shape=(self.d_embed, self.d_proj)		,    initializer="zeros"		,    trainable=__UpperCAmelCase		,    name=F"out_projs_._{i}"		,    )
																									self.out_projs.append(__UpperCAmelCase       )
																			else:
																									self.out_projs.append(__UpperCAmelCase       )
																			lowerCAmelCase__       =						self.add_weight(
																			    shape=(self.vocab_size, self.d_embed)		,    initializer="zeros"		,    trainable=__UpperCAmelCase		,    name=F"out_layers_._{i}_._weight"		,    )
																			lowerCAmelCase__       =						self.add_weight(
																			    shape=(self.vocab_size,)		,    initializer="zeros"		,    trainable=__UpperCAmelCase		,    name=F"out_layers_._{i}_._bias"		,    )
																			self.out_layers.append((weight, bias)       )
							else:
													for i in range(len(self.cutoffs       )       ):
																			lowerCAmelCase__      ,							lowerCAmelCase__       =						self.cutoff_ends[i], self.cutoff_ends[i + 1]
																			lowerCAmelCase__       =						self.d_embed // (self.div_val**i)

																			lowerCAmelCase__       =						self.add_weight(
																			    shape=(d_emb_i, self.d_proj)		,    initializer="zeros"		,    trainable=__UpperCAmelCase		,    name=F"out_projs_._{i}"       )
																			self.out_projs.append(__UpperCAmelCase       )
																			lowerCAmelCase__       =						self.add_weight(
																			    shape=(r_idx - l_idx, d_emb_i)		,    initializer="zeros"		,    trainable=__UpperCAmelCase		,    name=F"out_layers_._{i}_._weight"		,    )
																			lowerCAmelCase__       =						self.add_weight(
																			    shape=(r_idx - l_idx,)		,    initializer="zeros"		,    trainable=__UpperCAmelCase		,    name=F"out_layers_._{i}_._bias"		,    )
																			self.out_layers.append((weight, bias)       )
							super().build(__UpperCAmelCase       )

	@staticmethod
	def 		UpperCAmelCase  (							__UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase=None       )->					Union[str, Any]:







							'''simple docstring'''





							lowerCAmelCase__       =						x
							if proj is not None:
													lowerCAmelCase__       =						tf.einsum("ibd,ed->ibe"		,    __UpperCAmelCase		,    __UpperCAmelCase       )
							return tf.einsum("ibd,nd->ibn"		,    __UpperCAmelCase		,    __UpperCAmelCase       ) + b

	@staticmethod
	def 		UpperCAmelCase  (							__UpperCAmelCase		,    __UpperCAmelCase       )->					Any:







							'''simple docstring'''





							lowerCAmelCase__       =						shape_list(__UpperCAmelCase       )
							lowerCAmelCase__       =						tf.range(lp_size[0]		,    dtype=target.dtype       )
							lowerCAmelCase__       =						tf.stack([r, target]		,    1       )
							return tf.gather_nd(__UpperCAmelCase		,    __UpperCAmelCase       )


	def 		UpperCAmelCase  (							self		,    __UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase=True		,    __UpperCAmelCase=False       )->					Tuple:







							'''simple docstring'''





							lowerCAmelCase__       =						0
							if self.n_clusters == 0:
													lowerCAmelCase__       =						self._logit(__UpperCAmelCase		,    self.out_layers[0][0]		,    self.out_layers[0][1]		,    self.out_projs[0]       )
													if target is not None:
																			lowerCAmelCase__       =						tf.nn.sparse_softmax_cross_entropy_with_logits(labels=__UpperCAmelCase		,    logits=__UpperCAmelCase       )
													lowerCAmelCase__       =						tf.nn.log_softmax(__UpperCAmelCase		,    axis=-1       )
							else:
													lowerCAmelCase__       =						shape_list(__UpperCAmelCase       )
													lowerCAmelCase__       =						[]
													lowerCAmelCase__       =						tf.zeros(hidden_sizes[:2]       )
													for i in range(len(self.cutoffs       )       ):
																			lowerCAmelCase__      ,							lowerCAmelCase__       =						self.cutoff_ends[i], self.cutoff_ends[i + 1]
																			if target is not None:
																									lowerCAmelCase__       =						(target >= l_idx) & (target < r_idx)
																									lowerCAmelCase__       =						tf.where(__UpperCAmelCase       )
																									lowerCAmelCase__       =						tf.boolean_mask(__UpperCAmelCase		,    __UpperCAmelCase       ) - l_idx

																			if self.div_val == 1:
																									lowerCAmelCase__       =						self.out_layers[0][0][l_idx:r_idx]
																									lowerCAmelCase__       =						self.out_layers[0][1][l_idx:r_idx]
																			else:
																									lowerCAmelCase__       =						self.out_layers[i][0]
																									lowerCAmelCase__       =						self.out_layers[i][1]

																			if i == 0:
																									lowerCAmelCase__       =						tf.concat([cur_W, self.cluster_weight]		,    0       )
																									lowerCAmelCase__       =						tf.concat([cur_b, self.cluster_bias]		,    0       )

																									lowerCAmelCase__       =						self._logit(__UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    self.out_projs[0]       )
																									lowerCAmelCase__       =						tf.nn.log_softmax(__UpperCAmelCase       )
																									out.append(head_logprob[..., : self.cutoffs[0]]       )
																									if target is not None:
																															lowerCAmelCase__       =						tf.boolean_mask(__UpperCAmelCase		,    __UpperCAmelCase       )
																															lowerCAmelCase__       =						self._gather_logprob(__UpperCAmelCase		,    __UpperCAmelCase       )
																			else:
																									lowerCAmelCase__       =						self._logit(__UpperCAmelCase		,    __UpperCAmelCase		,    __UpperCAmelCase		,    self.out_projs[i]       )
																									lowerCAmelCase__       =						tf.nn.log_softmax(__UpperCAmelCase       )
																									lowerCAmelCase__       =						self.cutoffs[0] + i - 1  # No probability for the head cluster
																									lowerCAmelCase__       =						head_logprob[..., cluster_prob_idx, None] + tail_logprob
																									out.append(__UpperCAmelCase       )
																									if target is not None:
																															lowerCAmelCase__       =						tf.boolean_mask(__UpperCAmelCase		,    __UpperCAmelCase       )
																															lowerCAmelCase__       =						tf.boolean_mask(__UpperCAmelCase		,    __UpperCAmelCase       )
																															lowerCAmelCase__       =						self._gather_logprob(__UpperCAmelCase		,    __UpperCAmelCase       )
																															cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
																			if target is not None:
																									loss += tf.scatter_nd(__UpperCAmelCase		,    -cur_logprob		,    shape_list(__UpperCAmelCase       )       )
													lowerCAmelCase__       =						tf.concat(__UpperCAmelCase		,    axis=-1       )

							if target is not None:
													if return_mean:
																			lowerCAmelCase__       =						tf.reduce_mean(__UpperCAmelCase       )
													# Add the training-time loss value to the layer using `self.add_loss()`.
													self.add_loss(__UpperCAmelCase       )

													# Log the loss as a metric (we could log arbitrary metrics,
													# including different metrics for training and inference.
													self.add_metric(__UpperCAmelCase		,    name=self.name		,    aggregation="mean" if return_mean else ""       )

							return out














from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a_							=    {'''configuration_sew''': ['''SEW_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''SEWConfig''']}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	a_							=    [
	    '''SEW_PRETRAINED_MODEL_ARCHIVE_LIST''',
	    '''SEWForCTC''',
	    '''SEWForSequenceClassification''',
	    '''SEWModel''',
	    '''SEWPreTrainedModel''',
	]

if TYPE_CHECKING:
	from .configuration_sew import SEW_PRETRAINED_CONFIG_ARCHIVE_MAP, SEWConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_sew import (
		    SEW_PRETRAINED_MODEL_ARCHIVE_LIST,
		    SEWForCTC,
		    SEWForSequenceClassification,
		    SEWModel,
		    SEWPreTrainedModel,
		)


else:
	import sys

	a_							=    _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)






