





import tensorflow as tf

from ...tf_utils import shape_list






class  lowercase    (      tf.keras.layers.Layer       ):






					"""simple docstring"""



					def __init__(    self						:     List[str] ,					a_						:     List[str] ,					a_						:     List[Any] ,					a_						:     Tuple ,					a_						:     Optional[int] ,					a_						:     List[Any]=1 ,					a_						:     Any=False ,					**a_						:     Optional[Any]							):

						"""simple docstring"""



						super().__init__(**a_							)

						lowerCamelCase__										=			vocab_size
						lowerCamelCase__										=			d_embed
						lowerCamelCase__										=			d_proj

						lowerCamelCase__										=			cutoffs + [vocab_size]
						lowerCamelCase__										=			[0] + self.cutoffs
						lowerCamelCase__										=			div_val

						lowerCamelCase__										=			self.cutoffs[0]
						lowerCamelCase__										=			len(self.cutoffs							) - 1
						lowerCamelCase__										=			self.shortlist_size + self.n_clusters
						lowerCamelCase__										=			keep_order

						lowerCamelCase__										=			[]
						lowerCamelCase__										=			[]





					def       _UpperCamelCase       (    self						:     Optional[int] ,					a_						:     str							):

						"""simple docstring"""



						if self.n_clusters > 0:
							lowerCamelCase__										=			self.add_weight(
							    shape=(self.n_clusters, self.d_embed) ,					initializer="""zeros""" ,					trainable=a_ ,					name="""cluster_weight"""							)
							lowerCamelCase__										=			self.add_weight(
							    shape=(self.n_clusters,) ,					initializer="""zeros""" ,					trainable=a_ ,					name="""cluster_bias"""							)

						if self.div_val == 1:
							for i in range(len(self.cutoffs							)							):
								if self.d_proj != self.d_embed:
									lowerCamelCase__										=			self.add_weight(
									    shape=(self.d_embed, self.d_proj) ,					initializer="""zeros""" ,					trainable=a_ ,					name=F'''out_projs_._{i}''' ,					)
									self.out_projs.append(a_							)
								else:
									self.out_projs.append(a_							)
								lowerCamelCase__										=			self.add_weight(
								    shape=(self.vocab_size, self.d_embed) ,					initializer="""zeros""" ,					trainable=a_ ,					name=F'''out_layers_._{i}_._weight''' ,					)
								lowerCamelCase__										=			self.add_weight(
								    shape=(self.vocab_size,) ,					initializer="""zeros""" ,					trainable=a_ ,					name=F'''out_layers_._{i}_._bias''' ,					)
								self.out_layers.append((weight, bias)							)
						else:
							for i in range(len(self.cutoffs							)							):
								lowerCamelCase__										=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
								lowerCamelCase__										=			self.d_embed // (self.div_val**i)

								lowerCamelCase__										=			self.add_weight(
								    shape=(d_emb_i, self.d_proj) ,					initializer="""zeros""" ,					trainable=a_ ,					name=F'''out_projs_._{i}'''							)
								self.out_projs.append(a_							)
								lowerCamelCase__										=			self.add_weight(
								    shape=(r_idx - l_idx, d_emb_i) ,					initializer="""zeros""" ,					trainable=a_ ,					name=F'''out_layers_._{i}_._weight''' ,					)
								lowerCamelCase__										=			self.add_weight(
								    shape=(r_idx - l_idx,) ,					initializer="""zeros""" ,					trainable=a_ ,					name=F'''out_layers_._{i}_._bias''' ,					)
								self.out_layers.append((weight, bias)							)
						super().build(a_							)





					@staticmethod
					def       _UpperCamelCase       (    a_						:     Optional[int] ,					a_						:     Union[str, Any] ,					a_						:     Optional[int] ,					a_						:     List[Any]=None							):

						"""simple docstring"""



						lowerCamelCase__										=			x
						if proj is not None:
							lowerCamelCase__										=			tf.einsum("""ibd,ed->ibe""" ,					a_ ,					a_							)
						return tf.einsum("""ibd,nd->ibn""" ,					a_ ,					a_							) + b





					@staticmethod
					def       _UpperCamelCase       (    a_						:     List[Any] ,					a_						:     str							):

						"""simple docstring"""



						lowerCamelCase__										=			shape_list(a_							)
						lowerCamelCase__										=			tf.range(lp_size[0] ,					dtype=target.dtype							)
						lowerCamelCase__										=			tf.stack([r, target] ,					1							)
						return tf.gather_nd(a_ ,					a_							)





					def       _UpperCamelCase       (    self						:     Any ,					a_						:     Optional[Any] ,					a_						:     int ,					a_						:     int=True ,					a_						:     Tuple=False							):

						"""simple docstring"""



						lowerCamelCase__										=			0
						if self.n_clusters == 0:
							lowerCamelCase__										=			self._logit(a_ ,					self.out_layers[0][0] ,					self.out_layers[0][1] ,					self.out_projs[0]							)
							if target is not None:
								lowerCamelCase__										=			tf.nn.sparse_softmax_cross_entropy_with_logits(labels=a_ ,					logits=a_							)
							lowerCamelCase__										=			tf.nn.log_softmax(a_ ,					axis=-1							)
						else:
							lowerCamelCase__										=			shape_list(a_							)
							lowerCamelCase__										=			[]
							lowerCamelCase__										=			tf.zeros(hidden_sizes[:2]							)
							for i in range(len(self.cutoffs							)							):
								lowerCamelCase__										=			self.cutoff_ends[i], self.cutoff_ends[i + 1]
								if target is not None:
									lowerCamelCase__										=			(target >= l_idx) & (target < r_idx)
									lowerCamelCase__										=			tf.where(a_							)
									lowerCamelCase__										=			tf.boolean_mask(a_ ,					a_							) - l_idx

								if self.div_val == 1:
									lowerCamelCase__										=			self.out_layers[0][0][l_idx:r_idx]
									lowerCamelCase__										=			self.out_layers[0][1][l_idx:r_idx]
								else:
									lowerCamelCase__										=			self.out_layers[i][0]
									lowerCamelCase__										=			self.out_layers[i][1]

								if i == 0:
									lowerCamelCase__										=			tf.concat([cur_W, self.cluster_weight] ,					0							)
									lowerCamelCase__										=			tf.concat([cur_b, self.cluster_bias] ,					0							)

									lowerCamelCase__										=			self._logit(a_ ,					a_ ,					a_ ,					self.out_projs[0]							)
									lowerCamelCase__										=			tf.nn.log_softmax(a_							)
									out.append(head_logprob[..., : self.cutoffs[0]]							)
									if target is not None:
										lowerCamelCase__										=			tf.boolean_mask(a_ ,					a_							)
										lowerCamelCase__										=			self._gather_logprob(a_ ,					a_							)
								else:
									lowerCamelCase__										=			self._logit(a_ ,					a_ ,					a_ ,					self.out_projs[i]							)
									lowerCamelCase__										=			tf.nn.log_softmax(a_							)
									lowerCamelCase__										=			self.cutoffs[0] + i - 1  # No probability for the head cluster
									lowerCamelCase__										=			head_logprob[..., cluster_prob_idx, None] + tail_logprob
									out.append(a_							)
									if target is not None:
										lowerCamelCase__										=			tf.boolean_mask(a_ ,					a_							)
										lowerCamelCase__										=			tf.boolean_mask(a_ ,					a_							)
										lowerCamelCase__										=			self._gather_logprob(a_ ,					a_							)
										cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
								if target is not None:
									loss += tf.scatter_nd(a_ ,					-cur_logprob ,					shape_list(a_							)							)
							lowerCamelCase__										=			tf.concat(a_ ,					axis=-1							)

						if target is not None:
							if return_mean:
								lowerCamelCase__										=			tf.reduce_mean(a_							)
							# Add the training-time loss value to the layer using `self.add_loss()`.
							self.add_loss(a_							)

							# Log the loss as a metric (we could log arbitrary metrics,
							# including different metrics for training and inference.
							self.add_metric(a_ ,					name=self.name ,					aggregation="""mean""" if return_mean else """"""							)

						return out


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


lowercase    :   List[str]						=			{
    'configuration_gpt_bigcode': ['GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP', 'GPTBigCodeConfig'],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase    :   List[str]						=			[
		    'GPT_BIGCODE_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'GPTBigCodeForSequenceClassification',
		    'GPTBigCodeForTokenClassification',
		    'GPTBigCodeForCausalLM',
		    'GPTBigCodeModel',
		    'GPTBigCodePreTrainedModel',
		]

if TYPE_CHECKING:
		from .configuration_gpt_bigcode import GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTBigCodeConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_gpt_bigcode import (
				    GPT_BIGCODE_PRETRAINED_MODEL_ARCHIVE_LIST,
				    GPTBigCodeForCausalLM,
				    GPTBigCodeForSequenceClassification,
				    GPTBigCodeForTokenClassification,
				    GPTBigCodeModel,
				    GPTBigCodePreTrainedModel,
				)


else:
		import sys

		lowercase    :   Tuple						=			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)