



"""simple docstring"""







import tensorflow as tf

from ...tf_utils import shape_list
class 			__A			(      tf.keras.layers.Layer       ):







		'''simple docstring'''





		def __init__(      self		:    List[str]     ,_snake_case		:    List[str]     ,_snake_case		:    int     ,_snake_case		:    List[str]     ,_snake_case		:    int     ,_snake_case		:    List[str]=1     ,_snake_case		:    Optional[Any]=False     ,**_snake_case		:    Any		)   ->						Any:






				"""simple docstring"""



				super().__init__(**_snake_case		)

				lowercase__  :       Optional[int]												=     vocab_size
				lowercase__  :       Dict												=     d_embed
				lowercase__  :       Tuple												=     d_proj

				lowercase__  :       Union[str, Any]												=     cutoffs + [vocab_size]
				lowercase__  :       Any												=     [0] + self.cutoffs
				lowercase__  :       Tuple												=     div_val

				lowercase__  :       Any												=     self.cutoffs[0]
				lowercase__  :       Union[str, Any]												=     len(self.cutoffs		) - 1
				lowercase__  :       Optional[Any]												=     self.shortlist_size + self.n_clusters
				lowercase__  :       Union[str, Any]												=     keep_order

				lowercase__  :       List[Any]												=     []
				lowercase__  :       Union[str, Any]												=     []







		def    UpperCAmelCase       (      self		:    List[Any]     ,_snake_case		:    Optional[int]		)   ->						Union[str, Any]:






				"""simple docstring"""



				if self.n_clusters > 0:
						lowercase__  :       str												=     self.add_weight(
						    shape=(self.n_clusters, self.d_embed)     ,initializer='''zeros'''     ,trainable=_snake_case     ,name='''cluster_weight'''		)
						lowercase__  :       Optional[int]												=     self.add_weight(
						    shape=(self.n_clusters,)     ,initializer='''zeros'''     ,trainable=_snake_case     ,name='''cluster_bias'''		)

				if self.div_val == 1:
						for i in range(len(self.cutoffs		)		):
								if self.d_proj != self.d_embed:
										lowercase__  :       Any												=     self.add_weight(
										    shape=(self.d_embed, self.d_proj)     ,initializer='''zeros'''     ,trainable=_snake_case     ,name=f"""out_projs_._{i}"""     ,)
										self.out_projs.append(_snake_case		)
								else:
										self.out_projs.append(_snake_case		)
								lowercase__  :       int												=     self.add_weight(
								    shape=(self.vocab_size, self.d_embed)     ,initializer='''zeros'''     ,trainable=_snake_case     ,name=f"""out_layers_._{i}_._weight"""     ,)
								lowercase__  :       Union[str, Any]												=     self.add_weight(
								    shape=(self.vocab_size,)     ,initializer='''zeros'''     ,trainable=_snake_case     ,name=f"""out_layers_._{i}_._bias"""     ,)
								self.out_layers.append((weight, bias)		)
				else:
						for i in range(len(self.cutoffs		)		):
								lowercase__  :       Optional[Any]												=     self.cutoff_ends[i], self.cutoff_ends[i + 1]
								lowercase__  :       Optional[int]												=     self.d_embed // (self.div_val**i)

								lowercase__  :       int												=     self.add_weight(
								    shape=(d_emb_i, self.d_proj)     ,initializer='''zeros'''     ,trainable=_snake_case     ,name=f"""out_projs_._{i}"""		)
								self.out_projs.append(_snake_case		)
								lowercase__  :       Optional[int]												=     self.add_weight(
								    shape=(r_idx - l_idx, d_emb_i)     ,initializer='''zeros'''     ,trainable=_snake_case     ,name=f"""out_layers_._{i}_._weight"""     ,)
								lowercase__  :       Optional[int]												=     self.add_weight(
								    shape=(r_idx - l_idx,)     ,initializer='''zeros'''     ,trainable=_snake_case     ,name=f"""out_layers_._{i}_._bias"""     ,)
								self.out_layers.append((weight, bias)		)
				super().build(_snake_case		)







		@staticmethod
		def    UpperCAmelCase       (      _snake_case		:    Any     ,_snake_case		:    Tuple     ,_snake_case		:    Optional[Any]     ,_snake_case		:    Dict=None		)   ->						Optional[int]:






				"""simple docstring"""



				lowercase__  :       List[Any]												=     x
				if proj is not None:
						lowercase__  :       Union[str, Any]												=     tf.einsum('''ibd,ed->ibe'''     ,_snake_case     ,_snake_case		)
				return tf.einsum('''ibd,nd->ibn'''     ,_snake_case     ,_snake_case		) + b







		@staticmethod
		def    UpperCAmelCase       (      _snake_case		:    Dict     ,_snake_case		:    List[str]		)   ->						Union[str, Any]:






				"""simple docstring"""



				lowercase__  :       List[str]												=     shape_list(_snake_case		)
				lowercase__  :       Any												=     tf.range(lp_size[0]     ,dtype=target.dtype		)
				lowercase__  :       int												=     tf.stack([r, target]     ,1		)
				return tf.gather_nd(_snake_case     ,_snake_case		)







		def    UpperCAmelCase       (      self		:    Dict     ,_snake_case		:    Union[str, Any]     ,_snake_case		:    Optional[Any]     ,_snake_case		:    Any=True     ,_snake_case		:    Tuple=False		)   ->						Union[str, Any]:






				"""simple docstring"""



				lowercase__  :       Optional[int]												=     0
				if self.n_clusters == 0:
						lowercase__  :       Dict												=     self._logit(_snake_case     ,self.out_layers[0][0]     ,self.out_layers[0][1]     ,self.out_projs[0]		)
						if target is not None:
								lowercase__  :       Dict												=     tf.nn.sparse_softmax_cross_entropy_with_logits(labels=_snake_case     ,logits=_snake_case		)
						lowercase__  :       Dict												=     tf.nn.log_softmax(_snake_case     ,axis=-1		)
				else:
						lowercase__  :       Optional[Any]												=     shape_list(_snake_case		)
						lowercase__  :       Optional[int]												=     []
						lowercase__  :       List[Any]												=     tf.zeros(hidden_sizes[:2]		)
						for i in range(len(self.cutoffs		)		):
								lowercase__  :       Tuple												=     self.cutoff_ends[i], self.cutoff_ends[i + 1]
								if target is not None:
										lowercase__  :       Tuple												=     (target >= l_idx) & (target < r_idx)
										lowercase__  :       Union[str, Any]												=     tf.where(_snake_case		)
										lowercase__  :       Any												=     tf.boolean_mask(_snake_case     ,_snake_case		) - l_idx

								if self.div_val == 1:
										lowercase__  :       List[str]												=     self.out_layers[0][0][l_idx:r_idx]
										lowercase__  :       str												=     self.out_layers[0][1][l_idx:r_idx]
								else:
										lowercase__  :       Union[str, Any]												=     self.out_layers[i][0]
										lowercase__  :       Tuple												=     self.out_layers[i][1]

								if i == 0:
										lowercase__  :       Optional[int]												=     tf.concat([cur_W, self.cluster_weight]     ,0		)
										lowercase__  :       Tuple												=     tf.concat([cur_b, self.cluster_bias]     ,0		)

										lowercase__  :       Dict												=     self._logit(_snake_case     ,_snake_case     ,_snake_case     ,self.out_projs[0]		)
										lowercase__  :       Tuple												=     tf.nn.log_softmax(_snake_case		)
										out.append(head_logprob[..., : self.cutoffs[0]]		)
										if target is not None:
												lowercase__  :       Union[str, Any]												=     tf.boolean_mask(_snake_case     ,_snake_case		)
												lowercase__  :       List[str]												=     self._gather_logprob(_snake_case     ,_snake_case		)
								else:
										lowercase__  :       List[str]												=     self._logit(_snake_case     ,_snake_case     ,_snake_case     ,self.out_projs[i]		)
										lowercase__  :       Any												=     tf.nn.log_softmax(_snake_case		)
										lowercase__  :       List[Any]												=     self.cutoffs[0] + i - 1  # No probability for the head cluster
										lowercase__  :       Union[str, Any]												=     head_logprob[..., cluster_prob_idx, None] + tail_logprob
										out.append(_snake_case		)
										if target is not None:
												lowercase__  :       Optional[Any]												=     tf.boolean_mask(_snake_case     ,_snake_case		)
												lowercase__  :       Dict												=     tf.boolean_mask(_snake_case     ,_snake_case		)
												lowercase__  :       Optional[Any]												=     self._gather_logprob(_snake_case     ,_snake_case		)
												cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
								if target is not None:
										loss += tf.scatter_nd(_snake_case     ,-cur_logprob     ,shape_list(_snake_case		)		)
						lowercase__  :       Union[str, Any]												=     tf.concat(_snake_case     ,axis=-1		)

				if target is not None:
						if return_mean:
								lowercase__  :       Dict												=     tf.reduce_mean(_snake_case		)
						# Add the training-time loss value to the layer using `self.add_loss()`.
						self.add_loss(_snake_case		)

						# Log the loss as a metric (we could log arbitrary metrics,
						# including different metrics for training and inference.
						self.add_metric(_snake_case     ,name=self.name     ,aggregation='''mean''' if return_mean else ''''''		)

				return out




'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


_UpperCamelCase 							=  {"""configuration_vit""": ["""VIT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """ViTConfig""", """ViTOnnxConfig"""]}

try:
      if not is_vision_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _UpperCamelCase 							=  ["""ViTFeatureExtractor"""]
      _UpperCamelCase 							=  ["""ViTImageProcessor"""]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _UpperCamelCase 							=  [
          """VIT_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """ViTForImageClassification""",
          """ViTForMaskedImageModeling""",
          """ViTModel""",
          """ViTPreTrainedModel""",
      ]

try:
      if not is_tf_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _UpperCamelCase 							=  [
          """TFViTForImageClassification""",
          """TFViTModel""",
          """TFViTPreTrainedModel""",
      ]

try:
      if not is_flax_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      _UpperCamelCase 							=  [
          """FlaxViTForImageClassification""",
          """FlaxViTModel""",
          """FlaxViTPreTrainedModel""",
      ]

if TYPE_CHECKING:
      from .configuration_vit import VIT_PRETRAINED_CONFIG_ARCHIVE_MAP, ViTConfig, ViTOnnxConfig

      try:
            if not is_vision_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .feature_extraction_vit import ViTFeatureExtractor
            from .image_processing_vit import ViTImageProcessor

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_vit import (
                VIT_PRETRAINED_MODEL_ARCHIVE_LIST,
                ViTForImageClassification,
                ViTForMaskedImageModeling,
                ViTModel,
                ViTPreTrainedModel,
            )

      try:
            if not is_tf_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_tf_vit import TFViTForImageClassification, TFViTModel, TFViTPreTrainedModel

      try:
            if not is_flax_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_flax_vit import FlaxViTForImageClassification, FlaxViTModel, FlaxViTPreTrainedModel


else:
      import sys

      _UpperCamelCase 							=  _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

