

import tensorflow as tf

from ...tf_utils import shape_list



class        A__(tf.keras.layers.Layer							):
						"""simple docstring"""






						def __init__(    self  ,      _lowercase  ,      _lowercase  ,      _lowercase  ,      _lowercase  ,      _lowercase=1  ,      _lowercase=False  ,      **_lowercase    )     -> Union[str, Any]:
									super().__init__(**_lowercase    )

									a_						:						Dict 							= vocab_size
									a_						:						List[Any] 							= d_embed
									a_						:						Union[str, Any] 							= d_proj

									a_						:						Dict 							= cutoffs + [vocab_size]
									a_						:						Tuple 							= [0] + self.cutoffs
									a_						:						List[str] 							= div_val

									a_						:						Optional[int] 							= self.cutoffs[0]
									a_						:						Tuple 							= len(self.cutoffs    ) - 1
									a_						:						Optional[int] 							= self.shortlist_size + self.n_clusters
									a_						:						Dict 							= keep_order

									a_						:						Optional[Any] 							= []
									a_						:						Optional[Any] 							= []





						def 			UpperCamelCase__					(    self  ,      _lowercase    )     -> Optional[Any]:
									if self.n_clusters > 0:
												a_						:						Any 							= self.add_weight(
												    shape=(self.n_clusters, self.d_embed)  ,      initializer="""zeros"""  ,      trainable=_lowercase  ,      name="""cluster_weight"""    )
												a_						:						Optional[Any] 							= self.add_weight(
												    shape=(self.n_clusters,)  ,      initializer="""zeros"""  ,      trainable=_lowercase  ,      name="""cluster_bias"""    )

									if self.div_val == 1:
												for i in range(len(self.cutoffs    )    ):
															if self.d_proj != self.d_embed:
																		a_						:						Optional[Any] 							= self.add_weight(
																		    shape=(self.d_embed, self.d_proj)  ,      initializer="""zeros"""  ,      trainable=_lowercase  ,      name=F'''out_projs_._{i}'''  ,      )
																		self.out_projs.append(_lowercase    )
															else:
																		self.out_projs.append(_lowercase    )
															a_						:						Optional[Any] 							= self.add_weight(
															    shape=(self.vocab_size, self.d_embed)  ,      initializer="""zeros"""  ,      trainable=_lowercase  ,      name=F'''out_layers_._{i}_._weight'''  ,      )
															a_						:						List[Any] 							= self.add_weight(
															    shape=(self.vocab_size,)  ,      initializer="""zeros"""  ,      trainable=_lowercase  ,      name=F'''out_layers_._{i}_._bias'''  ,      )
															self.out_layers.append((weight, bias)    )
									else:
												for i in range(len(self.cutoffs    )    ):
															a_							,					a_						:						int 							= self.cutoff_ends[i], self.cutoff_ends[i + 1]
															a_						:						Optional[int] 							= self.d_embed // (self.div_val**i)

															a_						:						Union[str, Any] 							= self.add_weight(
															    shape=(d_emb_i, self.d_proj)  ,      initializer="""zeros"""  ,      trainable=_lowercase  ,      name=F'''out_projs_._{i}'''    )
															self.out_projs.append(_lowercase    )
															a_						:						Union[str, Any] 							= self.add_weight(
															    shape=(r_idx - l_idx, d_emb_i)  ,      initializer="""zeros"""  ,      trainable=_lowercase  ,      name=F'''out_layers_._{i}_._weight'''  ,      )
															a_						:						List[Any] 							= self.add_weight(
															    shape=(r_idx - l_idx,)  ,      initializer="""zeros"""  ,      trainable=_lowercase  ,      name=F'''out_layers_._{i}_._bias'''  ,      )
															self.out_layers.append((weight, bias)    )
									super().build(_lowercase    )





						@staticmethod
						def 			UpperCamelCase__					(    _lowercase  ,      _lowercase  ,      _lowercase  ,      _lowercase=None    )     -> str:
									a_						:						List[str] 							= x
									if proj is not None:
												a_						:						int 							= tf.einsum("""ibd,ed->ibe"""  ,      _lowercase  ,      _lowercase    )
									return tf.einsum("""ibd,nd->ibn"""  ,      _lowercase  ,      _lowercase    ) + b





						@staticmethod
						def 			UpperCamelCase__					(    _lowercase  ,      _lowercase    )     -> Optional[int]:
									a_						:						Optional[int] 							= shape_list(_lowercase    )
									a_						:						Tuple 							= tf.range(lp_size[0]  ,      dtype=target.dtype    )
									a_						:						str 							= tf.stack([r, target]  ,      1    )
									return tf.gather_nd(_lowercase  ,      _lowercase    )





						def 			UpperCamelCase__					(    self  ,      _lowercase  ,      _lowercase  ,      _lowercase=True  ,      _lowercase=False    )     -> int:
									a_						:						Dict 							= 0
									if self.n_clusters == 0:
												a_						:						Optional[Any] 							= self._logit(_lowercase  ,      self.out_layers[0][0]  ,      self.out_layers[0][1]  ,      self.out_projs[0]    )
												if target is not None:
															a_						:						str 							= tf.nn.sparse_softmax_cross_entropy_with_logits(labels=_lowercase  ,      logits=_lowercase    )
												a_						:						Any 							= tf.nn.log_softmax(_lowercase  ,      axis=-1    )
									else:
												a_						:						List[Any] 							= shape_list(_lowercase    )
												a_						:						Dict 							= []
												a_						:						str 							= tf.zeros(hidden_sizes[:2]    )
												for i in range(len(self.cutoffs    )    ):
															a_							,					a_						:						List[str] 							= self.cutoff_ends[i], self.cutoff_ends[i + 1]
															if target is not None:
																		a_						:						Tuple 							= (target >= l_idx) & (target < r_idx)
																		a_						:						Any 							= tf.where(_lowercase    )
																		a_						:						Optional[int] 							= tf.boolean_mask(_lowercase  ,      _lowercase    ) - l_idx

															if self.div_val == 1:
																		a_						:						Optional[int] 							= self.out_layers[0][0][l_idx:r_idx]
																		a_						:						int 							= self.out_layers[0][1][l_idx:r_idx]
															else:
																		a_						:						str 							= self.out_layers[i][0]
																		a_						:						Tuple 							= self.out_layers[i][1]

															if i == 0:
																		a_						:						Union[str, Any] 							= tf.concat([cur_W, self.cluster_weight]  ,      0    )
																		a_						:						Dict 							= tf.concat([cur_b, self.cluster_bias]  ,      0    )

																		a_						:						Optional[Any] 							= self._logit(_lowercase  ,      _lowercase  ,      _lowercase  ,      self.out_projs[0]    )
																		a_						:						Optional[int] 							= tf.nn.log_softmax(_lowercase    )
																		out.append(head_logprob[..., : self.cutoffs[0]]    )
																		if target is not None:
																					a_						:						Tuple 							= tf.boolean_mask(_lowercase  ,      _lowercase    )
																					a_						:						int 							= self._gather_logprob(_lowercase  ,      _lowercase    )
															else:
																		a_						:						List[str] 							= self._logit(_lowercase  ,      _lowercase  ,      _lowercase  ,      self.out_projs[i]    )
																		a_						:						Union[str, Any] 							= tf.nn.log_softmax(_lowercase    )
																		a_						:						List[str] 							= self.cutoffs[0] + i - 1  # No probability for the head cluster
																		a_						:						Dict 							= head_logprob[..., cluster_prob_idx, None] + tail_logprob
																		out.append(_lowercase    )
																		if target is not None:
																					a_						:						Dict 							= tf.boolean_mask(_lowercase  ,      _lowercase    )
																					a_						:						Optional[int] 							= tf.boolean_mask(_lowercase  ,      _lowercase    )
																					a_						:						Tuple 							= self._gather_logprob(_lowercase  ,      _lowercase    )
																					cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
															if target is not None:
																		loss += tf.scatter_nd(_lowercase  ,      -cur_logprob  ,      shape_list(_lowercase    )    )
												a_						:						Optional[int] 							= tf.concat(_lowercase  ,      axis=-1    )

									if target is not None:
												if return_mean:
															a_						:						List[str] 							= tf.reduce_mean(_lowercase    )
												# Add the training-time loss value to the layer using `self.add_loss()`.
												self.add_loss(_lowercase    )

												# Log the loss as a metric (we could log arbitrary metrics,
												# including different metrics for training and inference.
												self.add_metric(_lowercase  ,      name=self.name  ,      aggregation="""mean""" if return_mean else """"""    )

									return out





from ...configuration_utils import PretrainedConfig
from ...utils import logging


__snake_case				:      Any   		=			logging.get_logger(__name__)

__snake_case				:      Union[str, Any]   		=			{
    """microsoft/cvt-13""": """https://huggingface.co/microsoft/cvt-13/resolve/main/config.json""",
    # See all Cvt models at https://huggingface.co/models?filter=cvt
}



class        A__(a_							):
						"""simple docstring"""






						_A    :				Union[str, Any]       					=			'''cvt'''





						def __init__(    self  ,      _lowercase=3  ,      _lowercase=[7, 3, 3]  ,      _lowercase=[4, 2, 2]  ,      _lowercase=[2, 1, 1]  ,      _lowercase=[64, 192, 384]  ,      _lowercase=[1, 3, 6]  ,      _lowercase=[1, 2, 10]  ,      _lowercase=[4.0, 4.0, 4.0]  ,      _lowercase=[0.0, 0.0, 0.0]  ,      _lowercase=[0.0, 0.0, 0.0]  ,      _lowercase=[0.0, 0.0, 0.1]  ,      _lowercase=[True, True, True]  ,      _lowercase=[False, False, True]  ,      _lowercase=["dw_bn", "dw_bn", "dw_bn"]  ,      _lowercase=[3, 3, 3]  ,      _lowercase=[1, 1, 1]  ,      _lowercase=[2, 2, 2]  ,      _lowercase=[1, 1, 1]  ,      _lowercase=[1, 1, 1]  ,      _lowercase=0.0_2  ,      _lowercase=1e-12  ,      **_lowercase  ,      )     -> Tuple:
									super().__init__(**_lowercase    )
									a_						:						str 							= num_channels
									a_						:						Optional[int] 							= patch_sizes
									a_						:						Dict 							= patch_stride
									a_						:						int 							= patch_padding
									a_						:						int 							= embed_dim
									a_						:						Tuple 							= num_heads
									a_						:						Optional[Any] 							= depth
									a_						:						str 							= mlp_ratio
									a_						:						Optional[int] 							= attention_drop_rate
									a_						:						int 							= drop_rate
									a_						:						List[Any] 							= drop_path_rate
									a_						:						List[Any] 							= qkv_bias
									a_						:						Dict 							= cls_token
									a_						:						Union[str, Any] 							= qkv_projection_method
									a_						:						List[Any] 							= kernel_qkv
									a_						:						Optional[Any] 							= padding_kv
									a_						:						Any 							= stride_kv
									a_						:						Dict 							= padding_q
									a_						:						Optional[int] 							= stride_q
									a_						:						Tuple 							= initializer_range
									a_						:						int 							= layer_norm_eps


