





import tensorflow as tf

from ...tf_utils import shape_list

class        __SCREAMING_SNAKE_CASE					(     tf.keras.layers.Layer       ):






							def __init__(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase=1  , __lowerCAmelCase=False  , **__lowerCAmelCase  ):
														super().__init__(**__lowerCAmelCase  )

														UpperCamelCase__				     =			vocab_size
														UpperCamelCase__				     =			d_embed
														UpperCamelCase__				     =			d_proj

														UpperCamelCase__				     =			cutoffs + [vocab_size]
														UpperCamelCase__				     =			[0] + self.cutoffs
														UpperCamelCase__				     =			div_val

														UpperCamelCase__				     =			self.cutoffs[0]
														UpperCamelCase__				     =			len(self.cutoffs  ) - 1
														UpperCamelCase__				     =			self.shortlist_size + self.n_clusters
														UpperCamelCase__				     =			keep_order

														UpperCamelCase__				     =			[]
														UpperCamelCase__				     =			[]






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														if self.n_clusters > 0:
																					UpperCamelCase__				     =			self.add_weight(
																					    shape=(self.n_clusters, self.d_embed)  , initializer="""zeros"""  , trainable=__lowerCAmelCase  , name="""cluster_weight"""  )
																					UpperCamelCase__				     =			self.add_weight(
																					    shape=(self.n_clusters,)  , initializer="""zeros"""  , trainable=__lowerCAmelCase  , name="""cluster_bias"""  )

														if self.div_val == 1:
																					for i in range(len(self.cutoffs  )  ):
																												if self.d_proj != self.d_embed:
																																			UpperCamelCase__				     =			self.add_weight(
																																			    shape=(self.d_embed, self.d_proj)  , initializer="""zeros"""  , trainable=__lowerCAmelCase  , name=f"""out_projs_._{i}"""  , )
																																			self.out_projs.append(__lowerCAmelCase  )
																												else:
																																			self.out_projs.append(__lowerCAmelCase  )
																												UpperCamelCase__				     =			self.add_weight(
																												    shape=(self.vocab_size, self.d_embed)  , initializer="""zeros"""  , trainable=__lowerCAmelCase  , name=f"""out_layers_._{i}_._weight"""  , )
																												UpperCamelCase__				     =			self.add_weight(
																												    shape=(self.vocab_size,)  , initializer="""zeros"""  , trainable=__lowerCAmelCase  , name=f"""out_layers_._{i}_._bias"""  , )
																												self.out_layers.append((weight, bias)  )
														else:
																					for i in range(len(self.cutoffs  )  ):
																												UpperCamelCase__     ,     UpperCamelCase__				     =			self.cutoff_ends[i], self.cutoff_ends[i + 1]
																												UpperCamelCase__				     =			self.d_embed // (self.div_val**i)

																												UpperCamelCase__				     =			self.add_weight(
																												    shape=(d_emb_i, self.d_proj)  , initializer="""zeros"""  , trainable=__lowerCAmelCase  , name=f"""out_projs_._{i}"""  )
																												self.out_projs.append(__lowerCAmelCase  )
																												UpperCamelCase__				     =			self.add_weight(
																												    shape=(r_idx - l_idx, d_emb_i)  , initializer="""zeros"""  , trainable=__lowerCAmelCase  , name=f"""out_layers_._{i}_._weight"""  , )
																												UpperCamelCase__				     =			self.add_weight(
																												    shape=(r_idx - l_idx,)  , initializer="""zeros"""  , trainable=__lowerCAmelCase  , name=f"""out_layers_._{i}_._bias"""  , )
																												self.out_layers.append((weight, bias)  )
														super().build(__lowerCAmelCase  )






							@staticmethod
							def 				_lowerCamelCase		(						__lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase=None  ):
														UpperCamelCase__				     =			x
														if proj is not None:
																					UpperCamelCase__				     =			tf.einsum("""ibd,ed->ibe"""  , __lowerCAmelCase  , __lowerCAmelCase  )
														return tf.einsum("""ibd,nd->ibn"""  , __lowerCAmelCase  , __lowerCAmelCase  ) + b






							@staticmethod
							def 				_lowerCamelCase		(						__lowerCAmelCase  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			shape_list(__lowerCAmelCase  )
														UpperCamelCase__				     =			tf.range(lp_size[0]  , dtype=target.dtype  )
														UpperCamelCase__				     =			tf.stack([r, target]  , 1  )
														return tf.gather_nd(__lowerCAmelCase  , __lowerCAmelCase  )







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase=True  , __lowerCAmelCase=False  ):
														UpperCamelCase__				     =			0
														if self.n_clusters == 0:
																					UpperCamelCase__				     =			self._logit(__lowerCAmelCase  , self.out_layers[0][0]  , self.out_layers[0][1]  , self.out_projs[0]  )
																					if target is not None:
																												UpperCamelCase__				     =			tf.nn.sparse_softmax_cross_entropy_with_logits(labels=__lowerCAmelCase  , logits=__lowerCAmelCase  )
																					UpperCamelCase__				     =			tf.nn.log_softmax(__lowerCAmelCase  , axis=-1  )
														else:
																					UpperCamelCase__				     =			shape_list(__lowerCAmelCase  )
																					UpperCamelCase__				     =			[]
																					UpperCamelCase__				     =			tf.zeros(hidden_sizes[:2]  )
																					for i in range(len(self.cutoffs  )  ):
																												UpperCamelCase__     ,     UpperCamelCase__				     =			self.cutoff_ends[i], self.cutoff_ends[i + 1]
																												if target is not None:
																																			UpperCamelCase__				     =			(target >= l_idx) & (target < r_idx)
																																			UpperCamelCase__				     =			tf.where(__lowerCAmelCase  )
																																			UpperCamelCase__				     =			tf.boolean_mask(__lowerCAmelCase  , __lowerCAmelCase  ) - l_idx

																												if self.div_val == 1:
																																			UpperCamelCase__				     =			self.out_layers[0][0][l_idx:r_idx]
																																			UpperCamelCase__				     =			self.out_layers[0][1][l_idx:r_idx]
																												else:
																																			UpperCamelCase__				     =			self.out_layers[i][0]
																																			UpperCamelCase__				     =			self.out_layers[i][1]

																												if i == 0:
																																			UpperCamelCase__				     =			tf.concat([cur_W, self.cluster_weight]  , 0  )
																																			UpperCamelCase__				     =			tf.concat([cur_b, self.cluster_bias]  , 0  )

																																			UpperCamelCase__				     =			self._logit(__lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , self.out_projs[0]  )
																																			UpperCamelCase__				     =			tf.nn.log_softmax(__lowerCAmelCase  )
																																			out.append(head_logprob[..., : self.cutoffs[0]]  )
																																			if target is not None:
																																										UpperCamelCase__				     =			tf.boolean_mask(__lowerCAmelCase  , __lowerCAmelCase  )
																																										UpperCamelCase__				     =			self._gather_logprob(__lowerCAmelCase  , __lowerCAmelCase  )
																												else:
																																			UpperCamelCase__				     =			self._logit(__lowerCAmelCase  , __lowerCAmelCase  , __lowerCAmelCase  , self.out_projs[i]  )
																																			UpperCamelCase__				     =			tf.nn.log_softmax(__lowerCAmelCase  )
																																			UpperCamelCase__				     =			self.cutoffs[0] + i - 1  # No probability for the head cluster
																																			UpperCamelCase__				     =			head_logprob[..., cluster_prob_idx, None] + tail_logprob
																																			out.append(__lowerCAmelCase  )
																																			if target is not None:
																																										UpperCamelCase__				     =			tf.boolean_mask(__lowerCAmelCase  , __lowerCAmelCase  )
																																										UpperCamelCase__				     =			tf.boolean_mask(__lowerCAmelCase  , __lowerCAmelCase  )
																																										UpperCamelCase__				     =			self._gather_logprob(__lowerCAmelCase  , __lowerCAmelCase  )
																																										cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
																												if target is not None:
																																			loss += tf.scatter_nd(__lowerCAmelCase  , -cur_logprob  , shape_list(__lowerCAmelCase  )  )
																					UpperCamelCase__				     =			tf.concat(__lowerCAmelCase  , axis=-1  )

														if target is not None:
																					if return_mean:
																												UpperCamelCase__				     =			tf.reduce_mean(__lowerCAmelCase  )
																					# Add the training-time loss value to the layer using `self.add_loss()`.
																					self.add_loss(__lowerCAmelCase  )

																					# Log the loss as a metric (we could log arbitrary metrics,
																					# including different metrics for training and inference.
																					self.add_metric(__lowerCAmelCase  , name=self.name  , aggregation="""mean""" if return_mean else """"""  )

														return out











import math

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__											= logging.get_logger(__name__)

UpperCamelCase__											= {
    "facebook/data2vec-base-960h": "https://huggingface.co/facebook/data2vec-audio-base-960h/resolve/main/config.json",
    # See all Data2VecAudio models at https://huggingface.co/models?filter=data2vec-audio
}

class        __SCREAMING_SNAKE_CASE					(     _a       ):
							snake_case     :      Optional[Any]						     =       """data2vec-audio"""






							def __init__(						self  , __lowerCAmelCase=32  , __lowerCAmelCase=768  , __lowerCAmelCase=12  , __lowerCAmelCase=12  , __lowerCAmelCase=3072  , __lowerCAmelCase="gelu"  , __lowerCAmelCase=0.1  , __lowerCAmelCase=0.1  , __lowerCAmelCase=0.1  , __lowerCAmelCase=0.0  , __lowerCAmelCase=0.1  , __lowerCAmelCase=0.1  , __lowerCAmelCase=0.02  , __lowerCAmelCase=1E-5  , __lowerCAmelCase="gelu"  , __lowerCAmelCase=(512, 512, 512, 512, 512, 512, 512)  , __lowerCAmelCase=(5, 2, 2, 2, 2, 2, 2)  , __lowerCAmelCase=(10, 3, 3, 3, 3, 2, 2)  , __lowerCAmelCase=False  , __lowerCAmelCase=16  , __lowerCAmelCase=19  , __lowerCAmelCase=5  , __lowerCAmelCase=0.05  , __lowerCAmelCase=10  , __lowerCAmelCase=2  , __lowerCAmelCase=0.0  , __lowerCAmelCase=10  , __lowerCAmelCase=0  , __lowerCAmelCase="sum"  , __lowerCAmelCase=False  , __lowerCAmelCase=False  , __lowerCAmelCase=256  , __lowerCAmelCase=(512, 512, 512, 512, 1500)  , __lowerCAmelCase=(5, 3, 3, 1, 1)  , __lowerCAmelCase=(1, 2, 3, 1, 1)  , __lowerCAmelCase=512  , __lowerCAmelCase=0  , __lowerCAmelCase=1  , __lowerCAmelCase=2  , __lowerCAmelCase=False  , __lowerCAmelCase=3  , __lowerCAmelCase=2  , __lowerCAmelCase=3  , __lowerCAmelCase=None  , **__lowerCAmelCase  , ):
														super().__init__(**__lowerCAmelCase  , pad_token_id=__lowerCAmelCase  , bos_token_id=__lowerCAmelCase  , eos_token_id=__lowerCAmelCase  )
														UpperCamelCase__				     =			hidden_size
														UpperCamelCase__				     =			feat_extract_activation
														UpperCamelCase__				     =			list(__lowerCAmelCase  )
														UpperCamelCase__				     =			list(__lowerCAmelCase  )
														UpperCamelCase__				     =			list(__lowerCAmelCase  )
														UpperCamelCase__				     =			conv_bias
														UpperCamelCase__				     =			num_conv_pos_embeddings
														UpperCamelCase__				     =			num_conv_pos_embedding_groups
														UpperCamelCase__				     =			conv_pos_kernel_size
														UpperCamelCase__				     =			len(self.conv_dim  )
														UpperCamelCase__				     =			num_hidden_layers
														UpperCamelCase__				     =			intermediate_size
														UpperCamelCase__				     =			hidden_act
														UpperCamelCase__				     =			num_attention_heads
														UpperCamelCase__				     =			hidden_dropout
														UpperCamelCase__				     =			attention_dropout
														UpperCamelCase__				     =			activation_dropout
														UpperCamelCase__				     =			feat_proj_dropout
														UpperCamelCase__				     =			final_dropout
														UpperCamelCase__				     =			layerdrop
														UpperCamelCase__				     =			layer_norm_eps
														UpperCamelCase__				     =			initializer_range
														UpperCamelCase__				     =			vocab_size
														UpperCamelCase__				     =			use_weighted_layer_sum

														if (
														    (len(self.conv_stride  ) != self.num_feat_extract_layers)
														    or (len(self.conv_kernel  ) != self.num_feat_extract_layers)
														    or (len(self.conv_dim  ) != self.num_feat_extract_layers)
														):
																					raise ValueError(
																					    """Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =="""
																					    """ `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ="""
																					    f""" {len(self.conv_dim  )}`, `len(config.conv_stride) = {len(self.conv_stride  )}`,"""
																					    f""" `len(config.conv_kernel) = {len(self.conv_kernel  )}`."""  )

														# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
														UpperCamelCase__				     =			mask_time_prob
														UpperCamelCase__				     =			mask_time_length
														UpperCamelCase__				     =			mask_time_min_masks
														UpperCamelCase__				     =			mask_feature_prob
														UpperCamelCase__				     =			mask_feature_length
														UpperCamelCase__				     =			mask_feature_min_masks

														# ctc loss
														UpperCamelCase__				     =			ctc_loss_reduction
														UpperCamelCase__				     =			ctc_zero_infinity

														# adapter
														UpperCamelCase__				     =			add_adapter
														UpperCamelCase__				     =			adapter_kernel_size
														UpperCamelCase__				     =			adapter_stride
														UpperCamelCase__				     =			num_adapter_layers
														UpperCamelCase__				     =			output_hidden_size or hidden_size

														# SequenceClassification-specific parameter. Feel free to ignore for other classes.
														UpperCamelCase__				     =			classifier_proj_size

														# XVector-specific parameters. Feel free to ignore for other classes.
														UpperCamelCase__				     =			list(__lowerCAmelCase  )
														UpperCamelCase__				     =			list(__lowerCAmelCase  )
														UpperCamelCase__				     =			list(__lowerCAmelCase  )
														UpperCamelCase__				     =			xvector_output_dim







							@property
							def 				_lowerCamelCase		(						self  ):
														return math.prod(self.conv_stride  )




