





'''simple docstring'''







import torch
from torch import nn
class 							__SCREAMING_SNAKE_CASE    (nn.Module ):






				"""simple docstring"""







				def __init__(    self	:   List[str]   ,							__a	:   int   ,							__a	:   Dict   ,							__a	:   Dict   ,							__a	:   List[str]   ,							__a	:   int=1   ,							__a	:   Optional[int]=False   ):
								super().__init__()

								_a				      =					n_token
								_a				      =					d_embed
								_a				      =					d_proj

								_a				      =					cutoffs + [n_token]
								_a				      =					[0] + self.cutoffs
								_a				      =					div_val

								_a				      =					self.cutoffs[0]
								_a				      =					len(self.cutoffs   ) - 1
								_a				      =					self.shortlist_size + self.n_clusters

								if self.n_clusters > 0:
												_a				      =					nn.Parameter(torch.zeros(self.n_clusters   ,							self.d_embed   )   )
												_a				      =					nn.Parameter(torch.zeros(self.n_clusters   )   )

								_a				      =					nn.ModuleList()
								_a				      =					nn.ParameterList()

								if div_val == 1:
												for i in range(len(self.cutoffs   )   ):
																if d_proj != d_embed:
																				self.out_projs.append(nn.Parameter(torch.FloatTensor(__a   ,							__a   )   )   )
																else:
																				self.out_projs.append(__a   )

												self.out_layers.append(nn.Linear(__a   ,							__a   )   )
								else:
												for i in range(len(self.cutoffs   )   ):
																_a				      =					self.cutoff_ends[i], self.cutoff_ends[i + 1]
																_a				      =					d_embed // (div_val**i)

																self.out_projs.append(nn.Parameter(torch.FloatTensor(__a   ,							__a   )   )   )

																self.out_layers.append(nn.Linear(__a   ,							r_idx - l_idx   )   )

								_a				      =					keep_order




				def     UpperCamelCase__  (    self	:   Any   ,							__a	:   List[Any]   ,							__a	:   Tuple   ,							__a	:   Optional[Any]   ,							__a	:   Optional[Any]   ):
								if proj is None:
												_a				      =					nn.functional.linear(__a   ,							__a   ,							bias=__a   )
								else:
												# if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
												_a				      =					nn.functional.linear(__a   ,							proj.t().contiguous()   )
												_a				      =					nn.functional.linear(__a   ,							__a   ,							bias=__a   )
												# else:
												#     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
												#     if bias is not None:
												#         logit = logit + bias

								return logit




				def     UpperCamelCase__  (    self	:   Any   ,							__a	:   Tuple   ,							__a	:   List[str]=None   ,							__a	:   Optional[Any]=False   ):

								if labels is not None:
												# Shift so that tokens < n predict n
												_a				      =					hidden[..., :-1, :].contiguous()
												_a				      =					labels[..., 1:].contiguous()
												_a				      =					hidden.view(-1   ,							hidden.size(-1   )   )
												_a				      =					labels.view(-1   )
												if hidden.size(0   ) != labels.size(0   ):
																raise RuntimeError("Input and labels should have the same size in the batch dimension."   )
								else:
												_a				      =					hidden.view(-1   ,							hidden.size(-1   )   )

								if self.n_clusters == 0:
												_a				      =					self._compute_logit(__a   ,							self.out_layers[0].weight   ,							self.out_layers[0].bias   ,							self.out_projs[0]   )
												if labels is not None:
																_a				      =					labels != -1_00
																_a				      =					torch.zeros_like(__a   ,							dtype=hidden.dtype   ,							device=hidden.device   )
																_a				      =					(
																    -nn.functional.log_softmax(__a   ,							dim=-1   )[mask].gather(1   ,							labels[mask].unsqueeze(1   )   ).squeeze(1   )
																)
												else:
																_a				      =					nn.functional.log_softmax(__a   ,							dim=-1   )
								else:
												# construct weights and biases
												_a				      =					[], []
												for i in range(len(self.cutoffs   )   ):
																if self.div_val == 1:
																				_a				      =					self.cutoff_ends[i], self.cutoff_ends[i + 1]
																				_a				      =					self.out_layers[0].weight[l_idx:r_idx]
																				_a				      =					self.out_layers[0].bias[l_idx:r_idx]
																else:
																				_a				      =					self.out_layers[i].weight
																				_a				      =					self.out_layers[i].bias

																if i == 0:
																				_a				      =					torch.cat([weight_i, self.cluster_weight]   ,							dim=0   )
																				_a				      =					torch.cat([bias_i, self.cluster_bias]   ,							dim=0   )

																weights.append(__a   )
																biases.append(__a   )

												_a				      =					weights[0], biases[0], self.out_projs[0]

												_a				      =					self._compute_logit(__a   ,							__a   ,							__a   ,							__a   )
												_a				      =					nn.functional.log_softmax(__a   ,							dim=1   )

												if labels is None:
																_a				      =					hidden.new_empty((head_logit.size(0   ), self.n_token)   )
												else:
																_a				      =					torch.zeros_like(__a   ,							dtype=hidden.dtype   ,							device=hidden.device   )

												_a				      =					0
												_a				      =					[0] + self.cutoffs
												for i in range(len(__a   ) - 1   ):
																_a				      =					cutoff_values[i], cutoff_values[i + 1]

																if labels is not None:
																				_a				      =					(labels >= l_idx) & (labels < r_idx)
																				_a				      =					mask_i.nonzero().squeeze()

																				if indices_i.numel() == 0:
																								continue

																				_a				      =					labels.index_select(0   ,							__a   ) - l_idx
																				_a				      =					head_logprob.index_select(0   ,							__a   )
																				_a				      =					hidden.index_select(0   ,							__a   )
																else:
																				_a				      =					hidden

																if i == 0:
																				if labels is not None:
																								_a				      =					head_logprob_i.gather(1   ,							target_i[:, None]   ).squeeze(1   )
																				else:
																								_a				      =					head_logprob[:, : self.cutoffs[0]]
																else:
																				_a				      =					weights[i], biases[i], self.out_projs[i]

																				_a				      =					self._compute_logit(__a   ,							__a   ,							__a   ,							__a   )
																				_a				      =					nn.functional.log_softmax(__a   ,							dim=1   )
																				_a				      =					self.cutoffs[0] + i - 1  # No probability for the head cluster
																				if labels is not None:
																								_a				      =					head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
																								    1   ,							target_i[:, None]   ).squeeze(1   )
																				else:
																								_a				      =					head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
																								_a				      =					logprob_i

																if labels is not None:
																				if (hasattr(self   ,							"keep_order"   ) and self.keep_order) or keep_order:
																								out.index_copy_(0   ,							__a   ,							-logprob_i   )
																				else:
																								out[offset : offset + logprob_i.size(0   )].copy_(-logprob_i   )
																				offset += logprob_i.size(0   )

								return out




				def     UpperCamelCase__  (    self	:   Optional[Any]   ,							__a	:   Dict   ):
								if self.n_clusters == 0:
												_a				      =					self._compute_logit(__a   ,							self.out_layers[0].weight   ,							self.out_layers[0].bias   ,							self.out_projs[0]   )
												return nn.functional.log_softmax(__a   ,							dim=-1   )
								else:
												# construct weights and biases
												_a				      =					[], []
												for i in range(len(self.cutoffs   )   ):
																if self.div_val == 1:
																				_a				      =					self.cutoff_ends[i], self.cutoff_ends[i + 1]
																				_a				      =					self.out_layers[0].weight[l_idx:r_idx]
																				_a				      =					self.out_layers[0].bias[l_idx:r_idx]
																else:
																				_a				      =					self.out_layers[i].weight
																				_a				      =					self.out_layers[i].bias

																if i == 0:
																				_a				      =					torch.cat([weight_i, self.cluster_weight]   ,							dim=0   )
																				_a				      =					torch.cat([bias_i, self.cluster_bias]   ,							dim=0   )

																weights.append(__a   )
																biases.append(__a   )

												_a				      =					weights[0], biases[0], self.out_projs[0]
												_a				      =					self._compute_logit(__a   ,							__a   ,							__a   ,							__a   )

												_a				      =					hidden.new_empty((head_logit.size(0   ), self.n_token)   )
												_a				      =					nn.functional.log_softmax(__a   ,							dim=1   )

												_a				      =					[0] + self.cutoffs
												for i in range(len(__a   ) - 1   ):
																_a				      =					cutoff_values[i], cutoff_values[i + 1]

																if i == 0:
																				_a				      =					head_logprob[:, : self.cutoffs[0]]
																else:
																				_a				      =					weights[i], biases[i], self.out_projs[i]

																				_a				      =					self._compute_logit(__a   ,							__a   ,							__a   ,							__a   )
																				_a				      =					nn.functional.log_softmax(__a   ,							dim=1   )

																				_a				      =					head_logprob[:, -i] + tail_logprob_i
																				_a				      =					logprob_i

												return out




'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_speech_available, is_torch_available


__A   :    str											= {
    "configuration_audio_spectrogram_transformer": [
        "AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "ASTConfig",
    ]
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __A   :    List[str]											= [
      "AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST",
      "ASTForAudioClassification",
      "ASTModel",
      "ASTPreTrainedModel",
  ]

try:
  if not is_speech_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __A   :    Union[str, Any]											= ["ASTFeatureExtractor"]

if TYPE_CHECKING:
  from .configuration_audio_spectrogram_transformer import (
      AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
      ASTConfig,
  )

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_audio_spectrogram_transformer import (
        AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
        ASTForAudioClassification,
        ASTModel,
        ASTPreTrainedModel,
    )

  try:
    if not is_speech_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .feature_extraction_audio_spectrogram_transformer import ASTFeatureExtractor


else:
  import sys

  __A   :    List[Any]											= _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



