





import torch
from torch import nn

class 		__a				(						nn.Module   ):



       def __init__(  self						: Union[str, Any]       ,      UpperCAmelCase						: Tuple       ,      UpperCAmelCase						: Tuple       ,      UpperCAmelCase						: Tuple       ,      UpperCAmelCase						: Any       ,      UpperCAmelCase						: Dict=1       ,      UpperCAmelCase						: Optional[int]=False      ):
           super().__init__()

           lowerCAmelCase_       :						str							= n_token
           lowerCAmelCase_       :						Dict							= d_embed
           lowerCAmelCase_       :						Union[str, Any]							= d_proj

           lowerCAmelCase_       :						List[str]							= cutoffs + [n_token]
           lowerCAmelCase_       :						Optional[int]							= [0] + self.cutoffs
           lowerCAmelCase_       :						int							= div_val

           lowerCAmelCase_       :						Tuple							= self.cutoffs[0]
           lowerCAmelCase_       :						Dict							= len(self.cutoffs      ) - 1
           lowerCAmelCase_       :						Optional[Any]							= self.shortlist_size + self.n_clusters

           if self.n_clusters > 0:
               lowerCAmelCase_       :						str							= nn.Parameter(torch.zeros(self.n_clusters       ,      self.d_embed      )      )
               lowerCAmelCase_       :						List[str]							= nn.Parameter(torch.zeros(self.n_clusters      )      )

           lowerCAmelCase_       :						List[Any]							= nn.ModuleList()
           lowerCAmelCase_       :						List[Any]							= nn.ParameterList()

           if div_val == 1:
               for i in range(len(self.cutoffs      )      ):
                   if d_proj != d_embed:
                       self.out_projs.append(nn.Parameter(torch.FloatTensor(A_       ,      A_      )      )      )
                   else:
                       self.out_projs.append(A_      )

               self.out_layers.append(nn.Linear(A_       ,      A_      )      )
           else:
               for i in range(len(self.cutoffs      )      ):
                   lowerCAmelCase_		,		lowerCAmelCase_       :						Dict							= self.cutoff_ends[i], self.cutoff_ends[i + 1]
                   lowerCAmelCase_       :						Dict							= d_embed // (div_val**i)

                   self.out_projs.append(nn.Parameter(torch.FloatTensor(A_       ,      A_      )      )      )

                   self.out_layers.append(nn.Linear(A_       ,      r_idx - l_idx      )      )

           lowerCAmelCase_       :						Union[str, Any]							= keep_order



       def 			A				(  self						: Any       ,      UpperCAmelCase						: List[Any]       ,      UpperCAmelCase						: Tuple       ,      UpperCAmelCase						: int       ,      UpperCAmelCase						: Dict      ):

           if proj is None:
               lowerCAmelCase_       :						Dict							= nn.functional.linear(A_       ,      A_       ,      bias=A_      )
           else:
               # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
               lowerCAmelCase_       :						List[str]							= nn.functional.linear(A_       ,      proj.t().contiguous()      )
               lowerCAmelCase_       :						Optional[int]							= nn.functional.linear(A_       ,      A_       ,      bias=A_      )
               # else:
               #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
               #     if bias is not None:
               #         logit = logit + bias

           return logit



       def 			A				(  self						: Optional[int]       ,      UpperCAmelCase						: List[Any]       ,      UpperCAmelCase						: Tuple=None       ,      UpperCAmelCase						: List[str]=False      ):

           if labels is not None:
               # Shift so that tokens < n predict n
               lowerCAmelCase_       :						int							= hidden[..., :-1, :].contiguous()
               lowerCAmelCase_       :						Any							= labels[..., 1:].contiguous()
               lowerCAmelCase_       :						Dict							= hidden.view(-1       ,      hidden.size(-1      )      )
               lowerCAmelCase_       :						Dict							= labels.view(-1      )
               if hidden.size(0      ) != labels.size(0      ):
                   raise RuntimeError("""Input and labels should have the same size in the batch dimension."""      )
           else:
               lowerCAmelCase_       :						Any							= hidden.view(-1       ,      hidden.size(-1      )      )

           if self.n_clusters == 0:
               lowerCAmelCase_       :						Any							= self._compute_logit(A_       ,      self.out_layers[0].weight       ,      self.out_layers[0].bias       ,      self.out_projs[0]      )
               if labels is not None:
                   lowerCAmelCase_       :						Optional[int]							= labels != -1_00
                   lowerCAmelCase_       :						List[Any]							= torch.zeros_like(A_       ,      dtype=hidden.dtype       ,      device=hidden.device      )
                   lowerCAmelCase_       :						Tuple							= (
                       -nn.functional.log_softmax(A_       ,      dim=-1      )[mask].gather(1       ,      labels[mask].unsqueeze(1      )      ).squeeze(1      )
                   )
               else:
                   lowerCAmelCase_       :						int							= nn.functional.log_softmax(A_       ,      dim=-1      )
           else:
               # construct weights and biases
               lowerCAmelCase_		,		lowerCAmelCase_       :						Any							= [], []
               for i in range(len(self.cutoffs      )      ):
                   if self.div_val == 1:
                       lowerCAmelCase_		,		lowerCAmelCase_       :						int							= self.cutoff_ends[i], self.cutoff_ends[i + 1]
                       lowerCAmelCase_       :						Dict							= self.out_layers[0].weight[l_idx:r_idx]
                       lowerCAmelCase_       :						Dict							= self.out_layers[0].bias[l_idx:r_idx]
                   else:
                       lowerCAmelCase_       :						List[Any]							= self.out_layers[i].weight
                       lowerCAmelCase_       :						List[Any]							= self.out_layers[i].bias

                   if i == 0:
                       lowerCAmelCase_       :						Any							= torch.cat([weight_i, self.cluster_weight]       ,      dim=0      )
                       lowerCAmelCase_       :						Optional[int]							= torch.cat([bias_i, self.cluster_bias]       ,      dim=0      )

                   weights.append(A_      )
                   biases.append(A_      )

               lowerCAmelCase_		,		lowerCAmelCase_		,		lowerCAmelCase_       :						str							= weights[0], biases[0], self.out_projs[0]

               lowerCAmelCase_       :						List[str]							= self._compute_logit(A_       ,      A_       ,      A_       ,      A_      )
               lowerCAmelCase_       :						int							= nn.functional.log_softmax(A_       ,      dim=1      )

               if labels is None:
                   lowerCAmelCase_       :						List[str]							= hidden.new_empty((head_logit.size(0      ), self.n_token)      )
               else:
                   lowerCAmelCase_       :						Optional[int]							= torch.zeros_like(A_       ,      dtype=hidden.dtype       ,      device=hidden.device      )

               lowerCAmelCase_       :						Optional[int]							= 0
               lowerCAmelCase_       :						Dict							= [0] + self.cutoffs
               for i in range(len(A_      ) - 1      ):
                   lowerCAmelCase_		,		lowerCAmelCase_       :						Optional[Any]							= cutoff_values[i], cutoff_values[i + 1]

                   if labels is not None:
                       lowerCAmelCase_       :						List[str]							= (labels >= l_idx) & (labels < r_idx)
                       lowerCAmelCase_       :						Union[str, Any]							= mask_i.nonzero().squeeze()

                       if indices_i.numel() == 0:
                           continue

                       lowerCAmelCase_       :						Any							= labels.index_select(0       ,      A_      ) - l_idx
                       lowerCAmelCase_       :						Dict							= head_logprob.index_select(0       ,      A_      )
                       lowerCAmelCase_       :						Dict							= hidden.index_select(0       ,      A_      )
                   else:
                       lowerCAmelCase_       :						Optional[int]							= hidden

                   if i == 0:
                       if labels is not None:
                           lowerCAmelCase_       :						List[Any]							= head_logprob_i.gather(1       ,      target_i[:, None]      ).squeeze(1      )
                       else:
                           lowerCAmelCase_       :						List[Any]							= head_logprob[:, : self.cutoffs[0]]
                   else:
                       lowerCAmelCase_		,		lowerCAmelCase_		,		lowerCAmelCase_       :						Tuple							= weights[i], biases[i], self.out_projs[i]

                       lowerCAmelCase_       :						List[str]							= self._compute_logit(A_       ,      A_       ,      A_       ,      A_      )
                       lowerCAmelCase_       :						Any							= nn.functional.log_softmax(A_       ,      dim=1      )
                       lowerCAmelCase_       :						Union[str, Any]							= self.cutoffs[0] + i - 1  # No probability for the head cluster
                       if labels is not None:
                           lowerCAmelCase_       :						List[str]							= head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                               1       ,      target_i[:, None]      ).squeeze(1      )
                       else:
                           lowerCAmelCase_       :						Dict							= head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                           lowerCAmelCase_       :						Dict							= logprob_i

                   if labels is not None:
                       if (hasattr(self       ,      """keep_order"""      ) and self.keep_order) or keep_order:
                           out.index_copy_(0       ,      A_       ,      -logprob_i      )
                       else:
                           out[offset : offset + logprob_i.size(0      )].copy_(-logprob_i      )
                       offset += logprob_i.size(0      )

           return out



       def 			A				(  self						: Optional[Any]       ,      UpperCAmelCase						: Any      ):

           if self.n_clusters == 0:
               lowerCAmelCase_       :						Any							= self._compute_logit(A_       ,      self.out_layers[0].weight       ,      self.out_layers[0].bias       ,      self.out_projs[0]      )
               return nn.functional.log_softmax(A_       ,      dim=-1      )
           else:
               # construct weights and biases
               lowerCAmelCase_		,		lowerCAmelCase_       :						Union[str, Any]							= [], []
               for i in range(len(self.cutoffs      )      ):
                   if self.div_val == 1:
                       lowerCAmelCase_		,		lowerCAmelCase_       :						Dict							= self.cutoff_ends[i], self.cutoff_ends[i + 1]
                       lowerCAmelCase_       :						Dict							= self.out_layers[0].weight[l_idx:r_idx]
                       lowerCAmelCase_       :						List[str]							= self.out_layers[0].bias[l_idx:r_idx]
                   else:
                       lowerCAmelCase_       :						Union[str, Any]							= self.out_layers[i].weight
                       lowerCAmelCase_       :						Union[str, Any]							= self.out_layers[i].bias

                   if i == 0:
                       lowerCAmelCase_       :						Union[str, Any]							= torch.cat([weight_i, self.cluster_weight]       ,      dim=0      )
                       lowerCAmelCase_       :						List[str]							= torch.cat([bias_i, self.cluster_bias]       ,      dim=0      )

                   weights.append(A_      )
                   biases.append(A_      )

               lowerCAmelCase_		,		lowerCAmelCase_		,		lowerCAmelCase_       :						Tuple							= weights[0], biases[0], self.out_projs[0]
               lowerCAmelCase_       :						str							= self._compute_logit(A_       ,      A_       ,      A_       ,      A_      )

               lowerCAmelCase_       :						Any							= hidden.new_empty((head_logit.size(0      ), self.n_token)      )
               lowerCAmelCase_       :						List[Any]							= nn.functional.log_softmax(A_       ,      dim=1      )

               lowerCAmelCase_       :						Optional[int]							= [0] + self.cutoffs
               for i in range(len(A_      ) - 1      ):
                   lowerCAmelCase_		,		lowerCAmelCase_       :						Optional[int]							= cutoff_values[i], cutoff_values[i + 1]

                   if i == 0:
                       lowerCAmelCase_       :						Tuple							= head_logprob[:, : self.cutoffs[0]]
                   else:
                       lowerCAmelCase_		,		lowerCAmelCase_		,		lowerCAmelCase_       :						Any							= weights[i], biases[i], self.out_projs[i]

                       lowerCAmelCase_       :						Optional[Any]							= self._compute_logit(A_       ,      A_       ,      A_       ,      A_      )
                       lowerCAmelCase_       :						Optional[Any]							= nn.functional.log_softmax(A_       ,      dim=1      )

                       lowerCAmelCase_       :						Optional[int]							= head_logprob[:, -i] + tail_logprob_i
                       lowerCAmelCase_       :						List[Any]							= logprob_i

               return out









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_A					:				Any				=			{"""configuration_xglm""": ["""XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP""", """XGLMConfig"""]}

try:
  if not is_sentencepiece_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _A					:				Optional[Any]				=			["""XGLMTokenizer"""]

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _A					:				Any				=			["""XGLMTokenizerFast"""]

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _A					:				str				=			[
      """XGLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """XGLMForCausalLM""",
      """XGLMModel""",
      """XGLMPreTrainedModel""",
  ]


try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _A					:				Tuple				=			[
      """FlaxXGLMForCausalLM""",
      """FlaxXGLMModel""",
      """FlaxXGLMPreTrainedModel""",
  ]


try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _A					:				List[Any]				=			[
      """TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST""",
      """TFXGLMForCausalLM""",
      """TFXGLMModel""",
      """TFXGLMPreTrainedModel""",
  ]


if TYPE_CHECKING:
  from .configuration_xglm import XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XGLMConfig

  try:
    if not is_sentencepiece_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_xglm import XGLMTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_xglm_fast import XGLMTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_xglm import XGLM_PRETRAINED_MODEL_ARCHIVE_LIST, XGLMForCausalLM, XGLMModel, XGLMPreTrainedModel

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_flax_xglm import FlaxXGLMForCausalLM, FlaxXGLMModel, FlaxXGLMPreTrainedModel

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_xglm import (
        TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXGLMForCausalLM,
        TFXGLMModel,
        TFXGLMPreTrainedModel,
    )


else:
  import sys

  _A					:				List[Any]				=			_LazyModule(__name__, globals()["""__file__"""], _import_structure)
