

import torch
from torch import nn





class 				__lowerCAmelCase  (      nn.Module							):




  def __init__(       self							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase=1							,     lowerCAmelCase=False      )						->				Optional[int]:
      '''simple docstring'''






      super().__init__()

      _lowercase			 =n_token
      _lowercase			 =d_embed
      _lowercase			 =d_proj

      _lowercase			 =cutoffs + [n_token]
      _lowercase			 =[0] + self.cutoffs
      _lowercase			 =div_val

      _lowercase			 =self.cutoffs[0]
      _lowercase			 =len(self.cutoffs      ) - 1
      _lowercase			 =self.shortlist_size + self.n_clusters

      if self.n_clusters > 0:
          _lowercase			 =nn.Parameter(torch.zeros(self.n_clusters							,     self.d_embed      )      )
          _lowercase			 =nn.Parameter(torch.zeros(self.n_clusters      )      )

      _lowercase			 =nn.ModuleList()
      _lowercase			 =nn.ParameterList()

      if div_val == 1:
          for i in range(len(self.cutoffs      )      ):
              if d_proj != d_embed:
                  self.out_projs.append(nn.Parameter(torch.FloatTensor(UpperCAmelCase__							,     UpperCAmelCase__      )      )      )
              else:
                  self.out_projs.append(UpperCAmelCase__      )

          self.out_layers.append(nn.Linear(UpperCAmelCase__							,     UpperCAmelCase__      )      )
      else:
          for i in range(len(self.cutoffs      )      ):
              _lowercase			 =self.cutoff_ends[i], self.cutoff_ends[i + 1]
              _lowercase			 =d_embed // (div_val**i)

              self.out_projs.append(nn.Parameter(torch.FloatTensor(UpperCAmelCase__							,     UpperCAmelCase__      )      )      )

              self.out_layers.append(nn.Linear(UpperCAmelCase__							,     r_idx - l_idx      )      )

      _lowercase			 =keep_order




  def 						A__  (       self							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase      )						->				Optional[Any]:
      '''simple docstring'''






      if proj is None:
          _lowercase			 =nn.functional.linear(UpperCAmelCase__							,     UpperCAmelCase__							,     bias=UpperCAmelCase__      )
      else:
          # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
          _lowercase			 =nn.functional.linear(UpperCAmelCase__							,     proj.t().contiguous()      )
          _lowercase			 =nn.functional.linear(UpperCAmelCase__							,     UpperCAmelCase__							,     bias=UpperCAmelCase__      )
          # else:
          #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
          #     if bias is not None:
          #         logit = logit + bias

      return logit




  def 						A__  (       self							,     lowerCAmelCase							,     lowerCAmelCase=None							,     lowerCAmelCase=False      )						->				str:
      '''simple docstring'''






      if labels is not None:
          # Shift so that tokens < n predict n
          _lowercase			 =hidden[..., :-1, :].contiguous()
          _lowercase			 =labels[..., 1:].contiguous()
          _lowercase			 =hidden.view(-1							,     hidden.size(-1      )      )
          _lowercase			 =labels.view(-1      )
          if hidden.size(0      ) != labels.size(0      ):
              raise RuntimeError('Input and labels should have the same size in the batch dimension.'      )
      else:
          _lowercase			 =hidden.view(-1							,     hidden.size(-1      )      )

      if self.n_clusters == 0:
          _lowercase			 =self._compute_logit(UpperCAmelCase__							,     self.out_layers[0].weight							,     self.out_layers[0].bias							,     self.out_projs[0]      )
          if labels is not None:
              _lowercase			 =labels != -100
              _lowercase			 =torch.zeros_like(UpperCAmelCase__							,     dtype=hidden.dtype							,     device=hidden.device      )
              _lowercase			 =(
                  -nn.functional.log_softmax(UpperCAmelCase__							,     dim=-1      )[mask].gather(1							,     labels[mask].unsqueeze(1      )      ).squeeze(1      )
              )
          else:
              _lowercase			 =nn.functional.log_softmax(UpperCAmelCase__							,     dim=-1      )
      else:
          # construct weights and biases
          _lowercase			 =[], []
          for i in range(len(self.cutoffs      )      ):
              if self.div_val == 1:
                  _lowercase			 =self.cutoff_ends[i], self.cutoff_ends[i + 1]
                  _lowercase			 =self.out_layers[0].weight[l_idx:r_idx]
                  _lowercase			 =self.out_layers[0].bias[l_idx:r_idx]
              else:
                  _lowercase			 =self.out_layers[i].weight
                  _lowercase			 =self.out_layers[i].bias

              if i == 0:
                  _lowercase			 =torch.cat([weight_i, self.cluster_weight]							,     dim=0      )
                  _lowercase			 =torch.cat([bias_i, self.cluster_bias]							,     dim=0      )

              weights.append(UpperCAmelCase__      )
              biases.append(UpperCAmelCase__      )

          _lowercase			 =weights[0], biases[0], self.out_projs[0]

          _lowercase			 =self._compute_logit(UpperCAmelCase__							,     UpperCAmelCase__							,     UpperCAmelCase__							,     UpperCAmelCase__      )
          _lowercase			 =nn.functional.log_softmax(UpperCAmelCase__							,     dim=1      )

          if labels is None:
              _lowercase			 =hidden.new_empty((head_logit.size(0      ), self.n_token)      )
          else:
              _lowercase			 =torch.zeros_like(UpperCAmelCase__							,     dtype=hidden.dtype							,     device=hidden.device      )

          _lowercase			 =0
          _lowercase			 =[0] + self.cutoffs
          for i in range(len(UpperCAmelCase__      ) - 1      ):
              _lowercase			 =cutoff_values[i], cutoff_values[i + 1]

              if labels is not None:
                  _lowercase			 =(labels >= l_idx) & (labels < r_idx)
                  _lowercase			 =mask_i.nonzero().squeeze()

                  if indices_i.numel() == 0:
                      continue

                  _lowercase			 =labels.index_select(0							,     UpperCAmelCase__      ) - l_idx
                  _lowercase			 =head_logprob.index_select(0							,     UpperCAmelCase__      )
                  _lowercase			 =hidden.index_select(0							,     UpperCAmelCase__      )
              else:
                  _lowercase			 =hidden

              if i == 0:
                  if labels is not None:
                      _lowercase			 =head_logprob_i.gather(1							,     target_i[:, None]      ).squeeze(1      )
                  else:
                      _lowercase			 =head_logprob[:, : self.cutoffs[0]]
              else:
                  _lowercase			 =weights[i], biases[i], self.out_projs[i]

                  _lowercase			 =self._compute_logit(UpperCAmelCase__							,     UpperCAmelCase__							,     UpperCAmelCase__							,     UpperCAmelCase__      )
                  _lowercase			 =nn.functional.log_softmax(UpperCAmelCase__							,     dim=1      )
                  _lowercase			 =self.cutoffs[0] + i - 1  # No probability for the head cluster
                  if labels is not None:
                      _lowercase			 =head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                          1							,     target_i[:, None]      ).squeeze(1      )
                  else:
                      _lowercase			 =head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                      _lowercase			 =logprob_i

              if labels is not None:
                  if (hasattr(self							,     'keep_order'      ) and self.keep_order) or keep_order:
                      out.index_copy_(0							,     UpperCAmelCase__							,     -logprob_i      )
                  else:
                      out[offset : offset + logprob_i.size(0      )].copy_(-logprob_i      )
                  offset += logprob_i.size(0      )

      return out




  def 						A__  (       self							,     lowerCAmelCase      )						->				List[Any]:
      '''simple docstring'''






      if self.n_clusters == 0:
          _lowercase			 =self._compute_logit(UpperCAmelCase__							,     self.out_layers[0].weight							,     self.out_layers[0].bias							,     self.out_projs[0]      )
          return nn.functional.log_softmax(UpperCAmelCase__							,     dim=-1      )
      else:
          # construct weights and biases
          _lowercase			 =[], []
          for i in range(len(self.cutoffs      )      ):
              if self.div_val == 1:
                  _lowercase			 =self.cutoff_ends[i], self.cutoff_ends[i + 1]
                  _lowercase			 =self.out_layers[0].weight[l_idx:r_idx]
                  _lowercase			 =self.out_layers[0].bias[l_idx:r_idx]
              else:
                  _lowercase			 =self.out_layers[i].weight
                  _lowercase			 =self.out_layers[i].bias

              if i == 0:
                  _lowercase			 =torch.cat([weight_i, self.cluster_weight]							,     dim=0      )
                  _lowercase			 =torch.cat([bias_i, self.cluster_bias]							,     dim=0      )

              weights.append(UpperCAmelCase__      )
              biases.append(UpperCAmelCase__      )

          _lowercase			 =weights[0], biases[0], self.out_projs[0]
          _lowercase			 =self._compute_logit(UpperCAmelCase__							,     UpperCAmelCase__							,     UpperCAmelCase__							,     UpperCAmelCase__      )

          _lowercase			 =hidden.new_empty((head_logit.size(0      ), self.n_token)      )
          _lowercase			 =nn.functional.log_softmax(UpperCAmelCase__							,     dim=1      )

          _lowercase			 =[0] + self.cutoffs
          for i in range(len(UpperCAmelCase__      ) - 1      ):
              _lowercase			 =cutoff_values[i], cutoff_values[i + 1]

              if i == 0:
                  _lowercase			 =head_logprob[:, : self.cutoffs[0]]
              else:
                  _lowercase			 =weights[i], biases[i], self.out_projs[i]

                  _lowercase			 =self._compute_logit(UpperCAmelCase__							,     UpperCAmelCase__							,     UpperCAmelCase__							,     UpperCAmelCase__      )
                  _lowercase			 =nn.functional.log_softmax(UpperCAmelCase__							,     dim=1      )

                  _lowercase			 =head_logprob[:, -i] + tail_logprob_i
                  _lowercase			 =logprob_i

          return out





import unittest

from knapsack import knapsack as k





class 				__lowerCAmelCase  (      unittest.TestCase							):




  def 						A__  (       self      )						->				List[str]:
        '''simple docstring'''






        _lowercase			 =0
        _lowercase			 =[0]
        _lowercase			 =[0]
        _lowercase			 =len(lowerCAmelCase      )
        self.assertEqual(k.knapsack(lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase      )							,     0      )

        _lowercase			 =[60]
        _lowercase			 =[10]
        _lowercase			 =len(lowerCAmelCase      )
        self.assertEqual(k.knapsack(lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase      )							,     0      )




  def 						A__  (       self      )						->				List[Any]:
        '''simple docstring'''






        _lowercase			 =3
        _lowercase			 =[1, 2, 3]
        _lowercase			 =[3, 2, 1]
        _lowercase			 =len(lowerCAmelCase      )
        self.assertEqual(k.knapsack(lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase      )							,     5      )




  def 						A__  (       self      )						->				str:
        '''simple docstring'''






        _lowercase			 =50
        _lowercase			 =[60, 100, 120]
        _lowercase			 =[10, 20, 30]
        _lowercase			 =len(lowerCAmelCase      )
        self.assertEqual(k.knapsack(lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase							,     lowerCAmelCase      )							,     220      )


if __name__ == "__main__":
  unittest.main()


