"""simple docstring"""







import torch
from torch import nn







class 			UpperCamelCase						(     nn.Module    ):
   def __init__(     self	,__UpperCamelCase	,__UpperCamelCase	,__UpperCamelCase	,__UpperCamelCase	,__UpperCamelCase=1	,__UpperCamelCase=False     )      -> Union[str, Any]:


        '''simple docstring'''


        super().__init__()

        lowercase_      :							Tuple						=      n_token
        lowercase_      :							str						=      d_embed
        lowercase_      :							List[Any]						=      d_proj

        lowercase_      :							Dict						=      cutoffs + [n_token]
        lowercase_      :							str						=      [0] + self.cutoffs
        lowercase_      :							List[Any]						=      div_val

        lowercase_      :							List[Any]						=      self.cutoffs[0]
        lowercase_      :							List[Any]						=      len(self.cutoffs     ) - 1
        lowercase_      :							Tuple						=      self.shortlist_size + self.n_clusters

        if self.n_clusters > 0:
             lowercase_      :							Any						=      nn.Parameter(torch.zeros(self.n_clusters	,self.d_embed     )     )
             lowercase_      :							Union[str, Any]						=      nn.Parameter(torch.zeros(self.n_clusters     )     )

        lowercase_      :							str						=      nn.ModuleList()
        lowercase_      :							Tuple						=      nn.ParameterList()

        if div_val == 1:
             for i in range(len(self.cutoffs     )     ):
                  if d_proj != d_embed:
                       self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCamelCase__	,lowerCamelCase__     )     )     )
                  else:
                       self.out_projs.append(lowerCamelCase__     )

             self.out_layers.append(nn.Linear(lowerCamelCase__	,lowerCamelCase__     )     )
        else:
             for i in range(len(self.cutoffs     )     ):
                  lowercase_      :							int						=      self.cutoff_ends[i], self.cutoff_ends[i + 1]
                  lowercase_      :							Union[str, Any]						=      d_embed // (div_val**i)

                  self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCamelCase__	,lowerCamelCase__     )     )     )

                  self.out_layers.append(nn.Linear(lowerCamelCase__	,r_idx - l_idx     )     )

        lowercase_      :							Optional[int]						=      keep_order
   def      _UpperCAmelCase  (     self	,__UpperCamelCase	,__UpperCamelCase	,__UpperCamelCase	,__UpperCamelCase     )      -> str:


        '''simple docstring'''


        if proj is None:
             lowercase_      :							Optional[Any]						=      nn.functional.linear(lowerCamelCase__	,lowerCamelCase__	,bias=lowerCamelCase__     )
        else:
             # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
             lowercase_      :							Dict						=      nn.functional.linear(lowerCamelCase__	,proj.t().contiguous()     )
             lowercase_      :							Dict						=      nn.functional.linear(lowerCamelCase__	,lowerCamelCase__	,bias=lowerCamelCase__     )
             # else:
             #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
             #     if bias is not None:
             #         logit = logit + bias

        return logit
   def      _UpperCAmelCase  (     self	,__UpperCamelCase	,__UpperCamelCase=None	,__UpperCamelCase=False     )      -> List[Any]:


        '''simple docstring'''


        if labels is not None:
             # Shift so that tokens < n predict n
             lowercase_      :							Tuple						=      hidden[..., :-1, :].contiguous()
             lowercase_      :							int						=      labels[..., 1:].contiguous()
             lowercase_      :							Dict						=      hidden.view(-1	,hidden.size(-1     )     )
             lowercase_      :							Optional[Any]						=      labels.view(-1     )
             if hidden.size(0     ) != labels.size(0     ):
                  raise RuntimeError('Input and labels should have the same size in the batch dimension.'     )
        else:
             lowercase_      :							List[Any]						=      hidden.view(-1	,hidden.size(-1     )     )

        if self.n_clusters == 0:
             lowercase_      :							int						=      self._compute_logit(lowerCamelCase__	,self.out_layers[0].weight	,self.out_layers[0].bias	,self.out_projs[0]     )
             if labels is not None:
                  lowercase_      :							Dict						=      labels != -100
                  lowercase_      :							Optional[Any]						=      torch.zeros_like(lowerCamelCase__	,dtype=hidden.dtype	,device=hidden.device     )
                  lowercase_      :							int						=      (
                      -nn.functional.log_softmax(lowerCamelCase__	,dim=-1     )[mask].gather(1	,labels[mask].unsqueeze(1     )     ).squeeze(1     )
                  )
             else:
                  lowercase_      :							Any						=      nn.functional.log_softmax(lowerCamelCase__	,dim=-1     )
        else:
             # construct weights and biases
             lowercase_      :							int						=      [], []
             for i in range(len(self.cutoffs     )     ):
                  if self.div_val == 1:
                       lowercase_      :							Any						=      self.cutoff_ends[i], self.cutoff_ends[i + 1]
                       lowercase_      :							Optional[int]						=      self.out_layers[0].weight[l_idx:r_idx]
                       lowercase_      :							Optional[int]						=      self.out_layers[0].bias[l_idx:r_idx]
                  else:
                       lowercase_      :							Tuple						=      self.out_layers[i].weight
                       lowercase_      :							int						=      self.out_layers[i].bias

                  if i == 0:
                       lowercase_      :							Optional[int]						=      torch.cat([weight_i, self.cluster_weight]	,dim=0     )
                       lowercase_      :							Any						=      torch.cat([bias_i, self.cluster_bias]	,dim=0     )

                  weights.append(lowerCamelCase__     )
                  biases.append(lowerCamelCase__     )

             lowercase_      :							Optional[Any]						=      weights[0], biases[0], self.out_projs[0]

             lowercase_      :							List[Any]						=      self._compute_logit(lowerCamelCase__	,lowerCamelCase__	,lowerCamelCase__	,lowerCamelCase__     )
             lowercase_      :							int						=      nn.functional.log_softmax(lowerCamelCase__	,dim=1     )

             if labels is None:
                  lowercase_      :							Optional[int]						=      hidden.new_empty((head_logit.size(0     ), self.n_token)     )
             else:
                  lowercase_      :							Any						=      torch.zeros_like(lowerCamelCase__	,dtype=hidden.dtype	,device=hidden.device     )

             lowercase_      :							Tuple						=      0
             lowercase_      :							Optional[Any]						=      [0] + self.cutoffs
             for i in range(len(lowerCamelCase__     ) - 1     ):
                  lowercase_      :							List[str]						=      cutoff_values[i], cutoff_values[i + 1]

                  if labels is not None:
                       lowercase_      :							Any						=      (labels >= l_idx) & (labels < r_idx)
                       lowercase_      :							Union[str, Any]						=      mask_i.nonzero().squeeze()

                       if indices_i.numel() == 0:
                            continue

                       lowercase_      :							Tuple						=      labels.index_select(0	,lowerCamelCase__     ) - l_idx
                       lowercase_      :							List[str]						=      head_logprob.index_select(0	,lowerCamelCase__     )
                       lowercase_      :							Dict						=      hidden.index_select(0	,lowerCamelCase__     )
                  else:
                       lowercase_      :							List[Any]						=      hidden

                  if i == 0:
                       if labels is not None:
                            lowercase_      :							str						=      head_logprob_i.gather(1	,target_i[:, None]     ).squeeze(1     )
                       else:
                            lowercase_      :							Optional[int]						=      head_logprob[:, : self.cutoffs[0]]
                  else:
                       lowercase_      :							Tuple						=      weights[i], biases[i], self.out_projs[i]

                       lowercase_      :							Optional[Any]						=      self._compute_logit(lowerCamelCase__	,lowerCamelCase__	,lowerCamelCase__	,lowerCamelCase__     )
                       lowercase_      :							Tuple						=      nn.functional.log_softmax(lowerCamelCase__	,dim=1     )
                       lowercase_      :							List[str]						=      self.cutoffs[0] + i - 1  # No probability for the head cluster
                       if labels is not None:
                            lowercase_      :							Optional[Any]						=      head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                                1	,target_i[:, None]     ).squeeze(1     )
                       else:
                            lowercase_      :							Optional[int]						=      head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                            lowercase_      :							Optional[int]						=      logprob_i

                  if labels is not None:
                       if (hasattr(self	,'keep_order'     ) and self.keep_order) or keep_order:
                            out.index_copy_(0	,lowerCamelCase__	,-logprob_i     )
                       else:
                            out[offset : offset + logprob_i.size(0     )].copy_(-logprob_i     )
                       offset += logprob_i.size(0     )

        return out







   def      _UpperCAmelCase  (     self	,__UpperCamelCase     )      -> Tuple:


        '''simple docstring'''


        if self.n_clusters == 0:
             lowercase_      :							List[str]						=      self._compute_logit(lowerCamelCase__	,self.out_layers[0].weight	,self.out_layers[0].bias	,self.out_projs[0]     )
             return nn.functional.log_softmax(lowerCamelCase__	,dim=-1     )
        else:
             # construct weights and biases
             lowercase_      :							Optional[Any]						=      [], []
             for i in range(len(self.cutoffs     )     ):
                  if self.div_val == 1:
                       lowercase_      :							Dict						=      self.cutoff_ends[i], self.cutoff_ends[i + 1]
                       lowercase_      :							str						=      self.out_layers[0].weight[l_idx:r_idx]
                       lowercase_      :							Dict						=      self.out_layers[0].bias[l_idx:r_idx]
                  else:
                       lowercase_      :							Tuple						=      self.out_layers[i].weight
                       lowercase_      :							Optional[Any]						=      self.out_layers[i].bias

                  if i == 0:
                       lowercase_      :							Dict						=      torch.cat([weight_i, self.cluster_weight]	,dim=0     )
                       lowercase_      :							List[Any]						=      torch.cat([bias_i, self.cluster_bias]	,dim=0     )

                  weights.append(lowerCamelCase__     )
                  biases.append(lowerCamelCase__     )

             lowercase_      :							Union[str, Any]						=      weights[0], biases[0], self.out_projs[0]
             lowercase_      :							List[str]						=      self._compute_logit(lowerCamelCase__	,lowerCamelCase__	,lowerCamelCase__	,lowerCamelCase__     )

             lowercase_      :							Optional[int]						=      hidden.new_empty((head_logit.size(0     ), self.n_token)     )
             lowercase_      :							Dict						=      nn.functional.log_softmax(lowerCamelCase__	,dim=1     )

             lowercase_      :							List[Any]						=      [0] + self.cutoffs
             for i in range(len(lowerCamelCase__     ) - 1     ):
                  lowercase_      :							List[Any]						=      cutoff_values[i], cutoff_values[i + 1]

                  if i == 0:
                       lowercase_      :							Optional[Any]						=      head_logprob[:, : self.cutoffs[0]]
                  else:
                       lowercase_      :							Any						=      weights[i], biases[i], self.out_projs[i]

                       lowercase_      :							str						=      self._compute_logit(lowerCamelCase__	,lowerCamelCase__	,lowerCamelCase__	,lowerCamelCase__     )
                       lowercase_      :							Optional[int]						=      nn.functional.log_softmax(lowerCamelCase__	,dim=1     )

                       lowercase_      :							List[Any]						=      head_logprob[:, -i] + tail_logprob_i
                       lowercase_      :							Tuple						=      logprob_i

             return out






'''simple docstring'''
from typing import List

import numpy as np






def A__       (	UpperCAmelCase_				):
						_UpperCamelCase   :						Any												=  {key: len(UpperCAmelCase_				) for key, value in gen_kwargs.items() if isinstance(UpperCAmelCase_    ,    UpperCAmelCase_				)}
						if len(set(lists_lengths.values()				)				) > 1:
												raise RuntimeError(
												    (
												        'Sharding is ambiguous for this dataset: '
												        + 'we found several data sources lists of different lengths, and we don\'t know over which list we should parallelize:\n'
												        + '\n'.join(f'\t- key {key} has length {length}' for key, length in lists_lengths.items()				)
												        + '\nTo fix this, check the \'gen_kwargs\' and make sure to use lists only for data sources, '
												        + 'and use tuples otherwise. In the end there should only be one single list, or several lists with the same length.'
												    )				)
						_UpperCamelCase   :						Tuple												=  max(lists_lengths.values()    ,    default=0				)
						return max(1    ,    UpperCAmelCase_				)






def A__       (	UpperCAmelCase_    ,    UpperCAmelCase_				):
						_UpperCamelCase   :						Dict												=  []
						for group_idx in range(UpperCAmelCase_				):
												_UpperCamelCase   :						int												=  num_shards // max_num_jobs + (group_idx < (num_shards % max_num_jobs))
												if num_shards_to_add == 0:
																		break
												_UpperCamelCase   :						Tuple												=  shards_indices_per_group[-1].stop if shards_indices_per_group else 0
												_UpperCamelCase   :						Tuple												=  range(UpperCAmelCase_    ,    start + num_shards_to_add				)
												shards_indices_per_group.append(UpperCAmelCase_				)
						return shards_indices_per_group






def A__       (	UpperCAmelCase_    ,    UpperCAmelCase_				):
						_UpperCamelCase   :						Tuple												=  _number_of_shards_in_gen_kwargs(UpperCAmelCase_				)
						if num_shards == 1:
												return [dict(UpperCAmelCase_				)]
						else:
												_UpperCamelCase   :						str												=  _distribute_shards(num_shards=UpperCAmelCase_    ,    max_num_jobs=UpperCAmelCase_				)
												return [
												    {
												        key: [value[shard_idx] for shard_idx in shard_indices_per_group[group_idx]]
												        if isinstance(UpperCAmelCase_    ,    UpperCAmelCase_				)
												        else value
												        for key, value in gen_kwargs.items()
												    }
												    for group_idx in range(len(UpperCAmelCase_				)				)
												]






def A__       (	UpperCAmelCase_				):
						return {
						    key: [value for gen_kwargs in gen_kwargs_list for value in gen_kwargs[key]]
						    if isinstance(gen_kwargs_list[0][key]    ,    UpperCAmelCase_				)
						    else gen_kwargs_list[0][key]
						    for key in gen_kwargs_list[0]
						}






def A__       (	UpperCAmelCase_    ,    UpperCAmelCase_				):
						_UpperCamelCase   :						int												=  {len(UpperCAmelCase_				) for value in gen_kwargs.values() if isinstance(UpperCAmelCase_    ,    UpperCAmelCase_				)}
						_UpperCamelCase   :						Union[str, Any]												=  {}
						for size in list_sizes:
												_UpperCamelCase   :						str												=  list(range(UpperCAmelCase_				)				)
												rng.shuffle(indices_per_size[size]				)
						# Now let's copy the gen_kwargs and shuffle the lists based on their sizes
						_UpperCamelCase   :						Union[str, Any]												=  dict(UpperCAmelCase_				)
						for key, value in shuffled_kwargs.items():
												if isinstance(UpperCAmelCase_    ,    UpperCAmelCase_				):
																		_UpperCamelCase   :						Dict												=  [value[i] for i in indices_per_size[len(UpperCAmelCase_				)]]
						return shuffled_kwargs


