



"""simple docstring"""






from typing import Dict, List, Optional, Union

import numpy as np

from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy


snake_case   					=				logging.get_logger(__name__)



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):



   def __init__(       self    :	str							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	int							,       __lowerCamelCase    :	float							,       **__lowerCamelCase    :	str				):



     """simple docstring"""
     _snake_case   	=      feature_size
     _snake_case   	=      sampling_rate
     _snake_case   	=      padding_value

     _snake_case   	=      kwargs.pop('''padding_side'''							,       '''right'''				)
     _snake_case   	=      kwargs.pop('''return_attention_mask'''							,       __lowerCamelCase				)

     super().__init__(**__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Dict							,       __lowerCamelCase    :	Union[
           BatchFeature,
           List[BatchFeature],
           Dict[str, BatchFeature],
           Dict[str, List[BatchFeature]],
           List[Dict[str, BatchFeature]],
       ]							,       __lowerCamelCase    :	Union[bool, str, PaddingStrategy] = True							,       __lowerCamelCase    :	Optional[int] = None							,       __lowerCamelCase    :	bool = False							,       __lowerCamelCase    :	Optional[int] = None							,       __lowerCamelCase    :	Optional[bool] = None							,       __lowerCamelCase    :	Optional[Union[str, TensorType]] = None							,       ):



     """simple docstring"""
     # If we have a list of dicts, let's convert it in a dict of lists
     # We do this to allow using this method as a collate_fn function in PyTorch Dataloader
     if isinstance(__lowerCamelCase							,       (list, tuple)				) and isinstance(processed_features[0]							,       (dict, BatchFeature)				):
       _snake_case   	=      {
           key: [example[key] for example in processed_features] for key in processed_features[0].keys()
       }

     # The model's main input name, usually `input_values`, has be passed for padding
     if self.model_input_names[0] not in processed_features:
       raise ValueError(
           '''You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`'''
           f""" to this method that includes {self.model_input_names[0]}, but you provided"""
           f""" {list(processed_features.keys()				)}"""				)

     _snake_case   	=      processed_features[self.model_input_names[0]]
     _snake_case   	=      (
         return_attention_mask if return_attention_mask is not None else self.return_attention_mask
     )

     if len(__lowerCamelCase				) == 0:
       if return_attention_mask:
         _snake_case   	=      []
       return processed_features

     # If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays
     # and rebuild them afterwards if no return_tensors is specified
     # Note that we lose the specific device the tensor may be on for PyTorch

     _snake_case   	=      required_input[0]
     if isinstance(__lowerCamelCase							,       (list, tuple)				):
       # first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
       _snake_case   	=      0
       while len(required_input[index]				) == 0:
         index += 1
       if index < len(__lowerCamelCase				):
         _snake_case   	=      required_input[index][0]

     if return_tensors is None:
       if is_tf_tensor(__lowerCamelCase				):
         _snake_case   	=      '''tf'''
       elif is_torch_tensor(__lowerCamelCase				):
         _snake_case   	=      '''pt'''
       elif isinstance(__lowerCamelCase							,       (int, float, list, tuple, np.ndarray)				):
         _snake_case   	=      '''np'''
       else:
         raise ValueError(
             f"""type of {first_element} unknown: {type(__lowerCamelCase				)}. """
             '''Should be one of a python, numpy, pytorch or tensorflow object.'''				)

     for key, value in processed_features.items():
       if isinstance(value[0]							,       (int, float)				):
         _snake_case   	=      to_numpy(__lowerCamelCase				)
       else:
         _snake_case   	=      [to_numpy(__lowerCamelCase				) for v in value]

        # Convert padding_strategy in PaddingStrategy
     _snake_case   	=      self._get_padding_strategies(padding=__lowerCamelCase							,       max_length=__lowerCamelCase				)

     _snake_case   	=      processed_features[self.model_input_names[0]]

     _snake_case   	=      len(__lowerCamelCase				)
     if not all(len(__lowerCamelCase				) == batch_size for v in processed_features.values()				):
       raise ValueError('''Some items in the output dictionary have a different batch size than others.'''				)

     _snake_case   	=      []
     for i in range(__lowerCamelCase				):
       _snake_case   	=      {k: v[i] for k, v in processed_features.items()}
       # truncation
       _snake_case   	=      self._truncate(
           __lowerCamelCase							,       max_length=__lowerCamelCase							,       pad_to_multiple_of=__lowerCamelCase							,       truncation=__lowerCamelCase							,       )
       truncated_inputs.append(__lowerCamelCase				)

     if padding_strategy == PaddingStrategy.LONGEST:
       # make sure that `max_length` cannot be longer than the longest truncated length
       _snake_case   	=      max(len(input_slice[self.model_input_names[0]]				) for input_slice in truncated_inputs				)
       _snake_case   	=      PaddingStrategy.MAX_LENGTH

     _snake_case   	=      {}
     for i in range(__lowerCamelCase				):
       # padding
       _snake_case   	=      self._pad(
           truncated_inputs[i]							,       max_length=__lowerCamelCase							,       padding_strategy=__lowerCamelCase							,       pad_to_multiple_of=__lowerCamelCase							,       return_attention_mask=__lowerCamelCase							,       )

       for key, value in outputs.items():
         if key not in batch_outputs:
           _snake_case   	=      []
         if value.dtype is np.dtype(np.floataa				):
           _snake_case   	=      value.astype(np.floataa				)
         batch_outputs[key].append(__lowerCamelCase				)

     return BatchFeature(__lowerCamelCase							,       tensor_type=__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	int							,       __lowerCamelCase    :	Union[Dict[str, np.ndarray], BatchFeature]							,       __lowerCamelCase    :	Optional[int] = None							,       __lowerCamelCase    :	PaddingStrategy = PaddingStrategy.DO_NOT_PAD							,       __lowerCamelCase    :	Optional[int] = None							,       __lowerCamelCase    :	Optional[bool] = None							,       ):



     """simple docstring"""
     _snake_case   	=      processed_features[self.model_input_names[0]]

     if padding_strategy == PaddingStrategy.LONGEST:
       _snake_case   	=      len(__lowerCamelCase				)

     if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
       _snake_case   	=      ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

     _snake_case   	=      padding_strategy != PaddingStrategy.DO_NOT_PAD and len(__lowerCamelCase				) < max_length

     if return_attention_mask and "attention_mask" not in processed_features:
       _snake_case   	=      np.ones(len(__lowerCamelCase				)							,       dtype=np.intaa				)

     if needs_to_be_padded:
       _snake_case   	=      max_length - len(__lowerCamelCase				)
       if self.padding_side == "right":
         if return_attention_mask:
           _snake_case   	=      np.pad(
               processed_features['''attention_mask''']							,       (0, difference)				)
         _snake_case   	=      ((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)
         _snake_case   	=      np.pad(
             __lowerCamelCase							,       __lowerCamelCase							,       '''constant'''							,       constant_values=self.padding_value				)
       elif self.padding_side == "left":
         if return_attention_mask:
           _snake_case   	=      np.pad(
               processed_features['''attention_mask''']							,       (difference, 0)				)
         _snake_case   	=      ((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)
         _snake_case   	=      np.pad(
             __lowerCamelCase							,       __lowerCamelCase							,       '''constant'''							,       constant_values=self.padding_value				)
       else:
         raise ValueError('''Invalid padding strategy:''' + str(self.padding_side				)				)

     return processed_features



   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	Union[Dict[str, np.ndarray], BatchFeature]							,       __lowerCamelCase    :	Optional[int] = None							,       __lowerCamelCase    :	Optional[int] = None							,       __lowerCamelCase    :	Optional[bool] = None							,       ):



     """simple docstring"""
     if not truncation:
       return processed_features
     elif truncation and max_length is None:
       raise ValueError('''When setting ``truncation=True``, make sure that ``max_length`` is defined.'''				)

     _snake_case   	=      processed_features[self.model_input_names[0]]

     # find `max_length` that fits `pad_to_multiple_of`
     if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
       _snake_case   	=      ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

     _snake_case   	=      len(__lowerCamelCase				) > max_length

     if needs_to_be_truncated:
       _snake_case   	=      processed_features[self.model_input_names[0]][:max_length]
       if "attention_mask" in processed_features:
         _snake_case   	=      processed_features['''attention_mask'''][:max_length]

     return processed_features





   def    __UpperCAmelCase    (       self    :	List[Any]							,       __lowerCamelCase    :	str=False							,       __lowerCamelCase    :	Any=None				):



     """simple docstring"""

     # Get padding strategy
     if padding is not False:
       if padding is True:
         _snake_case   	=      PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch
       elif not isinstance(__lowerCamelCase							,       __lowerCamelCase				):
         _snake_case   	=      PaddingStrategy(__lowerCamelCase				)
       elif isinstance(__lowerCamelCase							,       __lowerCamelCase				):
         _snake_case   	=      padding
     else:
       _snake_case   	=      PaddingStrategy.DO_NOT_PAD

     # Set max length if needed
     if max_length is None:
       if padding_strategy == PaddingStrategy.MAX_LENGTH:
         raise ValueError(
             f"""When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined"""				)

        # Test if we have a padding value
     if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
       raise ValueError(
           '''Asking to pad but the feature_extractor does not have a padding value. Please select a value to use'''
           ''' as `padding_value`. For example: `feature_extractor.padding_value = 0.0`.'''				)

     return padding_strategy













import torch
from torch import nn





class 					A__     (     nn.Module ):

     '''simple docstring'''






     def __init__( self      :    List[str]   ,						_SCREAMING_SNAKE_CASE      :    List[Any]   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]   ,						_SCREAMING_SNAKE_CASE      :    Tuple   ,						_SCREAMING_SNAKE_CASE      :    str   ,						_SCREAMING_SNAKE_CASE      :    Optional[int]=1   ,						_SCREAMING_SNAKE_CASE      :    List[str]=False ):






           """simple docstring"""




           super().__init__()

           UpperCamelCase					       =  n_token
           UpperCamelCase					       =  d_embed
           UpperCamelCase					       =  d_proj

           UpperCamelCase					       =  cutoffs + [n_token]
           UpperCamelCase					       =  [0] + self.cutoffs
           UpperCamelCase					       =  div_val

           UpperCamelCase					       =  self.cutoffs[0]
           UpperCamelCase					       =  len(self.cutoffs ) - 1
           UpperCamelCase					       =  self.shortlist_size + self.n_clusters

           if self.n_clusters > 0:
                 UpperCamelCase					       =  nn.Parameter(torch.zeros(self.n_clusters   ,						self.d_embed ) )
                 UpperCamelCase					       =  nn.Parameter(torch.zeros(self.n_clusters ) )

           UpperCamelCase					       =  nn.ModuleList()
           UpperCamelCase					       =  nn.ParameterList()

           if div_val == 1:
                 for i in range(len(self.cutoffs ) ):
                       if d_proj != d_embed:
                             self.out_projs.append(nn.Parameter(torch.FloatTensor(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE ) ) )
                       else:
                             self.out_projs.append(_SCREAMING_SNAKE_CASE )

                 self.out_layers.append(nn.Linear(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE ) )
           else:
                 for i in range(len(self.cutoffs ) ):
                       UpperCamelCase       ,  UpperCamelCase					       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
                       UpperCamelCase					       =  d_embed // (div_val**i)

                       self.out_projs.append(nn.Parameter(torch.FloatTensor(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE ) ) )

                       self.out_layers.append(nn.Linear(_SCREAMING_SNAKE_CASE   ,						r_idx - l_idx ) )

           UpperCamelCase					       =  keep_order







     def   _SCREAMING_SNAKE_CASE     ( self      :    Dict   ,						_SCREAMING_SNAKE_CASE      :    Optional[Any]   ,						_SCREAMING_SNAKE_CASE      :    Dict   ,						_SCREAMING_SNAKE_CASE      :    List[Any]   ,						_SCREAMING_SNAKE_CASE      :    Dict ):






           """simple docstring"""




           if proj is None:
                 UpperCamelCase					       =  nn.functional.linear(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						bias=_SCREAMING_SNAKE_CASE )
           else:
                 # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
                 UpperCamelCase					       =  nn.functional.linear(_SCREAMING_SNAKE_CASE   ,						proj.t().contiguous() )
                 UpperCamelCase					       =  nn.functional.linear(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						bias=_SCREAMING_SNAKE_CASE )
                 # else:
                 #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
                 #     if bias is not None:
                 #         logit = logit + bias

           return logit







     def   _SCREAMING_SNAKE_CASE     ( self      :    List[Any]   ,						_SCREAMING_SNAKE_CASE      :    Union[str, Any]   ,						_SCREAMING_SNAKE_CASE      :    Optional[Any]=None   ,						_SCREAMING_SNAKE_CASE      :    Dict=False ):






           """simple docstring"""




           if labels is not None:
                 # Shift so that tokens < n predict n
                 UpperCamelCase					       =  hidden[..., :-1, :].contiguous()
                 UpperCamelCase					       =  labels[..., 1:].contiguous()
                 UpperCamelCase					       =  hidden.view(-1   ,						hidden.size(-1 ) )
                 UpperCamelCase					       =  labels.view(-1 )
                 if hidden.size(0 ) != labels.size(0 ):
                       raise RuntimeError('Input and labels should have the same size in the batch dimension.' )
           else:
                 UpperCamelCase					       =  hidden.view(-1   ,						hidden.size(-1 ) )

           if self.n_clusters == 0:
                 UpperCamelCase					       =  self._compute_logit(_SCREAMING_SNAKE_CASE   ,						self.out_layers[0].weight   ,						self.out_layers[0].bias   ,						self.out_projs[0] )
                 if labels is not None:
                       UpperCamelCase					       =  labels != -100
                       UpperCamelCase					       =  torch.zeros_like(_SCREAMING_SNAKE_CASE   ,						dtype=hidden.dtype   ,						device=hidden.device )
                       UpperCamelCase					       =  (
                           -nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=-1 )[mask].gather(1   ,						labels[mask].unsqueeze(1 ) ).squeeze(1 )
                       )
                 else:
                       UpperCamelCase					       =  nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=-1 )
           else:
                 # construct weights and biases
                 UpperCamelCase       ,  UpperCamelCase					       =  [], []
                 for i in range(len(self.cutoffs ) ):
                       if self.div_val == 1:
                             UpperCamelCase       ,  UpperCamelCase					       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
                             UpperCamelCase					       =  self.out_layers[0].weight[l_idx:r_idx]
                             UpperCamelCase					       =  self.out_layers[0].bias[l_idx:r_idx]
                       else:
                             UpperCamelCase					       =  self.out_layers[i].weight
                             UpperCamelCase					       =  self.out_layers[i].bias

                       if i == 0:
                             UpperCamelCase					       =  torch.cat([weight_i, self.cluster_weight]   ,						dim=0 )
                             UpperCamelCase					       =  torch.cat([bias_i, self.cluster_bias]   ,						dim=0 )

                       weights.append(_SCREAMING_SNAKE_CASE )
                       biases.append(_SCREAMING_SNAKE_CASE )

                 UpperCamelCase       ,  UpperCamelCase       ,  UpperCamelCase					       =  weights[0], biases[0], self.out_projs[0]

                 UpperCamelCase					       =  self._compute_logit(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
                 UpperCamelCase					       =  nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=1 )

                 if labels is None:
                       UpperCamelCase					       =  hidden.new_empty((head_logit.size(0 ), self.n_token) )
                 else:
                       UpperCamelCase					       =  torch.zeros_like(_SCREAMING_SNAKE_CASE   ,						dtype=hidden.dtype   ,						device=hidden.device )

                 UpperCamelCase					       =  0
                 UpperCamelCase					       =  [0] + self.cutoffs
                 for i in range(len(_SCREAMING_SNAKE_CASE ) - 1 ):
                       UpperCamelCase       ,  UpperCamelCase					       =  cutoff_values[i], cutoff_values[i + 1]

                       if labels is not None:
                             UpperCamelCase					       =  (labels >= l_idx) & (labels < r_idx)
                             UpperCamelCase					       =  mask_i.nonzero().squeeze()

                             if indices_i.numel() == 0:
                                   continue

                             UpperCamelCase					       =  labels.index_select(0   ,						_SCREAMING_SNAKE_CASE ) - l_idx
                             UpperCamelCase					       =  head_logprob.index_select(0   ,						_SCREAMING_SNAKE_CASE )
                             UpperCamelCase					       =  hidden.index_select(0   ,						_SCREAMING_SNAKE_CASE )
                       else:
                             UpperCamelCase					       =  hidden

                       if i == 0:
                             if labels is not None:
                                   UpperCamelCase					       =  head_logprob_i.gather(1   ,						target_i[:, None] ).squeeze(1 )
                             else:
                                   UpperCamelCase					       =  head_logprob[:, : self.cutoffs[0]]
                       else:
                             UpperCamelCase       ,  UpperCamelCase       ,  UpperCamelCase					       =  weights[i], biases[i], self.out_projs[i]

                             UpperCamelCase					       =  self._compute_logit(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
                             UpperCamelCase					       =  nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=1 )
                             UpperCamelCase					       =  self.cutoffs[0] + i - 1  # No probability for the head cluster
                             if labels is not None:
                                   UpperCamelCase					       =  head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                                       1   ,						target_i[:, None] ).squeeze(1 )
                             else:
                                   UpperCamelCase					       =  head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                                   UpperCamelCase					       =  logprob_i

                       if labels is not None:
                             if (hasattr(self   ,						'keep_order' ) and self.keep_order) or keep_order:
                                   out.index_copy_(0   ,						_SCREAMING_SNAKE_CASE   ,						-logprob_i )
                             else:
                                   out[offset : offset + logprob_i.size(0 )].copy_(-logprob_i )
                             offset += logprob_i.size(0 )

           return out







     def   _SCREAMING_SNAKE_CASE     ( self      :    int   ,						_SCREAMING_SNAKE_CASE      :    str ):






           """simple docstring"""




           if self.n_clusters == 0:
                 UpperCamelCase					       =  self._compute_logit(_SCREAMING_SNAKE_CASE   ,						self.out_layers[0].weight   ,						self.out_layers[0].bias   ,						self.out_projs[0] )
                 return nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=-1 )
           else:
                 # construct weights and biases
                 UpperCamelCase       ,  UpperCamelCase					       =  [], []
                 for i in range(len(self.cutoffs ) ):
                       if self.div_val == 1:
                             UpperCamelCase       ,  UpperCamelCase					       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
                             UpperCamelCase					       =  self.out_layers[0].weight[l_idx:r_idx]
                             UpperCamelCase					       =  self.out_layers[0].bias[l_idx:r_idx]
                       else:
                             UpperCamelCase					       =  self.out_layers[i].weight
                             UpperCamelCase					       =  self.out_layers[i].bias

                       if i == 0:
                             UpperCamelCase					       =  torch.cat([weight_i, self.cluster_weight]   ,						dim=0 )
                             UpperCamelCase					       =  torch.cat([bias_i, self.cluster_bias]   ,						dim=0 )

                       weights.append(_SCREAMING_SNAKE_CASE )
                       biases.append(_SCREAMING_SNAKE_CASE )

                 UpperCamelCase       ,  UpperCamelCase       ,  UpperCamelCase					       =  weights[0], biases[0], self.out_projs[0]
                 UpperCamelCase					       =  self._compute_logit(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )

                 UpperCamelCase					       =  hidden.new_empty((head_logit.size(0 ), self.n_token) )
                 UpperCamelCase					       =  nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=1 )

                 UpperCamelCase					       =  [0] + self.cutoffs
                 for i in range(len(_SCREAMING_SNAKE_CASE ) - 1 ):
                       UpperCamelCase       ,  UpperCamelCase					       =  cutoff_values[i], cutoff_values[i + 1]

                       if i == 0:
                             UpperCamelCase					       =  head_logprob[:, : self.cutoffs[0]]
                       else:
                             UpperCamelCase       ,  UpperCamelCase       ,  UpperCamelCase					       =  weights[i], biases[i], self.out_projs[i]

                             UpperCamelCase					       =  self._compute_logit(_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE   ,						_SCREAMING_SNAKE_CASE )
                             UpperCamelCase					       =  nn.functional.log_softmax(_SCREAMING_SNAKE_CASE   ,						dim=1 )

                             UpperCamelCase					       =  head_logprob[:, -i] + tail_logprob_i
                             UpperCamelCase					       =  logprob_i

                 return out



