
'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case_ :  Optional[int]     =					logging.get_logger(__name__)

snake_case_ :  int     =					'▁'

snake_case_ :  str     =					{'vocab_file': 'sentencepiece.bpe.model', 'monolingual_vocab_file': 'dict.txt'}

snake_case_ :  Optional[Any]     =					{
    'vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model',
    },
    'monolingual_vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt',
    },
}

snake_case_ :  Dict     =					{'vinai/bartpho-syllable': 1_024}


class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   _snake_case		   =							VOCAB_FILES_NAMES
   _snake_case		   =							PRETRAINED_VOCAB_FILES_MAP
   _snake_case		   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _snake_case		   =							['''input_ids''', '''attention_mask''']







   def __init__(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__="<s>" ,   lowerCamelCase__="</s>" ,   lowerCamelCase__="</s>" ,   lowerCamelCase__="<s>" ,   lowerCamelCase__="<unk>" ,   lowerCamelCase__="<pad>" ,   lowerCamelCase__="<mask>" ,   lowerCamelCase__ = None ,   **lowerCamelCase__ ,   ):

      '''simple docstring'''







      UpperCamelCase       =  AddedToken(lowerCamelCase__ ,   lstrip=lowerCamelCase__ ,   rstrip=lowerCamelCase__   ) if isinstance(lowerCamelCase__ ,   lowerCamelCase__   ) else mask_token

      UpperCamelCase       =  {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=lowerCamelCase__ ,   eos_token=lowerCamelCase__ ,   unk_token=lowerCamelCase__ ,   sep_token=lowerCamelCase__ ,   cls_token=lowerCamelCase__ ,   pad_token=lowerCamelCase__ ,   mask_token=lowerCamelCase__ ,   sp_model_kwargs=self.sp_model_kwargs ,   **lowerCamelCase__ ,   )

      UpperCamelCase       =  vocab_file
      UpperCamelCase       =  monolingual_vocab_file
      UpperCamelCase       =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(str(lowerCamelCase__   )   )

      # Load the reduced vocab

      # Keep order of special tokens for backward compatibility
      UpperCamelCase       =  {}
      UpperCamelCase       =  0
      for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
         if str(lowerCamelCase__   ) not in self.fairseq_tokens_to_ids:
            UpperCamelCase       =  cnt
            cnt += 1
      with open(lowerCamelCase__ ,   '''r''' ,   encoding='''utf-8'''   ) as f:
         for line in f.readlines():
            UpperCamelCase       =  line.strip().split()[0]
            UpperCamelCase       =  len(self.fairseq_tokens_to_ids   )
      if str(lowerCamelCase__   ) not in self.fairseq_tokens_to_ids:
         UpperCamelCase       =  len(self.fairseq_tokens_to_ids   )

      UpperCamelCase       =  {v: k for k, v in self.fairseq_tokens_to_ids.items()}







   def __getstate__(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.__dict__.copy()
      UpperCamelCase       =  None
      UpperCamelCase       =  self.sp_model.serialized_model_proto()
      return state







   def __setstate__(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  d

      # for backward compatibility
      if not hasattr(self ,   '''sp_model_kwargs'''   ):
         UpperCamelCase       =  {}

      UpperCamelCase       =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.LoadFromSerializedProto(self.sp_model_proto   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      if token_ids_a is None:
         return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      UpperCamelCase       =  [self.cls_token_id]
      UpperCamelCase       =  [self.sep_token_id]
      return cls + token_ids_a + sep + sep + token_ids_a + sep







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None ,   lowerCamelCase__ = False   ):

      '''simple docstring'''







      if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=lowerCamelCase__ ,   token_ids_a=lowerCamelCase__ ,   already_has_special_tokens=lowerCamelCase__   )

      if token_ids_a is None:
         return [1] + ([0] * len(lowerCamelCase__   )) + [1]
      return [1] + ([0] * len(lowerCamelCase__   )) + [1, 1] + ([0] * len(lowerCamelCase__   )) + [1]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      UpperCamelCase       =  [self.sep_token_id]
      UpperCamelCase       =  [self.cls_token_id]

      if token_ids_a is None:
         return len(cls + token_ids_a + sep   ) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]







   @property
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      return len(self.fairseq_ids_to_tokens   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  {self.convert_ids_to_tokens(lowerCamelCase__   ): i for i in range(self.vocab_size   )}
      vocab.update(self.added_tokens_encoder   )
      return vocab







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.sp_model.encode(lowerCamelCase__ ,   out_type=lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      if token in self.fairseq_tokens_to_ids:
         return self.fairseq_tokens_to_ids[token]
      else:
         return self.unk_token_id







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.fairseq_ids_to_tokens[index]







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  ''''''.join(lowerCamelCase__   ).replace(lowerCamelCase__ ,   ''' '''   ).strip()
      return out_string







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      if not os.path.isdir(lowerCamelCase__   ):
         logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
         return
      UpperCamelCase       =  os.path.join(
          lowerCamelCase__ ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )
      UpperCamelCase       =  os.path.join(
          lowerCamelCase__ ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''monolingual_vocab_file'''] ,   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCamelCase__   ) and os.path.isfile(self.vocab_file   ):
         copyfile(self.vocab_file ,   lowerCamelCase__   )
      elif not os.path.isfile(self.vocab_file   ):
         with open(lowerCamelCase__ ,   '''wb'''   ) as fi:
            UpperCamelCase       =  self.sp_model.serialized_model_proto()
            fi.write(lowerCamelCase__   )

      if os.path.abspath(self.monolingual_vocab_file   ) != os.path.abspath(
          lowerCamelCase__   ) and os.path.isfile(self.monolingual_vocab_file   ):
         copyfile(self.monolingual_vocab_file ,   lowerCamelCase__   )
      elif not os.path.isfile(self.monolingual_vocab_file   ):
         with open(lowerCamelCase__ ,   '''w''' ,   encoding='''utf-8'''   ) as fp:
            for token in self.fairseq_tokens_to_ids:
               if token not in self.all_special_tokens:
                  fp.write(f'{str(lowerCamelCase__   )} \n'   )

      return out_vocab_file, out_monolingual_vocab_file


'''simple docstring'''


import torch
from torch import nn


class 						lowercase__	(  nn.Module   ):






   '''simple docstring'''







   def __init__(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__=1 ,   lowerCamelCase__=False   ):

      '''simple docstring'''







      super().__init__()

      UpperCamelCase       =  n_token
      UpperCamelCase       =  d_embed
      UpperCamelCase       =  d_proj

      UpperCamelCase       =  cutoffs + [n_token]
      UpperCamelCase       =  [0] + self.cutoffs
      UpperCamelCase       =  div_val

      UpperCamelCase       =  self.cutoffs[0]
      UpperCamelCase       =  len(self.cutoffs   ) - 1
      UpperCamelCase       =  self.shortlist_size + self.n_clusters

      if self.n_clusters > 0:
         UpperCamelCase       =  nn.Parameter(torch.zeros(self.n_clusters ,   self.d_embed   )   )
         UpperCamelCase       =  nn.Parameter(torch.zeros(self.n_clusters   )   )

      UpperCamelCase       =  nn.ModuleList()
      UpperCamelCase       =  nn.ParameterList()

      if div_val == 1:
         for i in range(len(self.cutoffs   )   ):
            if d_proj != d_embed:
               self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCamelCase__ ,   lowerCamelCase__   )   )   )
            else:
               self.out_projs.append(lowerCamelCase__   )

         self.out_layers.append(nn.Linear(lowerCamelCase__ ,   lowerCamelCase__   )   )
      else:
         for i in range(len(self.cutoffs   )   ):
            UpperCamelCase     , UpperCamelCase       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
            UpperCamelCase       =  d_embed // (div_val**i)

            self.out_projs.append(nn.Parameter(torch.FloatTensor(lowerCamelCase__ ,   lowerCamelCase__   )   )   )

            self.out_layers.append(nn.Linear(lowerCamelCase__ ,   r_idx - l_idx   )   )

      UpperCamelCase       =  keep_order







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   ):

      '''simple docstring'''







      if proj is None:
         UpperCamelCase       =  nn.functional.linear(lowerCamelCase__ ,   lowerCamelCase__ ,   bias=lowerCamelCase__   )
      else:
         # if CUDA_MAJOR <= 9 and CUDA_MINOR <= 1:
         UpperCamelCase       =  nn.functional.linear(lowerCamelCase__ ,   proj.t().contiguous()   )
         UpperCamelCase       =  nn.functional.linear(lowerCamelCase__ ,   lowerCamelCase__ ,   bias=lowerCamelCase__   )
         # else:
         #     logit = torch.einsum('bd,de,ev->bv', (hidden, proj, weight.t()))
         #     if bias is not None:
         #         logit = logit + bias

      return logit







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__=None ,   lowerCamelCase__=False   ):

      '''simple docstring'''







      if labels is not None:
         # Shift so that tokens < n predict n
         UpperCamelCase       =  hidden[..., :-1, :].contiguous()
         UpperCamelCase       =  labels[..., 1:].contiguous()
         UpperCamelCase       =  hidden.view(-1 ,   hidden.size(-1   )   )
         UpperCamelCase       =  labels.view(-1   )
         if hidden.size(0   ) != labels.size(0   ):
            raise RuntimeError('''Input and labels should have the same size in the batch dimension.'''   )
      else:
         UpperCamelCase       =  hidden.view(-1 ,   hidden.size(-1   )   )

      if self.n_clusters == 0:
         UpperCamelCase       =  self._compute_logit(lowerCamelCase__ ,   self.out_layers[0].weight ,   self.out_layers[0].bias ,   self.out_projs[0]   )
         if labels is not None:
            UpperCamelCase       =  labels != -1_0_0
            UpperCamelCase       =  torch.zeros_like(lowerCamelCase__ ,   dtype=hidden.dtype ,   device=hidden.device   )
            UpperCamelCase       =  (
                -nn.functional.log_softmax(lowerCamelCase__ ,   dim=-1   )[mask].gather(1 ,   labels[mask].unsqueeze(1   )   ).squeeze(1   )
            )
         else:
            UpperCamelCase       =  nn.functional.log_softmax(lowerCamelCase__ ,   dim=-1   )
      else:
         # construct weights and biases
         UpperCamelCase     , UpperCamelCase       =  [], []
         for i in range(len(self.cutoffs   )   ):
            if self.div_val == 1:
               UpperCamelCase     , UpperCamelCase       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
               UpperCamelCase       =  self.out_layers[0].weight[l_idx:r_idx]
               UpperCamelCase       =  self.out_layers[0].bias[l_idx:r_idx]
            else:
               UpperCamelCase       =  self.out_layers[i].weight
               UpperCamelCase       =  self.out_layers[i].bias

            if i == 0:
               UpperCamelCase       =  torch.cat([weight_i, self.cluster_weight] ,   dim=0   )
               UpperCamelCase       =  torch.cat([bias_i, self.cluster_bias] ,   dim=0   )

            weights.append(lowerCamelCase__   )
            biases.append(lowerCamelCase__   )

         UpperCamelCase     , UpperCamelCase     , UpperCamelCase       =  weights[0], biases[0], self.out_projs[0]

         UpperCamelCase       =  self._compute_logit(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   )
         UpperCamelCase       =  nn.functional.log_softmax(lowerCamelCase__ ,   dim=1   )

         if labels is None:
            UpperCamelCase       =  hidden.new_empty((head_logit.size(0   ), self.n_token)   )
         else:
            UpperCamelCase       =  torch.zeros_like(lowerCamelCase__ ,   dtype=hidden.dtype ,   device=hidden.device   )

         UpperCamelCase       =  0
         UpperCamelCase       =  [0] + self.cutoffs
         for i in range(len(lowerCamelCase__   ) - 1   ):
            UpperCamelCase     , UpperCamelCase       =  cutoff_values[i], cutoff_values[i + 1]

            if labels is not None:
               UpperCamelCase       =  (labels >= l_idx) & (labels < r_idx)
               UpperCamelCase       =  mask_i.nonzero().squeeze()

               if indices_i.numel() == 0:
                  continue

               UpperCamelCase       =  labels.index_select(0 ,   lowerCamelCase__   ) - l_idx
               UpperCamelCase       =  head_logprob.index_select(0 ,   lowerCamelCase__   )
               UpperCamelCase       =  hidden.index_select(0 ,   lowerCamelCase__   )
            else:
               UpperCamelCase       =  hidden

            if i == 0:
               if labels is not None:
                  UpperCamelCase       =  head_logprob_i.gather(1 ,   target_i[:, None]   ).squeeze(1   )
               else:
                  UpperCamelCase       =  head_logprob[:, : self.cutoffs[0]]
            else:
               UpperCamelCase     , UpperCamelCase     , UpperCamelCase       =  weights[i], biases[i], self.out_projs[i]

               UpperCamelCase       =  self._compute_logit(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   )
               UpperCamelCase       =  nn.functional.log_softmax(lowerCamelCase__ ,   dim=1   )
               UpperCamelCase       =  self.cutoffs[0] + i - 1  # No probability for the head cluster
               if labels is not None:
                  UpperCamelCase       =  head_logprob_i[:, cluster_prob_idx] + tail_logprob_i.gather(
                      1 ,   target_i[:, None]   ).squeeze(1   )
               else:
                  UpperCamelCase       =  head_logprob[:, cluster_prob_idx, None] + tail_logprob_i
                  UpperCamelCase       =  logprob_i

            if labels is not None:
               if (hasattr(self ,   '''keep_order'''   ) and self.keep_order) or keep_order:
                  out.index_copy_(0 ,   lowerCamelCase__ ,   -logprob_i   )
               else:
                  out[offset : offset + logprob_i.size(0   )].copy_(-logprob_i   )
               offset += logprob_i.size(0   )

      return out







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      if self.n_clusters == 0:
         UpperCamelCase       =  self._compute_logit(lowerCamelCase__ ,   self.out_layers[0].weight ,   self.out_layers[0].bias ,   self.out_projs[0]   )
         return nn.functional.log_softmax(lowerCamelCase__ ,   dim=-1   )
      else:
         # construct weights and biases
         UpperCamelCase     , UpperCamelCase       =  [], []
         for i in range(len(self.cutoffs   )   ):
            if self.div_val == 1:
               UpperCamelCase     , UpperCamelCase       =  self.cutoff_ends[i], self.cutoff_ends[i + 1]
               UpperCamelCase       =  self.out_layers[0].weight[l_idx:r_idx]
               UpperCamelCase       =  self.out_layers[0].bias[l_idx:r_idx]
            else:
               UpperCamelCase       =  self.out_layers[i].weight
               UpperCamelCase       =  self.out_layers[i].bias

            if i == 0:
               UpperCamelCase       =  torch.cat([weight_i, self.cluster_weight] ,   dim=0   )
               UpperCamelCase       =  torch.cat([bias_i, self.cluster_bias] ,   dim=0   )

            weights.append(lowerCamelCase__   )
            biases.append(lowerCamelCase__   )

         UpperCamelCase     , UpperCamelCase     , UpperCamelCase       =  weights[0], biases[0], self.out_projs[0]
         UpperCamelCase       =  self._compute_logit(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   )

         UpperCamelCase       =  hidden.new_empty((head_logit.size(0   ), self.n_token)   )
         UpperCamelCase       =  nn.functional.log_softmax(lowerCamelCase__ ,   dim=1   )

         UpperCamelCase       =  [0] + self.cutoffs
         for i in range(len(lowerCamelCase__   ) - 1   ):
            UpperCamelCase     , UpperCamelCase       =  cutoff_values[i], cutoff_values[i + 1]

            if i == 0:
               UpperCamelCase       =  head_logprob[:, : self.cutoffs[0]]
            else:
               UpperCamelCase     , UpperCamelCase     , UpperCamelCase       =  weights[i], biases[i], self.out_projs[i]

               UpperCamelCase       =  self._compute_logit(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   )
               UpperCamelCase       =  nn.functional.log_softmax(lowerCamelCase__ ,   dim=1   )

               UpperCamelCase       =  head_logprob[:, -i] + tail_logprob_i
               UpperCamelCase       =  logprob_i

         return out





