



"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


snake_case   					=				'''▁'''

snake_case   					=				{'''vocab_file''': '''spiece.model'''}

snake_case   					=				{
    '''vocab_file''': {'''google/pegasus-xsum''': '''https://huggingface.co/google/pegasus-xsum/resolve/main/spiece.model'''}
}

snake_case   					=				{
    '''google/pegasus-xsum''': 5_1_2,
}


snake_case   					=				logging.get_logger(__name__)



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):
   A__		:	Tuple           =  VOCAB_FILES_NAMES

   A__		:	List[Any]           =  VOCAB_FILES_NAMES
   A__		:	Any           =  PRETRAINED_VOCAB_FILES_MAP
   A__		:	Tuple           =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A__		:	List[Any]           =  ['''input_ids''', '''attention_mask''']



   def __init__(       self    :	Union[str, Any]							,       __lowerCamelCase    :	List[str]							,       __lowerCamelCase    :	Optional[int]="<pad>"							,       __lowerCamelCase    :	Optional[Any]="</s>"							,       __lowerCamelCase    :	Any="<unk>"							,       __lowerCamelCase    :	Union[str, Any]="<mask_2>"							,       __lowerCamelCase    :	Optional[Any]="<mask_1>"							,       __lowerCamelCase    :	List[Any]=None							,       __lowerCamelCase    :	int=1_0_3							,       __lowerCamelCase    :	Optional[Dict[str, Any]] = None							,       **__lowerCamelCase    :	List[str]							,       ):



     """simple docstring"""
     _snake_case   	=      offset
     if additional_special_tokens is not None:
       if not isinstance(__lowerCamelCase							,       __lowerCamelCase				):
         raise TypeError(
             f"""additional_special_tokens should be of type {type(__lowerCamelCase				)}, but is"""
             f""" {type(__lowerCamelCase				)}"""				)

       _snake_case   	=      (
           ([mask_token_sent] + additional_special_tokens)
           if mask_token_sent not in additional_special_tokens and mask_token_sent is not None
           else additional_special_tokens
       )
       # fill additional tokens with ..., <unk_token_102> in case not all additional tokens are already taken
       additional_special_tokens_extended += [
           f"""<unk_{i}>""" for i in range(len(__lowerCamelCase				)							,       self.offset - 1				)
       ]

       if len(set(__lowerCamelCase				)				) != len(__lowerCamelCase				):
         raise ValueError(
             '''Please make sure that the provided additional_special_tokens do not contain an incorrectly'''
             f""" shifted list of <unk_x> tokens. Found {additional_special_tokens_extended}."""				)
       _snake_case   	=      additional_special_tokens_extended
     else:
       _snake_case   	=      [mask_token_sent] if mask_token_sent is not None else []
       additional_special_tokens += [f"""<unk_{i}>""" for i in range(2							,       self.offset				)]

     _snake_case   	=      {} if sp_model_kwargs is None else sp_model_kwargs

     super().__init__(
         eos_token=__lowerCamelCase							,       unk_token=__lowerCamelCase							,       mask_token=__lowerCamelCase							,       pad_token=__lowerCamelCase							,       mask_token_sent=__lowerCamelCase							,       offset=__lowerCamelCase							,       additional_special_tokens=__lowerCamelCase							,       sp_model_kwargs=self.sp_model_kwargs							,       **__lowerCamelCase							,       )
     _snake_case   	=      mask_token_sent
     _snake_case   	=      vocab_file
     _snake_case   	=      spm.SentencePieceProcessor(**self.sp_model_kwargs				)
     self.sp_model.Load(__lowerCamelCase				)

     # add special tokens to encoder dict
     _snake_case   	=      {
         0: self.pad_token,
         1: self.eos_token,
     }

     if self.mask_token_sent is not None:
       self.encoder.update(
           {
               2: self.mask_token_sent,
               3: self.mask_token,
           }				)

     if self.offset > 0:
       # entries 2-104 are only used for pretraining and called <mask_1>, <mask_2>, unk_2, ...unk_102
       # mask_token_sent is already added to list -> so start at 1
       self.encoder.update({i + 3: additional_special_tokens[i] for i in range(1							,       self.offset - 1				)}				)

     _snake_case   	=      {v: k for k, v in self.encoder.items()}



   @property
   def    __UpperCAmelCase    (       self    :	List[str]				):



     """simple docstring"""
     return len(self.sp_model				) + self.offset



   def    __UpperCAmelCase    (       self    :	Optional[Any]				):



     """simple docstring"""
     _snake_case   	=      {self.convert_ids_to_tokens(__lowerCamelCase				): i for i in range(self.vocab_size				)}
     vocab.update(self.added_tokens_encoder				)
     return vocab



   def __getstate__(       self    :	Dict				):



     """simple docstring"""
     _snake_case   	=      self.__dict__.copy()
     _snake_case   	=      None
     return state



   def __setstate__(       self    :	Union[str, Any]							,       __lowerCamelCase    :	Any				):



     """simple docstring"""
     _snake_case   	=      d

     # for backward compatibility
     if not hasattr(self							,       '''sp_model_kwargs'''				):
       _snake_case   	=      {}

     _snake_case   	=      spm.SentencePieceProcessor(**self.sp_model_kwargs				)
     self.sp_model.Load(self.vocab_file				)



   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	str				):



     """simple docstring"""
     return self.sp_model.encode(__lowerCamelCase							,       out_type=__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	str				):



     """simple docstring"""
     if token in self.decoder:
       return self.decoder[token]
     elif token in self.added_tokens_decoder:
       return self.added_tokens_decoder[token]
     _snake_case   	=      self.sp_model.piece_to_id(__lowerCamelCase				)
     return sp_id + self.offset



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     if index in self.encoder:
       return self.encoder[index]
     elif index in self.added_tokens_encoder:
       return self.added_tokens_encoder[index]
     else:
       _snake_case   	=      self.sp_model.IdToPiece(index - self.offset				)
     return token



   def    __UpperCAmelCase    (       self    :	List[str]							,       __lowerCamelCase    :	List[str]				):



     """simple docstring"""
     _snake_case   	=      []
     _snake_case   	=      ''''''
     for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
         out_string += self.sp_model.decode(__lowerCamelCase				) + token
         _snake_case   	=      []
       else:
         current_sub_tokens.append(__lowerCamelCase				)
     out_string += self.sp_model.decode(__lowerCamelCase				)
     return out_string.strip()



   def    __UpperCAmelCase    (       self    :	int							,       __lowerCamelCase    :	Optional[int]=False				):



     """simple docstring"""
     return 1



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       __lowerCamelCase    :	str				):



     """simple docstring"""
     _snake_case   	=      set(self.all_special_ids				)  # call it once instead of inside list comp
     all_special_ids.remove(self.unk_token_id				)  # <unk> is only sometimes special

     return [1 if x in all_special_ids else 0 for x in seq]



   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	List							,       __lowerCamelCase    :	Optional[List] = None							,       __lowerCamelCase    :	bool = False				):



     """simple docstring"""
     if already_has_special_tokens:
       return self._special_token_mask(__lowerCamelCase				)
     elif token_ids_a is None:
       return self._special_token_mask(__lowerCamelCase				) + [1]
     else:
       return self._special_token_mask(token_ids_a + token_ids_a				) + [1]



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       __lowerCamelCase    :	Tuple							,       __lowerCamelCase    :	Any=None				):



     """simple docstring"""
     if token_ids_a is None:
       return token_ids_a + [self.eos_token_id]
     # We don't expect to process pairs, but leave the pair logic for API consistency
     return token_ids_a + token_ids_a + [self.eos_token_id]





   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	Optional[str] = None				):



     """simple docstring"""
     if not os.path.isdir(__lowerCamelCase				):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
       return
     _snake_case   	=      os.path.join(
         __lowerCamelCase							,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

     if os.path.abspath(self.vocab_file				) != os.path.abspath(__lowerCamelCase				) and os.path.isfile(self.vocab_file				):
       copyfile(self.vocab_file							,       __lowerCamelCase				)
     elif not os.path.isfile(self.vocab_file				):
       with open(__lowerCamelCase							,       '''wb'''				) as fi:
         _snake_case   	=      self.sp_model.serialized_model_proto()
         fi.write(__lowerCamelCase				)

     return (out_vocab_file,)










"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available


snake_case   					=				{'''tokenization_herbert''': ['''HerbertTokenizer''']}

try:
     if not is_tokenizers_available():
          raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
     pass
else:
     snake_case   					=				['''HerbertTokenizerFast''']


if TYPE_CHECKING:
     from .tokenization_herbert import HerbertTokenizer

     try:
          if not is_tokenizers_available():
               raise OptionalDependencyNotAvailable()
     except OptionalDependencyNotAvailable:
          pass
     else:
          from .tokenization_herbert_fast import HerbertTokenizerFast

else:
     import sys

     snake_case   					=				_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





