




import os
import re
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase_  						=   logging.get_logger(__name__)
lowerCamelCase_  						=   {"vocab_file": "spiece.model"}

lowerCamelCase_  						=   {
    "vocab_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/spiece.model",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/spiece.model",
    },
}

lowerCamelCase_  						=   {
    "google/fnet-base": 512,
    "google/fnet-large": 512,
}

lowerCamelCase_  						=   "▁"
class       __a					( __lowerCamelCase     ):


     """simple docstring"""


     _A				:    List[str]									=  VOCAB_FILES_NAMES
     _A				:    str									=  PRETRAINED_VOCAB_FILES_MAP
     _A				:    Union[str, Any]									=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _A				:    Optional[Any]									=  ["input_ids", "token_type_ids"]




     def __init__(	self			: Tuple				,_UpperCamelCase			: Union[str, Any]				,_UpperCamelCase			: List[str]=False				,_UpperCamelCase			: Dict=True				,_UpperCamelCase			: List[str]=True				,_UpperCamelCase			: Dict="<unk>"				,_UpperCamelCase			: Dict="[SEP]"				,_UpperCamelCase			: List[str]="<pad>"				,_UpperCamelCase			: Union[str, Any]="[CLS]"				,_UpperCamelCase			: Optional[int]="[MASK]"				,_UpperCamelCase			: Optional[Dict[str, Any]] = None				,**_UpperCamelCase			: Union[str, Any]				,)		->   None:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__								=(
             AddedToken(_UpperCamelCase				,lstrip=_UpperCamelCase				,rstrip=_UpperCamelCase				,normalized=_UpperCamelCase						)
             if isinstance(_UpperCamelCase				,_UpperCamelCase						)
             else mask_token
         )

         SCREAMING_SNAKE_CASE__								={} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             do_lower_case=_UpperCamelCase				,remove_space=_UpperCamelCase				,keep_accents=_UpperCamelCase				,unk_token=_UpperCamelCase				,sep_token=_UpperCamelCase				,pad_token=_UpperCamelCase				,cls_token=_UpperCamelCase				,mask_token=_UpperCamelCase				,sp_model_kwargs=self.sp_model_kwargs				,**_UpperCamelCase				,)

         SCREAMING_SNAKE_CASE__								=do_lower_case
         SCREAMING_SNAKE_CASE__								=remove_space
         SCREAMING_SNAKE_CASE__								=keep_accents
         SCREAMING_SNAKE_CASE__								=vocab_file

         SCREAMING_SNAKE_CASE__								=spm.SentencePieceProcessor(**self.sp_model_kwargs						)
         self.sp_model.Load(_UpperCamelCase						)




     @property
     def   __A				(	self			: Tuple						)		->   Any:


         '''simple docstring'''





         return len(self.sp_model						)




     def   __A				(	self			: Optional[int]						)		->   Optional[Any]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__								={self.convert_ids_to_tokens(_UpperCamelCase						): i for i in range(self.vocab_size						)}
         vocab.update(self.added_tokens_encoder						)
         return vocab




     def __getstate__(	self			: List[str]						)		->   List[str]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__								=self.__dict__.copy()
         SCREAMING_SNAKE_CASE__								=None
         return state




     def __setstate__(	self			: Tuple				,_UpperCamelCase			: int						)		->   Tuple:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__								=d

         # for backward compatibility
         if not hasattr(self				,"""sp_model_kwargs"""						):
             SCREAMING_SNAKE_CASE__								={}

         SCREAMING_SNAKE_CASE__								=spm.SentencePieceProcessor(**self.sp_model_kwargs						)
         self.sp_model.Load(self.vocab_file						)




     def   __A				(	self			: int				,_UpperCamelCase			: Optional[Any]						)		->   str:


         '''simple docstring'''





         if self.remove_space:
             SCREAMING_SNAKE_CASE__								=""" """.join(inputs.strip().split()						)
         else:
             SCREAMING_SNAKE_CASE__								=inputs
         SCREAMING_SNAKE_CASE__								=outputs.replace("""``"""				,"""\""""						).replace("""''"""				,"""\""""						)

         if not self.keep_accents:
             SCREAMING_SNAKE_CASE__								=unicodedata.normalize("""NFKD"""				,_UpperCamelCase						)
             SCREAMING_SNAKE_CASE__								="""""".join([c for c in outputs if not unicodedata.combining(_UpperCamelCase						)]						)
         if self.do_lower_case:
             SCREAMING_SNAKE_CASE__								=outputs.lower()

         return outputs




     def   __A				(	self			: Tuple				,_UpperCamelCase			: str						)		->   List[str]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__								=self.preprocess_text(_UpperCamelCase						)
         SCREAMING_SNAKE_CASE__								=self.sp_model.encode(_UpperCamelCase				,out_type=_UpperCamelCase						)
         SCREAMING_SNAKE_CASE__								=[]
         for piece in pieces:
             if len(_UpperCamelCase						) > 1 and piece[-1] == str(""","""						) and piece[-2].isdigit():
                 SCREAMING_SNAKE_CASE__								=self.sp_model.EncodeAsPieces(piece[:-1].replace(_UpperCamelCase				,""""""						)						)
                 if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                     if len(cur_pieces[0]						) == 1:
                         SCREAMING_SNAKE_CASE__								=cur_pieces[1:]
                     else:
                         SCREAMING_SNAKE_CASE__								=cur_pieces[0][1:]
                 cur_pieces.append(piece[-1]						)
                 new_pieces.extend(_UpperCamelCase						)
             else:
                 new_pieces.append(_UpperCamelCase						)

         return new_pieces




     def   __A				(	self			: Optional[int]				,_UpperCamelCase			: Optional[Any]						)		->   str:


         '''simple docstring'''





         return self.sp_model.PieceToId(_UpperCamelCase						)




     def   __A				(	self			: List[str]				,_UpperCamelCase			: List[str]						)		->   Union[str, Any]:


         '''simple docstring'''





         return self.sp_model.IdToPiece(_UpperCamelCase						)




     def   __A				(	self			: List[Any]				,_UpperCamelCase			: Union[str, Any]						)		->   Dict:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__								=[]
         SCREAMING_SNAKE_CASE__								=""""""
         SCREAMING_SNAKE_CASE__								=False
         for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                 if not prev_is_special:
                     out_string += " "
                 out_string += self.sp_model.decode(_UpperCamelCase						) + token
                 SCREAMING_SNAKE_CASE__								=True
                 SCREAMING_SNAKE_CASE__								=[]
             else:
                 current_sub_tokens.append(_UpperCamelCase						)
                 SCREAMING_SNAKE_CASE__								=False
         out_string += self.sp_model.decode(_UpperCamelCase						)
         return out_string.strip()




     def   __A				(	self			: Tuple				,_UpperCamelCase			: List[int]				,_UpperCamelCase			: bool = False				,_UpperCamelCase			: bool = None				,_UpperCamelCase			: bool = True				,**_UpperCamelCase			: List[str]				,)		->   str:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__								=kwargs.pop("""use_source_tokenizer"""				,_UpperCamelCase						)

         SCREAMING_SNAKE_CASE__								=self.convert_ids_to_tokens(_UpperCamelCase				,skip_special_tokens=_UpperCamelCase						)

         # To avoid mixing byte-level and unicode for byte-level BPT
         # we need to build string separately for added tokens and byte-level tokens
         # cf. https://github.com/huggingface/transformers/issues/1133
         SCREAMING_SNAKE_CASE__								=[]
         SCREAMING_SNAKE_CASE__								=[]
         for token in filtered_tokens:
             if skip_special_tokens and token in self.all_special_ids:
                 continue
             if token in self.added_tokens_encoder:
                 if current_sub_text:
                     sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase						)						)
                     SCREAMING_SNAKE_CASE__								=[]
                 sub_texts.append(_UpperCamelCase						)
             else:
                 current_sub_text.append(_UpperCamelCase						)
         if current_sub_text:
             sub_texts.append(self.convert_tokens_to_string(_UpperCamelCase						)						)

         # Mimic the behavior of the Rust tokenizer:
         # No space after <unk>
         if spaces_between_special_tokens:
             SCREAMING_SNAKE_CASE__								=re.sub(r"""(<unk>) """				,r"""\1"""				,""" """.join(_UpperCamelCase						)						)
         else:
             SCREAMING_SNAKE_CASE__								="""""".join(_UpperCamelCase						)

         SCREAMING_SNAKE_CASE__								=(
             clean_up_tokenization_spaces
             if clean_up_tokenization_spaces is not None
             else self.clean_up_tokenization_spaces
         )
         if clean_up_tokenization_spaces:
             SCREAMING_SNAKE_CASE__								=self.clean_up_tokenization(_UpperCamelCase						)
             return clean_text
         else:
             return text




     def   __A				(	self			: str				,_UpperCamelCase			: List[int]				,_UpperCamelCase			: Optional[List[int]] = None						)		->   List[int]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__								=[self.sep_token_id]
         SCREAMING_SNAKE_CASE__								=[self.cls_token_id]
         if token_ids_a is None:
             return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep




     def   __A				(	self			: Any				,_UpperCamelCase			: List[int]				,_UpperCamelCase			: Optional[List[int]] = None				,_UpperCamelCase			: bool = False						)		->   List[int]:


         '''simple docstring'''





         if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=_UpperCamelCase				,token_ids_a=_UpperCamelCase				,already_has_special_tokens=_UpperCamelCase						)

         if token_ids_a is not None:
             return [1] + ([0] * len(_UpperCamelCase						)) + [1] + ([0] * len(_UpperCamelCase						)) + [1]
         return [1] + ([0] * len(_UpperCamelCase						)) + [1]




     def   __A				(	self			: Union[str, Any]				,_UpperCamelCase			: List[int]				,_UpperCamelCase			: Optional[List[int]] = None						)		->   List[int]:


         '''simple docstring'''





         SCREAMING_SNAKE_CASE__								=[self.sep_token_id]
         SCREAMING_SNAKE_CASE__								=[self.cls_token_id]

         if token_ids_a is None:
             return len(cls + token_ids_a + sep						) * [0]
         return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]




     def   __A				(	self			: List[str]				,_UpperCamelCase			: str				,_UpperCamelCase			: Optional[str] = None						)		->   Tuple[str]:


         '''simple docstring'''





         if not os.path.isdir(_UpperCamelCase						):
             logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
             return
         SCREAMING_SNAKE_CASE__								=os.path.join(
             _UpperCamelCase				,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

         if os.path.abspath(self.vocab_file						) != os.path.abspath(_UpperCamelCase						) and os.path.isfile(self.vocab_file						):
             copyfile(self.vocab_file				,_UpperCamelCase						)
         elif not os.path.isfile(self.vocab_file						):
             with open(_UpperCamelCase				,"""wb"""						) as fi:
                 SCREAMING_SNAKE_CASE__								=self.sp_model.serialized_model_proto()
                 fi.write(_UpperCamelCase						)

         return (out_vocab_file,)












def 	UpperCAmelCase_    (	__UpperCamelCase	):
    if not isinstance(__UpperCamelCase,				__UpperCamelCase	):
        raise ValueError("""double_factorial() only accepts integral values"""	)
    if n < 0:
        raise ValueError("""double_factorial() not defined for negative values"""	)
    return 1 if n <= 1 else n * double_factorial(n - 2	)


if __name__ == "__main__":
    import doctest

    doctest.testmod()






