'''simple docstring'''

import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__  :						List[str]					=      logging.get_logger(__name__)

UpperCamelCase__  :						Any					=      {"vocab_file": "sentencepiece.model"}

UpperCamelCase__  :						Any					=      {
    "vocab_file": {
        "google/rembert": "https://huggingface.co/google/rembert/resolve/main/sentencepiece.model",
    },
}

UpperCamelCase__  :						Optional[int]					=      {
    "google/rembert": 256,
}


class 					_lowercase						(   snake_case__ ):
      '''simple docstring'''


      UpperCAmelCase_   :   List[str]   				= VOCAB_FILES_NAMES
      UpperCAmelCase_   :   Union[str, Any]   				= PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase_   :   List[Any]   				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




      def __init__(					self       ,lowerCamelCase_       ,lowerCamelCase_=False       ,lowerCamelCase_=True       ,lowerCamelCase_=True       ,lowerCamelCase_="[CLS]"       ,lowerCamelCase_="[SEP]"       ,lowerCamelCase_="[UNK]"       ,lowerCamelCase_="[SEP]"       ,lowerCamelCase_="[PAD]"       ,lowerCamelCase_="[CLS]"       ,lowerCamelCase_="[MASK]"       ,**lowerCamelCase_       ,)      ->      Union[str, Any]:



         '''simple docstring'''



         super().__init__(
             do_lower_case=__snake_case       ,remove_space=__snake_case       ,keep_accents=__snake_case       ,bos_token=__snake_case       ,eos_token=__snake_case       ,unk_token=__snake_case       ,sep_token=__snake_case       ,pad_token=__snake_case       ,cls_token=__snake_case       ,mask_token=__snake_case       ,**__snake_case       ,)

         UpperCAmelCase__				:					Tuple 					=	do_lower_case
         UpperCAmelCase__				:					Union[str, Any] 					=	remove_space
         UpperCAmelCase__				:					Optional[int] 					=	keep_accents
         UpperCAmelCase__				:					int 					=	vocab_file

         UpperCAmelCase__				:					Union[str, Any] 					=	spm.SentencePieceProcessor()
         self.sp_model.Load(__snake_case				)




      @property
      def 		lowerCAmelCase__       (					self				)      ->      Optional[Any]:



         '''simple docstring'''



         return len(self.sp_model				)




      def 		lowerCAmelCase__       (					self				)      ->      Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Tuple 					=	{self.convert_ids_to_tokens(__snake_case				): i for i in range(self.vocab_size				)}
         vocab.update(self.added_tokens_encoder				)
         return vocab




      def __getstate__(					self				)      ->      str:



         '''simple docstring'''



         UpperCAmelCase__				:					Dict 					=	self.__dict__.copy()
         UpperCAmelCase__				:					List[str] 					=	None
         return state




      def __setstate__(					self       ,lowerCamelCase_				)      ->      Optional[int]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	d
         UpperCAmelCase__				:					Dict 					=	spm.SentencePieceProcessor()
         self.sp_model.Load(self.vocab_file				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_=False				)      ->      Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	self.sp_model.EncodeAsPieces(__snake_case				)
         return pieces




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_				)      ->      List[Any]:



         '''simple docstring'''



         return self.sp_model.PieceToId(__snake_case				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_				)      ->      str:



         '''simple docstring'''



         return self.sp_model.IdToPiece(__snake_case				)




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_				)      ->      Any:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[Any] 					=	self.sp_model.decode_pieces(__snake_case				)
         return out_string




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_ = None				)      ->      Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					List[Any] 					=	[self.sep_token_id]
         UpperCAmelCase__				:					List[str] 					=	[self.cls_token_id]
         if token_ids_a is None:
            return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_ = None       ,lowerCamelCase_ = False				)      ->      Union[str, Any]:



         '''simple docstring'''



         if already_has_special_tokens:
            if token_ids_a is not None:
               raise ValueError(
                   '''You should not supply a second sequence if the provided sequence of '''
                   '''ids is already formatted with special tokens for the model.'''				)
            return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

         if token_ids_a is not None:
            return [1] + ([0] * len(__snake_case				)) + [1] + ([0] * len(__snake_case				)) + [1]
         return [1] + ([0] * len(__snake_case				)) + [1]




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_ = None				)      ->      str:



         '''simple docstring'''



         UpperCAmelCase__				:					Union[str, Any] 					=	[self.sep_token_id]
         UpperCAmelCase__				:					Dict 					=	[self.cls_token_id]

         if token_ids_a is None:
            return len(cls + token_ids_a + sep				) * [0]
         return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]




      def 		lowerCAmelCase__       (					self       ,lowerCamelCase_       ,lowerCamelCase_ = None				)      ->      Tuple:



         '''simple docstring'''



         if not os.path.isdir(__snake_case				):
            logger.error('''Vocabulary path ({}) should be a directory'''.format(__snake_case				)				)
            return
         UpperCAmelCase__				:					List[str] 					=	os.path.join(
             __snake_case       ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

         if os.path.abspath(self.vocab_file				) != os.path.abspath(__snake_case				):
            copyfile(self.vocab_file       ,__snake_case				)

         return (out_vocab_file,)








from ..utils import DummyObject, requires_backends

class     A__   (       metaclass=snake_case__      ):







    """simple docstring"""

    __magic_name__         =						['flax', 'transformers']
    def __init__(			self     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(self     ,   ['''flax''', '''transformers''']       )
    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )




    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )






class     A__   (       metaclass=snake_case__      ):







    """simple docstring"""

    __magic_name__         =						['flax', 'transformers']
    def __init__(			self     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(self     ,   ['''flax''', '''transformers''']       )
    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )




    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )






class     A__   (       metaclass=snake_case__      ):







    """simple docstring"""

    __magic_name__         =						['flax', 'transformers']
    def __init__(			self     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(self     ,   ['''flax''', '''transformers''']       )
    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )




    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )






class     A__   (       metaclass=snake_case__      ):







    """simple docstring"""

    __magic_name__         =						['flax', 'transformers']
    def __init__(			self     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(self     ,   ['''flax''', '''transformers''']       )
    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )




    @classmethod
    def 						a_     (			cls     ,   *__snake_case     ,   **__snake_case       ):
         requires_backends(cls     ,   ['''flax''', '''transformers''']       )




