



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowercase		:				Optional[Any]      		=logging.get_logger(__name__)
_lowercase		:				Tuple      		={"vocab_file": "spiece.model"}

_lowercase		:				Optional[int]      		={
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

_lowercase		:				List[str]      		={
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

_lowercase		:				Optional[int]      		="▁"

class        snake_case__    (__lowerCamelCase		):







      """simple docstring"""




      __lowerCAmelCase					:Any      =					VOCAB_FILES_NAMES
      __lowerCAmelCase					:int      =					PRETRAINED_VOCAB_FILES_MAP
      __lowerCAmelCase					:Dict      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

      def __init__(       self ,       __lowercase ,       __lowercase=True ,       __lowercase=True ,       __lowercase=False ,       __lowercase="[CLS]" ,       __lowercase="[SEP]" ,       __lowercase="<unk>" ,       __lowercase="[SEP]" ,       __lowercase="<pad>" ,       __lowercase="[CLS]" ,       __lowercase="[MASK]" ,       __lowercase = None ,       **__lowercase ,       ) ->				Dict:
         """simple docstring"""




         a__       :						Optional[int]							=  (
             AddedToken(UpperCamelCase_ ,       lstrip=UpperCamelCase_ ,       rstrip=UpperCamelCase_ ,       normalized=UpperCamelCase_  )
             if isinstance(UpperCamelCase_ ,       UpperCamelCase_  )
             else mask_token
         )

         a__       :						Optional[int]							=  {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             do_lower_case=UpperCamelCase_ ,       remove_space=UpperCamelCase_ ,       keep_accents=UpperCamelCase_ ,       bos_token=UpperCamelCase_ ,       eos_token=UpperCamelCase_ ,       unk_token=UpperCamelCase_ ,       sep_token=UpperCamelCase_ ,       pad_token=UpperCamelCase_ ,       cls_token=UpperCamelCase_ ,       mask_token=UpperCamelCase_ ,       sp_model_kwargs=self.sp_model_kwargs ,       **UpperCamelCase_ ,       )

         a__       :						Dict							=  do_lower_case
         a__       :						int							=  remove_space
         a__       :						Optional[int]							=  keep_accents
         a__       :						Any							=  vocab_file

         a__       :						Optional[Any]							=  spm.SentencePieceProcessor(**self.sp_model_kwargs  )
         self.sp_model.Load(UpperCamelCase_  )

      @property
      def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[Any]:
         """simple docstring"""




         return len(self.sp_model  )

      def 							SCREAMING_SNAKE_CASE__(       self  ) ->				Optional[Any]:
         """simple docstring"""




         a__       :						Optional[int]							=  {self.convert_ids_to_tokens(UpperCamelCase_  ): i for i in range(self.vocab_size  )}
         vocab.update(self.added_tokens_encoder  )
         return vocab

      def __getstate__(       self  ) ->				Optional[int]:
         """simple docstring"""




         a__       :						List[str]							=  self.__dict__.copy()
         a__       :						Any							=  None
         return state

      def __setstate__(       self ,       __lowercase  ) ->				str:
         """simple docstring"""




         a__       :						Optional[Any]							=  d

         # for backward compatibility
         if not hasattr(self ,       """sp_model_kwargs"""  ):
            a__       :						Tuple							=  {}

         a__       :						List[Any]							=  spm.SentencePieceProcessor(**self.sp_model_kwargs  )
         self.sp_model.Load(self.vocab_file  )

      def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase  ) ->				int:
         """simple docstring"""




         if self.remove_space:
            a__       :						int							=  """ """.join(inputs.strip().split()  )
         else:
            a__       :						str							=  inputs
         a__       :						Any							=  outputs.replace("""``""" ,       """\""""  ).replace("""''""" ,       """\""""  )

         if not self.keep_accents:
            a__       :						Any							=  unicodedata.normalize("""NFKD""" ,       UpperCamelCase_  )
            a__       :						Optional[int]							=  """""".join([c for c in outputs if not unicodedata.combining(UpperCamelCase_  )]  )
         if self.do_lower_case:
            a__       :						List[Any]							=  outputs.lower()

         return outputs

      def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase  ) ->				str:
         """simple docstring"""




         a__       :						Any							=  self.preprocess_text(UpperCamelCase_  )
         a__       :						Optional[Any]							=  self.sp_model.encode(UpperCamelCase_ ,       out_type=UpperCamelCase_  )
         a__       :						Optional[Any]							=  []
         for piece in pieces:
            if len(UpperCamelCase_  ) > 1 and piece[-1] == str(""","""  ) and piece[-2].isdigit():
               a__       :						int							=  self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase_ ,       """"""  )  )
               if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                  if len(cur_pieces[0]  ) == 1:
                     a__       :						Any							=  cur_pieces[1:]
                  else:
                     a__       :						List[Any]							=  cur_pieces[0][1:]
               cur_pieces.append(piece[-1]  )
               new_pieces.extend(UpperCamelCase_  )
            else:
               new_pieces.append(UpperCamelCase_  )

         return new_pieces

      def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase  ) ->				List[str]:
         """simple docstring"""




         return self.sp_model.PieceToId(UpperCamelCase_  )

      def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase  ) ->				Dict:
         """simple docstring"""




         return self.sp_model.IdToPiece(UpperCamelCase_  )

      def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase  ) ->				int:
         """simple docstring"""




         a__       :						Tuple							=  []
         a__       :						Union[str, Any]							=  """"""
         a__       :						Union[str, Any]							=  False
         for token in tokens:
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
               if not prev_is_special:
                  out_string += " "
               out_string += self.sp_model.decode(UpperCamelCase_  ) + token
               a__       :						Any							=  True
               a__       :						Optional[int]							=  []
            else:
               current_sub_tokens.append(UpperCamelCase_  )
               a__       :						List[str]							=  False
         out_string += self.sp_model.decode(UpperCamelCase_  )
         return out_string.strip()

      def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase ,       __lowercase = None  ) ->				Tuple:
         """simple docstring"""




         a__       :						List[Any]							=  [self.sep_token_id]
         a__       :						Dict							=  [self.cls_token_id]
         if token_ids_a is None:
            return cls + token_ids_a + sep
         return cls + token_ids_a + sep + token_ids_a + sep

      def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase ,       __lowercase = None ,       __lowercase = False  ) ->				Optional[int]:
         """simple docstring"""




         if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=UpperCamelCase_ ,       token_ids_a=UpperCamelCase_ ,       already_has_special_tokens=UpperCamelCase_  )

         if token_ids_a is not None:
            return [1] + ([0] * len(UpperCamelCase_  )) + [1] + ([0] * len(UpperCamelCase_  )) + [1]
         return [1] + ([0] * len(UpperCamelCase_  )) + [1]

      def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase ,       __lowercase = None  ) ->				Any:
         """simple docstring"""




         a__       :						Optional[int]							=  [self.sep_token_id]
         a__       :						List[str]							=  [self.cls_token_id]

         if token_ids_a is None:
            return len(cls + token_ids_a + sep  ) * [0]
         return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]

      def 							SCREAMING_SNAKE_CASE__(       self ,       __lowercase ,       __lowercase = None  ) ->				Optional[int]:
         """simple docstring"""




         if not os.path.isdir(UpperCamelCase_  ):
            logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
            return
         a__       :						Any							=  os.path.join(
             UpperCamelCase_ ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )

         if os.path.abspath(self.vocab_file  ) != os.path.abspath(UpperCamelCase_  ) and os.path.isfile(self.vocab_file  ):
            copyfile(self.vocab_file ,       UpperCamelCase_  )
         elif not os.path.isfile(self.vocab_file  ):
            with open(UpperCamelCase_ ,       """wb"""  ) as fi:
               a__       :						Union[str, Any]							=  self.sp_model.serialized_model_proto()
               fi.write(UpperCamelCase_  )

         return (out_vocab_file,)









def      lowerCAmelCase_					(  _lowercase    :       int)							->	int:







			"""simple docstring"""

			if not isinstance(_lowercase						,       _lowercase):
						raise TypeError("""only integers accepted as input""")
			else:
						a__       :						Any							=  str(abs(_lowercase))
						a__       :						str							=  [list(_lowercase) for char in range(len(_lowercase))]
						for index in range(len(_lowercase)):
									num_transpositions[index].pop(_lowercase)
						return max(
						    int("""""".join(list(_lowercase))) for transposition in num_transpositions)


if __name__ == "__main__":
		__import__("doctest").testmod()




