


"""simple docstring"""







import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a__       :							Any        =  logging.get_logger(__name__)
a__       :							Any        =  {'''vocab_file''': '''spiece.model'''}

a__       :							Union[str, Any]        =  {
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

a__       :							List[Any]        =  {
    '''albert-base-v1''': 5_1_2,
    '''albert-large-v1''': 5_1_2,
    '''albert-xlarge-v1''': 5_1_2,
    '''albert-xxlarge-v1''': 5_1_2,
    '''albert-base-v2''': 5_1_2,
    '''albert-large-v2''': 5_1_2,
    '''albert-xlarge-v2''': 5_1_2,
    '''albert-xxlarge-v2''': 5_1_2,
}

a__       :							Any        =  '''▁'''




class 				UpperCamelCase_			(				__snake_case):

 """simple docstring"""





 snake_case__       :						Any					      =    VOCAB_FILES_NAMES
 snake_case__       :						Union[str, Any]					      =    PRETRAINED_VOCAB_FILES_MAP
 snake_case__       :						Union[str, Any]					      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




 def __init__(				self    :       Dict    ,    UpperCAmelCase__    :       List[str]    ,    UpperCAmelCase__    :       List[str]=True    ,    UpperCAmelCase__    :       Any=True    ,    UpperCAmelCase__    :       Optional[Any]=False    ,    UpperCAmelCase__    :       Tuple="[CLS]"    ,    UpperCAmelCase__    :       List[Any]="[SEP]"    ,    UpperCAmelCase__    :       str="<unk>"    ,    UpperCAmelCase__    :       Dict="[SEP]"    ,    UpperCAmelCase__    :       List[Any]="<pad>"    ,    UpperCAmelCase__    :       str="[CLS]"    ,    UpperCAmelCase__    :       Dict="[MASK]"    ,    UpperCAmelCase__    :       List[str] = None    ,    **UpperCAmelCase__    :       List[str]    ,    ) ->			List[str]:
      __SCREAMING_SNAKE_CASE		     =       (
          AddedToken(__UpperCamelCase    ,    lstrip=__UpperCamelCase    ,    rstrip=__UpperCamelCase    ,    normalized=__UpperCamelCase     )
          if isinstance(__UpperCamelCase    ,    __UpperCamelCase     )
          else mask_token
      )

      __SCREAMING_SNAKE_CASE		     =       {} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          do_lower_case=__UpperCamelCase    ,    remove_space=__UpperCamelCase    ,    keep_accents=__UpperCamelCase    ,    bos_token=__UpperCamelCase    ,    eos_token=__UpperCamelCase    ,    unk_token=__UpperCamelCase    ,    sep_token=__UpperCamelCase    ,    pad_token=__UpperCamelCase    ,    cls_token=__UpperCamelCase    ,    mask_token=__UpperCamelCase    ,    sp_model_kwargs=self.sp_model_kwargs    ,    **__UpperCamelCase    ,    )

      __SCREAMING_SNAKE_CASE		     =       do_lower_case
      __SCREAMING_SNAKE_CASE		     =       remove_space
      __SCREAMING_SNAKE_CASE		     =       keep_accents
      __SCREAMING_SNAKE_CASE		     =       vocab_file

      __SCREAMING_SNAKE_CASE		     =       spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(__UpperCamelCase     )




 @property
 def 						UpperCAmelCase_   (				self    :       Union[str, Any]     ) ->			Optional[Any]:
      return len(self.sp_model     )




 def 						UpperCAmelCase_   (				self    :       int     ) ->			str:
      __SCREAMING_SNAKE_CASE		     =       {self.convert_ids_to_tokens(__UpperCamelCase     ): i for i in range(self.vocab_size     )}
      vocab.update(self.added_tokens_encoder     )
      return vocab




 def __getstate__(				self    :       Optional[int]     ) ->			Optional[Any]:
      __SCREAMING_SNAKE_CASE		     =       self.__dict__.copy()
      __SCREAMING_SNAKE_CASE		     =       None
      return state




 def __setstate__(				self    :       Optional[int]    ,    UpperCAmelCase__    :       List[Any]     ) ->			Optional[Any]:
      __SCREAMING_SNAKE_CASE		     =       d

      # for backward compatibility
      if not hasattr(self    ,    "sp_model_kwargs"     ):
           __SCREAMING_SNAKE_CASE		     =       {}

      __SCREAMING_SNAKE_CASE		     =       spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(self.vocab_file     )




 def 						UpperCAmelCase_   (				self    :       str    ,    UpperCAmelCase__    :       int     ) ->			Optional[Any]:


      if self.remove_space:
           __SCREAMING_SNAKE_CASE		     =       " ".join(inputs.strip().split()     )
      else:
           __SCREAMING_SNAKE_CASE		     =       inputs
      __SCREAMING_SNAKE_CASE		     =       outputs.replace("``"    ,    "\""     ).replace("\'\'"    ,    "\""     )

      if not self.keep_accents:
           __SCREAMING_SNAKE_CASE		     =       unicodedata.normalize("NFKD"    ,    __UpperCamelCase     )
           __SCREAMING_SNAKE_CASE		     =       "".join([c for c in outputs if not unicodedata.combining(__UpperCamelCase     )]     )
      if self.do_lower_case:
           __SCREAMING_SNAKE_CASE		     =       outputs.lower()

      return outputs




 def 						UpperCAmelCase_   (				self    :       Optional[int]    ,    UpperCAmelCase__    :       List[str]     ) ->			Tuple:
      __SCREAMING_SNAKE_CASE		     =       self.preprocess_text(__UpperCamelCase     )
      __SCREAMING_SNAKE_CASE		     =       self.sp_model.encode(__UpperCamelCase    ,    out_type=__UpperCamelCase     )
      __SCREAMING_SNAKE_CASE		     =       []
      for piece in pieces:
           if len(__UpperCamelCase     ) > 1 and piece[-1] == str(","     ) and piece[-2].isdigit():
                __SCREAMING_SNAKE_CASE		     =       self.sp_model.EncodeAsPieces(piece[:-1].replace(__UpperCamelCase    ,    ""     )     )
                if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                     if len(cur_pieces[0]     ) == 1:
                          __SCREAMING_SNAKE_CASE		     =       cur_pieces[1:]
                     else:
                          __SCREAMING_SNAKE_CASE		     =       cur_pieces[0][1:]
                cur_pieces.append(piece[-1]     )
                new_pieces.extend(__UpperCamelCase     )
           else:
                new_pieces.append(__UpperCamelCase     )

      return new_pieces




 def 						UpperCAmelCase_   (				self    :       Tuple    ,    UpperCAmelCase__    :       Union[str, Any]     ) ->			List[str]:
      return self.sp_model.PieceToId(__UpperCamelCase     )




 def 						UpperCAmelCase_   (				self    :       Dict    ,    UpperCAmelCase__    :       int     ) ->			int:
      return self.sp_model.IdToPiece(__UpperCamelCase     )




 def 						UpperCAmelCase_   (				self    :       int    ,    UpperCAmelCase__    :       int     ) ->			List[Any]:
      __SCREAMING_SNAKE_CASE		     =       []
      __SCREAMING_SNAKE_CASE		     =       ""
      __SCREAMING_SNAKE_CASE		     =       False
      for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
                if not prev_is_special:
                     out_string += " "
                out_string += self.sp_model.decode(__UpperCamelCase     ) + token
                __SCREAMING_SNAKE_CASE		     =       True
                __SCREAMING_SNAKE_CASE		     =       []
           else:
                current_sub_tokens.append(__UpperCamelCase     )
                __SCREAMING_SNAKE_CASE		     =       False
      out_string += self.sp_model.decode(__UpperCamelCase     )
      return out_string.strip()




 def 						UpperCAmelCase_   (				self    :       Dict    ,    UpperCAmelCase__    :       List[str]    ,    UpperCAmelCase__    :       Optional[int] = None     ) ->			str:
      __SCREAMING_SNAKE_CASE		     =       [self.sep_token_id]
      __SCREAMING_SNAKE_CASE		     =       [self.cls_token_id]
      if token_ids_a is None:
           return cls + token_ids_a + sep
      return cls + token_ids_a + sep + token_ids_a + sep




 def 						UpperCAmelCase_   (				self    :       Tuple    ,    UpperCAmelCase__    :       Tuple    ,    UpperCAmelCase__    :       List[Any] = None    ,    UpperCAmelCase__    :       int = False     ) ->			str:


      if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=__UpperCamelCase    ,    token_ids_a=__UpperCamelCase    ,    already_has_special_tokens=__UpperCamelCase     )

      if token_ids_a is not None:
           return [1] + ([0] * len(__UpperCamelCase     )) + [1] + ([0] * len(__UpperCamelCase     )) + [1]
      return [1] + ([0] * len(__UpperCamelCase     )) + [1]




 def 						UpperCAmelCase_   (				self    :       Optional[Any]    ,    UpperCAmelCase__    :       List[str]    ,    UpperCAmelCase__    :       List[Any] = None     ) ->			Union[str, Any]:
      __SCREAMING_SNAKE_CASE		     =       [self.sep_token_id]
      __SCREAMING_SNAKE_CASE		     =       [self.cls_token_id]

      if token_ids_a is None:
           return len(cls + token_ids_a + sep     ) * [0]
      return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]




 def 						UpperCAmelCase_   (				self    :       int    ,    UpperCAmelCase__    :       Tuple    ,    UpperCAmelCase__    :       str = None     ) ->			Tuple:


      if not os.path.isdir(__UpperCamelCase     ):
           logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
           return
      __SCREAMING_SNAKE_CASE		     =       os.path.join(
          __UpperCamelCase    ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

      if os.path.abspath(self.vocab_file     ) != os.path.abspath(__UpperCamelCase     ) and os.path.isfile(self.vocab_file     ):
           copyfile(self.vocab_file    ,    __UpperCamelCase     )
      elif not os.path.isfile(self.vocab_file     ):
           with open(__UpperCamelCase    ,    "wb"     ) as fi:
                __SCREAMING_SNAKE_CASE		     =       self.sp_model.serialized_model_proto()
                fi.write(__UpperCamelCase     )

      return (out_vocab_file,)




"""simple docstring"""







from abc import ABC, abstractmethod
from argparse import ArgumentParser




class 				UpperCamelCase_			(				UpperCamelCase):

	"""simple docstring"""





	@staticmethod
	@abstractmethod
	def 						UpperCAmelCase_   (				UpperCAmelCase__    :       ArgumentParser     ) ->			Optional[Any]:
						raise NotImplementedError()




	@abstractmethod
	def 						UpperCAmelCase_   (				self    :       Optional[Any]     ) ->			List[str]:
						raise NotImplementedError()
