


"""simple docstring"""






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase							:    List[str]					   =	logging.get_logger(__name__)
_lowerCAmelCase							:    Any					   =	{'''vocab_file''': '''spiece.model'''}

_lowerCAmelCase							:    Optional[int]					   =	{
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

_lowerCAmelCase							:    Tuple					   =	{
    '''albert-base-v1''': 512,
    '''albert-large-v1''': 512,
    '''albert-xlarge-v1''': 512,
    '''albert-xxlarge-v1''': 512,
    '''albert-base-v2''': 512,
    '''albert-large-v2''': 512,
    '''albert-xlarge-v2''': 512,
    '''albert-xxlarge-v2''': 512,
}

_lowerCAmelCase							:    int					   =	'''▁'''




class 							A_      (       _a ):
  lowerCAmelCase__      			=	VOCAB_FILES_NAMES
  lowerCAmelCase__      			=	PRETRAINED_VOCAB_FILES_MAP
  lowerCAmelCase__      			=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


  def __init__(			self:						Optional[Any]							,__lowerCAmelCase:						Dict							,__lowerCAmelCase:						int=True							,__lowerCAmelCase:						Union[str, Any]=True							,__lowerCAmelCase:						Dict=False							,__lowerCAmelCase:						int="[CLS]"							,__lowerCAmelCase:						Optional[Any]="[SEP]"							,__lowerCAmelCase:						List[str]="<unk>"							,__lowerCAmelCase:						Optional[Any]="[SEP]"							,__lowerCAmelCase:						Optional[Any]="<pad>"							,__lowerCAmelCase:						Optional[int]="[CLS]"							,__lowerCAmelCase:						str="[MASK]"							,__lowerCAmelCase:						Optional[Dict[str, Any]] = None							,**__lowerCAmelCase:						Dict							,):







     '''simple docstring'''







     _lowerCamelCase   :				Tuple							=     (
         AddedToken(__lowerCAmelCase							,lstrip=__lowerCAmelCase							,rstrip=__lowerCAmelCase							,normalized=__lowerCAmelCase    )
         if isinstance(__lowerCAmelCase							,__lowerCAmelCase    )
         else mask_token
     )

     _lowerCamelCase   :				Dict							=     {} if sp_model_kwargs is None else sp_model_kwargs

     super().__init__(
         do_lower_case=__lowerCAmelCase							,remove_space=__lowerCAmelCase							,keep_accents=__lowerCAmelCase							,bos_token=__lowerCAmelCase							,eos_token=__lowerCAmelCase							,unk_token=__lowerCAmelCase							,sep_token=__lowerCAmelCase							,pad_token=__lowerCAmelCase							,cls_token=__lowerCAmelCase							,mask_token=__lowerCAmelCase							,sp_model_kwargs=self.sp_model_kwargs							,**__lowerCAmelCase							,)

     _lowerCamelCase   :				Optional[int]							=     do_lower_case
     _lowerCamelCase   :				Optional[int]							=     remove_space
     _lowerCamelCase   :				int							=     keep_accents
     _lowerCamelCase   :				Any							=     vocab_file

     _lowerCamelCase   :				int							=     spm.SentencePieceProcessor(**self.sp_model_kwargs    )
     self.sp_model.Load(__lowerCAmelCase    )


  @property
  def 							_lowercase					(			self:						Dict    ):







     '''simple docstring'''







     return len(self.sp_model    )


  def 							_lowercase					(			self:						Union[str, Any]    ):







     '''simple docstring'''







     _lowerCamelCase   :				str							=     {self.convert_ids_to_tokens(__lowerCAmelCase    ): i for i in range(self.vocab_size    )}
     vocab.update(self.added_tokens_encoder    )
     return vocab


  def __getstate__(			self:						List[str]    ):







     '''simple docstring'''







     _lowerCamelCase   :				Optional[int]							=     self.__dict__.copy()
     _lowerCamelCase   :				List[Any]							=     None
     return state


  def __setstate__(			self:						Any							,__lowerCAmelCase:						Any    ):







     '''simple docstring'''







     _lowerCamelCase   :				Union[str, Any]							=     d

     # for backward compatibility
     if not hasattr(self							,"sp_model_kwargs"    ):
        _lowerCamelCase   :				Tuple							=     {}

     _lowerCamelCase   :				List[str]							=     spm.SentencePieceProcessor(**self.sp_model_kwargs    )
     self.sp_model.Load(self.vocab_file    )


  def 							_lowercase					(			self:						Any							,__lowerCAmelCase:						Tuple    ):







     '''simple docstring'''







     if self.remove_space:
        _lowerCamelCase   :				Dict							=     " ".join(inputs.strip().split()    )
     else:
        _lowerCamelCase   :				str							=     inputs
     _lowerCamelCase   :				Union[str, Any]							=     outputs.replace("``"							,"\""    ).replace("''"							,"\""    )

     if not self.keep_accents:
        _lowerCamelCase   :				List[str]							=     unicodedata.normalize("NFKD"							,__lowerCAmelCase    )
        _lowerCamelCase   :				List[Any]							=     "".join([c for c in outputs if not unicodedata.combining(__lowerCAmelCase    )]    )
     if self.do_lower_case:
        _lowerCamelCase   :				Any							=     outputs.lower()

     return outputs


  def 							_lowercase					(			self:						Dict							,__lowerCAmelCase:						str    ):







     '''simple docstring'''







     _lowerCamelCase   :				Union[str, Any]							=     self.preprocess_text(__lowerCAmelCase    )
     _lowerCamelCase   :				List[str]							=     self.sp_model.encode(__lowerCAmelCase							,out_type=__lowerCAmelCase    )
     _lowerCamelCase   :				Dict							=     []
     for piece in pieces:
        if len(__lowerCAmelCase    ) > 1 and piece[-1] == str(","    ) and piece[-2].isdigit():
           _lowerCamelCase   :				Any							=     self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCAmelCase							,""    )    )
           if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
              if len(cur_pieces[0]    ) == 1:
                 _lowerCamelCase   :				Optional[Any]							=     cur_pieces[1:]
              else:
                 _lowerCamelCase   :				Any							=     cur_pieces[0][1:]
           cur_pieces.append(piece[-1]    )
           new_pieces.extend(__lowerCAmelCase    )
        else:
           new_pieces.append(__lowerCAmelCase    )

     return new_pieces


  def 							_lowercase					(			self:						Optional[Any]							,__lowerCAmelCase:						str    ):







     '''simple docstring'''







     return self.sp_model.PieceToId(__lowerCAmelCase    )


  def 							_lowercase					(			self:						Optional[Any]							,__lowerCAmelCase:						List[Any]    ):







     '''simple docstring'''







     return self.sp_model.IdToPiece(__lowerCAmelCase    )


  def 							_lowercase					(			self:						Optional[Any]							,__lowerCAmelCase:						Any    ):







     '''simple docstring'''







     _lowerCamelCase   :				Tuple							=     []
     _lowerCamelCase   :				Dict							=     ""
     _lowerCamelCase   :				Tuple							=     False
     for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
           if not prev_is_special:
              out_string += " "
           out_string += self.sp_model.decode(__lowerCAmelCase    ) + token
           _lowerCamelCase   :				List[Any]							=     True
           _lowerCamelCase   :				List[Any]							=     []
        else:
           current_sub_tokens.append(__lowerCAmelCase    )
           _lowerCamelCase   :				Any							=     False
     out_string += self.sp_model.decode(__lowerCAmelCase    )
     return out_string.strip()


  def 							_lowercase					(			self:						Any							,__lowerCAmelCase:						List[int]							,__lowerCAmelCase:						Optional[List[int]] = None    ):







     '''simple docstring'''







     _lowerCamelCase   :				str							=     [self.sep_token_id]
     _lowerCamelCase   :				Tuple							=     [self.cls_token_id]
     if token_ids_a is None:
        return cls + token_ids_a + sep
     return cls + token_ids_a + sep + token_ids_a + sep


  def 							_lowercase					(			self:						Union[str, Any]							,__lowerCAmelCase:						List[int]							,__lowerCAmelCase:						Optional[List[int]] = None							,__lowerCAmelCase:						bool = False    ):







     '''simple docstring'''







     if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__lowerCAmelCase							,token_ids_a=__lowerCAmelCase							,already_has_special_tokens=__lowerCAmelCase    )

     if token_ids_a is not None:
        return [1] + ([0] * len(__lowerCAmelCase    )) + [1] + ([0] * len(__lowerCAmelCase    )) + [1]
     return [1] + ([0] * len(__lowerCAmelCase    )) + [1]


  def 							_lowercase					(			self:						str							,__lowerCAmelCase:						List[int]							,__lowerCAmelCase:						Optional[List[int]] = None    ):







     '''simple docstring'''







     _lowerCamelCase   :				Tuple							=     [self.sep_token_id]
     _lowerCamelCase   :				int							=     [self.cls_token_id]

     if token_ids_a is None:
        return len(cls + token_ids_a + sep    ) * [0]
     return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]






  def 							_lowercase					(			self:						Any							,__lowerCAmelCase:						str							,__lowerCAmelCase:						Optional[str] = None    ):







     '''simple docstring'''







     if not os.path.isdir(__lowerCAmelCase    ):
        logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""    )
        return
     _lowerCamelCase   :				str							=     os.path.join(
         __lowerCAmelCase							,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]    )

     if os.path.abspath(self.vocab_file    ) != os.path.abspath(__lowerCAmelCase    ) and os.path.isfile(self.vocab_file    ):
        copyfile(self.vocab_file							,__lowerCAmelCase    )
     elif not os.path.isfile(self.vocab_file    ):
        with open(__lowerCAmelCase							,"wb"    ) as fi:
           _lowerCamelCase   :				Dict							=     self.sp_model.serialized_model_proto()
           fi.write(__lowerCAmelCase    )

     return (out_vocab_file,)






'''simple docstring'''

def    a__							(							lowercase   :	str )						->						int:





     """simple docstring"""

     assert column_title.isupper()
     _UpperCamelCase							       =			0
     _UpperCamelCase							       =			len(lowercase ) - 1
     _UpperCamelCase							       =			0

     while index >= 0:
          _UpperCamelCase							       =			(ord(column_title[index] ) - 64) * pow(26,   lowercase )
          answer += value
          power += 1
          index -= 1

     return answer


if __name__ == "__main__":
   from doctest import testmod

   testmod()



