






'''simple docstring'''





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__      :			List[str]							=		logging.get_logger(__name__)
SCREAMING_SNAKE_CASE__      :			Any							=		{"""vocab_file""": """spiece.model"""}

SCREAMING_SNAKE_CASE__      :			Optional[Any]							=		{
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

SCREAMING_SNAKE_CASE__      :			List[Any]							=		{
    """albert-base-v1""": 5_12,
    """albert-large-v1""": 5_12,
    """albert-xlarge-v1""": 5_12,
    """albert-xxlarge-v1""": 5_12,
    """albert-base-v2""": 5_12,
    """albert-large-v2""": 5_12,
    """albert-xlarge-v2""": 5_12,
    """albert-xxlarge-v2""": 5_12,
}

SCREAMING_SNAKE_CASE__      :			Optional[int]							=		"""▁"""
class 						lowerCamelCase_				(      lowerCamelCase ):
				a__					       =  VOCAB_FILES_NAMES
				a__					       =  PRETRAINED_VOCAB_FILES_MAP
				a__					       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







				def __init__(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase=True       ,					__lowerCAmelCase=True       ,					__lowerCAmelCase=False       ,					__lowerCAmelCase="[CLS]"       ,					__lowerCAmelCase="[SEP]"       ,					__lowerCAmelCase="<unk>"       ,					__lowerCAmelCase="[SEP]"       ,					__lowerCAmelCase="<pad>"       ,					__lowerCAmelCase="[CLS]"       ,					__lowerCAmelCase="[MASK]"       ,					__lowerCAmelCase = None       ,					**__lowerCAmelCase       ,					):
								"""simple docstring"""
								__magic_name__		:Optional[int]        =      (
								    AddedToken(__lowerCAmelCase       ,					lstrip=__lowerCAmelCase       ,					rstrip=__lowerCAmelCase       ,					normalized=__lowerCAmelCase     )
								    if isinstance(__lowerCAmelCase       ,					__lowerCAmelCase     )
								    else mask_token
								)

								__magic_name__		:Optional[Any]        =      {} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    do_lower_case=__lowerCAmelCase       ,					remove_space=__lowerCAmelCase       ,					keep_accents=__lowerCAmelCase       ,					bos_token=__lowerCAmelCase       ,					eos_token=__lowerCAmelCase       ,					unk_token=__lowerCAmelCase       ,					sep_token=__lowerCAmelCase       ,					pad_token=__lowerCAmelCase       ,					cls_token=__lowerCAmelCase       ,					mask_token=__lowerCAmelCase       ,					sp_model_kwargs=self.sp_model_kwargs       ,					**__lowerCAmelCase       ,					)

								__magic_name__		:Optional[Any]        =      do_lower_case
								__magic_name__		:int        =      remove_space
								__magic_name__		:Tuple        =      keep_accents
								__magic_name__		:Tuple        =      vocab_file

								__magic_name__		:int        =      spm.SentencePieceProcessor(**self.sp_model_kwargs     )
								self.sp_model.Load(__lowerCAmelCase     )







				@property
				def 			A	(						self     ):
								"""simple docstring"""
								return len(self.sp_model     )







				def 			A	(						self     ):
								"""simple docstring"""
								__magic_name__		:str        =      {self.convert_ids_to_tokens(__lowerCAmelCase     ): i for i in range(self.vocab_size     )}
								vocab.update(self.added_tokens_encoder     )
								return vocab







				def __getstate__(						self     ):
								"""simple docstring"""
								__magic_name__		:List[Any]        =      self.__dict__.copy()
								__magic_name__		:Optional[Any]        =      None
								return state







				def __setstate__(						self       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								__magic_name__		:List[str]        =      d

								# for backward compatibility
								if not hasattr(self       ,					'''sp_model_kwargs'''     ):
												__magic_name__		:Optional[Any]        =      {}

								__magic_name__		:Dict        =      spm.SentencePieceProcessor(**self.sp_model_kwargs     )
								self.sp_model.Load(self.vocab_file     )







				def 			A	(						self       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								if self.remove_space:
												__magic_name__		:str        =      ''' '''.join(inputs.strip().split()     )
								else:
												__magic_name__		:int        =      inputs
								__magic_name__		:Optional[Any]        =      outputs.replace('''``'''       ,					'''"'''     ).replace('''\'\''''       ,					'''"'''     )

								if not self.keep_accents:
												__magic_name__		:Tuple        =      unicodedata.normalize('''NFKD'''       ,					__lowerCAmelCase     )
												__magic_name__		:Any        =      ''''''.join([c for c in outputs if not unicodedata.combining(__lowerCAmelCase     )]     )
								if self.do_lower_case:
												__magic_name__		:Optional[Any]        =      outputs.lower()

								return outputs







				def 			A	(						self       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]        =      self.preprocess_text(__lowerCAmelCase     )
								__magic_name__		:List[str]        =      self.sp_model.encode(__lowerCAmelCase       ,					out_type=__lowerCAmelCase     )
								__magic_name__		:List[str]        =      []
								for piece in pieces:
												if len(__lowerCAmelCase     ) > 1 and piece[-1] == str(''','''     ) and piece[-2].isdigit():
																__magic_name__		:str        =      self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCAmelCase       ,					''''''     )     )
																if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																				if len(cur_pieces[0]     ) == 1:
																								__magic_name__		:Tuple        =      cur_pieces[1:]
																				else:
																								__magic_name__		:List[str]        =      cur_pieces[0][1:]
																cur_pieces.append(piece[-1]     )
																new_pieces.extend(__lowerCAmelCase     )
												else:
																new_pieces.append(__lowerCAmelCase     )

								return new_pieces







				def 			A	(						self       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								return self.sp_model.PieceToId(__lowerCAmelCase     )







				def 			A	(						self       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								return self.sp_model.IdToPiece(__lowerCAmelCase     )







				def 			A	(						self       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								__magic_name__		:List[Any]        =      []
								__magic_name__		:List[Any]        =      ''''''
								__magic_name__		:Dict        =      False
								for token in tokens:
												# make sure that special tokens are not decoded using sentencepiece model
												if token in self.all_special_tokens:
																if not prev_is_special:
																				out_string += " "
																out_string += self.sp_model.decode(__lowerCAmelCase     ) + token
																__magic_name__		:int        =      True
																__magic_name__		:Dict        =      []
												else:
																current_sub_tokens.append(__lowerCAmelCase     )
																__magic_name__		:Tuple        =      False
								out_string += self.sp_model.decode(__lowerCAmelCase     )
								return out_string.strip()







				def 			A	(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase = None     ):
								"""simple docstring"""
								__magic_name__		:str        =      [self.sep_token_id]
								__magic_name__		:Union[str, Any]        =      [self.cls_token_id]
								if token_ids_a is None:
												return cls + token_ids_a + sep
								return cls + token_ids_a + sep + token_ids_a + sep







				def 			A	(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase = None       ,					__lowerCAmelCase = False     ):
								"""simple docstring"""

								if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=__lowerCAmelCase       ,					token_ids_a=__lowerCAmelCase       ,					already_has_special_tokens=__lowerCAmelCase     )

								if token_ids_a is not None:
												return [1] + ([0] * len(__lowerCAmelCase     )) + [1] + ([0] * len(__lowerCAmelCase     )) + [1]
								return [1] + ([0] * len(__lowerCAmelCase     )) + [1]







				def 			A	(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase = None     ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]        =      [self.sep_token_id]
								__magic_name__		:List[str]        =      [self.cls_token_id]

								if token_ids_a is None:
												return len(cls + token_ids_a + sep     ) * [0]
								return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]







				def 			A	(						self       ,					__lowerCAmelCase       ,					__lowerCAmelCase = None     ):
								"""simple docstring"""
								if not os.path.isdir(__lowerCAmelCase     ):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''     )
												return
								__magic_name__		:str        =      os.path.join(
								    __lowerCAmelCase       ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )

								if os.path.abspath(self.vocab_file     ) != os.path.abspath(__lowerCAmelCase     ) and os.path.isfile(self.vocab_file     ):
												copyfile(self.vocab_file       ,					__lowerCAmelCase     )
								elif not os.path.isfile(self.vocab_file     ):
												with open(__lowerCAmelCase       ,					'''wb'''     ) as fi:
																__magic_name__		:Any        =      self.sp_model.serialized_model_proto()
																fi.write(__lowerCAmelCase     )

								return (out_vocab_file,)














from typing import List

from .keymap import KEYMAP, get_character

def      __lowercase					( snake_case ):







				"""simple docstring"""






				def decorator(snake_case ):
								__magic_name__		:int        =      getattr(snake_case,  '''handle_key''',  [] )
								handle += [key]
								setattr(snake_case,  '''handle_key''',  snake_case )
								return func

				return decorator

def      __lowercase					( *snake_case ):







				"""simple docstring"""






				def decorator(snake_case ):
								__magic_name__		:Union[str, Any]        =      getattr(snake_case,  '''handle_key''',  [] )
								handle += keys
								setattr(snake_case,  '''handle_key''',  snake_case )
								return func

				return decorator
class 						lowerCamelCase_				(      lowerCamelCase ):







				def __new__(						cls       ,					__lowerCAmelCase       ,					__lowerCAmelCase       ,					__lowerCAmelCase     ):
								"""simple docstring"""
								__magic_name__		:Optional[int]        =      super().__new__(cls       ,					__lowerCAmelCase       ,					__lowerCAmelCase       ,					__lowerCAmelCase     )
								if not hasattr(__lowerCAmelCase       ,					'''key_handler'''     ):
												setattr(__lowerCAmelCase       ,					'''key_handler'''       ,					{}     )
								setattr(__lowerCAmelCase       ,					'''handle_input'''       ,					KeyHandler.handle_input     )

								for value in attrs.values():
												__magic_name__		:Dict        =      getattr(__lowerCAmelCase       ,					'''handle_key'''       ,					[]     )
												for key in handled_keys:
																__magic_name__		:List[Any]        =      value
								return new_cls







				@staticmethod
				def 			A	(						cls     ):
								"""simple docstring"""
								__magic_name__		:Tuple        =      get_character()
								if char != KEYMAP["undefined"]:
												__magic_name__		:Dict        =      ord(__lowerCAmelCase     )
								__magic_name__		:str        =      cls.key_handler.get(__lowerCAmelCase     )
								if handler:
												__magic_name__		:Optional[Any]        =      char
												return handler(cls     )
								else:
												return None







def      __lowercase					( cls ):







				"""simple docstring"""






				return KeyHandler(cls.__name__,  cls.__bases__,  cls.__dict__.copy() )






