



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_lowercase						: Optional[int]     						=logging.get_logger(__name__)

_lowercase						: int     						={"vocab_file": "spiece.model"}

_lowercase						: Optional[int]     						={
    "vocab_file": {
        "TsinghuaAI/CPM-Generate": "https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model",
    }
}

class   snake_case__							(__lowercase						):







						"""simple docstring"""




						def __init__(  self   ,  __lowercase   ,  __lowercase=False   ,  __lowercase=True   ,  __lowercase=False   ,  __lowercase="<s>"   ,  __lowercase="</s>"   ,  __lowercase="<unk>"   ,  __lowercase="<sep>"   ,  __lowercase="<pad>"   ,  __lowercase="<cls>"   ,  __lowercase="<mask>"   ,  __lowercase=["<eop>", "<eod>"]   ,  __lowercase = None   ,  **__lowercase   ,  )   -> int:
									"""simple docstring"""




									a__  :		Dict					=    AddedToken(_A   ,  lstrip=_A   ,  rstrip=_A    ) if isinstance(_A   ,  _A    ) else mask_token

									a__  :		List[str]					=    {} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    do_lower_case=_A   ,  remove_space=_A   ,  keep_accents=_A   ,  bos_token=_A   ,  eos_token=_A   ,  unk_token=_A   ,  sep_token=_A   ,  pad_token=_A   ,  cls_token=_A   ,  mask_token=_A   ,  additional_special_tokens=_A   ,  sp_model_kwargs=self.sp_model_kwargs   ,  **_A   ,  )

									a__  :		Dict					=    3

									a__  :		Union[str, Any]					=    do_lower_case
									a__  :		int					=    remove_space
									a__  :		List[Any]					=    keep_accents
									a__  :		Optional[int]					=    vocab_file

									a__  :		Union[str, Any]					=    spm.SentencePieceProcessor(**self.sp_model_kwargs    )
									self.sp_model.Load(_A    )

									try:
												import jieba
									except ModuleNotFoundError as error:
												raise error.__class__(
												    """You need to install jieba to use CpmTokenizer or CpmTokenizerFast. """
												    """See https://pypi.org/project/jieba/ for installation."""    )
									a__  :		Union[str, Any]					=    jieba
									a__  :		Optional[int]					=    str.maketrans(""" \n"""   ,  """\u2582\u2583"""    )

						@property
						# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
						def 				SCREAMING_SNAKE_CASE__(  self    )   -> List[Any]:
									"""simple docstring"""




									return len(self.sp_model    )

						def 				SCREAMING_SNAKE_CASE__(  self    )   -> Any:
									"""simple docstring"""




									a__  :		Any					=    {self.convert_ids_to_tokens(_A    ): i for i in range(self.vocab_size    )}
									vocab.update(self.added_tokens_encoder    )
									return vocab

						def __getstate__(  self    )   -> str:
									"""simple docstring"""




									a__  :		List[Any]					=    self.__dict__.copy()
									a__  :		Optional[int]					=    None
									return state

						def __setstate__(  self   ,  __lowercase    )   -> str:
									"""simple docstring"""




									a__  :		Union[str, Any]					=    d

									# for backward compatibility
									if not hasattr(self   ,  """sp_model_kwargs"""    ):
												a__  :		List[str]					=    {}

									a__  :		Dict					=    spm.SentencePieceProcessor(**self.sp_model_kwargs    )
									self.sp_model.Load(self.vocab_file    )

						def 				SCREAMING_SNAKE_CASE__(  self   ,  __lowercase    )   -> str:
									"""simple docstring"""




									if self.remove_space:
												a__  :		Optional[Any]					=    """ """.join(inputs.strip().split()    )
									else:
												a__  :		Optional[int]					=    inputs
									a__  :		Tuple					=    outputs.replace("""``"""   ,  """\""""    ).replace("""\'\'"""   ,  """\""""    )

									if not self.keep_accents:
												a__  :		Optional[Any]					=    unicodedata.normalize("""NFKD"""   ,  _A    )
												a__  :		str					=    """""".join([c for c in outputs if not unicodedata.combining(_A    )]    )
									if self.do_lower_case:
												a__  :		Union[str, Any]					=    outputs.lower()

									return outputs

						def 				SCREAMING_SNAKE_CASE__(  self   ,  __lowercase    )   -> Tuple:
									"""simple docstring"""




									a__  :		List[Any]					=    self.preprocess_text(_A    )
									a__  :		int					=    self.sp_model.encode(_A   ,  out_type=_A    )
									a__  :		Optional[int]					=    []
									for piece in pieces:
												if len(_A    ) > 1 and piece[-1] == str(""","""    ) and piece[-2].isdigit():
															a__  :		Dict					=    self.sp_model.EncodeAsPieces(piece[:-1].replace(_A   ,  """"""    )    )
															if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																		if len(cur_pieces[0]    ) == 1:
																					a__  :		Union[str, Any]					=    cur_pieces[1:]
																		else:
																					a__  :		List[str]					=    cur_pieces[0][1:]
															cur_pieces.append(piece[-1]    )
															new_pieces.extend(_A    )
												else:
															new_pieces.append(_A    )

									return new_pieces

						def 				SCREAMING_SNAKE_CASE__(  self   ,  __lowercase    )   -> str:
									"""simple docstring"""




									return self.sp_model.PieceToId(_A    )

						def 				SCREAMING_SNAKE_CASE__(  self   ,  __lowercase    )   -> int:
									"""simple docstring"""




									return self.sp_model.IdToPiece(_A    )

						def 				SCREAMING_SNAKE_CASE__(  self   ,  __lowercase    )   -> List[Any]:
									"""simple docstring"""




									a__  :		Optional[int]					=    """""".join(_A    ).replace(_A   ,  """ """    ).strip()
									return out_string

						def 				SCREAMING_SNAKE_CASE__(  self   ,  __lowercase   ,  __lowercase = None    )   -> Any:
									"""simple docstring"""




									a__  :		List[Any]					=    [self.sep_token_id]
									a__  :		Dict					=    [self.cls_token_id]
									if token_ids_a is None:
												return token_ids_a + sep + cls
									return token_ids_a + sep + token_ids_a + sep + cls

						def 				SCREAMING_SNAKE_CASE__(  self   ,  __lowercase   ,  __lowercase = None   ,  __lowercase = False    )   -> Optional[int]:
									"""simple docstring"""




									if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=_A   ,  token_ids_a=_A   ,  already_has_special_tokens=_A    )

									if token_ids_a is not None:
												return ([0] * len(_A    )) + [1] + ([0] * len(_A    )) + [1, 1]
									return ([0] * len(_A    )) + [1, 1]

						def 				SCREAMING_SNAKE_CASE__(  self   ,  __lowercase   ,  __lowercase = None    )   -> int:
									"""simple docstring"""




									a__  :		str					=    [self.sep_token_id]
									a__  :		List[str]					=    [2]

									if token_ids_a is None:
												return len(token_ids_a + sep    ) * [0] + cls_segment_id
									return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id

						def 				SCREAMING_SNAKE_CASE__(  self   ,  __lowercase   ,  __lowercase = None    )   -> Any:
									"""simple docstring"""




									if not os.path.isdir(_A    ):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
												return
									a__  :		Tuple					=    os.path.join(
									    _A   ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

									if os.path.abspath(self.vocab_file    ) != os.path.abspath(_A    ) and os.path.isfile(self.vocab_file    ):
												copyfile(self.vocab_file   ,  _A    )
									elif not os.path.isfile(self.vocab_file    ):
												with open(_A   ,  """wb"""    ) as fi:
															a__  :		List[Any]					=    self.sp_model.serialized_model_proto()
															fi.write(_A    )

									return (out_vocab_file,)

						def 				SCREAMING_SNAKE_CASE__(  self   ,  *__lowercase   ,  **__lowercase    )   -> List[Any]:
									"""simple docstring"""




									a__  :		Optional[int]					=    super()._decode(*_A   ,  **_A    )
									a__  :		Union[str, Any]					=    text.replace(""" """   ,  """"""    ).replace("""\u2582"""   ,  """ """    ).replace("""\u2583"""   ,  """\n"""    )
									return text





from typing import List

from .keymap import KEYMAP, get_character
def 					_snake_case	(  __snake_case						):

					def decorator(__snake_case						):
										_UpperCamelCase			     =     getattr(__snake_case  ,     '''handle_key'''  ,     []						)
										handle += [key]
										setattr(__snake_case  ,     '''handle_key'''  ,     __snake_case						)
										return func

					return decorator
def 					_snake_case	(  *__snake_case						):

					def decorator(__snake_case						):
										_UpperCamelCase			     =     getattr(__snake_case  ,     '''handle_key'''  ,     []						)
										handle += keys
										setattr(__snake_case  ,     '''handle_key'''  ,     __snake_case						)
										return func

					return decorator
class     lowerCAmelCase_     (       __lowercase ):







							def __new__(   cls      :     Optional[Any] ,			_A      :     Optional[Any] ,			_A      :     Optional[int] ,			_A      :     Union[str, Any]		):
												_UpperCamelCase			     =     super().__new__(cls ,			_A ,			_A ,			_A		)
												if not hasattr(_A ,			'''key_handler'''		):
																	setattr(_A ,			'''key_handler''' ,			{}		)
												setattr(_A ,			'''handle_input''' ,			KeyHandler.handle_input		)

												for value in attrs.values():
																	_UpperCamelCase			     =     getattr(_A ,			'''handle_key''' ,			[]		)
																	for key in handled_keys:
																						_UpperCamelCase			     =     value
												return new_cls







							@staticmethod
							def  UpperCamelCase_						(   cls      :     str		):
												_UpperCamelCase			     =     get_character()
												if char != KEYMAP["undefined"]:
																	_UpperCamelCase			     =     ord(_A		)
												_UpperCamelCase			     =     cls.key_handler.get(_A		)
												if handler:
																	_UpperCamelCase			     =     char
																	return handler(cls		)
												else:
																	return None




def 					_snake_case	(  cls						):
					return KeyHandler(cls.__name__  ,     cls.__bases__  ,     cls.__dict__.copy()						)
