


"""simple docstring"""



import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case         =					logging.get_logger(__name__)

__snake_case         =					"""▁"""

__snake_case         =					{"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

__snake_case         =					{
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

__snake_case         =					{
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

__snake_case         =					{
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

__snake_case         =					{
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}







class _lowerCAmelCase       (   snake_case_			):
   __UpperCAmelCase				: List[str]													=						["input_ids"]

   __UpperCAmelCase				: Any													=						VOCAB_FILES_NAMES
   __UpperCAmelCase				: Optional[int]													=						PRETRAINED_INIT_CONFIGURATION
   __UpperCAmelCase				: Optional[Any]													=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCAmelCase				: Optional[Any]													=						PRETRAINED_VOCAB_FILES_MAP
   __UpperCAmelCase				: int													=						RESOURCE_FILES_NAMES







   def __init__(			self		,			UpperCamelCase__		,			UpperCamelCase__=None		,			UpperCamelCase__=False		,			UpperCamelCase__="utf8"		,			UpperCamelCase__="[UNK]"		,			UpperCamelCase__="[SEP]"		,			UpperCamelCase__="[PAD]"		,			UpperCamelCase__="[CLS]"		,			UpperCamelCase__="[MASK]"		,			UpperCamelCase__ = None		,			**UpperCamelCase__		,			)			->	None:

       '''simple docstring'''




       snake_case					:   Optional[int]         =					{} if sp_model_kwargs is None else sp_model_kwargs
       super().__init__(
           do_lower_case=UpperCamelCase__		,			unk_token=UpperCamelCase__		,			sep_token=UpperCamelCase__		,			pad_token=UpperCamelCase__		,			cls_token=UpperCamelCase__		,			mask_token=UpperCamelCase__		,			vocab_file=UpperCamelCase__		,			encoding=UpperCamelCase__		,			sp_model_kwargs=self.sp_model_kwargs		,			**UpperCamelCase__		,			)
       snake_case					:   str         =					do_lower_case
       snake_case					:   Union[str, Any]         =					sentencepiece_model_ckpt
       snake_case					:   Optional[int]         =					spm.SentencePieceProcessor(**self.sp_model_kwargs		)
       self.sp_model.Load(UpperCamelCase__		)

       # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
       if vocab_file is not None:
           snake_case					:   Optional[int]         =					self.load_vocab(filepath=UpperCamelCase__		)
       else:
           snake_case					:   List[Any]         =					{self.sp_model.id_to_piece(UpperCamelCase__		): id for id in range(self.sp_model.get_piece_size()		)}
       snake_case					:   Dict         =					{v: k for k, v in self.vocab.items()}







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		)			->	List[str]:

       '''simple docstring'''




       if text is None:
           return None

       snake_case					:   Dict         =					self.tokenize(UpperCamelCase__		)
       snake_case      ,snake_case					:   Tuple         =					"", []

       for i, ch in enumerate(UpperCamelCase__		):
           if ch in self.SP_CHAR_MAPPING:
               snake_case					:   int         =					self.SP_CHAR_MAPPING.get(UpperCamelCase__		)
           else:
               snake_case					:   Tuple         =					unicodedata.normalize("NFKC"		,			UpperCamelCase__		)
           if self.is_whitespace(UpperCamelCase__		):
               continue
           normalized_text += ch
           char_mapping.extend([i] * len(UpperCamelCase__		)		)

       snake_case      ,snake_case      ,snake_case					:   List[str]         =					normalized_text, [], 0

       if self.do_lower_case:
           snake_case					:   Union[str, Any]         =					text.lower()

       for token in split_tokens:
           if token[:1] == "▁":
               snake_case					:   Tuple         =					token[1:]
           snake_case					:   List[Any]         =					text[offset:].index(UpperCamelCase__		) + offset
           snake_case					:   List[Any]         =					start + len(UpperCamelCase__		)

           token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)		)
           snake_case					:   int         =					end
       return token_mapping







   @property
   def 				lowerCamelCase	(			self		)			->	int:

       '''simple docstring'''




       return len(self.vocab		)







   def 				lowerCamelCase	(			self		)			->	List[Any]:

       '''simple docstring'''




       return dict(self.vocab		,			**self.added_tokens_encoder		)







   def __getstate__(			self		)			->	Any:

       '''simple docstring'''




       snake_case					:   Tuple         =					self.__dict__.copy()
       snake_case					:   Optional[int]         =					None
       return state







   def __setstate__(			self		,			UpperCamelCase__		)			->	Tuple:

       '''simple docstring'''




       snake_case					:   int         =					d

       # for backward compatibility
       if not hasattr(self		,			"sp_model_kwargs"		):
           snake_case					:   List[Any]         =					{}

       snake_case					:   int         =					spm.SentencePieceProcessor(**self.sp_model_kwargs		)
       self.sp_model.Load(self.sentencepiece_model_ckpt		)







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		)			->	int:

       '''simple docstring'''




       return "".join((self.SP_CHAR_MAPPING.get(UpperCamelCase__		,			UpperCamelCase__		) for c in text)		)







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		,			UpperCamelCase__=False		,			UpperCamelCase__=64		,			UpperCamelCase__=0.1		)			->	Tuple:

       '''simple docstring'''




       if self.sp_model_kwargs.get("enable_sampling"		) is True:
           snake_case					:   Tuple         =					True
       if self.sp_model_kwargs.get("alpha"		) is not None:
           snake_case					:   Dict         =					self.sp_model_kwargs.get("alpha"		)
       if self.sp_model_kwargs.get("nbest_size"		) is not None:
           snake_case					:   List[Any]         =					self.sp_model_kwargs.get("nbest_size"		)

       if not enable_sampling:
           snake_case					:   Any         =					self.sp_model.EncodeAsPieces(UpperCamelCase__		)
       else:
           snake_case					:   Dict         =					self.sp_model.SampleEncodeAsPieces(UpperCamelCase__		,			UpperCamelCase__		,			UpperCamelCase__		)
       snake_case					:   Tuple         =					[]
       for pi, piece in enumerate(UpperCamelCase__		):
           if piece == SPIECE_UNDERLINE:
               if not pieces[pi + 1].startswith(UpperCamelCase__		) and pi != 0:
                   new_pieces.append(UpperCamelCase__		)
                   continue
               else:
                   continue
           snake_case					:   int         =					0
           for i, chunk in enumerate(UpperCamelCase__		):
               if chunk == SPIECE_UNDERLINE:
                   continue
               if self.is_ch_char(UpperCamelCase__		) or self.is_punct(UpperCamelCase__		):
                   if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                       new_pieces.append(piece[lst_i:i]		)
                   new_pieces.append(UpperCamelCase__		)
                   snake_case					:   Union[str, Any]         =					i + 1
               elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                   if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                       new_pieces.append(piece[lst_i:i]		)
                   snake_case					:   str         =					i
               elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                   if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                       new_pieces.append(piece[lst_i:i]		)
                   snake_case					:   List[Any]         =					i
           if len(UpperCamelCase__		) > lst_i:
               new_pieces.append(piece[lst_i:]		)
       return new_pieces







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		)			->	Any:

       '''simple docstring'''




       snake_case					:   Any         =					"".join(UpperCamelCase__		).replace(UpperCamelCase__		,			" "		).strip()
       return out_string







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		)			->	int:

       '''simple docstring'''




       snake_case					:   Optional[Any]         =					self.convert_ids_to_tokens(UpperCamelCase__		)
       snake_case					:   str         =					"".join(UpperCamelCase__		).replace(UpperCamelCase__		,			" "		).strip()
       return out_string







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		)			->	str:

       '''simple docstring'''




       return self.vocab.get(UpperCamelCase__		,			self.vocab.get(self.unk_token		)		)







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		)			->	Optional[Any]:

       '''simple docstring'''




       return self.reverse_vocab.get(UpperCamelCase__		,			self.unk_token		)







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		,			UpperCamelCase__=None		)			->	Any:

       '''simple docstring'''




       if token_ids_a is None:
           return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       snake_case					:   Optional[Any]         =					[self.cls_token_id]
       snake_case					:   str         =					[self.sep_token_id]
       return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		,			UpperCamelCase__=None		)			->	Dict:

       '''simple docstring'''




       if offset_mapping_a is None:
           return [(0, 0)] + offset_mapping_a + [(0, 0)]

       return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		,			UpperCamelCase__=None		,			UpperCamelCase__=False		)			->	Any:

       '''simple docstring'''




       if already_has_special_tokens:
           if token_ids_a is not None:
               raise ValueError(
                   "You should not supply a second sequence if the provided sequence of "
                   "ids is already formatted with special tokens for the model."		)
           return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

       if token_ids_a is not None:
           return [1] + ([0] * len(UpperCamelCase__		)) + [1, 1] + ([0] * len(UpperCamelCase__		)) + [1]
       return [1] + ([0] * len(UpperCamelCase__		)) + [1]







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		,			UpperCamelCase__ = None		)			->	List[int]:

       '''simple docstring'''




       if token_ids_a is None:
           # [CLS] X [SEP]
           return (len(UpperCamelCase__		) + 2) * [0]

       # [CLS] A [SEP] [SEP] B [SEP]
       return [0] * (len(UpperCamelCase__		) + 1) + [1] * (len(UpperCamelCase__		) + 3)







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		)			->	Dict:

       '''simple docstring'''




       if "\u4e00" <= char <= "\u9fff":
           return True
       return False







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		)			->	List[str]:

       '''simple docstring'''




       if ("a" <= char <= "z") or ("A" <= char <= "Z"):
           return True
       return False







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		)			->	Union[str, Any]:

       '''simple docstring'''




       if char in ",;:.?!~，；：。？！《》【】":
           return True
       return False







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		)			->	List[Any]:

       '''simple docstring'''




       if char == " " or char == "\t" or char == "\n" or char == "\r":
           return True
       if len(UpperCamelCase__		) == 1:
           snake_case					:   Any         =					unicodedata.category(UpperCamelCase__		)
           if cat == "Zs":
               return True
       return False







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		)			->	List[Any]:

       '''simple docstring'''




       snake_case					:   Any         =					{}
       with io.open(UpperCamelCase__		,			"r"		,			encoding="utf-8"		) as f:
           for index, line in enumerate(UpperCamelCase__		):
               snake_case					:   Any         =					line.rstrip("\n"		)
               snake_case					:   Optional[Any]         =					int(UpperCamelCase__		)

       return token_to_idx







   def 				lowerCamelCase	(			self		,			UpperCamelCase__		,			UpperCamelCase__ = None		)			->	Tuple[str]:

       '''simple docstring'''




       snake_case					:   Any         =					0
       if os.path.isdir(UpperCamelCase__		):
           snake_case					:   str         =					os.path.join(
               UpperCamelCase__		,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
       else:
           snake_case					:   Optional[Any]         =					(filename_prefix + "-" if filename_prefix else "") + save_directory
       with open(UpperCamelCase__		,			"w"		,			encoding="utf-8"		) as writer:
           for token, token_index in sorted(self.vocab.items()		,			key=lambda UpperCamelCase__		: kv[1]		):
               if index != token_index:
                   logger.warning(
                       F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
                       " Please check that the vocabulary is not corrupted!"		)
                   snake_case					:   str         =					token_index
               writer.write(token + "\n"		)
               index += 1

       snake_case					:   str         =					os.path.join(UpperCamelCase__		,			"sentencepiece.bpe.model"		)
       with open(UpperCamelCase__		,			"wb"		) as fi:
           snake_case					:   List[Any]         =					self.sp_model.serialized_model_proto()
           fi.write(UpperCamelCase__		)

       return (vocab_file,)









"""simple docstring"""



import baseaa







def 						__lowerCAmelCase		(		lowercase						:						str				)			->		bytes:


    """simple docstring"""





    return baseaa.aaaencode(string.encode("utf-8"				)				)







def 						__lowerCAmelCase		(		lowercase						:						bytes				)			->		str:


    """simple docstring"""





    return baseaa.aaadecode(lowercase				).decode("utf-8"				)


if __name__ == "__main__":
  import doctest

  doctest.testmod()





