

import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a    :   List[Any]        =			logging.get_logger(__name__)

_a    :   str        =			"""▁"""

_a    :   Dict        =			{"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

_a    :   int        =			{
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

_a    :   Tuple        =			{
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

_a    :   List[Any]        =			{
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

_a    :   str        =			{
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}







class 	_UpperCAmelCase  (  _A		):




      """simple docstring"""



      A          =    ['''input_ids''']

      A          =    VOCAB_FILES_NAMES
      A          =    PRETRAINED_INIT_CONFIGURATION
      A          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      A          =    PRETRAINED_VOCAB_FILES_MAP
      A          =    RESOURCE_FILES_NAMES
      def __init__(   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase=None    ,	_lowerCAmelCase=False    ,	_lowerCAmelCase="utf8"    ,	_lowerCAmelCase="[UNK]"    ,	_lowerCAmelCase="[SEP]"    ,	_lowerCAmelCase="[PAD]"    ,	_lowerCAmelCase="[CLS]"    ,	_lowerCAmelCase="[MASK]"    ,	_lowerCAmelCase = None    ,	**_lowerCAmelCase    ,	):



          '''simple docstring'''
          lowerCAmelCase__   :int      							=				{} if sp_model_kwargs is None else sp_model_kwargs
          super().__init__(
              do_lower_case=__lowerCAmelCase    ,	unk_token=__lowerCAmelCase    ,	sep_token=__lowerCAmelCase    ,	pad_token=__lowerCAmelCase    ,	cls_token=__lowerCAmelCase    ,	mask_token=__lowerCAmelCase    ,	vocab_file=__lowerCAmelCase    ,	encoding=__lowerCAmelCase    ,	sp_model_kwargs=self.sp_model_kwargs    ,	**__lowerCAmelCase    ,	)
          lowerCAmelCase__   :int      							=				do_lower_case
          lowerCAmelCase__   :Any      							=				sentencepiece_model_ckpt
          lowerCAmelCase__   :str      							=				spm.SentencePieceProcessor(**self.sp_model_kwargs		)
          self.sp_model.Load(__lowerCAmelCase		)

          # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
          if vocab_file is not None:
              lowerCAmelCase__   :Dict      							=				self.load_vocab(filepath=__lowerCAmelCase		)
          else:
              lowerCAmelCase__   :Any      							=				{self.sp_model.id_to_piece(__lowerCAmelCase		): id for id in range(self.sp_model.get_piece_size()		)}
          lowerCAmelCase__   :Tuple      							=				{v: k for k, v in self.vocab.items()}
      def 			snake_case_      (   self    ,	_lowerCAmelCase		):



          '''simple docstring'''







          if text is None:
              return None

          lowerCAmelCase__   :Tuple      							=				self.tokenize(__lowerCAmelCase		)
          lowerCAmelCase__							,lowerCAmelCase__   :Optional[Any]      							=				"", []

          for i, ch in enumerate(__lowerCAmelCase		):
              if ch in self.SP_CHAR_MAPPING:
                  lowerCAmelCase__   :List[str]      							=				self.SP_CHAR_MAPPING.get(__lowerCAmelCase		)
              else:
                  lowerCAmelCase__   :Tuple      							=				unicodedata.normalize("NFKC"    ,	__lowerCAmelCase		)
              if self.is_whitespace(__lowerCAmelCase		):
                  continue
              normalized_text += ch
              char_mapping.extend([i] * len(__lowerCAmelCase		)		)

          lowerCAmelCase__							,lowerCAmelCase__							,lowerCAmelCase__   :Tuple      							=				normalized_text, [], 0

          if self.do_lower_case:
              lowerCAmelCase__   :int      							=				text.lower()

          for token in split_tokens:
              if token[:1] == "▁":
                  lowerCAmelCase__   :int      							=				token[1:]
              lowerCAmelCase__   :Tuple      							=				text[offset:].index(__lowerCAmelCase		) + offset
              lowerCAmelCase__   :Optional[int]      							=				start + len(__lowerCAmelCase		)

              token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)		)
              lowerCAmelCase__   :Optional[Any]      							=				end
          return token_mapping
      @property
      def 			snake_case_      (   self		):



          '''simple docstring'''
          return len(self.vocab		)
      def 			snake_case_      (   self		):



          '''simple docstring'''
          return dict(self.vocab    ,	**self.added_tokens_encoder		)
      def __getstate__(   self		):



          '''simple docstring'''
          lowerCAmelCase__   :str      							=				self.__dict__.copy()
          lowerCAmelCase__   :List[Any]      							=				None
          return state
      def __setstate__(   self    ,	_lowerCAmelCase		):



          '''simple docstring'''
          lowerCAmelCase__   :int      							=				d

          # for backward compatibility
          if not hasattr(self    ,	"sp_model_kwargs"		):
              lowerCAmelCase__   :str      							=				{}

          lowerCAmelCase__   :Union[str, Any]      							=				spm.SentencePieceProcessor(**self.sp_model_kwargs		)
          self.sp_model.Load(self.sentencepiece_model_ckpt		)
      def 			snake_case_      (   self    ,	_lowerCAmelCase		):



          '''simple docstring'''
          return "".join((self.SP_CHAR_MAPPING.get(__lowerCAmelCase    ,	__lowerCAmelCase		) for c in text)		)
      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase=False    ,	_lowerCAmelCase=64    ,	_lowerCAmelCase=0.1		):



          '''simple docstring'''







          if self.sp_model_kwargs.get("enable_sampling"		) is True:
              lowerCAmelCase__   :Tuple      							=				True
          if self.sp_model_kwargs.get("alpha"		) is not None:
              lowerCAmelCase__   :List[Any]      							=				self.sp_model_kwargs.get("alpha"		)
          if self.sp_model_kwargs.get("nbest_size"		) is not None:
              lowerCAmelCase__   :str      							=				self.sp_model_kwargs.get("nbest_size"		)

          if not enable_sampling:
              lowerCAmelCase__   :Optional[Any]      							=				self.sp_model.EncodeAsPieces(__lowerCAmelCase		)
          else:
              lowerCAmelCase__   :List[str]      							=				self.sp_model.SampleEncodeAsPieces(__lowerCAmelCase    ,	__lowerCAmelCase    ,	__lowerCAmelCase		)
          lowerCAmelCase__   :Optional[Any]      							=				[]
          for pi, piece in enumerate(__lowerCAmelCase		):
              if piece == SPIECE_UNDERLINE:
                  if not pieces[pi + 1].startswith(__lowerCAmelCase		) and pi != 0:
                      new_pieces.append(__lowerCAmelCase		)
                      continue
                  else:
                      continue
              lowerCAmelCase__   :Dict      							=				0
              for i, chunk in enumerate(__lowerCAmelCase		):
                  if chunk == SPIECE_UNDERLINE:
                      continue
                  if self.is_ch_char(__lowerCAmelCase		) or self.is_punct(__lowerCAmelCase		):
                      if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                          new_pieces.append(piece[lst_i:i]		)
                      new_pieces.append(__lowerCAmelCase		)
                      lowerCAmelCase__   :Optional[int]      							=				i + 1
                  elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                      if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                          new_pieces.append(piece[lst_i:i]		)
                      lowerCAmelCase__   :List[Any]      							=				i
                  elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                      if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                          new_pieces.append(piece[lst_i:i]		)
                      lowerCAmelCase__   :Optional[int]      							=				i
              if len(__lowerCAmelCase		) > lst_i:
                  new_pieces.append(piece[lst_i:]		)
          return new_pieces
      def 			snake_case_      (   self    ,	_lowerCAmelCase		):



          '''simple docstring'''
          lowerCAmelCase__   :str      							=				"".join(__lowerCAmelCase		).replace(__lowerCAmelCase    ,	" "		).strip()
          return out_string
      def 			snake_case_      (   self    ,	_lowerCAmelCase		):



          '''simple docstring'''
          lowerCAmelCase__   :str      							=				self.convert_ids_to_tokens(__lowerCAmelCase		)
          lowerCAmelCase__   :Optional[int]      							=				"".join(__lowerCAmelCase		).replace(__lowerCAmelCase    ,	" "		).strip()
          return out_string
      def 			snake_case_      (   self    ,	_lowerCAmelCase		):



          '''simple docstring'''
          return self.vocab.get(__lowerCAmelCase    ,	self.vocab.get(self.unk_token		)		)
      def 			snake_case_      (   self    ,	_lowerCAmelCase		):



          '''simple docstring'''
          return self.reverse_vocab.get(__lowerCAmelCase    ,	self.unk_token		)
      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase=None		):



          '''simple docstring'''







          if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          lowerCAmelCase__   :Dict      							=				[self.cls_token_id]
          lowerCAmelCase__   :Any      							=				[self.sep_token_id]
          return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep
      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase=None		):



          '''simple docstring'''







          if offset_mapping_a is None:
              return [(0, 0)] + offset_mapping_a + [(0, 0)]

          return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]
      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase=None    ,	_lowerCAmelCase=False		):



          '''simple docstring'''







          if already_has_special_tokens:
              if token_ids_a is not None:
                  raise ValueError(
                      "You should not supply a second sequence if the provided sequence of "
                      "ids is already formatted with special tokens for the model."		)
              return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

          if token_ids_a is not None:
              return [1] + ([0] * len(__lowerCAmelCase		)) + [1, 1] + ([0] * len(__lowerCAmelCase		)) + [1]
          return [1] + ([0] * len(__lowerCAmelCase		)) + [1]
      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase = None		):



          '''simple docstring'''







          if token_ids_a is None:
              # [CLS] X [SEP]
              return (len(__lowerCAmelCase		) + 2) * [0]

          # [CLS] A [SEP] [SEP] B [SEP]
          return [0] * (len(__lowerCAmelCase		) + 1) + [1] * (len(__lowerCAmelCase		) + 3)
      def 			snake_case_      (   self    ,	_lowerCAmelCase		):



          '''simple docstring'''







          if "\u4e00" <= char <= "\u9fff":
              return True
          return False
      def 			snake_case_      (   self    ,	_lowerCAmelCase		):



          '''simple docstring'''







          if ("a" <= char <= "z") or ("A" <= char <= "Z"):
              return True
          return False
      def 			snake_case_      (   self    ,	_lowerCAmelCase		):



          '''simple docstring'''







          if char in ",;:.?!~，；：。？！《》【】":
              return True
          return False
      def 			snake_case_      (   self    ,	_lowerCAmelCase		):



          '''simple docstring'''







          if char == " " or char == "\t" or char == "\n" or char == "\r":
              return True
          if len(__lowerCAmelCase		) == 1:
              lowerCAmelCase__   :Any      							=				unicodedata.category(__lowerCAmelCase		)
              if cat == "Zs":
                  return True
          return False
      def 			snake_case_      (   self    ,	_lowerCAmelCase		):



          '''simple docstring'''
          lowerCAmelCase__   :str      							=				{}
          with io.open(__lowerCAmelCase    ,	"r"    ,	encoding="utf-8"		) as f:
              for index, line in enumerate(__lowerCAmelCase		):
                  lowerCAmelCase__   :List[Any]      							=				line.rstrip("\n"		)
                  lowerCAmelCase__   :Dict      							=				int(__lowerCAmelCase		)

          return token_to_idx



      def 			snake_case_      (   self    ,	_lowerCAmelCase    ,	_lowerCAmelCase = None		):



          '''simple docstring'''
          lowerCAmelCase__   :Tuple      							=				0
          if os.path.isdir(__lowerCAmelCase		):
              lowerCAmelCase__   :Optional[Any]      							=				os.path.join(
                  __lowerCAmelCase    ,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
          else:
              lowerCAmelCase__   :Optional[Any]      							=				(filename_prefix + "-" if filename_prefix else "") + save_directory
          with open(__lowerCAmelCase    ,	"w"    ,	encoding="utf-8"		) as writer:
              for token, token_index in sorted(self.vocab.items()    ,	key=lambda _lowerCAmelCase		: kv[1]		):
                  if index != token_index:
                      logger.warning(
                          f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                          " Please check that the vocabulary is not corrupted!"		)
                      lowerCAmelCase__   :Tuple      							=				token_index
                  writer.write(token + "\n"		)
                  index += 1

          lowerCAmelCase__   :Tuple      							=				os.path.join(__lowerCAmelCase    ,	"sentencepiece.bpe.model"		)
          with open(__lowerCAmelCase    ,	"wb"		) as fi:
              lowerCAmelCase__   :List[str]      							=				self.sp_model.serialized_model_proto()
              fi.write(__lowerCAmelCase		)

          return (vocab_file,)











def 				snake_case__							(							UpperCAmelCase							:  str       ):
    return "".join(chr(ord(UpperCAmelCase       ) - 3_2       ) if "a" <= char <= "z" else char for char in word       )


if __name__ == "__main__":
 from doctest import testmod

 testmod()



