


'''simple docstring'''
import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_			:  Tuple										=   logging.get_logger(__name__)

UpperCAmelCase_			:  Tuple										=   '''▁'''

UpperCAmelCase_			:  str										=   {'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

UpperCAmelCase_			:  Optional[int]										=   {
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

UpperCAmelCase_			:  List[Any]										=   {
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

UpperCAmelCase_			:  List[Any]										=   {
    '''ernie-m-base''': 5_1_4,
    '''ernie-m-large''': 5_1_4,
}

UpperCAmelCase_			:  Optional[int]										=   {
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}

class     _lowerCamelCase       (     lowerCamelCase__					):




 '''simple docstring'''
 __lowercase					:		Dict     				=    ['''input_ids''']

 __lowercase					:		Tuple     				=    VOCAB_FILES_NAMES
 __lowercase					:		List[Any]     				=    PRETRAINED_INIT_CONFIGURATION
 __lowercase					:		Any     				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase					:		Optional[int]     				=    PRETRAINED_VOCAB_FILES_MAP
 __lowercase					:		Optional[int]     				=    RESOURCE_FILES_NAMES


 def __init__(   self   ,   __lowercase   ,   __lowercase=None   ,   __lowercase=False   ,   __lowercase="utf8"   ,   __lowercase="[UNK]"   ,   __lowercase="[SEP]"   ,   __lowercase="[PAD]"   ,   __lowercase="[CLS]"   ,   __lowercase="[MASK]"   ,   __lowercase = None   ,   **__lowercase   ,   ):

  """simple docstring"""







  __A	:		Tuple	  =      {} if sp_model_kwargs is None else sp_model_kwargs
  super().__init__(
      do_lower_case=__lowercase   ,   unk_token=__lowercase   ,   sep_token=__lowercase   ,   pad_token=__lowercase   ,   cls_token=__lowercase   ,   mask_token=__lowercase   ,   vocab_file=__lowercase   ,   encoding=__lowercase   ,   sp_model_kwargs=self.sp_model_kwargs   ,   **__lowercase   ,   )
  __A	:		int	  =      do_lower_case
  __A	:		str	  =      sentencepiece_model_ckpt
  __A	:		Any	  =      spm.SentencePieceProcessor(**self.sp_model_kwargs    )
  self.sp_model.Load(__lowercase    )

  # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
  if vocab_file is not None:
   __A	:		Union[str, Any]	  =      self.load_vocab(filepath=__lowercase    )
  else:
   __A	:		List[str]	  =      {self.sp_model.id_to_piece(__lowercase    ): id for id in range(self.sp_model.get_piece_size()    )}
  __A	:		Any	  =      {v: k for k, v in self.vocab.items()}


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  if text is None:
   return None

  __A	:		List[Any]	  =      self.tokenize(__lowercase    )
  __A	:		Dict	  =      "", []

  for i, ch in enumerate(__lowercase    ):
   if ch in self.SP_CHAR_MAPPING:
    __A	:		Dict	  =      self.SP_CHAR_MAPPING.get(__lowercase    )
   else:
    __A	:		Any	  =      unicodedata.normalize('NFKC'   ,   __lowercase    )
   if self.is_whitespace(__lowercase    ):
    continue
   normalized_text += ch
   char_mapping.extend([i] * len(__lowercase    )    )

  __A	:		Any	  =      normalized_text, [], 0

  if self.do_lower_case:
   __A	:		Optional[int]	  =      text.lower()

  for token in split_tokens:
   if token[:1] == "▁":
    __A	:		Tuple	  =      token[1:]
   __A	:		List[Any]	  =      text[offset:].index(__lowercase    ) + offset
   __A	:		Union[str, Any]	  =      start + len(__lowercase    )

   token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)    )
   __A	:		Optional[int]	  =      end
  return token_mapping


 @property
 def 			snake_case__ (   self    ):

  """simple docstring"""







  return len(self.vocab    )


 def 			snake_case__ (   self    ):

  """simple docstring"""







  return dict(self.vocab   ,   **self.added_tokens_encoder    )


 def __getstate__(   self    ):

  """simple docstring"""







  __A	:		str	  =      self.__dict__.copy()
  __A	:		List[Any]	  =      None
  return state


 def __setstate__(   self   ,   __lowercase    ):

  """simple docstring"""







  __A	:		str	  =      d

  # for backward compatibility
  if not hasattr(self   ,   'sp_model_kwargs'    ):
   __A	:		List[str]	  =      {}

  __A	:		Dict	  =      spm.SentencePieceProcessor(**self.sp_model_kwargs    )
  self.sp_model.Load(self.sentencepiece_model_ckpt    )


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  return "".join((self.SP_CHAR_MAPPING.get(__lowercase   ,   __lowercase    ) for c in text)    )


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase=False   ,   __lowercase=64   ,   __lowercase=0.1    ):

  """simple docstring"""







  if self.sp_model_kwargs.get('enable_sampling'    ) is True:
   __A	:		Union[str, Any]	  =      True
  if self.sp_model_kwargs.get('alpha'    ) is not None:
   __A	:		Union[str, Any]	  =      self.sp_model_kwargs.get('alpha'    )
  if self.sp_model_kwargs.get('nbest_size'    ) is not None:
   __A	:		List[Any]	  =      self.sp_model_kwargs.get('nbest_size'    )

  if not enable_sampling:
   __A	:		Union[str, Any]	  =      self.sp_model.EncodeAsPieces(__lowercase    )
  else:
   __A	:		Union[str, Any]	  =      self.sp_model.SampleEncodeAsPieces(__lowercase   ,   __lowercase   ,   __lowercase    )
  __A	:		int	  =      []
  for pi, piece in enumerate(__lowercase    ):
   if piece == SPIECE_UNDERLINE:
    if not pieces[pi + 1].startswith(__lowercase    ) and pi != 0:
     new_pieces.append(__lowercase    )
     continue
    else:
     continue
   __A	:		int	  =      0
   for i, chunk in enumerate(__lowercase    ):
    if chunk == SPIECE_UNDERLINE:
     continue
    if self.is_ch_char(__lowercase    ) or self.is_punct(__lowercase    ):
     if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
      new_pieces.append(piece[lst_i:i]    )
     new_pieces.append(__lowercase    )
     __A	:		Any	  =      i + 1
    elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
     if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
      new_pieces.append(piece[lst_i:i]    )
     __A	:		str	  =      i
    elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
     if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
      new_pieces.append(piece[lst_i:i]    )
     __A	:		List[Any]	  =      i
   if len(__lowercase    ) > lst_i:
    new_pieces.append(piece[lst_i:]    )
  return new_pieces


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  __A	:		Any	  =      "".join(__lowercase    ).replace(__lowercase   ,   ' '    ).strip()
  return out_string


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  __A	:		int	  =      self.convert_ids_to_tokens(__lowercase    )
  __A	:		Optional[Any]	  =      "".join(__lowercase    ).replace(__lowercase   ,   ' '    ).strip()
  return out_string


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  return self.vocab.get(__lowercase   ,   self.vocab.get(self.unk_token    )    )


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  return self.reverse_vocab.get(__lowercase   ,   self.unk_token    )


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase=None    ):

  """simple docstring"""







  if token_ids_a is None:
   return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  __A	:		List[Any]	  =      [self.cls_token_id]
  __A	:		List[Any]	  =      [self.sep_token_id]
  return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase=None    ):

  """simple docstring"""







  if offset_mapping_a is None:
   return [(0, 0)] + offset_mapping_a + [(0, 0)]

  return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase=None   ,   __lowercase=False    ):

  """simple docstring"""







  if already_has_special_tokens:
   if token_ids_a is not None:
    raise ValueError(
        'You should not supply a second sequence if the provided sequence of '
        'ids is already formatted with special tokens for the model.'    )
   return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

  if token_ids_a is not None:
   return [1] + ([0] * len(__lowercase    )) + [1, 1] + ([0] * len(__lowercase    )) + [1]
  return [1] + ([0] * len(__lowercase    )) + [1]


 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase = None    ):

  """simple docstring"""







  if token_ids_a is None:
   # [CLS] X [SEP]
   return (len(__lowercase    ) + 2) * [0]

  # [CLS] A [SEP] [SEP] B [SEP]
  return [0] * (len(__lowercase    ) + 1) + [1] * (len(__lowercase    ) + 3)


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  if "\u4e00" <= char <= "\u9fff":
   return True
  return False


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  if ("a" <= char <= "z") or ("A" <= char <= "Z"):
   return True
  return False


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  if char in ",;:.?!~，；：。？！《》【】":
   return True
  return False


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  if char == " " or char == "\t" or char == "\n" or char == "\r":
   return True
  if len(__lowercase    ) == 1:
   __A	:		Tuple	  =      unicodedata.category(__lowercase    )
   if cat == "Zs":
    return True
  return False


 def 			snake_case__ (   self   ,   __lowercase    ):

  """simple docstring"""







  __A	:		Tuple	  =      {}
  with io.open(__lowercase   ,   'r'   ,   encoding='utf-8'    ) as f:
   for index, line in enumerate(__lowercase    ):
    __A	:		Optional[int]	  =      line.rstrip('\n'    )
    __A	:		Union[str, Any]	  =      int(__lowercase    )

  return token_to_idx







 def 			snake_case__ (   self   ,   __lowercase   ,   __lowercase = None    ):

  """simple docstring"""







  __A	:		Union[str, Any]	  =      0
  if os.path.isdir(__lowercase    ):
   __A	:		str	  =      os.path.join(
       __lowercase   ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
  else:
   __A	:		Optional[int]	  =      (filename_prefix + "-" if filename_prefix else "") + save_directory
  with open(__lowercase   ,   'w'   ,   encoding='utf-8'    ) as writer:
   for token, token_index in sorted(self.vocab.items()   ,   key=lambda __lowercase    : kv[1]    ):
    if index != token_index:
     logger.warning(
         F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
         ' Please check that the vocabulary is not corrupted!'    )
     __A	:		str	  =      token_index
    writer.write(token + '\n'    )
    index += 1

  __A	:		Optional[Any]	  =      os.path.join(__lowercase   ,   'sentencepiece.bpe.model'    )
  with open(__lowercase   ,   'wb'    ) as fi:
   __A	:		Optional[int]	  =      self.sp_model.serialized_model_proto()
   fi.write(__lowercase    )

  return (vocab_file,)













def   __snake_case    (       _lowerCAmelCase			:			int						)			->    bool:
   if num < 0:
      return False

   A_       :					int        =      num
   A_       :					int        =      0
   while num > 0:
      A_       :					str        =      rev_num * 10 + (num % 10)
      num //= 10

   return num_copy == rev_num


if __name__ == "__main__":
 import doctest

 doctest.testmod()
