"""simple docstring"""






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__A       =					logging.get_logger(__name__)

__A       =					"▁"

__A       =					{"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

__A       =					{
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

__A       =					{
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

__A       =					{
    "ernie-m-base": 5_1_4,
    "ernie-m-large": 5_1_4,
}

__A       =					{
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}


class        UpperCAmelCase		(snake_case_    ):



    """simple docstring"""

    _UpperCAmelCase      :Optional[Any]		       =					["input_ids"]

    _UpperCAmelCase      :List[Any]		       =					VOCAB_FILES_NAMES
    _UpperCAmelCase      :Tuple		       =					PRETRAINED_INIT_CONFIGURATION
    _UpperCAmelCase      :int		       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _UpperCAmelCase      :List[Any]		       =					PRETRAINED_VOCAB_FILES_MAP
    _UpperCAmelCase      :int		       =					RESOURCE_FILES_NAMES






    def __init__(  self       ,     _UpperCAmelCase       ,     _UpperCAmelCase=None       ,     _UpperCAmelCase=False       ,     _UpperCAmelCase="utf8"       ,     _UpperCAmelCase="[UNK]"       ,     _UpperCAmelCase="[SEP]"       ,     _UpperCAmelCase="[PAD]"       ,     _UpperCAmelCase="[CLS]"       ,     _UpperCAmelCase="[MASK]"       ,     _UpperCAmelCase = None       ,     **_UpperCAmelCase       ,     ):
       # Mask token behave like a normal word, i.e. include the space before it and
       # is included in the raw text, there should be a match in a non-normalized sentence.

       lowercase__:  Any   = {} if sp_model_kwargs is None else sp_model_kwargs
       super().__init__(
           do_lower_case=_UpperCAmelCase       ,     unk_token=_UpperCAmelCase       ,     sep_token=_UpperCAmelCase       ,     pad_token=_UpperCAmelCase       ,     cls_token=_UpperCAmelCase       ,     mask_token=_UpperCAmelCase       ,     vocab_file=_UpperCAmelCase       ,     encoding=_UpperCAmelCase       ,     sp_model_kwargs=self.sp_model_kwargs       ,     **_UpperCAmelCase       ,     )
       lowercase__:  Union[str, Any]   = do_lower_case
       lowercase__:  str   = sentencepiece_model_ckpt
       lowercase__:  str   = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
       self.sp_model.Load(_UpperCAmelCase				)

       # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
       if vocab_file is not None:
          lowercase__:  Optional[Any]   = self.load_vocab(filepath=_UpperCAmelCase				)
       else:
          lowercase__:  Union[str, Any]   = {self.sp_model.id_to_piece(_UpperCAmelCase				): id for id in range(self.sp_model.get_piece_size()				)}
       lowercase__:  Dict   = {v: k for k, v in self.vocab.items()}






    def _snake_case     (  self       ,     _UpperCAmelCase				):
       if text is None:
          return None

       lowercase__:  Optional[Any]   = self.tokenize(_UpperCAmelCase				)
       lowercase__,					lowercase__:  Any   = '''''', []

       for i, ch in enumerate(_UpperCAmelCase				):
          if ch in self.SP_CHAR_MAPPING:
             lowercase__:  List[Any]   = self.SP_CHAR_MAPPING.get(_UpperCAmelCase				)
          else:
             lowercase__:  Any   = unicodedata.normalize('''NFKC'''       ,     _UpperCAmelCase				)
          if self.is_whitespace(_UpperCAmelCase				):
             continue
          normalized_text += ch
          char_mapping.extend([i] * len(_UpperCAmelCase				)				)

       lowercase__,					lowercase__,					lowercase__:  List[str]   = normalized_text, [], 0

       if self.do_lower_case:
          lowercase__:  Optional[int]   = text.lower()

       for token in split_tokens:
          if token[:1] == "▁":
             lowercase__:  Union[str, Any]   = token[1:]
          lowercase__:  List[str]   = text[offset:].index(_UpperCAmelCase				) + offset
          lowercase__:  Optional[Any]   = start + len(_UpperCAmelCase				)

          token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)				)
          lowercase__:  Dict   = end
       return token_mapping






    @property
    def _snake_case     (  self				):
       return len(self.vocab				)






    def _snake_case     (  self				):
       return dict(self.vocab       ,     **self.added_tokens_encoder				)






    def __getstate__(  self				):
       lowercase__:  Any   = self.__dict__.copy()
       lowercase__:  Dict   = None
       return state






    def __setstate__(  self       ,     _UpperCAmelCase				):
       lowercase__:  Union[str, Any]   = d

       # for backward compatibility
       if not hasattr(self       ,     '''sp_model_kwargs'''				):
          lowercase__:  List[str]   = {}

       lowercase__:  Dict   = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
       self.sp_model.Load(self.sentencepiece_model_ckpt				)






    def _snake_case     (  self       ,     _UpperCAmelCase				):
       return "".join((self.SP_CHAR_MAPPING.get(_UpperCAmelCase       ,     _UpperCAmelCase				) for c in text)				)






    def _snake_case     (  self       ,     _UpperCAmelCase       ,     _UpperCAmelCase=False       ,     _UpperCAmelCase=64       ,     _UpperCAmelCase=0.1				):

       if self.sp_model_kwargs.get('''enable_sampling'''				) is True:
          lowercase__:  List[Any]   = True
       if self.sp_model_kwargs.get('''alpha'''				) is not None:
          lowercase__:  Dict   = self.sp_model_kwargs.get('''alpha'''				)
       if self.sp_model_kwargs.get('''nbest_size'''				) is not None:
          lowercase__:  Dict   = self.sp_model_kwargs.get('''nbest_size'''				)

       if not enable_sampling:
          lowercase__:  int   = self.sp_model.EncodeAsPieces(_UpperCAmelCase				)
       else:
          lowercase__:  Union[str, Any]   = self.sp_model.SampleEncodeAsPieces(_UpperCAmelCase       ,     _UpperCAmelCase       ,     _UpperCAmelCase				)
       lowercase__:  Tuple   = []
       for pi, piece in enumerate(_UpperCAmelCase				):
          if piece == SPIECE_UNDERLINE:
             if not pieces[pi + 1].startswith(_UpperCAmelCase				) and pi != 0:
                new_pieces.append(_UpperCAmelCase				)
                continue
             else:
                continue
          lowercase__:  str   = 0
          for i, chunk in enumerate(_UpperCAmelCase				):
             if chunk == SPIECE_UNDERLINE:
                continue
             if self.is_ch_char(_UpperCAmelCase				) or self.is_punct(_UpperCAmelCase				):
                if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                   new_pieces.append(piece[lst_i:i]				)
                new_pieces.append(_UpperCAmelCase				)
                lowercase__:  Union[str, Any]   = i + 1
             elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                   new_pieces.append(piece[lst_i:i]				)
                lowercase__:  Optional[Any]   = i
             elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                   new_pieces.append(piece[lst_i:i]				)
                lowercase__:  str   = i
          if len(_UpperCAmelCase				) > lst_i:
             new_pieces.append(piece[lst_i:]				)
       return new_pieces






    def _snake_case     (  self       ,     _UpperCAmelCase				):
       lowercase__:  List[Any]   = ''''''.join(_UpperCAmelCase				).replace(_UpperCAmelCase       ,     ''' '''				).strip()
       return out_string






    def _snake_case     (  self       ,     _UpperCAmelCase				):
       lowercase__:  List[str]   = self.convert_ids_to_tokens(_UpperCAmelCase				)
       lowercase__:  int   = ''''''.join(_UpperCAmelCase				).replace(_UpperCAmelCase       ,     ''' '''				).strip()
       return out_string






    def _snake_case     (  self       ,     _UpperCAmelCase				):
       return self.vocab.get(_UpperCAmelCase       ,     self.vocab.get(self.unk_token				)				)






    def _snake_case     (  self       ,     _UpperCAmelCase				):
       return self.reverse_vocab.get(_UpperCAmelCase       ,     self.unk_token				)






    def _snake_case     (  self       ,     _UpperCAmelCase       ,     _UpperCAmelCase=None				):
       if token_ids_a is None:
          return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       lowercase__:  List[Any]   = [self.cls_token_id]
       lowercase__:  Any   = [self.sep_token_id]
       return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






    def _snake_case     (  self       ,     _UpperCAmelCase       ,     _UpperCAmelCase=None				):
       if offset_mapping_a is None:
          return [(0, 0)] + offset_mapping_a + [(0, 0)]

       return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






    def _snake_case     (  self       ,     _UpperCAmelCase       ,     _UpperCAmelCase=None       ,     _UpperCAmelCase=False				):

       if already_has_special_tokens:
          if token_ids_a is not None:
             raise ValueError(
                 '''You should not supply a second sequence if the provided sequence of '''
                 '''ids is already formatted with special tokens for the model.'''				)
          return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

       if token_ids_a is not None:
          return [1] + ([0] * len(_UpperCAmelCase				)) + [1, 1] + ([0] * len(_UpperCAmelCase				)) + [1]
       return [1] + ([0] * len(_UpperCAmelCase				)) + [1]






    def _snake_case     (  self       ,     _UpperCAmelCase       ,     _UpperCAmelCase = None				):
       # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
       if token_ids_a is None:
          # [CLS] X [SEP]
          return (len(_UpperCAmelCase				) + 2) * [0]

       # [CLS] A [SEP] [SEP] B [SEP]
       return [0] * (len(_UpperCAmelCase				) + 1) + [1] * (len(_UpperCAmelCase				) + 3)






    def _snake_case     (  self       ,     _UpperCAmelCase				):
       if "\u4e00" <= char <= "\u9fff":
          return True
       return False






    def _snake_case     (  self       ,     _UpperCAmelCase				):
       if ("a" <= char <= "z") or ("A" <= char <= "Z"):
          return True
       return False






    def _snake_case     (  self       ,     _UpperCAmelCase				):
       if char in ",;:.?!~，；：。？！《》【】":
          return True
       return False






    def _snake_case     (  self       ,     _UpperCAmelCase				):
       if char == " " or char == "\t" or char == "\n" or char == "\r":
          return True
       if len(_UpperCAmelCase				) == 1:
          lowercase__:  Optional[Any]   = unicodedata.category(_UpperCAmelCase				)
          if cat == "Zs":
             return True
       return False






    def _snake_case     (  self       ,     _UpperCAmelCase				):
       lowercase__:  Optional[int]   = {}
       with io.open(_UpperCAmelCase       ,     '''r'''       ,     encoding='''utf-8'''				) as f:
          for index, line in enumerate(_UpperCAmelCase				):
             lowercase__:  Union[str, Any]   = line.rstrip('''\n'''				)
             lowercase__:  Optional[int]   = int(_UpperCAmelCase				)

       return token_to_idx






    def _snake_case     (  self       ,     _UpperCAmelCase       ,     _UpperCAmelCase = None				):
       lowercase__:  Optional[Any]   = 0
       if os.path.isdir(_UpperCAmelCase				):
          lowercase__:  Tuple   = os.path.join(
              _UpperCAmelCase       ,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
       else:
          lowercase__:  Union[str, Any]   = (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
       with open(_UpperCAmelCase       ,     '''w'''       ,     encoding='''utf-8'''				) as writer:
          for token, token_index in sorted(self.vocab.items()       ,     key=lambda _UpperCAmelCase				: kv[1]				):
             if index != token_index:
                logger.warning(
                    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                    ''' Please check that the vocabulary is not corrupted!'''				)
                lowercase__:  Optional[Any]   = token_index
             writer.write(token + '''\n'''				)
             index += 1

       lowercase__:  Dict   = os.path.join(_UpperCAmelCase       ,     '''sentencepiece.bpe.model'''				)
       with open(_UpperCAmelCase       ,     '''wb'''				) as fi:
          lowercase__:  Union[str, Any]   = self.sp_model.serialized_model_proto()
          fi.write(_UpperCAmelCase				)

       return (vocab_file,)








import copy
from dataclasses import dataclass
from pathlib import Path
from typing import Dict, Optional, Union




@dataclass
class 					UpperCamelCase_ :



	'''simple docstring'''




	lowerCAmelCase						=					None
	lowerCAmelCase						=					False
	lowerCAmelCase						=					False
	lowerCAmelCase						=					False
	lowerCAmelCase						=					None
	lowerCAmelCase						=					None
	lowerCAmelCase						=					False
	lowerCAmelCase						=					False
	lowerCAmelCase						=					False
	lowerCAmelCase						=					True
	lowerCAmelCase						=					None
	lowerCAmelCase						=					1
	lowerCAmelCase						=					None
	lowerCAmelCase						=					False
	lowerCAmelCase						=					None
	lowerCAmelCase						=					None




	def 				_UpperCamelCase     (			self   )			->       "DownloadConfig":
								return self.__class__(**{k: copy.deepcopy(a   ) for k, v in self.__dict__.items()}   )





