




"""simple docstring"""





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A				=							logging.get_logger(__name__)

A				=							"""▁"""

A				=							{"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

A				=							{
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

A				=							{
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

A				=							{
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

A				=							{
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}



class   a__					(						__magic_name__ ):
			lowercase_		=						["input_ids"]

			lowercase_		=						VOCAB_FILES_NAMES
			lowercase_		=						PRETRAINED_INIT_CONFIGURATION
			lowercase_		=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowercase_		=						PRETRAINED_VOCAB_FILES_MAP
			lowercase_		=						RESOURCE_FILES_NAMES


			def __init__(   self   :     Dict   ,	UpperCamelCase_   :     Tuple   ,	UpperCamelCase_   :     List[str]=None   ,	UpperCamelCase_   :     List[str]=False   ,	UpperCamelCase_   :     List[str]="utf8"   ,	UpperCamelCase_   :     Tuple="[UNK]"   ,	UpperCamelCase_   :     Optional[int]="[SEP]"   ,	UpperCamelCase_   :     str="[PAD]"   ,	UpperCamelCase_   :     int="[CLS]"   ,	UpperCamelCase_   :     str="[MASK]"   ,	UpperCamelCase_   :     Optional[Dict[str, Any]] = None   ,	**UpperCamelCase_   :     int   ,	):


						"""simple docstring"""




						__UpperCAmelCase						:			Any												=       {} if sp_model_kwargs is None else sp_model_kwargs
						super().__init__(
						    do_lower_case=UpperCamelCase_   ,	unk_token=UpperCamelCase_   ,	sep_token=UpperCamelCase_   ,	pad_token=UpperCamelCase_   ,	cls_token=UpperCamelCase_   ,	mask_token=UpperCamelCase_   ,	vocab_file=UpperCamelCase_   ,	encoding=UpperCamelCase_   ,	sp_model_kwargs=self.sp_model_kwargs   ,	**UpperCamelCase_   ,	)
						__UpperCAmelCase						:			Tuple												=       do_lower_case
						__UpperCAmelCase						:			Any												=       sentencepiece_model_ckpt
						__UpperCAmelCase						:			Optional[Any]												=       spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(UpperCamelCase_)

						# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
						if vocab_file is not None:
									__UpperCAmelCase						:			List[Any]												=       self.load_vocab(filepath=UpperCamelCase_)
						else:
									__UpperCAmelCase						:			str												=       {self.sp_model.id_to_piece(UpperCamelCase_): id for id in range(self.sp_model.get_piece_size())}
						__UpperCAmelCase						:			Optional[int]												=       {v: k for k, v in self.vocab.items()}


			def    a_  (   self   :     Tuple   ,	UpperCamelCase_   :     Union[str, Any]):


						"""simple docstring"""




						if text is None:
									return None

						__UpperCAmelCase						:			str												=       self.tokenize(UpperCamelCase_)
						__UpperCAmelCase						,  __UpperCAmelCase						:			List[Any]												=       "", []

						for i, ch in enumerate(UpperCamelCase_):
									if ch in self.SP_CHAR_MAPPING:
												__UpperCAmelCase						:			int												=       self.SP_CHAR_MAPPING.get(UpperCamelCase_)
									else:
												__UpperCAmelCase						:			List[str]												=       unicodedata.normalize("NFKC"   ,	UpperCamelCase_)
									if self.is_whitespace(UpperCamelCase_):
												continue
									normalized_text += ch
									char_mapping.extend([i] * len(UpperCamelCase_))

						__UpperCAmelCase						,  __UpperCAmelCase						,  __UpperCAmelCase						:			List[str]												=       normalized_text, [], 0

						if self.do_lower_case:
									__UpperCAmelCase						:			List[Any]												=       text.lower()

						for token in split_tokens:
									if token[:1] == "▁":
												__UpperCAmelCase						:			Tuple												=       token[1:]
									__UpperCAmelCase						:			Tuple												=       text[offset:].index(UpperCamelCase_) + offset
									__UpperCAmelCase						:			Union[str, Any]												=       start + len(UpperCamelCase_)

									token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1))
									__UpperCAmelCase						:			Dict												=       end
						return token_mapping


			@property
			def    a_  (   self   :     Union[str, Any]):


						"""simple docstring"""




						return len(self.vocab)


			def    a_  (   self   :     Any):


						"""simple docstring"""




						return dict(self.vocab   ,	**self.added_tokens_encoder)


			def __getstate__(   self   :     int):


						"""simple docstring"""




						__UpperCAmelCase						:			Union[str, Any]												=       self.__dict__.copy()
						__UpperCAmelCase						:			Tuple												=       None
						return state


			def __setstate__(   self   :     str   ,	UpperCamelCase_   :     Any):


						"""simple docstring"""




						__UpperCAmelCase						:			int												=       d

						# for backward compatibility
						if not hasattr(self   ,	"sp_model_kwargs"):
									__UpperCAmelCase						:			List[str]												=       {}

						__UpperCAmelCase						:			Tuple												=       spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(self.sentencepiece_model_ckpt)


			def    a_  (   self   :     Any   ,	UpperCamelCase_   :     int):


						"""simple docstring"""




						return "".join((self.SP_CHAR_MAPPING.get(UpperCamelCase_   ,	UpperCamelCase_) for c in text))


			def    a_  (   self   :     Dict   ,	UpperCamelCase_   :     str   ,	UpperCamelCase_   :     Dict=False   ,	UpperCamelCase_   :     Dict=64   ,	UpperCamelCase_   :     Optional[Any]=0.1):


						"""simple docstring"""




						if self.sp_model_kwargs.get("enable_sampling") is True:
									__UpperCAmelCase						:			Optional[int]												=       True
						if self.sp_model_kwargs.get("alpha") is not None:
									__UpperCAmelCase						:			List[str]												=       self.sp_model_kwargs.get("alpha")
						if self.sp_model_kwargs.get("nbest_size") is not None:
									__UpperCAmelCase						:			Optional[Any]												=       self.sp_model_kwargs.get("nbest_size")

						if not enable_sampling:
									__UpperCAmelCase						:			List[str]												=       self.sp_model.EncodeAsPieces(UpperCamelCase_)
						else:
									__UpperCAmelCase						:			List[str]												=       self.sp_model.SampleEncodeAsPieces(UpperCamelCase_   ,	UpperCamelCase_   ,	UpperCamelCase_)
						__UpperCAmelCase						:			Dict												=       []
						for pi, piece in enumerate(UpperCamelCase_):
									if piece == SPIECE_UNDERLINE:
												if not pieces[pi + 1].startswith(UpperCamelCase_) and pi != 0:
															new_pieces.append(UpperCamelCase_)
															continue
												else:
															continue
									__UpperCAmelCase						:			Optional[Any]												=       0
									for i, chunk in enumerate(UpperCamelCase_):
												if chunk == SPIECE_UNDERLINE:
															continue
												if self.is_ch_char(UpperCamelCase_) or self.is_punct(UpperCamelCase_):
															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																		new_pieces.append(piece[lst_i:i])
															new_pieces.append(UpperCamelCase_)
															__UpperCAmelCase						:			int												=       i + 1
												elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																		new_pieces.append(piece[lst_i:i])
															__UpperCAmelCase						:			Tuple												=       i
												elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
															if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																		new_pieces.append(piece[lst_i:i])
															__UpperCAmelCase						:			Dict												=       i
									if len(UpperCamelCase_) > lst_i:
												new_pieces.append(piece[lst_i:])
						return new_pieces


			def    a_  (   self   :     Tuple   ,	UpperCamelCase_   :     Optional[int]):


						"""simple docstring"""




						__UpperCAmelCase						:			int												=       "".join(UpperCamelCase_).replace(UpperCamelCase_   ,	" ").strip()
						return out_string


			def    a_  (   self   :     Optional[Any]   ,	UpperCamelCase_   :     List[Any]):


						"""simple docstring"""




						__UpperCAmelCase						:			Dict												=       self.convert_ids_to_tokens(UpperCamelCase_)
						__UpperCAmelCase						:			str												=       "".join(UpperCamelCase_).replace(UpperCamelCase_   ,	" ").strip()
						return out_string


			def    a_  (   self   :     Union[str, Any]   ,	UpperCamelCase_   :     int):


						"""simple docstring"""




						return self.vocab.get(UpperCamelCase_   ,	self.vocab.get(self.unk_token))


			def    a_  (   self   :     Optional[int]   ,	UpperCamelCase_   :     Optional[int]):


						"""simple docstring"""




						return self.reverse_vocab.get(UpperCamelCase_   ,	self.unk_token)


			def    a_  (   self   :     Any   ,	UpperCamelCase_   :     List[Any]   ,	UpperCamelCase_   :     List[str]=None):


						"""simple docstring"""




						if token_ids_a is None:
									return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
						__UpperCAmelCase						:			str												=       [self.cls_token_id]
						__UpperCAmelCase						:			Optional[Any]												=       [self.sep_token_id]
						return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep


			def    a_  (   self   :     List[Any]   ,	UpperCamelCase_   :     Optional[Any]   ,	UpperCamelCase_   :     str=None):


						"""simple docstring"""




						if offset_mapping_a is None:
									return [(0, 0)] + offset_mapping_a + [(0, 0)]

						return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]


			def    a_  (   self   :     str   ,	UpperCamelCase_   :     str   ,	UpperCamelCase_   :     int=None   ,	UpperCamelCase_   :     str=False):


						"""simple docstring"""




						if already_has_special_tokens:
									if token_ids_a is not None:
												raise ValueError(
												    "You should not supply a second sequence if the provided sequence of "
												    "ids is already formatted with special tokens for the model.")
									return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

						if token_ids_a is not None:
									return [1] + ([0] * len(UpperCamelCase_)) + [1, 1] + ([0] * len(UpperCamelCase_)) + [1]
						return [1] + ([0] * len(UpperCamelCase_)) + [1]


			def    a_  (   self   :     Tuple   ,	UpperCamelCase_   :     List[int]   ,	UpperCamelCase_   :     Optional[List[int]] = None):


						"""simple docstring"""




						if token_ids_a is None:
									# [CLS] X [SEP]
									return (len(UpperCamelCase_) + 2) * [0]

						# [CLS] A [SEP] [SEP] B [SEP]
						return [0] * (len(UpperCamelCase_) + 1) + [1] * (len(UpperCamelCase_) + 3)


			def    a_  (   self   :     Optional[int]   ,	UpperCamelCase_   :     Optional[int]):


						"""simple docstring"""




						if "\u4e00" <= char <= "\u9fff":
									return True
						return False


			def    a_  (   self   :     List[str]   ,	UpperCamelCase_   :     Optional[Any]):


						"""simple docstring"""




						if ("a" <= char <= "z") or ("A" <= char <= "Z"):
									return True
						return False


			def    a_  (   self   :     List[Any]   ,	UpperCamelCase_   :     Any):


						"""simple docstring"""




						if char in ",;:.?!~，；：。？！《》【】":
									return True
						return False


			def    a_  (   self   :     Optional[int]   ,	UpperCamelCase_   :     List[Any]):


						"""simple docstring"""




						if char == " " or char == "\t" or char == "\n" or char == "\r":
									return True
						if len(UpperCamelCase_) == 1:
									__UpperCAmelCase						:			Dict												=       unicodedata.category(UpperCamelCase_)
									if cat == "Zs":
												return True
						return False


			def    a_  (   self   :     Union[str, Any]   ,	UpperCamelCase_   :     int):


						"""simple docstring"""




						__UpperCAmelCase						:			Optional[int]												=       {}
						with io.open(UpperCamelCase_   ,	"r"   ,	encoding="utf-8") as f:
									for index, line in enumerate(UpperCamelCase_):
												__UpperCAmelCase						:			Optional[int]												=       line.rstrip("\n")
												__UpperCAmelCase						:			int												=       int(UpperCamelCase_)

						return token_to_idx


			def    a_  (   self   :     List[str]   ,	UpperCamelCase_   :     str   ,	UpperCamelCase_   :     Optional[str] = None):


						"""simple docstring"""




						__UpperCAmelCase						:			Optional[Any]												=       0
						if os.path.isdir(UpperCamelCase_):
									__UpperCAmelCase						:			Tuple												=       os.path.join(
									    UpperCamelCase_   ,	(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
						else:
									__UpperCAmelCase						:			int												=       (filename_prefix + "-" if filename_prefix else "") + save_directory
						with open(UpperCamelCase_   ,	"w"   ,	encoding="utf-8") as writer:
									for token, token_index in sorted(self.vocab.items()   ,	key=lambda UpperCamelCase_: kv[1]):
												if index != token_index:
															logger.warning(
															    F"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
															    " Please check that the vocabulary is not corrupted!")
															__UpperCAmelCase						:			List[Any]												=       token_index
												writer.write(token + "\n")
												index += 1

						__UpperCAmelCase						:			Optional[Any]												=       os.path.join(UpperCamelCase_   ,	"sentencepiece.bpe.model")
						with open(UpperCamelCase_   ,	"wb") as fi:
									__UpperCAmelCase						:			Union[str, Any]												=       self.sp_model.serialized_model_proto()
									fi.write(UpperCamelCase_)

						return (vocab_file,)









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


a__: List[Any]							   =						{
    'configuration_x_clip': [
        'XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'XCLIPConfig',
        'XCLIPTextConfig',
        'XCLIPVisionConfig',
    ],
    'processing_x_clip': ['XCLIPProcessor'],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a__: Any							   =						[
           'XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST',
           'XCLIPModel',
           'XCLIPPreTrainedModel',
           'XCLIPTextModel',
           'XCLIPVisionModel',
       ]

if TYPE_CHECKING:
       from .configuration_x_clip import (
           XCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
           XCLIPConfig,
           XCLIPTextConfig,
           XCLIPVisionConfig,
       )
       from .processing_x_clip import XCLIPProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_x_clip import (
                  XCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
                  XCLIPModel,
                  XCLIPPreTrainedModel,
                  XCLIPTextModel,
                  XCLIPVisionModel,
              )

else:
       import sys

       a__: Optional[Any]							   =						_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





