





'''simple docstring'''


import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_   :   Union[str, Any]       =					logging.get_logger(__name__)

lowerCAmelCase_   :   Any       =					"▁"

lowerCAmelCase_   :   Tuple       =					{"vocab_file": "vocab.txt", "sentencepiece_model_ckpt": "sentencepiece.bpe.model"}

lowerCAmelCase_   :   List[Any]       =					{
    "sentencepiece_model_file": "sentencepiece.bpe.model",
    "vocab_file": "vocab.txt",
}

lowerCAmelCase_   :   Union[str, Any]       =					{
    "vocab_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt",
    },
    "sentencepiece_model_file": {
        "ernie-m-base": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
        "ernie-m-large": "https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model",
    },
}

lowerCAmelCase_   :   str       =					{
    "ernie-m-base": 514,
    "ernie-m-large": 514,
}

lowerCAmelCase_   :   List[Any]       =					{
    "ernie-m-base": {"do_lower_case": False},
    "ernie-m-large": {"do_lower_case": False},
}






class       UpperCamelCase__  (		__lowerCAmelCase     ):
	lowerCAmelCase__					:			List[str]								=						["input_ids"]

	lowerCAmelCase__					:			Any								=						VOCAB_FILES_NAMES
	lowerCAmelCase__					:			Dict								=						PRETRAINED_INIT_CONFIGURATION
	lowerCAmelCase__					:			Optional[Any]								=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCAmelCase__					:			List[str]								=						PRETRAINED_VOCAB_FILES_MAP
	lowerCAmelCase__					:			Dict								=						RESOURCE_FILES_NAMES







	def __init__(				self			:   Any , lowerCamelCase			:   Any , lowerCamelCase			:   Any=None , lowerCamelCase			:   Union[str, Any]=False , lowerCamelCase			:   Any="utf8" , lowerCamelCase			:   List[Any]="[UNK]" , lowerCamelCase			:   Optional[Any]="[SEP]" , lowerCamelCase			:   Optional[int]="[PAD]" , lowerCamelCase			:   Dict="[CLS]" , lowerCamelCase			:   Optional[int]="[MASK]" , lowerCamelCase			:   Optional[Dict[str, Any]] = None , **lowerCamelCase			:   Any , ):




				'''simple docstring'''
				# Mask token behave like a normal word, i.e. include the space before it and
				# is included in the raw text, there should be a match in a non-normalized sentence.

				a__     = {} if sp_model_kwargs is None else sp_model_kwargs
				super().__init__(
				    do_lower_case=lowerCamelCase , unk_token=lowerCamelCase , sep_token=lowerCamelCase , pad_token=lowerCamelCase , cls_token=lowerCamelCase , mask_token=lowerCamelCase , vocab_file=lowerCamelCase , encoding=lowerCamelCase , sp_model_kwargs=self.sp_model_kwargs , **lowerCamelCase , )
				a__     = do_lower_case
				a__     = sentencepiece_model_ckpt
				a__     = spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.Load(lowerCamelCase		)

				# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
				if vocab_file is not None:
							a__     = self.load_vocab(filepath=lowerCamelCase		)
				else:
							a__     = {self.sp_model.id_to_piece(lowerCamelCase		): id for id in range(self.sp_model.get_piece_size()		)}
				a__     = {v: k for k, v in self.vocab.items()}







	def 						__a   (				self			:   Tuple , lowerCamelCase			:   Optional[int]		):




				'''simple docstring'''
				if text is None:
							return None

				a__     = self.tokenize(lowerCamelCase		)
				a__   ,				a__     = "", []

				for i, ch in enumerate(lowerCamelCase		):
							if ch in self.SP_CHAR_MAPPING:
										a__     = self.SP_CHAR_MAPPING.get(lowerCamelCase		)
							else:
										a__     = unicodedata.normalize("NFKC" , lowerCamelCase		)
							if self.is_whitespace(lowerCamelCase		):
										continue
							normalized_text += ch
							char_mapping.extend([i] * len(lowerCamelCase		)		)

				a__   ,				a__   ,				a__     = normalized_text, [], 0

				if self.do_lower_case:
							a__     = text.lower()

				for token in split_tokens:
							if token[:1] == "▁":
										a__     = token[1:]
							a__     = text[offset:].index(lowerCamelCase		) + offset
							a__     = start + len(lowerCamelCase		)

							token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)		)
							a__     = end
				return token_mapping







	@property
	def 						__a   (				self			:   int		):




				'''simple docstring'''
				return len(self.vocab		)







	def 						__a   (				self			:   List[str]		):




				'''simple docstring'''
				return dict(self.vocab , **self.added_tokens_encoder		)







	def __getstate__(				self			:   str		):




				'''simple docstring'''
				a__     = self.__dict__.copy()
				a__     = None
				return state







	def __setstate__(				self			:   int , lowerCamelCase			:   List[str]		):




				'''simple docstring'''
				a__     = d

				# for backward compatibility
				if not hasattr(self , "sp_model_kwargs"		):
							a__     = {}

				a__     = spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.Load(self.sentencepiece_model_ckpt		)







	def 						__a   (				self			:   Any , lowerCamelCase			:   str		):




				'''simple docstring'''
				return "".join((self.SP_CHAR_MAPPING.get(lowerCamelCase , lowerCamelCase		) for c in text)		)







	def 						__a   (				self			:   Union[str, Any] , lowerCamelCase			:   List[str] , lowerCamelCase			:   Optional[int]=False , lowerCamelCase			:   Optional[int]=6_4 , lowerCamelCase			:   Optional[int]=0.1		):




				'''simple docstring'''

				if self.sp_model_kwargs.get("enable_sampling"		) is True:
							a__     = True
				if self.sp_model_kwargs.get("alpha"		) is not None:
							a__     = self.sp_model_kwargs.get("alpha"		)
				if self.sp_model_kwargs.get("nbest_size"		) is not None:
							a__     = self.sp_model_kwargs.get("nbest_size"		)

				if not enable_sampling:
							a__     = self.sp_model.EncodeAsPieces(lowerCamelCase		)
				else:
							a__     = self.sp_model.SampleEncodeAsPieces(lowerCamelCase , lowerCamelCase , lowerCamelCase		)
				a__     = []
				for pi, piece in enumerate(lowerCamelCase		):
							if piece == SPIECE_UNDERLINE:
										if not pieces[pi + 1].startswith(lowerCamelCase		) and pi != 0:
													new_pieces.append(lowerCamelCase		)
													continue
										else:
													continue
							a__     = 0
							for i, chunk in enumerate(lowerCamelCase		):
										if chunk == SPIECE_UNDERLINE:
													continue
										if self.is_ch_char(lowerCamelCase		) or self.is_punct(lowerCamelCase		):
													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																new_pieces.append(piece[lst_i:i]		)
													new_pieces.append(lowerCamelCase		)
													a__     = i + 1
										elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																new_pieces.append(piece[lst_i:i]		)
													a__     = i
										elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																new_pieces.append(piece[lst_i:i]		)
													a__     = i
							if len(lowerCamelCase		) > lst_i:
										new_pieces.append(piece[lst_i:]		)
				return new_pieces







	def 						__a   (				self			:   Optional[Any] , lowerCamelCase			:   List[str]		):




				'''simple docstring'''
				a__     = "".join(lowerCamelCase		).replace(lowerCamelCase , " "		).strip()
				return out_string







	def 						__a   (				self			:   Optional[Any] , lowerCamelCase			:   Any		):




				'''simple docstring'''
				a__     = self.convert_ids_to_tokens(lowerCamelCase		)
				a__     = "".join(lowerCamelCase		).replace(lowerCamelCase , " "		).strip()
				return out_string







	def 						__a   (				self			:   List[Any] , lowerCamelCase			:   Any		):




				'''simple docstring'''
				return self.vocab.get(lowerCamelCase , self.vocab.get(self.unk_token		)		)







	def 						__a   (				self			:   Optional[Any] , lowerCamelCase			:   int		):




				'''simple docstring'''
				return self.reverse_vocab.get(lowerCamelCase , self.unk_token		)







	def 						__a   (				self			:   Optional[Any] , lowerCamelCase			:   str , lowerCamelCase			:   Any=None		):




				'''simple docstring'''
				if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				a__     = [self.cls_token_id]
				a__     = [self.sep_token_id]
				return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep







	def 						__a   (				self			:   int , lowerCamelCase			:   List[Any] , lowerCamelCase			:   str=None		):




				'''simple docstring'''
				if offset_mapping_a is None:
							return [(0, 0)] + offset_mapping_a + [(0, 0)]

				return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]







	def 						__a   (				self			:   Dict , lowerCamelCase			:   Tuple , lowerCamelCase			:   Tuple=None , lowerCamelCase			:   Dict=False		):




				'''simple docstring'''

				if already_has_special_tokens:
							if token_ids_a is not None:
										raise ValueError(
										    "You should not supply a second sequence if the provided sequence of "
										    "ids is already formatted with special tokens for the model."		)
							return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

				if token_ids_a is not None:
							return [1] + ([0] * len(lowerCamelCase		)) + [1, 1] + ([0] * len(lowerCamelCase		)) + [1]
				return [1] + ([0] * len(lowerCamelCase		)) + [1]







	def 						__a   (				self			:   Union[str, Any] , lowerCamelCase			:   List[int] , lowerCamelCase			:   Optional[List[int]] = None		):




				'''simple docstring'''
				# called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
				if token_ids_a is None:
							# [CLS] X [SEP]
							return (len(lowerCamelCase		) + 2) * [0]

				# [CLS] A [SEP] [SEP] B [SEP]
				return [0] * (len(lowerCamelCase		) + 1) + [1] * (len(lowerCamelCase		) + 3)







	def 						__a   (				self			:   Any , lowerCamelCase			:   Tuple		):




				'''simple docstring'''
				if "\u4e00" <= char <= "\u9fff":
							return True
				return False







	def 						__a   (				self			:   Any , lowerCamelCase			:   Any		):




				'''simple docstring'''
				if ("a" <= char <= "z") or ("A" <= char <= "Z"):
							return True
				return False







	def 						__a   (				self			:   Tuple , lowerCamelCase			:   List[str]		):




				'''simple docstring'''
				if char in ",;:.?!~，；：。？！《》【】":
							return True
				return False







	def 						__a   (				self			:   Tuple , lowerCamelCase			:   Dict		):




				'''simple docstring'''
				if char == " " or char == "\t" or char == "\n" or char == "\r":
							return True
				if len(lowerCamelCase		) == 1:
							a__     = unicodedata.category(lowerCamelCase		)
							if cat == "Zs":
										return True
				return False







	def 						__a   (				self			:   int , lowerCamelCase			:   List[Any]		):




				'''simple docstring'''
				a__     = {}
				with io.open(lowerCamelCase , "r" , encoding="utf-8"		) as f:
							for index, line in enumerate(lowerCamelCase		):
										a__     = line.rstrip("\n"		)
										a__     = int(lowerCamelCase		)

				return token_to_idx







	def 						__a   (				self			:   int , lowerCamelCase			:   str , lowerCamelCase			:   Optional[str] = None		):




				'''simple docstring'''
				a__     = 0
				if os.path.isdir(lowerCamelCase		):
							a__     = os.path.join(
							    lowerCamelCase , (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
				else:
							a__     = (filename_prefix + "-" if filename_prefix else "") + save_directory
				with open(lowerCamelCase , "w" , encoding="utf-8"		) as writer:
							for token, token_index in sorted(self.vocab.items() , key=lambda lowerCamelCase		: kv[1]		):
										if index != token_index:
													logger.warning(
													    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
													    " Please check that the vocabulary is not corrupted!"		)
													a__     = token_index
										writer.write(token + "\n"		)
										index += 1

				a__     = os.path.join(lowerCamelCase , "sentencepiece.bpe.model"		)
				with open(lowerCamelCase , "wb"		) as fi:
							a__     = self.sp_model.serialized_model_proto()
							fi.write(lowerCamelCase		)

				return (vocab_file,)










'''simple docstring'''


# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from ...utils.dataclasses import (
    ComputeEnvironment,
    DistributedType,
    DynamoBackend,
    PrecisionType,
    SageMakerDistributedType,
)
from ..menu import BulletMenu


lowerCAmelCase_   :   List[Any]       =					[
    "EAGER",
    "AOT_EAGER",
    "INDUCTOR",
    "NVFUSER",
    "AOT_NVFUSER",
    "AOT_CUDAGRAPHS",
    "OFI",
    "FX2TRT",
    "ONNXRT",
    "IPEX",
]







def    _lowerCamelCase  (__lowerCamelCase  :     Tuple					,  __lowerCamelCase  :     Tuple=None					,  __lowerCamelCase  :     Tuple=None					,  __lowerCamelCase  :     List[Any]=None     )   ->    Any:
			a__     = True
			while ask_again:
						a__     = input(__lowerCamelCase     )
						try:
									if default is not None and len(__lowerCamelCase     ) == 0:
												return default
									return convert_value(__lowerCamelCase     ) if convert_value is not None else result
						except Exception:
									if error_message is not None:
												print(__lowerCamelCase     )







def    _lowerCamelCase  (__lowerCamelCase  :     Dict					,  __lowerCamelCase  :     List[Any]=[]					,  __lowerCamelCase  :     str=None					,  __lowerCamelCase  :     List[Any]=0     )   ->    List[Any]:
			a__     = BulletMenu(__lowerCamelCase					,  __lowerCamelCase     )
			a__     = menu.run(default_choice=__lowerCamelCase     )
			return convert_value(__lowerCamelCase     ) if convert_value is not None else result







def    _lowerCamelCase  (__lowerCamelCase  :     Tuple     )   ->    Optional[Any]:
			a__     = int(__lowerCamelCase     )
			return ComputeEnvironment(["LOCAL_MACHINE", "AMAZON_SAGEMAKER"][value]     )







def    _lowerCamelCase  (__lowerCamelCase  :     List[str]     )   ->    Optional[int]:
			a__     = int(__lowerCamelCase     )
			return DistributedType(["NO", "MULTI_CPU", "MULTI_XPU", "MULTI_GPU", "MULTI_NPU", "TPU"][value]     )







def    _lowerCamelCase  (__lowerCamelCase  :     List[Any]     )   ->    Optional[Any]:
			a__     = int(__lowerCamelCase     )
			return DynamoBackend(DYNAMO_BACKENDS[value]     ).value







def    _lowerCamelCase  (__lowerCamelCase  :     str     )   ->    str:
			a__     = int(__lowerCamelCase     )
			return PrecisionType(["no", "fp16", "bf16", "fp8"][value]     )







def    _lowerCamelCase  (__lowerCamelCase  :     int     )   ->    Optional[Any]:
			a__     = int(__lowerCamelCase     )
			return SageMakerDistributedType(["NO", "DATA_PARALLEL", "MODEL_PARALLEL"][value]     )







def    _lowerCamelCase  (__lowerCamelCase  :     Any     )   ->    List[Any]:
			return {"yes": True, "no": False}[value.lower()]






class       UpperCamelCase__  (		argparse.RawDescriptionHelpFormatter     ):







	def 						__a   (				self			:   Dict , lowerCamelCase			:   int , lowerCamelCase			:   Optional[Any] , lowerCamelCase			:   List[Any] , lowerCamelCase			:   Dict		):




				'''simple docstring'''
				a__     = super()._format_usage(lowerCamelCase , lowerCamelCase , lowerCamelCase , lowerCamelCase		)
				a__     = usage.replace("<command> [<args>] " , ""		)
				return usage



