





'''simple docstring'''






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A   : str             =			logging.get_logger(__name__)

A   : List[str]             =			'▁'

A   : Union[str, Any]             =			{'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

A   : Optional[int]             =			{
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

A   : Any             =			{
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

A   : int             =			{
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

A   : Dict             =			{
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}







class 			lowerCamelCase					(						__UpperCAmelCase  ):
					_SCREAMING_SNAKE_CASE							   =    ["input_ids"]

					_SCREAMING_SNAKE_CASE							   =    VOCAB_FILES_NAMES
					_SCREAMING_SNAKE_CASE							   =    PRETRAINED_INIT_CONFIGURATION
					_SCREAMING_SNAKE_CASE							   =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_SCREAMING_SNAKE_CASE							   =    PRETRAINED_VOCAB_FILES_MAP
					_SCREAMING_SNAKE_CASE							   =    RESOURCE_FILES_NAMES


					def __init__(					self	:  Dict       ,     __snake_case	:  Union[str, Any]       ,     __snake_case	:  Tuple=None       ,     __snake_case	:  Any=False       ,     __snake_case	:  Optional[int]="utf8"       ,     __snake_case	:  List[str]="[UNK]"       ,     __snake_case	:  List[str]="[SEP]"       ,     __snake_case	:  Tuple="[PAD]"       ,     __snake_case	:  Optional[Any]="[CLS]"       ,     __snake_case	:  Optional[Any]="[MASK]"       ,     __snake_case	:  Optional[Dict[str, Any]] = None       ,     **__snake_case	:  List[Any]       ,     ):





											'''simple docstring'''





											_snake_case:   List[Any]  					=		{} if sp_model_kwargs is None else sp_model_kwargs
											super().__init__(
											    do_lower_case=__snake_case       ,     unk_token=__snake_case       ,     sep_token=__snake_case       ,     pad_token=__snake_case       ,     cls_token=__snake_case       ,     mask_token=__snake_case       ,     vocab_file=__snake_case       ,     encoding=__snake_case       ,     sp_model_kwargs=self.sp_model_kwargs       ,     **__snake_case       ,     )
											_snake_case:   Any  					=		do_lower_case
											_snake_case:   Optional[Any]  					=		sentencepiece_model_ckpt
											_snake_case:   Union[str, Any]  					=		spm.SentencePieceProcessor(**self.sp_model_kwargs       )
											self.sp_model.Load(__snake_case       )

											# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
											if vocab_file is not None:
																	_snake_case:   Optional[Any]  					=		self.load_vocab(filepath=__snake_case       )
											else:
																	_snake_case:   Optional[Any]  					=		{self.sp_model.id_to_piece(__snake_case       ): id for id in range(self.sp_model.get_piece_size()       )}
											_snake_case:   Optional[Any]  					=		{v: k for k, v in self.vocab.items()}


					def    SCREAMING_SNAKE_CASE_		(					self	:  Any       ,     __snake_case	:  List[Any]       ):





											'''simple docstring'''





											if text is None:
																	return None

											_snake_case:   Optional[int]  					=		self.tokenize(__snake_case       )
											_snake_case					,						_snake_case:   List[Any]  					=		'', []

											for i, ch in enumerate(__snake_case       ):
																	if ch in self.SP_CHAR_MAPPING:
																							_snake_case:   Dict  					=		self.SP_CHAR_MAPPING.get(__snake_case       )
																	else:
																							_snake_case:   str  					=		unicodedata.normalize('NFKC'       ,     __snake_case       )
																	if self.is_whitespace(__snake_case       ):
																							continue
																	normalized_text += ch
																	char_mapping.extend([i] * len(__snake_case       )       )

											_snake_case					,						_snake_case					,						_snake_case:   List[Any]  					=		normalized_text, [], 0

											if self.do_lower_case:
																	_snake_case:   Optional[int]  					=		text.lower()

											for token in split_tokens:
																	if token[:1] == "▁":
																							_snake_case:   Dict  					=		token[1:]
																	_snake_case:   List[Any]  					=		text[offset:].index(__snake_case       ) + offset
																	_snake_case:   List[Any]  					=		start + len(__snake_case       )

																	token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)       )
																	_snake_case:   Union[str, Any]  					=		end
											return token_mapping


					@property
					def    SCREAMING_SNAKE_CASE_		(					self	:  str       ):





											'''simple docstring'''





											return len(self.vocab       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  str       ):





											'''simple docstring'''





											return dict(self.vocab       ,     **self.added_tokens_encoder       )


					def __getstate__(					self	:  List[str]       ):





											'''simple docstring'''





											_snake_case:   Union[str, Any]  					=		self.__dict__.copy()
											_snake_case:   Any  					=		None
											return state


					def __setstate__(					self	:  Union[str, Any]       ,     __snake_case	:  Optional[Any]       ):





											'''simple docstring'''





											_snake_case:   Any  					=		d

											# for backward compatibility
											if not hasattr(self       ,     'sp_model_kwargs'       ):
																	_snake_case:   Union[str, Any]  					=		{}

											_snake_case:   List[Any]  					=		spm.SentencePieceProcessor(**self.sp_model_kwargs       )
											self.sp_model.Load(self.sentencepiece_model_ckpt       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ,     __snake_case	:  Optional[int]       ):





											'''simple docstring'''





											return "".join((self.SP_CHAR_MAPPING.get(__snake_case       ,     __snake_case       ) for c in text)       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  str       ,     __snake_case	:  List[str]       ,     __snake_case	:  Tuple=False       ,     __snake_case	:  int=64       ,     __snake_case	:  List[str]=0.1       ):





											'''simple docstring'''





											if self.sp_model_kwargs.get('enable_sampling'       ) is True:
																	_snake_case:   Tuple  					=		True
											if self.sp_model_kwargs.get('alpha'       ) is not None:
																	_snake_case:   Any  					=		self.sp_model_kwargs.get('alpha'       )
											if self.sp_model_kwargs.get('nbest_size'       ) is not None:
																	_snake_case:   Any  					=		self.sp_model_kwargs.get('nbest_size'       )

											if not enable_sampling:
																	_snake_case:   Dict  					=		self.sp_model.EncodeAsPieces(__snake_case       )
											else:
																	_snake_case:   Optional[Any]  					=		self.sp_model.SampleEncodeAsPieces(__snake_case       ,     __snake_case       ,     __snake_case       )
											_snake_case:   Optional[int]  					=		[]
											for pi, piece in enumerate(__snake_case       ):
																	if piece == SPIECE_UNDERLINE:
																							if not pieces[pi + 1].startswith(__snake_case       ) and pi != 0:
																													new_pieces.append(__snake_case       )
																													continue
																							else:
																													continue
																	_snake_case:   Any  					=		0
																	for i, chunk in enumerate(__snake_case       ):
																							if chunk == SPIECE_UNDERLINE:
																													continue
																							if self.is_ch_char(__snake_case       ) or self.is_punct(__snake_case       ):
																													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																			new_pieces.append(piece[lst_i:i]       )
																													new_pieces.append(__snake_case       )
																													_snake_case:   List[Any]  					=		i + 1
																							elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																			new_pieces.append(piece[lst_i:i]       )
																													_snake_case:   str  					=		i
																							elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																													if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																																			new_pieces.append(piece[lst_i:i]       )
																													_snake_case:   Optional[int]  					=		i
																	if len(__snake_case       ) > lst_i:
																							new_pieces.append(piece[lst_i:]       )
											return new_pieces


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[Any]       ,     __snake_case	:  Union[str, Any]       ):





											'''simple docstring'''





											_snake_case:   Optional[int]  					=		''.join(__snake_case       ).replace(__snake_case       ,     ' '       ).strip()
											return out_string


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[str]       ,     __snake_case	:  Any       ):





											'''simple docstring'''





											_snake_case:   Optional[Any]  					=		self.convert_ids_to_tokens(__snake_case       )
											_snake_case:   Optional[int]  					=		''.join(__snake_case       ).replace(__snake_case       ,     ' '       ).strip()
											return out_string


					def    SCREAMING_SNAKE_CASE_		(					self	:  int       ,     __snake_case	:  List[str]       ):





											'''simple docstring'''





											return self.vocab.get(__snake_case       ,     self.vocab.get(self.unk_token       )       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ,     __snake_case	:  str       ):





											'''simple docstring'''





											return self.reverse_vocab.get(__snake_case       ,     self.unk_token       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[str]       ,     __snake_case	:  Any       ,     __snake_case	:  Optional[int]=None       ):





											'''simple docstring'''





											if token_ids_a is None:
																	return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											_snake_case:   Union[str, Any]  					=		[self.cls_token_id]
											_snake_case:   Dict  					=		[self.sep_token_id]
											return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep


					def    SCREAMING_SNAKE_CASE_		(					self	:  Any       ,     __snake_case	:  Union[str, Any]       ,     __snake_case	:  Dict=None       ):





											'''simple docstring'''





											if offset_mapping_a is None:
																	return [(0, 0)] + offset_mapping_a + [(0, 0)]

											return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[Any]       ,     __snake_case	:  Optional[Any]       ,     __snake_case	:  int=None       ,     __snake_case	:  List[Any]=False       ):





											'''simple docstring'''





											if already_has_special_tokens:
																	if token_ids_a is not None:
																							raise ValueError(
																							    'You should not supply a second sequence if the provided sequence of '
																							    'ids is already formatted with special tokens for the model.'       )
																	return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

											if token_ids_a is not None:
																	return [1] + ([0] * len(__snake_case       )) + [1, 1] + ([0] * len(__snake_case       )) + [1]
											return [1] + ([0] * len(__snake_case       )) + [1]


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[Any]       ,     __snake_case	:  List[int]       ,     __snake_case	:  Optional[List[int]] = None       ):





											'''simple docstring'''





											if token_ids_a is None:
																	# [CLS] X [SEP]
																	return (len(__snake_case       ) + 2) * [0]

											# [CLS] A [SEP] [SEP] B [SEP]
											return [0] * (len(__snake_case       ) + 1) + [1] * (len(__snake_case       ) + 3)


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ,     __snake_case	:  int       ):





											'''simple docstring'''





											if "\u4e00" <= char <= "\u9fff":
																	return True
											return False


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ,     __snake_case	:  Optional[int]       ):





											'''simple docstring'''





											if ("a" <= char <= "z") or ("A" <= char <= "Z"):
																	return True
											return False


					def    SCREAMING_SNAKE_CASE_		(					self	:  str       ,     __snake_case	:  Dict       ):





											'''simple docstring'''





											if char in ",;:.?!~，；：。？！《》【】":
																	return True
											return False


					def    SCREAMING_SNAKE_CASE_		(					self	:  Any       ,     __snake_case	:  Dict       ):





											'''simple docstring'''





											if char == " " or char == "\t" or char == "\n" or char == "\r":
																	return True
											if len(__snake_case       ) == 1:
																	_snake_case:   Dict  					=		unicodedata.category(__snake_case       )
																	if cat == "Zs":
																							return True
											return False


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ,     __snake_case	:  Tuple       ):





											'''simple docstring'''





											_snake_case:   List[str]  					=		{}
											with io.open(__snake_case       ,     'r'       ,     encoding='utf-8'       ) as f:
																	for index, line in enumerate(__snake_case       ):
																							_snake_case:   List[Any]  					=		line.rstrip('\n'       )
																							_snake_case:   Any  					=		int(__snake_case       )

											return token_to_idx




					def    SCREAMING_SNAKE_CASE_		(					self	:  int       ,     __snake_case	:  str       ,     __snake_case	:  Optional[str] = None       ):





											'''simple docstring'''





											_snake_case:   List[str]  					=		0
											if os.path.isdir(__snake_case       ):
																	_snake_case:   Tuple  					=		os.path.join(
																	    __snake_case       ,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']       )
											else:
																	_snake_case:   int  					=		(filename_prefix + '-' if filename_prefix else '') + save_directory
											with open(__snake_case       ,     'w'       ,     encoding='utf-8'       ) as writer:
																	for token, token_index in sorted(self.vocab.items()       ,     key=lambda __snake_case       : kv[1]       ):
																							if index != token_index:
																													logger.warning(
																													    f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																													    ' Please check that the vocabulary is not corrupted!'       )
																													_snake_case:   Any  					=		token_index
																							writer.write(token + '\n'       )
																							index += 1

											_snake_case:   List[Any]  					=		os.path.join(__snake_case       ,     'sentencepiece.bpe.model'       )
											with open(__snake_case       ,     'wb'       ) as fi:
																	_snake_case:   str  					=		self.sp_model.serialized_model_proto()
																	fi.write(__snake_case       )

											return (vocab_file,)












'''simple docstring'''






from __future__ import annotations

import sys
from collections import deque
from typing import Generic, TypeVar

A   : str             =			TypeVar('T')







class 			lowerCamelCase					(						Generic[T]  ):
					_SCREAMING_SNAKE_CASE							   =    42  # Cache store of keys
					_SCREAMING_SNAKE_CASE							   =    42  # References of the keys in cache
					_SCREAMING_SNAKE_CASE							   =    10  # Maximum capacity of cache


					def __init__(					self	:  List[Any]       ,     __snake_case	:  int       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		deque()
											_snake_case:   Union[str, Any]  					=		set()
											if not n:
																	_snake_case:   Optional[int]  					=		sys.maxsize
											elif n < 0:
																	raise ValueError('n should be an integer greater than 0.'       )
											else:
																	_snake_case:   Tuple  					=		n


					def    SCREAMING_SNAKE_CASE_		(					self	:  Any       ,     __snake_case	:  T       ):





											'''simple docstring'''





											if x not in self.key_reference:
																	if len(self.dq_store       ) == LRUCache._MAX_CAPACITY:
																							_snake_case:   int  					=		self.dq_store.pop()
																							self.key_reference.remove(__snake_case       )
											else:
																	self.dq_store.remove(__snake_case       )

											self.dq_store.appendleft(__snake_case       )
											self.key_reference.add(__snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  int       ):





											'''simple docstring'''





											for k in self.dq_store:
																	print(__snake_case       )




					def __repr__(					self	:  List[Any]       ):





											'''simple docstring'''





											return f'''LRUCache({self._MAX_CAPACITY}) => {list(self.dq_store       )}'''


if __name__ == "__main__":
		import doctest

		doctest.testmod()

		A   : LRUCache[str | int]             =			LRUCache(4)
		lru_cache.refer('A')
		lru_cache.refer(2)
		lru_cache.refer(3)
		lru_cache.refer('A')
		lru_cache.refer(4)
		lru_cache.refer(5)
		lru_cache.display()

		print(lru_cache)
		assert str(lru_cache) == "LRUCache(4) => [5, 4, 'A', 3]"





