






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


_UpperCAmelCase			:		Any   				=    logging.get_logger(__name__)

_UpperCAmelCase			:		Dict   				=    {'''vocab_file''': '''spiece.model'''}

_UpperCAmelCase			:		Union[str, Any]   				=    {
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

_UpperCAmelCase			:		Tuple   				=    {
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
_UpperCAmelCase			:		List[str]   				=    0
_UpperCAmelCase			:		List[str]   				=    1
_UpperCAmelCase			:		List[str]   				=    2
_UpperCAmelCase			:		Tuple   				=    3
_UpperCAmelCase			:		Optional[int]   				=    4
class  lowerCAmelCase_       (				snake_case__					):
							UpperCamelCase_		:Optional[int]		      =    VOCAB_FILES_NAMES
							UpperCamelCase_		:Optional[Any]		      =    PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase_		:Optional[Any]		      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase_		:str		      =    'left'




							def __init__(   self     :   Dict ,							SCREAMING_SNAKE_CASE_     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Dict=False ,							SCREAMING_SNAKE_CASE_     :   int=True ,							SCREAMING_SNAKE_CASE_     :   List[Any]=False ,							SCREAMING_SNAKE_CASE_     :   List[str]="<s>" ,							SCREAMING_SNAKE_CASE_     :   List[Any]="</s>" ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]="<unk>" ,							SCREAMING_SNAKE_CASE_     :   int="<sep>" ,							SCREAMING_SNAKE_CASE_     :   int="<pad>" ,							SCREAMING_SNAKE_CASE_     :   Optional[int]="<cls>" ,							SCREAMING_SNAKE_CASE_     :   int="<mask>" ,							SCREAMING_SNAKE_CASE_     :   Optional[int]=["<eop>", "<eod>"] ,							SCREAMING_SNAKE_CASE_     :   Optional[Dict[str, Any]] = None ,							**SCREAMING_SNAKE_CASE_     :   List[Any] ,							):
											lowerCAmelCase__      			=						AddedToken(UpperCamelCase_ ,							lstrip=UpperCamelCase_ ,							rstrip=UpperCamelCase_	) if isinstance(UpperCamelCase_ ,							UpperCamelCase_	) else mask_token

											lowerCAmelCase__      			=						{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    do_lower_case=UpperCamelCase_ ,							remove_space=UpperCamelCase_ ,							keep_accents=UpperCamelCase_ ,							bos_token=UpperCamelCase_ ,							eos_token=UpperCamelCase_ ,							unk_token=UpperCamelCase_ ,							sep_token=UpperCamelCase_ ,							pad_token=UpperCamelCase_ ,							cls_token=UpperCamelCase_ ,							mask_token=UpperCamelCase_ ,							additional_special_tokens=UpperCamelCase_ ,							sp_model_kwargs=self.sp_model_kwargs ,							**UpperCamelCase_ ,							)

											lowerCAmelCase__      			=						3

											lowerCAmelCase__      			=						do_lower_case
											lowerCAmelCase__      			=						remove_space
											lowerCAmelCase__      			=						keep_accents
											lowerCAmelCase__      			=						vocab_file

											lowerCAmelCase__      			=						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
											self.sp_model.Load(UpperCamelCase_	)




							@property
							def 						__snake_case					(   self     :   List[Any]	):
											return len(self.sp_model	)




							def 						__snake_case					(   self     :   Union[str, Any]	):
											lowerCAmelCase__      			=						{self.convert_ids_to_tokens(UpperCamelCase_	): i for i in range(self.vocab_size	)}
											vocab.update(self.added_tokens_encoder	)
											return vocab




							def __getstate__(   self     :   List[str]	):
											lowerCAmelCase__      			=						self.__dict__.copy()
											lowerCAmelCase__      			=						None
											return state




							def __setstate__(   self     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Tuple	):
											lowerCAmelCase__      			=						d

											# for backward compatibility
											if not hasattr(self ,							'''sp_model_kwargs'''	):
															lowerCAmelCase__      			=						{}

											lowerCAmelCase__      			=						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
											self.sp_model.Load(self.vocab_file	)




							def 						__snake_case					(   self     :   Dict ,							SCREAMING_SNAKE_CASE_     :   Union[str, Any]	):





											if self.remove_space:
															lowerCAmelCase__      			=						' '.join(inputs.strip().split()	)
											else:
															lowerCAmelCase__      			=						inputs
											lowerCAmelCase__      			=						outputs.replace('''``''' ,							'''"'''	).replace('''\'\'''' ,							'''"'''	)

											if not self.keep_accents:
															lowerCAmelCase__      			=						unicodedata.normalize('''NFKD''' ,							UpperCamelCase_	)
															lowerCAmelCase__      			=						''.join([c for c in outputs if not unicodedata.combining(UpperCamelCase_	)]	)
											if self.do_lower_case:
															lowerCAmelCase__      			=						outputs.lower()

											return outputs




							def 						__snake_case					(   self     :   Optional[Any] ,							SCREAMING_SNAKE_CASE_     :   str	):
											lowerCAmelCase__      			=						self.preprocess_text(UpperCamelCase_	)
											lowerCAmelCase__      			=						self.sp_model.encode(UpperCamelCase_ ,							out_type=UpperCamelCase_	)
											lowerCAmelCase__      			=						[]
											for piece in pieces:
															if len(UpperCamelCase_	) > 1 and piece[-1] == str(''','''	) and piece[-2].isdigit():
																			lowerCAmelCase__      			=						self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase_ ,							''''''	)	)
																			if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																							if len(cur_pieces[0]	) == 1:
																											lowerCAmelCase__      			=						cur_pieces[1:]
																							else:
																											lowerCAmelCase__      			=						cur_pieces[0][1:]
																			cur_pieces.append(piece[-1]	)
																			new_pieces.extend(UpperCamelCase_	)
															else:
																			new_pieces.append(UpperCamelCase_	)

											return new_pieces




							def 						__snake_case					(   self     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]	):
											return self.sp_model.PieceToId(UpperCamelCase_	)




							def 						__snake_case					(   self     :   List[Any] ,							SCREAMING_SNAKE_CASE_     :   Optional[Any]	):
											return self.sp_model.IdToPiece(UpperCamelCase_	)




							def 						__snake_case					(   self     :   int ,							SCREAMING_SNAKE_CASE_     :   Any	):
											lowerCAmelCase__      			=						''.join(UpperCamelCase_	).replace(UpperCamelCase_ ,							''' '''	).strip()
											return out_string




							def 						__snake_case					(   self     :   Any ,							SCREAMING_SNAKE_CASE_     :   List[int] ,							SCREAMING_SNAKE_CASE_     :   bool = False ,							SCREAMING_SNAKE_CASE_     :   bool = None ,							SCREAMING_SNAKE_CASE_     :   bool = True ,							**SCREAMING_SNAKE_CASE_     :   Union[str, Any] ,							):
											lowerCAmelCase__      			=						kwargs.pop('''use_source_tokenizer''' ,							UpperCamelCase_	)

											lowerCAmelCase__      			=						self.convert_ids_to_tokens(UpperCamelCase_ ,							skip_special_tokens=UpperCamelCase_	)

											# To avoid mixing byte-level and unicode for byte-level BPT
											# we need to build string separately for added tokens and byte-level tokens
											# cf. https://github.com/huggingface/transformers/issues/1133
											lowerCAmelCase__      			=						[]
											lowerCAmelCase__      			=						[]
											for token in filtered_tokens:
															if skip_special_tokens and token in self.all_special_ids:
																			continue
															if token in self.added_tokens_encoder:
																			if current_sub_text:
																							sub_texts.append(self.convert_tokens_to_string(UpperCamelCase_	)	)
																							lowerCAmelCase__      			=						[]
																			sub_texts.append(UpperCamelCase_	)
															else:
																			current_sub_text.append(UpperCamelCase_	)
											if current_sub_text:
															sub_texts.append(self.convert_tokens_to_string(UpperCamelCase_	)	)

											# Mimic the behavior of the Rust tokenizer:
											# By default, there are no spaces between special tokens
											lowerCAmelCase__      			=						''.join(UpperCamelCase_	)

											lowerCAmelCase__      			=						(
											    clean_up_tokenization_spaces
											    if clean_up_tokenization_spaces is not None
											    else self.clean_up_tokenization_spaces
											)
											if clean_up_tokenization_spaces:
															lowerCAmelCase__      			=						self.clean_up_tokenization(UpperCamelCase_	)
															return clean_text
											else:
															return text




							def 						__snake_case					(   self     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   List[int] ,							SCREAMING_SNAKE_CASE_     :   Optional[List[int]] = None	):
											lowerCAmelCase__      			=						[self.sep_token_id]
											lowerCAmelCase__      			=						[self.cls_token_id]
											if token_ids_a is None:
															return token_ids_a + sep + cls
											return token_ids_a + sep + token_ids_a + sep + cls




							def 						__snake_case					(   self     :   Optional[Any] ,							SCREAMING_SNAKE_CASE_     :   List[int] ,							SCREAMING_SNAKE_CASE_     :   Optional[List[int]] = None ,							SCREAMING_SNAKE_CASE_     :   bool = False	):





											if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=UpperCamelCase_ ,							token_ids_a=UpperCamelCase_ ,							already_has_special_tokens=UpperCamelCase_	)

											if token_ids_a is not None:
															return ([0] * len(UpperCamelCase_	)) + [1] + ([0] * len(UpperCamelCase_	)) + [1, 1]
											return ([0] * len(UpperCamelCase_	)) + [1, 1]




							def 						__snake_case					(   self     :   List[str] ,							SCREAMING_SNAKE_CASE_     :   List[int] ,							SCREAMING_SNAKE_CASE_     :   Optional[List[int]] = None	):
											lowerCAmelCase__      			=						[self.sep_token_id]
											lowerCAmelCase__      			=						[2]

											if token_ids_a is None:
															return len(token_ids_a + sep	) * [0] + cls_segment_id
											return len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1] + cls_segment_id




							def 						__snake_case					(   self     :   Union[str, Any] ,							SCREAMING_SNAKE_CASE_     :   str ,							SCREAMING_SNAKE_CASE_     :   Optional[str] = None	):





											if not os.path.isdir(UpperCamelCase_	):
															logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
															return
											lowerCAmelCase__      			=						os.path.join(
											    UpperCamelCase_ ,							(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)

											if os.path.abspath(self.vocab_file	) != os.path.abspath(UpperCamelCase_	) and os.path.isfile(self.vocab_file	):
															copyfile(self.vocab_file ,							UpperCamelCase_	)
											elif not os.path.isfile(self.vocab_file	):
															with open(UpperCamelCase_ ,							'''wb'''	) as fi:
																			lowerCAmelCase__      			=						self.sp_model.serialized_model_proto()
																			fi.write(UpperCamelCase_	)

											return (out_vocab_file,)













from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tensorflow_text_available, is_torch_available


_UpperCAmelCase			:		Optional[Any]   				=    {
    "configuration_ernie": ["ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP", "ErnieConfig", "ErnieOnnxConfig"],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_UpperCAmelCase			:		Any   				=    [
			    "ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "ErnieForCausalLM",
			    "ErnieForMaskedLM",
			    "ErnieForMultipleChoice",
			    "ErnieForNextSentencePrediction",
			    "ErnieForPreTraining",
			    "ErnieForQuestionAnswering",
			    "ErnieForSequenceClassification",
			    "ErnieForTokenClassification",
			    "ErnieModel",
			    "ErniePreTrainedModel",
			]

if TYPE_CHECKING:
			from .configuration_ernie import ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP, ErnieConfig, ErnieOnnxConfig

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_ernie import (
						    ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST,
						    ErnieForCausalLM,
						    ErnieForMaskedLM,
						    ErnieForMultipleChoice,
						    ErnieForNextSentencePrediction,
						    ErnieForPreTraining,
						    ErnieForQuestionAnswering,
						    ErnieForSequenceClassification,
						    ErnieForTokenClassification,
						    ErnieModel,
						    ErniePreTrainedModel,
						)

else:
			import sys

			_UpperCAmelCase			:		Dict   				=    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





