





from __future__ import annotations

from typing import Dict

from ...configuration_utils import PretrainedConfig


UpperCAmelCase__  :     Optional[Any]   			={
    '''susnato/ernie-m-base_pytorch''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/config.json''',
    '''susnato/ernie-m-large_pytorch''': '''https://huggingface.co/susnato/ernie-m-large_pytorch/blob/main/config.json''',
}

class  __A       (  a      ):






				"""simple docstring"""




				__A								=			"""ernie_m"""
				__A								=			{"""dropout""": """classifier_dropout""", """num_classes""": """num_labels"""}


				def __init__(						self				,	UpperCAmelCase_ = 250002				,	UpperCAmelCase_ = 768				,	UpperCAmelCase_ = 12				,	UpperCAmelCase_ = 12				,	UpperCAmelCase_ = 3072				,	UpperCAmelCase_ = "gelu"				,	UpperCAmelCase_ = 0.1				,	UpperCAmelCase_ = 0.1				,	UpperCAmelCase_ = 514				,	UpperCAmelCase_ = 0.0_2				,	UpperCAmelCase_ = 1				,	UpperCAmelCase_ = 1E-05				,	UpperCAmelCase_=None				,	UpperCAmelCase_=False				,	UpperCAmelCase_=0.0				,	**UpperCAmelCase_				,	):
								super().__init__(pad_token_id=UpperCAmelCase_				,	**UpperCAmelCase_							)
								lowerCamelCase			=vocab_size
								lowerCamelCase			=hidden_size
								lowerCamelCase			=num_hidden_layers
								lowerCamelCase			=num_attention_heads
								lowerCamelCase			=intermediate_size
								lowerCamelCase			=hidden_act
								lowerCamelCase			=hidden_dropout_prob
								lowerCamelCase			=attention_probs_dropout_prob
								lowerCamelCase			=max_position_embeddings
								lowerCamelCase			=initializer_range
								lowerCamelCase			=layer_norm_eps
								lowerCamelCase			=classifier_dropout
								lowerCamelCase			=is_decoder
								lowerCamelCase			=act_dropout












import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCAmelCase__  :     List[Any]   			=logging.get_logger(__name__)

UpperCAmelCase__  :     Dict   			={'''vocab_file''': '''spiece.model'''}

UpperCAmelCase__  :     Dict   			={
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

UpperCAmelCase__  :     List[str]   			={
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
UpperCAmelCase__  :     Any   			=0
UpperCAmelCase__  :     List[Any]   			=1
UpperCAmelCase__  :     Union[str, Any]   			=2
UpperCAmelCase__  :     Tuple   			=3
UpperCAmelCase__  :     int   			=4

class  __A       (  a      ):
				__A								=			VOCAB_FILES_NAMES
				__A								=			PRETRAINED_VOCAB_FILES_MAP
				__A								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A								=			"""left"""


				def __init__(						self				,	UpperCAmelCase_				,	UpperCAmelCase_=False				,	UpperCAmelCase_=True				,	UpperCAmelCase_=False				,	UpperCAmelCase_="<s>"				,	UpperCAmelCase_="</s>"				,	UpperCAmelCase_="<unk>"				,	UpperCAmelCase_="<sep>"				,	UpperCAmelCase_="<pad>"				,	UpperCAmelCase_="<cls>"				,	UpperCAmelCase_="<mask>"				,	UpperCAmelCase_=["<eop>", "<eod>"]				,	UpperCAmelCase_ = None				,	**UpperCAmelCase_				,	):
								# Mask token behave like a normal word, i.e. include the space before it
								lowerCamelCase			=AddedToken(UpperCAmelCase_				,	lstrip=UpperCAmelCase_				,	rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_				,	UpperCAmelCase_							) else mask_token

								lowerCamelCase			={} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    do_lower_case=UpperCAmelCase_				,	remove_space=UpperCAmelCase_				,	keep_accents=UpperCAmelCase_				,	bos_token=UpperCAmelCase_				,	eos_token=UpperCAmelCase_				,	unk_token=UpperCAmelCase_				,	sep_token=UpperCAmelCase_				,	pad_token=UpperCAmelCase_				,	cls_token=UpperCAmelCase_				,	mask_token=UpperCAmelCase_				,	additional_special_tokens=UpperCAmelCase_				,	sp_model_kwargs=self.sp_model_kwargs				,	**UpperCAmelCase_				,	)

								lowerCamelCase			=3

								lowerCamelCase			=do_lower_case
								lowerCamelCase			=remove_space
								lowerCamelCase			=keep_accents
								lowerCamelCase			=vocab_file

								lowerCamelCase			=spm.SentencePieceProcessor(**self.sp_model_kwargs							)
								self.sp_model.Load(UpperCAmelCase_							)


				@property
				def _snake_case    (						self							):
								return len(self.sp_model							)


				def _snake_case    (						self							):
								lowerCamelCase			={self.convert_ids_to_tokens(UpperCAmelCase_							): i for i in range(self.vocab_size							)}
								vocab.update(self.added_tokens_encoder							)
								return vocab


				def __getstate__(						self							):
								lowerCamelCase			=self.__dict__.copy()
								lowerCamelCase			=None
								return state


				def __setstate__(						self				,	UpperCAmelCase_							):
								lowerCamelCase			=d

								# for backward compatibility
								if not hasattr(self				,	"""sp_model_kwargs"""							):
												lowerCamelCase			={}

								lowerCamelCase			=spm.SentencePieceProcessor(**self.sp_model_kwargs							)
								self.sp_model.Load(self.vocab_file							)


				def _snake_case    (						self				,	UpperCAmelCase_							):
								if self.remove_space:
												lowerCamelCase			=""" """.join(inputs.strip().split()							)
								else:
												lowerCamelCase			=inputs
								lowerCamelCase			=outputs.replace("""``"""				,	"""\""""							).replace("""''"""				,	"""\""""							)

								if not self.keep_accents:
												lowerCamelCase			=unicodedata.normalize("""NFKD"""				,	UpperCAmelCase_							)
												lowerCamelCase			="""""".join([c for c in outputs if not unicodedata.combining(UpperCAmelCase_							)]							)
								if self.do_lower_case:
												lowerCamelCase			=outputs.lower()

								return outputs


				def _snake_case    (						self				,	UpperCAmelCase_							):
								lowerCamelCase			=self.preprocess_text(UpperCAmelCase_							)
								lowerCamelCase			=self.sp_model.encode(UpperCAmelCase_				,	out_type=UpperCAmelCase_							)
								lowerCamelCase			=[]
								for piece in pieces:
												if len(UpperCAmelCase_							) > 1 and piece[-1] == str(""","""							) and piece[-2].isdigit():
																lowerCamelCase			=self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase_				,	""""""							)							)
																if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																				if len(cur_pieces[0]							) == 1:
																								lowerCamelCase			=cur_pieces[1:]
																				else:
																								lowerCamelCase			=cur_pieces[0][1:]
																cur_pieces.append(piece[-1]							)
																new_pieces.extend(UpperCAmelCase_							)
												else:
																new_pieces.append(UpperCAmelCase_							)

								return new_pieces


				def _snake_case    (						self				,	UpperCAmelCase_							):
								return self.sp_model.PieceToId(UpperCAmelCase_							)


				def _snake_case    (						self				,	UpperCAmelCase_							):
								return self.sp_model.IdToPiece(UpperCAmelCase_							)


				def _snake_case    (						self				,	UpperCAmelCase_							):
								lowerCamelCase			="""""".join(UpperCAmelCase_							).replace(UpperCAmelCase_				,	""" """							).strip()
								return out_string


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = False				,	UpperCAmelCase_ = None				,	UpperCAmelCase_ = True				,	**UpperCAmelCase_				,	):
								lowerCamelCase			=kwargs.pop("""use_source_tokenizer"""				,	UpperCAmelCase_							)

								lowerCamelCase			=self.convert_ids_to_tokens(UpperCAmelCase_				,	skip_special_tokens=UpperCAmelCase_							)

								# To avoid mixing byte-level and unicode for byte-level BPT
								# we need to build string separately for added tokens and byte-level tokens
								# cf. https://github.com/huggingface/transformers/issues/1133
								lowerCamelCase			=[]
								lowerCamelCase			=[]
								for token in filtered_tokens:
												if skip_special_tokens and token in self.all_special_ids:
																continue
												if token in self.added_tokens_encoder:
																if current_sub_text:
																				sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase_							)							)
																				lowerCamelCase			=[]
																sub_texts.append(UpperCAmelCase_							)
												else:
																current_sub_text.append(UpperCAmelCase_							)
								if current_sub_text:
												sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase_							)							)

								# Mimic the behavior of the Rust tokenizer:
								# By default, there are no spaces between special tokens
								lowerCamelCase			="""""".join(UpperCAmelCase_							)

								lowerCamelCase			=(
								    clean_up_tokenization_spaces
								    if clean_up_tokenization_spaces is not None
								    else self.clean_up_tokenization_spaces
								)
								if clean_up_tokenization_spaces:
												lowerCamelCase			=self.clean_up_tokenization(UpperCAmelCase_							)
												return clean_text
								else:
												return text


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None							):
								lowerCamelCase			=[self.sep_token_id]
								lowerCamelCase			=[self.cls_token_id]
								if token_ids_a is None:
												return token_ids_a + sep + cls
								return token_ids_a + sep + token_ids_a + sep + cls


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None				,	UpperCAmelCase_ = False							):

								if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=UpperCAmelCase_				,	token_ids_a=UpperCAmelCase_				,	already_has_special_tokens=UpperCAmelCase_							)

								if token_ids_a is not None:
												return ([0] * len(UpperCAmelCase_							)) + [1] + ([0] * len(UpperCAmelCase_							)) + [1, 1]
								return ([0] * len(UpperCAmelCase_							)) + [1, 1]


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None							):
								lowerCamelCase			=[self.sep_token_id]
								lowerCamelCase			=[2]

								if token_ids_a is None:
												return len(token_ids_a + sep							) * [0] + cls_segment_id
								return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None							):
								if not os.path.isdir(UpperCAmelCase_							):
												logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
												return
								lowerCamelCase			=os.path.join(
								    UpperCAmelCase_				,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

								if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCAmelCase_							) and os.path.isfile(self.vocab_file							):
												copyfile(self.vocab_file				,	UpperCAmelCase_							)
								elif not os.path.isfile(self.vocab_file							):
												with open(UpperCAmelCase_				,	"""wb"""							) as fi:
																lowerCamelCase			=self.sp_model.serialized_model_proto()
																fi.write(UpperCAmelCase_							)

								return (out_vocab_file,)





