





import unittest

from transformers import MPNetConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
							import torch

							from transformers import (
							    MPNetForMaskedLM,
							    MPNetForMultipleChoice,
							    MPNetForQuestionAnswering,
							    MPNetForSequenceClassification,
							    MPNetForTokenClassification,
							    MPNetModel,
							)

class  __A       :






				"""simple docstring"""




				def __init__(						self				,	UpperCAmelCase_				,	UpperCAmelCase_=13				,	UpperCAmelCase_=7				,	UpperCAmelCase_=True				,	UpperCAmelCase_=True				,	UpperCAmelCase_=False				,	UpperCAmelCase_=True				,	UpperCAmelCase_=99				,	UpperCAmelCase_=64				,	UpperCAmelCase_=5				,	UpperCAmelCase_=4				,	UpperCAmelCase_=64				,	UpperCAmelCase_="gelu"				,	UpperCAmelCase_=0.1				,	UpperCAmelCase_=0.1				,	UpperCAmelCase_=512				,	UpperCAmelCase_=16				,	UpperCAmelCase_=2				,	UpperCAmelCase_=0.0_2				,	UpperCAmelCase_=3				,	UpperCAmelCase_=4				,	UpperCAmelCase_=None				,	):
								lowerCamelCase			=parent
								lowerCamelCase			=batch_size
								lowerCamelCase			=seq_length
								lowerCamelCase			=is_training
								lowerCamelCase			=use_input_mask
								lowerCamelCase			=use_token_type_ids
								lowerCamelCase			=use_labels
								lowerCamelCase			=vocab_size
								lowerCamelCase			=hidden_size
								lowerCamelCase			=num_hidden_layers
								lowerCamelCase			=num_attention_heads
								lowerCamelCase			=intermediate_size
								lowerCamelCase			=hidden_act
								lowerCamelCase			=hidden_dropout_prob
								lowerCamelCase			=attention_probs_dropout_prob
								lowerCamelCase			=max_position_embeddings
								lowerCamelCase			=type_vocab_size
								lowerCamelCase			=type_sequence_label_size
								lowerCamelCase			=initializer_range
								lowerCamelCase			=num_labels
								lowerCamelCase			=num_choices
								lowerCamelCase			=scope


				def _snake_case    (						self							):
								return MPNetConfig.from_pretrained("""microsoft/mpnet-base"""							)


				def _snake_case    (						self							):
								lowerCamelCase			=ids_tensor([self.batch_size, self.seq_length]				,	self.vocab_size							)

								lowerCamelCase			=None
								if self.use_input_mask:
												lowerCamelCase			=random_attention_mask([self.batch_size, self.seq_length]							)

								lowerCamelCase			=None
								lowerCamelCase			=None
								lowerCamelCase			=None
								if self.use_labels:
												lowerCamelCase			=ids_tensor([self.batch_size]				,	self.type_sequence_label_size							)
												lowerCamelCase			=ids_tensor([self.batch_size, self.seq_length]				,	self.num_labels							)
												lowerCamelCase			=ids_tensor([self.batch_size]				,	self.num_choices							)

								lowerCamelCase			=self.get_config()
								return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels


				def _snake_case    (						self							):
								return MPNetConfig(
								    vocab_size=self.vocab_size				,	hidden_size=self.hidden_size				,	num_hidden_layers=self.num_hidden_layers				,	num_attention_heads=self.num_attention_heads				,	intermediate_size=self.intermediate_size				,	hidden_act=self.hidden_act				,	hidden_dropout_prob=self.hidden_dropout_prob				,	attention_probs_dropout_prob=self.attention_probs_dropout_prob				,	max_position_embeddings=self.max_position_embeddings				,	initializer_range=self.initializer_range				,	)


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_							):
								lowerCamelCase			=MPNetModel(config=_a							)
								model.to(_a							)
								model.eval()
								lowerCamelCase			=model(_a				,	_a							)
								lowerCamelCase			=model(_a							)
								self.parent.assertEqual(result.last_hidden_state.shape				,	(self.batch_size, self.seq_length, self.hidden_size)							)
								self.parent.assertEqual(result.pooler_output.shape				,	(self.batch_size, self.hidden_size)							)


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_							):
								lowerCamelCase			=MPNetForQuestionAnswering(config=_a							)
								model.to(_a							)
								model.eval()
								lowerCamelCase			=model(
								    _a				,	attention_mask=_a				,	start_positions=_a				,	end_positions=_a				,	)
								self.parent.assertEqual(result.start_logits.shape				,	(self.batch_size, self.seq_length)							)
								self.parent.assertEqual(result.end_logits.shape				,	(self.batch_size, self.seq_length)							)


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_							):
								lowerCamelCase			=self.num_labels
								lowerCamelCase			=MPNetForSequenceClassification(_a							)
								model.to(_a							)
								model.eval()
								lowerCamelCase			=model(_a				,	attention_mask=_a				,	labels=_a							)
								self.parent.assertEqual(result.logits.shape				,	(self.batch_size, self.num_labels)							)


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_							):
								lowerCamelCase			=self.num_choices
								lowerCamelCase			=MPNetForMultipleChoice(config=_a							)
								model.to(_a							)
								model.eval()
								lowerCamelCase			=input_ids.unsqueeze(1							).expand(-1				,	self.num_choices				,	-1							).contiguous()
								lowerCamelCase			=input_mask.unsqueeze(1							).expand(-1				,	self.num_choices				,	-1							).contiguous()
								lowerCamelCase			=model(
								    _a				,	attention_mask=_a				,	labels=_a				,	)
								self.parent.assertEqual(result.logits.shape				,	(self.batch_size, self.num_choices)							)


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_				,	UpperCAmelCase_							):
								lowerCamelCase			=self.num_labels
								lowerCamelCase			=MPNetForTokenClassification(config=_a							)
								model.to(_a							)
								model.eval()
								lowerCamelCase			=model(_a				,	attention_mask=_a				,	labels=_a							)
								self.parent.assertEqual(result.logits.shape				,	(self.batch_size, self.seq_length, self.num_labels)							)


				def _snake_case    (						self							):
								lowerCamelCase			=self.prepare_config_and_inputs()
								(lowerCamelCase)			=config_and_inputs
								lowerCamelCase			={"""input_ids""": input_ids, """attention_mask""": input_mask}
								return config, inputs_dict




@require_torch
class  __A       (  UpperCamelCase_      ,   UpperCamelCase_      ,   unittest.TestCase      ):






				"""simple docstring"""




				__A								=			(
				    (
				        MPNetForMaskedLM,
				        MPNetForMultipleChoice,
				        MPNetForQuestionAnswering,
				        MPNetForSequenceClassification,
				        MPNetForTokenClassification,
				        MPNetModel,
				    )
				    if is_torch_available()
				    else ()
				)
				__A								=			(
				    {
				        """feature-extraction""": MPNetModel,
				        """fill-mask""": MPNetForMaskedLM,
				        """question-answering""": MPNetForQuestionAnswering,
				        """text-classification""": MPNetForSequenceClassification,
				        """token-classification""": MPNetForTokenClassification,
				        """zero-shot""": MPNetForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				__A								=			False
				__A								=			True


				def _snake_case    (						self							):
								lowerCamelCase			=MPNetModelTester(self							)
								lowerCamelCase			=ConfigTester(self				,	config_class=_a				,	hidden_size=37							)


				def _snake_case    (						self							):
								self.config_tester.run_common_tests()


				def _snake_case    (						self							):
								lowerCamelCase			=self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_mpnet_model(*_a							)


				def _snake_case    (						self							):
								lowerCamelCase			=self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_mpnet_for_sequence_classification(*_a							)


				def _snake_case    (						self							):
								lowerCamelCase			=self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_mpnet_for_multiple_choice(*_a							)


				def _snake_case    (						self							):
								lowerCamelCase			=self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_mpnet_for_token_classification(*_a							)


				def _snake_case    (						self							):
								lowerCamelCase			=self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_mpnet_for_question_answering(*_a							)




@require_torch
class  __A       (  unittest.TestCase      ):






				"""simple docstring"""




				@slow
				def _snake_case    (						self							):
								lowerCamelCase			=MPNetModel.from_pretrained("""microsoft/mpnet-base"""							)
								lowerCamelCase			=torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]]							)
								lowerCamelCase			=model(_a							)[0]
								lowerCamelCase			=torch.Size((1, 11, 768)							)
								self.assertEqual(output.shape				,	_a							)
								lowerCamelCase			=torch.tensor(
								    [[[-0.0_5_5_0, 0.1_9_4_3, -0.0_7_4_0], [-0.0_5_6_2, 0.2_2_1_1, -0.0_5_7_9], [-0.0_4_3_7, 0.3_3_3_7, -0.0_6_4_1]]]							)
								# compare the actual values for a slice.
								self.assertTrue(torch.allclose(output[:, :3, :3]				,	_a				,	atol=1E-4							)							)












import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCAmelCase__  :     List[Any]   			=logging.get_logger(__name__)

UpperCAmelCase__  :     Dict   			={'''vocab_file''': '''spiece.model'''}

UpperCAmelCase__  :     Dict   			={
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    }
}

UpperCAmelCase__  :     List[str]   			={
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

# Segments (not really needed)
UpperCAmelCase__  :     Any   			=0
UpperCAmelCase__  :     List[Any]   			=1
UpperCAmelCase__  :     Union[str, Any]   			=2
UpperCAmelCase__  :     Tuple   			=3
UpperCAmelCase__  :     int   			=4

class  __A       (  a      ):
				__A								=			VOCAB_FILES_NAMES
				__A								=			PRETRAINED_VOCAB_FILES_MAP
				__A								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				__A								=			"""left"""


				def __init__(						self				,	UpperCAmelCase_				,	UpperCAmelCase_=False				,	UpperCAmelCase_=True				,	UpperCAmelCase_=False				,	UpperCAmelCase_="<s>"				,	UpperCAmelCase_="</s>"				,	UpperCAmelCase_="<unk>"				,	UpperCAmelCase_="<sep>"				,	UpperCAmelCase_="<pad>"				,	UpperCAmelCase_="<cls>"				,	UpperCAmelCase_="<mask>"				,	UpperCAmelCase_=["<eop>", "<eod>"]				,	UpperCAmelCase_ = None				,	**UpperCAmelCase_				,	):
								# Mask token behave like a normal word, i.e. include the space before it
								lowerCamelCase			=AddedToken(UpperCAmelCase_				,	lstrip=UpperCAmelCase_				,	rstrip=UpperCAmelCase_							) if isinstance(UpperCAmelCase_				,	UpperCAmelCase_							) else mask_token

								lowerCamelCase			={} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    do_lower_case=UpperCAmelCase_				,	remove_space=UpperCAmelCase_				,	keep_accents=UpperCAmelCase_				,	bos_token=UpperCAmelCase_				,	eos_token=UpperCAmelCase_				,	unk_token=UpperCAmelCase_				,	sep_token=UpperCAmelCase_				,	pad_token=UpperCAmelCase_				,	cls_token=UpperCAmelCase_				,	mask_token=UpperCAmelCase_				,	additional_special_tokens=UpperCAmelCase_				,	sp_model_kwargs=self.sp_model_kwargs				,	**UpperCAmelCase_				,	)

								lowerCamelCase			=3

								lowerCamelCase			=do_lower_case
								lowerCamelCase			=remove_space
								lowerCamelCase			=keep_accents
								lowerCamelCase			=vocab_file

								lowerCamelCase			=spm.SentencePieceProcessor(**self.sp_model_kwargs							)
								self.sp_model.Load(UpperCAmelCase_							)


				@property
				def _snake_case    (						self							):
								return len(self.sp_model							)


				def _snake_case    (						self							):
								lowerCamelCase			={self.convert_ids_to_tokens(UpperCAmelCase_							): i for i in range(self.vocab_size							)}
								vocab.update(self.added_tokens_encoder							)
								return vocab


				def __getstate__(						self							):
								lowerCamelCase			=self.__dict__.copy()
								lowerCamelCase			=None
								return state


				def __setstate__(						self				,	UpperCAmelCase_							):
								lowerCamelCase			=d

								# for backward compatibility
								if not hasattr(self				,	"""sp_model_kwargs"""							):
												lowerCamelCase			={}

								lowerCamelCase			=spm.SentencePieceProcessor(**self.sp_model_kwargs							)
								self.sp_model.Load(self.vocab_file							)


				def _snake_case    (						self				,	UpperCAmelCase_							):
								if self.remove_space:
												lowerCamelCase			=""" """.join(inputs.strip().split()							)
								else:
												lowerCamelCase			=inputs
								lowerCamelCase			=outputs.replace("""``"""				,	"""\""""							).replace("""''"""				,	"""\""""							)

								if not self.keep_accents:
												lowerCamelCase			=unicodedata.normalize("""NFKD"""				,	UpperCAmelCase_							)
												lowerCamelCase			="""""".join([c for c in outputs if not unicodedata.combining(UpperCAmelCase_							)]							)
								if self.do_lower_case:
												lowerCamelCase			=outputs.lower()

								return outputs


				def _snake_case    (						self				,	UpperCAmelCase_							):
								lowerCamelCase			=self.preprocess_text(UpperCAmelCase_							)
								lowerCamelCase			=self.sp_model.encode(UpperCAmelCase_				,	out_type=UpperCAmelCase_							)
								lowerCamelCase			=[]
								for piece in pieces:
												if len(UpperCAmelCase_							) > 1 and piece[-1] == str(""","""							) and piece[-2].isdigit():
																lowerCamelCase			=self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase_				,	""""""							)							)
																if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																				if len(cur_pieces[0]							) == 1:
																								lowerCamelCase			=cur_pieces[1:]
																				else:
																								lowerCamelCase			=cur_pieces[0][1:]
																cur_pieces.append(piece[-1]							)
																new_pieces.extend(UpperCAmelCase_							)
												else:
																new_pieces.append(UpperCAmelCase_							)

								return new_pieces


				def _snake_case    (						self				,	UpperCAmelCase_							):
								return self.sp_model.PieceToId(UpperCAmelCase_							)


				def _snake_case    (						self				,	UpperCAmelCase_							):
								return self.sp_model.IdToPiece(UpperCAmelCase_							)


				def _snake_case    (						self				,	UpperCAmelCase_							):
								lowerCamelCase			="""""".join(UpperCAmelCase_							).replace(UpperCAmelCase_				,	""" """							).strip()
								return out_string


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = False				,	UpperCAmelCase_ = None				,	UpperCAmelCase_ = True				,	**UpperCAmelCase_				,	):
								lowerCamelCase			=kwargs.pop("""use_source_tokenizer"""				,	UpperCAmelCase_							)

								lowerCamelCase			=self.convert_ids_to_tokens(UpperCAmelCase_				,	skip_special_tokens=UpperCAmelCase_							)

								# To avoid mixing byte-level and unicode for byte-level BPT
								# we need to build string separately for added tokens and byte-level tokens
								# cf. https://github.com/huggingface/transformers/issues/1133
								lowerCamelCase			=[]
								lowerCamelCase			=[]
								for token in filtered_tokens:
												if skip_special_tokens and token in self.all_special_ids:
																continue
												if token in self.added_tokens_encoder:
																if current_sub_text:
																				sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase_							)							)
																				lowerCamelCase			=[]
																sub_texts.append(UpperCAmelCase_							)
												else:
																current_sub_text.append(UpperCAmelCase_							)
								if current_sub_text:
												sub_texts.append(self.convert_tokens_to_string(UpperCAmelCase_							)							)

								# Mimic the behavior of the Rust tokenizer:
								# By default, there are no spaces between special tokens
								lowerCamelCase			="""""".join(UpperCAmelCase_							)

								lowerCamelCase			=(
								    clean_up_tokenization_spaces
								    if clean_up_tokenization_spaces is not None
								    else self.clean_up_tokenization_spaces
								)
								if clean_up_tokenization_spaces:
												lowerCamelCase			=self.clean_up_tokenization(UpperCAmelCase_							)
												return clean_text
								else:
												return text


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None							):
								lowerCamelCase			=[self.sep_token_id]
								lowerCamelCase			=[self.cls_token_id]
								if token_ids_a is None:
												return token_ids_a + sep + cls
								return token_ids_a + sep + token_ids_a + sep + cls


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None				,	UpperCAmelCase_ = False							):

								if already_has_special_tokens:
												return super().get_special_tokens_mask(
												    token_ids_a=UpperCAmelCase_				,	token_ids_a=UpperCAmelCase_				,	already_has_special_tokens=UpperCAmelCase_							)

								if token_ids_a is not None:
												return ([0] * len(UpperCAmelCase_							)) + [1] + ([0] * len(UpperCAmelCase_							)) + [1, 1]
								return ([0] * len(UpperCAmelCase_							)) + [1, 1]


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None							):
								lowerCamelCase			=[self.sep_token_id]
								lowerCamelCase			=[2]

								if token_ids_a is None:
												return len(token_ids_a + sep							) * [0] + cls_segment_id
								return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id


				def _snake_case    (						self				,	UpperCAmelCase_				,	UpperCAmelCase_ = None							):
								if not os.path.isdir(UpperCAmelCase_							):
												logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
												return
								lowerCamelCase			=os.path.join(
								    UpperCAmelCase_				,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

								if os.path.abspath(self.vocab_file							) != os.path.abspath(UpperCAmelCase_							) and os.path.isfile(self.vocab_file							):
												copyfile(self.vocab_file				,	UpperCAmelCase_							)
								elif not os.path.isfile(self.vocab_file							):
												with open(UpperCAmelCase_				,	"""wb"""							) as fi:
																lowerCamelCase			=self.sp_model.serialized_model_proto()
																fi.write(UpperCAmelCase_							)

								return (out_vocab_file,)





