

"""simple docstring"""






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


snake_case_   :   Optional[int]        =       get_tests_dir("""fixtures/test_sentencepiece.model""")







@require_sentencepiece
class       snake_case__  ( lowerCAmelCase_					,  unittest.TestCase     ):
			SCREAMING_SNAKE_CASE__     				=						XLMProphetNetTokenizer
			SCREAMING_SNAKE_CASE__     				=						False
			SCREAMING_SNAKE_CASE__     				=						True







			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								super().setUp()

								# We have a SentencePiece fixture for testing
								UpperCAmelCase			:   Tuple      =				XLMProphetNetTokenizer(lowercase			,  keep_accents=lowercase  )
								tokenizer.save_pretrained(self.tmpdirname  )







			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Dict      =				"[PAD]"
								UpperCAmelCase			:   List[str]      =				0

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase  )			,  lowercase  )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase  )			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: Tuple  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[Any]      =				list(self.get_tokenizer().get_vocab().keys()  )

								self.assertEqual(vocab_keys[0]			,  "[PAD]"  )
								self.assertEqual(vocab_keys[1]			,  "[CLS]"  )
								self.assertEqual(vocab_keys[-1]			,  "j"  )
								self.assertEqual(len(lowercase  )			,  10_12  )







			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								self.assertEqual(self.get_tokenizer().vocab_size			,  10_12  )







			def 			__lowerCAmelCase				(			self			: Tuple  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[str]      =				XLMProphetNetTokenizer(lowercase			,  keep_accents=lowercase  )

								UpperCAmelCase			:   Optional[Any]      =				tokenizer.tokenize("This is a test"  )
								self.assertListEqual(lowercase			,  ["▁This", "▁is", "▁a", "▁t", "est"]  )

								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(lowercase  )			,  [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]			,  )

								UpperCAmelCase			:   str      =				tokenizer.tokenize("I was born in 92000, and this is falsé."  )
								self.assertListEqual(
								    lowercase			,  [
								        SPIECE_UNDERLINE + "I",
								        SPIECE_UNDERLINE + "was",
								        SPIECE_UNDERLINE + "b",
								        "or",
								        "n",
								        SPIECE_UNDERLINE + "in",
								        SPIECE_UNDERLINE + "",
								        "9",
								        "2",
								        "0",
								        "0",
								        "0",
								        ",",
								        SPIECE_UNDERLINE + "and",
								        SPIECE_UNDERLINE + "this",
								        SPIECE_UNDERLINE + "is",
								        SPIECE_UNDERLINE + "f",
								        "al",
								        "s",
								        "é",
								        ".",
								    ]			,  )
								UpperCAmelCase			:   Tuple      =				tokenizer.convert_tokens_to_ids(lowercase  )
								self.assertListEqual(
								    lowercase			,  [
								        value + tokenizer.fairseq_offset
								        for value in [8, 21, 84, 55, 24, 19, 7, -9, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, -9, 4]
								    ]			,  )

								UpperCAmelCase			:   Tuple      =				tokenizer.convert_ids_to_tokens(lowercase  )
								self.assertListEqual(
								    lowercase			,  [
								        SPIECE_UNDERLINE + "I",
								        SPIECE_UNDERLINE + "was",
								        SPIECE_UNDERLINE + "b",
								        "or",
								        "n",
								        SPIECE_UNDERLINE + "in",
								        SPIECE_UNDERLINE + "",
								        "[UNK]",
								        "2",
								        "0",
								        "0",
								        "0",
								        ",",
								        SPIECE_UNDERLINE + "and",
								        SPIECE_UNDERLINE + "this",
								        SPIECE_UNDERLINE + "is",
								        SPIECE_UNDERLINE + "f",
								        "al",
								        "s",
								        "[UNK]",
								        ".",
								    ]			,  )







			@cached_property
			def 			__lowerCAmelCase				(			self			: Tuple  ):





								'''simple docstring'''



								return XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"  )







			@slow
			def 			__lowerCAmelCase				(			self			: Tuple  ):





								'''simple docstring'''



								UpperCAmelCase			:   int      =				"Hello World!"
								UpperCAmelCase			:   str      =				[3_53_89, 66_72, 49, 2]
								self.assertListEqual(lowercase			,  self.big_tokenizer.encode(lowercase  )  )







			@slow
			def 			__lowerCAmelCase				(			self			: int  ):





								'''simple docstring'''



								UpperCAmelCase			:   Any      =				{"input_ids": [[1_10_73, 8_27_83, 18, 26, 8_27_83, 5_49, 5_15_40, 2_48, 1_72_09, 13_01, 2_17, 20, 21_51_86, 13_25, 1_47, 1_72_09, 13_01, 2_17, 20, 5_63_70, 53, 12_20_20, 20, 1_64_77, 27, 8_73_55, 45_48, 20, 47_28, 7_83_92, 17, 15_99_69, 18, 26, 2_44_91, 6_29, 15, 5_38, 2_27_04, 54_39, 15, 27_88, 2_44_91, 98_85, 15, 4_35_34, 6_05, 15, 8_14, 1_84_03, 3_32_00, 29, 15, 4_35_34, 2_44_58, 1_24_10, 1_11, 2_49_66, 8_36_69, 96_37, 14_40_68, 26, 8_50, 2_23_46, 27, 1_47, 2_49_66, 8_36_69, 8_34_90, 26, 3_91_13, 7_35, 27, 6_89, 6_56, 28_00, 13_39, 46_00, 53, 12_20_20, 11_57_85, 34, 8_16, 13_39, 4_68_87, 18, 1_47, 5_39_05, 19_51, 4_22_38, 4_11_70, 1_77_32, 8_34, 4_36, 15, 2_75_23, 9_87_33, 2_17, 1_47, 55_42, 49_81, 9_30, 1_73_47, 16, 2], [2_00_91, 6_29, 94, 8_27_86, 58, 4_90, 20, 15_28, 84, 5_39_05, 3_44, 8_05_92, 11_01_28, 1_88_22, 52_67, 13_06, 62, 15_25_37, 3_08, 79_97, 4_01, 12_44_27, 5_49, 3_54_42, 2_25, 1_09, 1_50_55, 2_57_48, 1_47, 71_19, 4_37_12, 34, 7_67, 13_53_66, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_92, 6_37_84, 11_94_66, 17, 14_78_08, 8_82_14, 18, 6_56, 81, 32, 32_96, 1_02_80, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=lowercase			,  model_name="microsoft/xprophetnet-large-wiki100-cased"			,  revision="1acad1643ddd54a44df6a1b797ada8373685d90e"			,  )





"""simple docstring"""






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case_   :   Union[str, Any]        =       logging.get_logger(__name__)

snake_case_   :   Tuple        =       {"""vocab_file""": """sentencepiece.bpe.model"""}

snake_case_   :   List[str]        =       {
    """vocab_file""": {
        """camembert-base""": """https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model""",
    }
}

snake_case_   :   List[Any]        =       {
    """camembert-base""": 5_1_2,
}

snake_case_   :   Any        =       """▁"""







class       snake_case__  ( lowerCAmelCase_     ):
			SCREAMING_SNAKE_CASE__     				=						VOCAB_FILES_NAMES
			SCREAMING_SNAKE_CASE__     				=						PRETRAINED_VOCAB_FILES_MAP
			SCREAMING_SNAKE_CASE__     				=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			SCREAMING_SNAKE_CASE__     				=						['''input_ids''', '''attention_mask''']







			def __init__(			self			: int			,  lowercase			: Union[str, Any]			,  lowercase			: str="<s>"			,  lowercase			: str="</s>"			,  lowercase			: Optional[int]="</s>"			,  lowercase			: Dict="<s>"			,  lowercase			: Optional[Any]="<unk>"			,  lowercase			: List[Any]="<pad>"			,  lowercase			: Any="<mask>"			,  lowercase			: Optional[int]=["<s>NOTUSED", "</s>NOTUSED"]			,  lowercase			: Optional[Dict[str, Any]] = None			,  **lowercase			: Dict			,  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[Any]      =				AddedToken(lowercase			,  lstrip=lowercase			,  rstrip=lowercase  ) if isinstance(lowercase			,  lowercase  ) else mask_token

								UpperCAmelCase			:   int      =				{} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    bos_token=lowercase			,  eos_token=lowercase			,  unk_token=lowercase			,  sep_token=lowercase			,  cls_token=lowercase			,  pad_token=lowercase			,  mask_token=lowercase			,  additional_special_tokens=lowercase			,  sp_model_kwargs=self.sp_model_kwargs			,  **lowercase			,  )
								UpperCAmelCase			:   Tuple      =				spm.SentencePieceProcessor(**self.sp_model_kwargs  )
								self.sp_model.Load(str(lowercase  )  )
								UpperCAmelCase			:   str      =				vocab_file
								# HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
								# sentencepiece vocabulary (this is the case for <s> and </s>
								UpperCAmelCase			:   Union[str, Any]      =				{"<s>NOTUSED": 0, "<pad>": 1, "</s>NOTUSED": 2, "<unk>": 3}
								UpperCAmelCase			:   Union[str, Any]      =				len(self.fairseq_tokens_to_ids  )
								UpperCAmelCase			:   Dict      =				len(self.sp_model  ) + len(self.fairseq_tokens_to_ids  )
								UpperCAmelCase			:   List[Any]      =				{v: k for k, v in self.fairseq_tokens_to_ids.items()}







			def 			__lowerCAmelCase				(			self			: Union[str, Any]			,  lowercase			: List[int]			,  lowercase			: Optional[List[int]] = None  ):





								'''simple docstring'''



								if token_ids_a is None:
													return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								UpperCAmelCase			:   Dict      =				[self.cls_token_id]
								UpperCAmelCase			:   List[str]      =				[self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep







			def 			__lowerCAmelCase				(			self			: Dict			,  lowercase			: List[int]			,  lowercase			: Optional[List[int]] = None			,  lowercase			: bool = False  ):





								'''simple docstring'''



								if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=lowercase			,  token_ids_a=lowercase			,  already_has_special_tokens=lowercase  )

								if token_ids_a is None:
													return [1] + ([0] * len(lowercase  )) + [1]
								return [1] + ([0] * len(lowercase  )) + [1, 1] + ([0] * len(lowercase  )) + [1]







			def 			__lowerCAmelCase				(			self			: Tuple			,  lowercase			: List[int]			,  lowercase			: Optional[List[int]] = None  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[Any]      =				[self.sep_token_id]
								UpperCAmelCase			:   List[str]      =				[self.cls_token_id]

								if token_ids_a is None:
													return len(cls + token_ids_a + sep  ) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]







			@property
			def 			__lowerCAmelCase				(			self			: Optional[int]  ):





								'''simple docstring'''



								return len(self.fairseq_tokens_to_ids  ) + len(self.sp_model  )







			def 			__lowerCAmelCase				(			self			: List[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[Any]      =				{self.convert_ids_to_tokens(lowercase  ): i for i in range(self.vocab_size  )}
								vocab.update(self.added_tokens_encoder  )
								return vocab







			def 			__lowerCAmelCase				(			self			: Optional[Any]			,  lowercase			: str  ):





								'''simple docstring'''



								return self.sp_model.encode(lowercase			,  out_type=lowercase  )







			def 			__lowerCAmelCase				(			self			: int			,  lowercase			: Optional[Any]  ):





								'''simple docstring'''



								if token in self.fairseq_tokens_to_ids:
													return self.fairseq_tokens_to_ids[token]
								elif self.sp_model.PieceToId(lowercase  ) == 0:
													# Convert sentence piece unk token to fairseq unk token index
													return self.unk_token_id
								return self.fairseq_offset + self.sp_model.PieceToId(lowercase  )







			def 			__lowerCAmelCase				(			self			: Any			,  lowercase			: Dict  ):





								'''simple docstring'''



								if index in self.fairseq_ids_to_tokens:
													return self.fairseq_ids_to_tokens[index]
								return self.sp_model.IdToPiece(index - self.fairseq_offset  )







			def 			__lowerCAmelCase				(			self			: Tuple			,  lowercase			: Optional[int]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Tuple      =				[]
								UpperCAmelCase			:   Tuple      =				""
								UpperCAmelCase			:   Union[str, Any]      =				False
								for token in tokens:
													# make sure that special tokens are not decoded using sentencepiece model
													if token in self.all_special_tokens:
																		if not prev_is_special:
																							out_string += " "
																		out_string += self.sp_model.decode(lowercase  ) + token
																		UpperCAmelCase			:   Optional[int]      =				True
																		UpperCAmelCase			:   str      =				[]
													else:
																		current_sub_tokens.append(lowercase  )
																		UpperCAmelCase			:   Any      =				False
								out_string += self.sp_model.decode(lowercase  )
								return out_string.strip()







			def __getstate__(			self			: str  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[str]      =				self.__dict__.copy()
								UpperCAmelCase			:   Dict      =				None
								return state







			def __setstate__(			self			: List[str]			,  lowercase			: str  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[Any]      =				d

								# for backward compatibility
								if not hasattr(self			,  "sp_model_kwargs"  ):
													UpperCAmelCase			:   Any      =				{}

								UpperCAmelCase			:   List[Any]      =				spm.SentencePieceProcessor(**self.sp_model_kwargs  )
								self.sp_model.Load(self.vocab_file  )







			def 			__lowerCAmelCase				(			self			: Optional[Any]			,  lowercase			: str			,  lowercase			: Optional[str] = None  ):





								'''simple docstring'''



								if not os.path.isdir(lowercase  ):
													logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
													return
								UpperCAmelCase			:   Optional[Any]      =				os.path.join(
								    lowercase			,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

								if os.path.abspath(self.vocab_file  ) != os.path.abspath(lowercase  ) and os.path.isfile(self.vocab_file  ):
													copyfile(self.vocab_file			,  lowercase  )
								elif not os.path.isfile(self.vocab_file  ):
													with open(lowercase			,  "wb"  ) as fi:
																		UpperCAmelCase			:   List[Any]      =				self.sp_model.serialized_model_proto()
																		fi.write(lowercase  )

								return (out_vocab_file,)


