

"""simple docstring"""






import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


snake_case_   :   Optional[Any]        =       get_tests_dir("""fixtures/spiece.model""")







@require_sentencepiece
@require_tokenizers
class       snake_case__  ( lowerCAmelCase_					,  unittest.TestCase     ):
			SCREAMING_SNAKE_CASE__     				=						DebertaVaTokenizer
			SCREAMING_SNAKE_CASE__     				=						DebertaVaTokenizerFast
			SCREAMING_SNAKE_CASE__     				=						True
			SCREAMING_SNAKE_CASE__     				=						True







			def 			__lowerCAmelCase				(			self			: int  ):





								'''simple docstring'''



								super().setUp()

								# We have a SentencePiece fixture for testing
								UpperCAmelCase			:   List[str]      =				DebertaVaTokenizer(lowercase			,  unk_token="<unk>"  )
								tokenizer.save_pretrained(self.tmpdirname  )







			def 			__lowerCAmelCase				(			self			: Tuple			,  lowercase			: List[str]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[int]      =				"this is a test"
								UpperCAmelCase			:   Dict      =				"this is a test"
								return input_text, output_text







			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   str      =				"<pad>"
								UpperCAmelCase			:   Optional[int]      =				0

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase  )			,  lowercase  )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase  )			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[Any]      =				list(self.get_tokenizer().get_vocab().keys()  )
								self.assertEqual(vocab_keys[0]			,  "<pad>"  )
								self.assertEqual(vocab_keys[1]			,  "<unk>"  )
								self.assertEqual(vocab_keys[-1]			,  "[PAD]"  )
								self.assertEqual(len(lowercase  )			,  3_00_01  )







			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								self.assertEqual(self.get_tokenizer().vocab_size			,  3_00_00  )







			def 			__lowerCAmelCase				(			self			: int  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[Any]      =				" \tHeLLo!how  \n Are yoU?  "
								UpperCAmelCase			:   Optional[int]      =				["▁hello", "!", "how", "▁are", "▁you", "?"]
								# fmt: on

								UpperCAmelCase			:   int      =				DebertaVaTokenizer(lowercase			,  do_lower_case=lowercase  )
								UpperCAmelCase			:   Any      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )

								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   Optional[Any]      =				DebertaVaTokenizerFast(lowercase			,  do_lower_case=lowercase  )
								UpperCAmelCase			:   str      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )

								self.assertListEqual(lowercase			,  lowercase  )







			@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."  )
			def 			__lowerCAmelCase				(			self			: Any  ):





								'''simple docstring'''



								pass







			@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."  )
			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								pass







			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   int      =				"I was born in 92000, and this is falsé."
								UpperCAmelCase			:   Optional[Any]      =				["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
								# fmt: on

								UpperCAmelCase			:   Dict      =				DebertaVaTokenizer(lowercase			,  split_by_punct=lowercase  )
								UpperCAmelCase			:   Tuple      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )

								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   Dict      =				DebertaVaTokenizerFast(lowercase			,  split_by_punct=lowercase  )
								UpperCAmelCase			:   Any      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )

								self.assertListEqual(lowercase			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Union[str, Any]      =				"I was born in 92000, and this is falsé."
								UpperCAmelCase			:   List[Any]      =				["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
								# fmt: on

								UpperCAmelCase			:   Any      =				DebertaVaTokenizer(lowercase			,  do_lower_case=lowercase			,  split_by_punct=lowercase  )
								UpperCAmelCase			:   Dict      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )
								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   Tuple      =				DebertaVaTokenizerFast(lowercase			,  do_lower_case=lowercase			,  split_by_punct=lowercase  )
								UpperCAmelCase			:   int      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )
								self.assertListEqual(lowercase			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: List[str]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[str]      =				"I was born in 92000, and this is falsé."
								UpperCAmelCase			:   Dict      =				["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
								# fmt: on

								UpperCAmelCase			:   Tuple      =				DebertaVaTokenizer(lowercase			,  do_lower_case=lowercase			,  split_by_punct=lowercase  )
								UpperCAmelCase			:   Optional[Any]      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )

								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   Any      =				DebertaVaTokenizerFast(lowercase			,  do_lower_case=lowercase			,  split_by_punct=lowercase  )
								UpperCAmelCase			:   int      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )

								self.assertListEqual(lowercase			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								UpperCAmelCase			:   Tuple      =				"I was born in 92000, and this is falsé."
								UpperCAmelCase			:   List[Any]      =				["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
								# fmt: on

								UpperCAmelCase			:   Any      =				DebertaVaTokenizer(lowercase			,  do_lower_case=lowercase			,  split_by_punct=lowercase  )
								UpperCAmelCase			:   int      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )

								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   List[Any]      =				DebertaVaTokenizerFast(lowercase			,  do_lower_case=lowercase			,  split_by_punct=lowercase  )
								UpperCAmelCase			:   List[Any]      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )

								self.assertListEqual(lowercase			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: Optional[int]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Tuple      =				" \tHeLLo!how  \n Are yoU?  "
								UpperCAmelCase			:   str      =				["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
								# fmt: on

								UpperCAmelCase			:   int      =				DebertaVaTokenizer(lowercase			,  do_lower_case=lowercase			,  split_by_punct=lowercase  )
								UpperCAmelCase			:   List[Any]      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )

								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   Optional[Any]      =				DebertaVaTokenizerFast(lowercase			,  do_lower_case=lowercase			,  split_by_punct=lowercase  )
								UpperCAmelCase			:   Dict      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )

								self.assertListEqual(lowercase			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Any      =				self.get_tokenizer()
								UpperCAmelCase			:   Union[str, Any]      =				self.get_rust_tokenizer()

								UpperCAmelCase			:   Optional[int]      =				"I was born in 92000, and this is falsé."

								UpperCAmelCase			:   Dict      =				tokenizer.convert_ids_to_tokens(tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )
								UpperCAmelCase			:   Tuple      =				rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )  )
								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   Optional[int]      =				tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )
								UpperCAmelCase			:   str      =				rust_tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   Union[str, Any]      =				self.get_rust_tokenizer()
								UpperCAmelCase			:   List[str]      =				tokenizer.encode(lowercase  )
								UpperCAmelCase			:   Tuple      =				rust_tokenizer.encode(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[Any]      =				"This is a test"
								UpperCAmelCase			:   Tuple      =				[13, 1, 43_98, 25, 21, 12_89]
								UpperCAmelCase			:   int      =				["▁", "T", "his", "▁is", "▁a", "▁test"]
								UpperCAmelCase			:   List[Any]      =				["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

								UpperCAmelCase			:   Optional[int]      =				DebertaVaTokenizer(lowercase			,  keep_accents=lowercase  )
								UpperCAmelCase			:   str      =				DebertaVaTokenizerFast(lowercase			,  keep_accents=lowercase  )

								UpperCAmelCase			:   str      =				tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )
								UpperCAmelCase			:   Tuple      =				tokenizer.tokenize(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )
								UpperCAmelCase			:   List[str]      =				tokenizer.convert_ids_to_tokens(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   Union[str, Any]      =				rust_tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )
								UpperCAmelCase			:   Dict      =				rust_tokenizer.tokenize(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )
								UpperCAmelCase			:   Any      =				rust_tokenizer.convert_ids_to_tokens(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )

								# fmt: off
								UpperCAmelCase			:   List[Any]      =				"I was born in 92000, and this is falsé."
								UpperCAmelCase			:   str      =				[13, 1, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]
								UpperCAmelCase			:   Optional[int]      =				["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
								UpperCAmelCase			:   Tuple      =				["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
								# fmt: on

								UpperCAmelCase			:   Any      =				tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )
								UpperCAmelCase			:   Optional[int]      =				tokenizer.tokenize(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )
								UpperCAmelCase			:   List[Any]      =				tokenizer.convert_ids_to_tokens(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )

								UpperCAmelCase			:   int      =				rust_tokenizer.encode(lowercase			,  add_special_tokens=lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )
								UpperCAmelCase			:   Optional[Any]      =				rust_tokenizer.tokenize(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )
								UpperCAmelCase			:   Tuple      =				rust_tokenizer.convert_ids_to_tokens(lowercase  )
								self.assertListEqual(lowercase			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: Optional[int]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[int]      =				DebertaVaTokenizer(lowercase  )

								UpperCAmelCase			:   Optional[int]      =				tokenizer.encode("sequence builders"  )
								UpperCAmelCase			:   Dict      =				tokenizer.encode("multi-sequence build"  )

								UpperCAmelCase			:   Optional[int]      =				tokenizer.build_inputs_with_special_tokens(lowercase  )
								UpperCAmelCase			:   Tuple      =				tokenizer.build_inputs_with_special_tokens(lowercase			,  lowercase  )

								self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]			,  lowercase  )
								self.assertEqual(
								    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]			,  lowercase			,  )







			@slow
			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								UpperCAmelCase			:   Any      =				{"input_ids": [[1, 3_98_67, 36, 1_93_90, 4_86, 27, 3_50_52, 8_14_36, 18, 6_06_85, 12_25, 7, 3_50_52, 8_14_36, 18, 93_67, 1_68_99, 18, 1_59_37, 53, 5_94, 7_73, 18, 1_62_87, 3_04_65, 36, 1_59_37, 6, 4_11_39, 38, 3_69_79, 6_07_63, 1_91, 6, 3_41_32, 99, 6, 5_05_38, 3_90, 4_32_30, 6, 3_41_32, 27_79, 2_08_50, 14, 6_99, 10_72, 11_94, 36, 3_82, 1_09_01, 53, 7, 6_99, 10_72, 20_84, 36, 2_04_22, 6_30, 53, 19, 1_05, 30_49, 18_96, 10_53, 1_68_99, 15_06, 11, 3_79_78, 42_43, 7, 12_37, 3_18_69, 2_00, 1_65_66, 6_54, 6, 3_50_52, 8_14_36, 7, 5_56_30, 1_35_93, 4, 2], [1, 26, 1_50_11, 13, 6_67, 8, 10_53, 18, 2_36_11, 12_37, 7_23_56, 1_28_20, 34, 10_41_34, 12_09, 35, 1_33_13, 66_27, 21, 2_02, 3_47, 7, 1_64, 23_99, 11, 46, 44_85, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 12_32, 28_64, 1_57_85, 1_49_51, 1_05, 5, 85_81, 12_50, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=lowercase			,  model_name="microsoft/deberta-v2-xlarge"			,  revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"			,  )





"""simple docstring"""






import math



def   lowercase_	(    _lowercase       :		int	):




					'''simple docstring'''






					return math.sqrt(_lowercase	) * math.sqrt(_lowercase	) == num



def   lowercase_	(    _lowercase       :		int	):




					'''simple docstring'''






					UpperCAmelCase			:   List[Any]      =				0
					UpperCAmelCase			:   Tuple      =				n
					while left <= right:
										UpperCAmelCase			:   Tuple      =				(left + right) // 2
										if mid**2 == n:
															return True
										elif mid**2 > n:
															UpperCAmelCase			:   List[str]      =				mid - 1
										else:
															UpperCAmelCase			:   Any      =				mid + 1
					return False


if __name__ == "__main__":
	import doctest

	doctest.testmod()


