

'''simple docstring'''


import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase__    :							Union[str, Any]					    =    get_tests_dir("""fixtures/spiece.model""")




@require_sentencepiece
@require_tokenizers
class      a      (					SCREAMING_SNAKE_CASE_      ,      unittest.TestCase							):






						"""simple docstring"""




						__UpperCAmelCase        =     DebertaVaTokenizer
						__UpperCAmelCase        =     DebertaVaTokenizerFast
						__UpperCAmelCase        =     True
						__UpperCAmelCase        =     True







						def    __magic_name__				(				self       :   Optional[int]	):



								'''simple docstring'''



								super().setUp()

								# We have a SentencePiece fixture for testing
								snake_case__    :							Any							    =	DebertaVaTokenizer(snake_case_      ,      unk_token='''<unk>'''	)
								tokenizer.save_pretrained(self.tmpdirname	)







						def    __magic_name__				(				self       :   Union[str, Any]      ,      snake_case_       :   str	):



								'''simple docstring'''



								snake_case__    :							Tuple							    =	'this is a test'
								snake_case__    :							Optional[int]							    =	'this is a test'
								return input_text, output_text







						def    __magic_name__				(				self       :   Dict	):



								'''simple docstring'''



								snake_case__    :							Any							    =	'<pad>'
								snake_case__    :							Optional[Any]							    =	0

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case_	)      ,      snake_case_	)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case_	)      ,      snake_case_	)







						def    __magic_name__				(				self       :   Dict	):



								'''simple docstring'''



								snake_case__    :							List[Any]							    =	list(self.get_tokenizer().get_vocab().keys()	)
								self.assertEqual(vocab_keys[0]      ,      '''<pad>'''	)
								self.assertEqual(vocab_keys[1]      ,      '''<unk>'''	)
								self.assertEqual(vocab_keys[-1]      ,      '''[PAD]'''	)
								self.assertEqual(len(snake_case_	)      ,      3_0_0_0_1	)







						def    __magic_name__				(				self       :   Union[str, Any]	):



								'''simple docstring'''



								self.assertEqual(self.get_tokenizer().vocab_size      ,      3_0_0_0_0	)







						def    __magic_name__				(				self       :   List[Any]	):



								'''simple docstring'''



								snake_case__    :							Tuple							    =	' \tHeLLo!how  \n Are yoU?  '
								snake_case__    :							List[Any]							    =	['▁hello', '!', 'how', '▁are', '▁you', '?']
								# fmt: on

								snake_case__    :							Dict							    =	DebertaVaTokenizer(snake_case_      ,      do_lower_case=snake_case_	)
								snake_case__    :							Optional[Any]							    =	tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)

								self.assertListEqual(snake_case_      ,      snake_case_	)

								snake_case__    :							Optional[Any]							    =	DebertaVaTokenizerFast(snake_case_      ,      do_lower_case=snake_case_	)
								snake_case__    :							Tuple							    =	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)

								self.assertListEqual(snake_case_      ,      snake_case_	)







						@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''	)
						def    __magic_name__				(				self       :   Optional[Any]	):



								'''simple docstring'''



								pass







						@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''	)
						def    __magic_name__				(				self       :   Optional[int]	):



								'''simple docstring'''



								pass







						def    __magic_name__				(				self       :   str	):



								'''simple docstring'''



								snake_case__    :							int							    =	'I was born in 92000, and this is falsé.'
								snake_case__    :							Union[str, Any]							    =	['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
								# fmt: on

								snake_case__    :							Tuple							    =	DebertaVaTokenizer(snake_case_      ,      split_by_punct=snake_case_	)
								snake_case__    :							List[str]							    =	tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)

								self.assertListEqual(snake_case_      ,      snake_case_	)

								snake_case__    :							Union[str, Any]							    =	DebertaVaTokenizerFast(snake_case_      ,      split_by_punct=snake_case_	)
								snake_case__    :							Any							    =	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)

								self.assertListEqual(snake_case_      ,      snake_case_	)







						def    __magic_name__				(				self       :   Tuple	):



								'''simple docstring'''



								snake_case__    :							Optional[Any]							    =	'I was born in 92000, and this is falsé.'
								snake_case__    :							Optional[Any]							    =	['▁i', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
								# fmt: on

								snake_case__    :							Optional[int]							    =	DebertaVaTokenizer(snake_case_      ,      do_lower_case=snake_case_      ,      split_by_punct=snake_case_	)
								snake_case__    :							int							    =	tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)
								self.assertListEqual(snake_case_      ,      snake_case_	)

								snake_case__    :							List[str]							    =	DebertaVaTokenizerFast(snake_case_      ,      do_lower_case=snake_case_      ,      split_by_punct=snake_case_	)
								snake_case__    :							List[Any]							    =	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)
								self.assertListEqual(snake_case_      ,      snake_case_	)







						def    __magic_name__				(				self       :   Tuple	):



								'''simple docstring'''



								snake_case__    :							Dict							    =	'I was born in 92000, and this is falsé.'
								snake_case__    :							Dict							    =	['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
								# fmt: on

								snake_case__    :							Union[str, Any]							    =	DebertaVaTokenizer(snake_case_      ,      do_lower_case=snake_case_      ,      split_by_punct=snake_case_	)
								snake_case__    :							Tuple							    =	tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)

								self.assertListEqual(snake_case_      ,      snake_case_	)

								snake_case__    :							List[Any]							    =	DebertaVaTokenizerFast(snake_case_      ,      do_lower_case=snake_case_      ,      split_by_punct=snake_case_	)
								snake_case__    :							Union[str, Any]							    =	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)

								self.assertListEqual(snake_case_      ,      snake_case_	)







						def    __magic_name__				(				self       :   Any	):



								'''simple docstring'''



								snake_case__    :							Dict							    =	'I was born in 92000, and this is falsé.'
								snake_case__    :							List[Any]							    =	['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
								# fmt: on

								snake_case__    :							Optional[Any]							    =	DebertaVaTokenizer(snake_case_      ,      do_lower_case=snake_case_      ,      split_by_punct=snake_case_	)
								snake_case__    :							Union[str, Any]							    =	tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)

								self.assertListEqual(snake_case_      ,      snake_case_	)

								snake_case__    :							List[str]							    =	DebertaVaTokenizerFast(snake_case_      ,      do_lower_case=snake_case_      ,      split_by_punct=snake_case_	)
								snake_case__    :							Any							    =	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)

								self.assertListEqual(snake_case_      ,      snake_case_	)







						def    __magic_name__				(				self       :   Any	):



								'''simple docstring'''



								snake_case__    :							Tuple							    =	' \tHeLLo!how  \n Are yoU?  '
								snake_case__    :							Tuple							    =	['▁', '<unk>', 'e', '<unk>', 'o', '!', 'how', '▁', '<unk>', 're', '▁yo', '<unk>', '?']
								# fmt: on

								snake_case__    :							Dict							    =	DebertaVaTokenizer(snake_case_      ,      do_lower_case=snake_case_      ,      split_by_punct=snake_case_	)
								snake_case__    :							Tuple							    =	tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)

								self.assertListEqual(snake_case_      ,      snake_case_	)

								snake_case__    :							Optional[Any]							    =	DebertaVaTokenizerFast(snake_case_      ,      do_lower_case=snake_case_      ,      split_by_punct=snake_case_	)
								snake_case__    :							Any							    =	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)

								self.assertListEqual(snake_case_      ,      snake_case_	)







						def    __magic_name__				(				self       :   Union[str, Any]	):



								'''simple docstring'''



								snake_case__    :							str							    =	self.get_tokenizer()
								snake_case__    :							Dict							    =	self.get_rust_tokenizer()

								snake_case__    :							int							    =	'I was born in 92000, and this is falsé.'

								snake_case__    :							Tuple							    =	tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)
								snake_case__    :							Tuple							    =	rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)	)
								self.assertListEqual(snake_case_      ,      snake_case_	)

								snake_case__    :							Optional[Any]							    =	tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)
								snake_case__    :							Optional[int]							    =	rust_tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)

								snake_case__    :							Optional[int]							    =	self.get_rust_tokenizer()
								snake_case__    :							Any							    =	tokenizer.encode(snake_case_	)
								snake_case__    :							int							    =	rust_tokenizer.encode(snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)







						def    __magic_name__				(				self       :   Union[str, Any]	):



								'''simple docstring'''



								snake_case__    :							str							    =	'This is a test'
								snake_case__    :							str							    =	[1_3, 1, 4_3_9_8, 2_5, 2_1, 1_2_8_9]
								snake_case__    :							str							    =	['▁', 'T', 'his', '▁is', '▁a', '▁test']
								snake_case__    :							Any							    =	['▁', '<unk>', 'his', '▁is', '▁a', '▁test']

								snake_case__    :							Optional[int]							    =	DebertaVaTokenizer(snake_case_      ,      keep_accents=snake_case_	)
								snake_case__    :							int							    =	DebertaVaTokenizerFast(snake_case_      ,      keep_accents=snake_case_	)

								snake_case__    :							Any							    =	tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)
								snake_case__    :							int							    =	tokenizer.tokenize(snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)
								snake_case__    :							str							    =	tokenizer.convert_ids_to_tokens(snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)

								snake_case__    :							List[Any]							    =	rust_tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)
								snake_case__    :							List[str]							    =	rust_tokenizer.tokenize(snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)
								snake_case__    :							List[Any]							    =	rust_tokenizer.convert_ids_to_tokens(snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)

								# fmt: off
								snake_case__    :							int							    =	'I was born in 92000, and this is falsé.'
								snake_case__    :							Union[str, Any]							    =	[1_3, 1, 2_3, 3_8_6, 1_9, 5_6_1, 3_0_5_0, 1_5, 1_7, 4_8, 2_5, 8_2_5_6, 1_8, 1, 9]
								snake_case__    :							int							    =	['▁', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.', ]
								snake_case__    :							Dict							    =	['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
								# fmt: on

								snake_case__    :							Union[str, Any]							    =	tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)
								snake_case__    :							Union[str, Any]							    =	tokenizer.tokenize(snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)
								snake_case__    :							int							    =	tokenizer.convert_ids_to_tokens(snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)

								snake_case__    :							Any							    =	rust_tokenizer.encode(snake_case_      ,      add_special_tokens=snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)
								snake_case__    :							Any							    =	rust_tokenizer.tokenize(snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)
								snake_case__    :							Tuple							    =	rust_tokenizer.convert_ids_to_tokens(snake_case_	)
								self.assertListEqual(snake_case_      ,      snake_case_	)







						def    __magic_name__				(				self       :   Optional[Any]	):



								'''simple docstring'''



								snake_case__    :							List[Any]							    =	DebertaVaTokenizer(snake_case_	)

								snake_case__    :							Dict							    =	tokenizer.encode('''sequence builders'''	)
								snake_case__    :							int							    =	tokenizer.encode('''multi-sequence build'''	)

								snake_case__    :							List[Any]							    =	tokenizer.build_inputs_with_special_tokens(snake_case_	)
								snake_case__    :							Optional[Any]							    =	tokenizer.build_inputs_with_special_tokens(snake_case_      ,      snake_case_	)

								self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]      ,      snake_case_	)
								self.assertEqual(
								    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]      ,      snake_case_      ,      )







						@slow
						def    __magic_name__				(				self       :   Union[str, Any]	):



								'''simple docstring'''



								snake_case__    :							List[str]							    =	{'input_ids': [[1, 3_9_8_6_7, 3_6, 1_9_3_9_0, 4_8_6, 2_7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 6_0_6_8_5, 1_2_2_5, 7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 9_3_6_7, 1_6_8_9_9, 1_8, 1_5_9_3_7, 5_3, 5_9_4, 7_7_3, 1_8, 1_6_2_8_7, 3_0_4_6_5, 3_6, 1_5_9_3_7, 6, 4_1_1_3_9, 3_8, 3_6_9_7_9, 6_0_7_6_3, 1_9_1, 6, 3_4_1_3_2, 9_9, 6, 5_0_5_3_8, 3_9_0, 4_3_2_3_0, 6, 3_4_1_3_2, 2_7_7_9, 2_0_8_5_0, 1_4, 6_9_9, 1_0_7_2, 1_1_9_4, 3_6, 3_8_2, 1_0_9_0_1, 5_3, 7, 6_9_9, 1_0_7_2, 2_0_8_4, 3_6, 2_0_4_2_2, 6_3_0, 5_3, 1_9, 1_0_5, 3_0_4_9, 1_8_9_6, 1_0_5_3, 1_6_8_9_9, 1_5_0_6, 1_1, 3_7_9_7_8, 4_2_4_3, 7, 1_2_3_7, 3_1_8_6_9, 2_0_0, 1_6_5_6_6, 6_5_4, 6, 3_5_0_5_2, 8_1_4_3_6, 7, 5_5_6_3_0, 1_3_5_9_3, 4, 2], [1, 2_6, 1_5_0_1_1, 1_3, 6_6_7, 8, 1_0_5_3, 1_8, 2_3_6_1_1, 1_2_3_7, 7_2_3_5_6, 1_2_8_2_0, 3_4, 1_0_4_1_3_4, 1_2_0_9, 3_5, 1_3_3_1_3, 6_6_2_7, 2_1, 2_0_2, 3_4_7, 7, 1_6_4, 2_3_9_9, 1_1, 4_6, 4_4_8_5, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_2_3_2, 2_8_6_4, 1_5_7_8_5, 1_4_9_5_1, 1_0_5, 5, 8_5_8_1, 1_2_5_0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=snake_case_      ,      model_name='''microsoft/deberta-v2-xlarge'''      ,      revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670'''      ,      )








"""simple docstring"""







def     UpperCAmelCase_					(				__a   :			int						):




			'''simple docstring'''



			_lowerCamelCase					:					Optional[Any]	       =    int(__a						)
			if decimal in (0, 1):  # Exit cases for the recursion
						return str(__a						)
			_lowerCamelCase       ,							_lowerCamelCase					:					Union[str, Any]	       =    divmod(__a      ,							2						)
			return binary_recursive(__a						) + str(__a						)

def     UpperCAmelCase_					(				__a   :			str						):




			'''simple docstring'''



			_lowerCamelCase					:					int	       =    str(__a						).strip()
			if not number:
						raise ValueError('No input value was provided'						)
			_lowerCamelCase					:					Tuple	       =    '-' if number.startswith('-'						) else ''
			_lowerCamelCase					:					List[Any]	       =    number.lstrip('-'						)
			if not number.isnumeric():
						raise ValueError('Input value is not an integer'						)
			return f"{negative}0b{binary_recursive(int(__a						)						)}"


if __name__ == "__main__":
	from doctest import testmod

	testmod()





