
import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_						:     List[Any]	       =							get_tests_dir("""fixtures/spiece.model""")




@require_sentencepiece
@require_tokenizers
class   __lowercase						(						__snake_case   , unittest.TestCase       ):
	_A       =   DebertaVaTokenizer
	_A       =   DebertaVaTokenizerFast
	_A       =   True
	_A       =   True






	def       _a(self				: str    ) ->			Tuple:
							super().setUp()

							# We have a SentencePiece fixture for testing
							_lowercase       :	Tuple           =   DebertaVaTokenizer(snake_case      ,	unk_token="<unk>"    )
							tokenizer.save_pretrained(self.tmpdirname    )






	def       _a(self				: Optional[Any]      ,	snake_case				: Optional[int]    ) ->			Union[str, Any]:
							_lowercase       :	Union[str, Any]           =   "this is a test"
							_lowercase       :	Optional[int]           =   "this is a test"
							return input_text, output_text






	def       _a(self				: List[Any]    ) ->			Union[str, Any]:
							_lowercase       :	int           =   "<pad>"
							_lowercase       :	Optional[Any]           =   0

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case    )      ,	snake_case    )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case    )      ,	snake_case    )






	def       _a(self				: Any    ) ->			Optional[Any]:
							_lowercase       :	Optional[int]           =   list(self.get_tokenizer().get_vocab().keys()    )
							self.assertEqual(vocab_keys[0]      ,	"<pad>"    )
							self.assertEqual(vocab_keys[1]      ,	"<unk>"    )
							self.assertEqual(vocab_keys[-1]      ,	"[PAD]"    )
							self.assertEqual(len(snake_case    )      ,	3_0001    )






	def       _a(self				: Optional[int]    ) ->			List[Any]:
							self.assertEqual(self.get_tokenizer().vocab_size      ,	3_0000    )






	def       _a(self				: str    ) ->			Optional[Any]:
							# fmt: off
							_lowercase       :	Optional[int]           =   " \tHeLLo!how  \n Are yoU?  "
							_lowercase       :	Union[str, Any]           =   ["▁hello", "!", "how", "▁are", "▁you", "?"]
							# fmt: on

							_lowercase       :	Any           =   DebertaVaTokenizer(snake_case      ,	do_lower_case=snake_case    )
							_lowercase       :	Dict           =   tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )

							self.assertListEqual(snake_case      ,	snake_case    )

							_lowercase       :	List[str]           =   DebertaVaTokenizerFast(snake_case      ,	do_lower_case=snake_case    )
							_lowercase       :	Union[str, Any]           =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )

							self.assertListEqual(snake_case      ,	snake_case    )






	@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."    )
	def       _a(self				: Union[str, Any]    ) ->			List[Any]:
							pass






	@unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."    )
	def       _a(self				: Tuple    ) ->			Union[str, Any]:
							pass






	def       _a(self				: Tuple    ) ->			Optional[Any]:
							# fmt: off
							_lowercase       :	Any           =   "I was born in 92000, and this is falsé."
							_lowercase       :	Optional[int]           =   ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
							# fmt: on

							_lowercase       :	int           =   DebertaVaTokenizer(snake_case      ,	split_by_punct=snake_case    )
							_lowercase       :	Tuple           =   tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )

							self.assertListEqual(snake_case      ,	snake_case    )

							_lowercase       :	Any           =   DebertaVaTokenizerFast(snake_case      ,	split_by_punct=snake_case    )
							_lowercase       :	List[str]           =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )

							self.assertListEqual(snake_case      ,	snake_case    )






	def       _a(self				: List[Any]    ) ->			List[Any]:
							# fmt: off
							_lowercase       :	List[str]           =   "I was born in 92000, and this is falsé."
							_lowercase       :	Optional[Any]           =   ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
							# fmt: on

							_lowercase       :	Dict           =   DebertaVaTokenizer(snake_case      ,	do_lower_case=snake_case      ,	split_by_punct=snake_case    )
							_lowercase       :	Optional[int]           =   tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )
							self.assertListEqual(snake_case      ,	snake_case    )

							_lowercase       :	Optional[Any]           =   DebertaVaTokenizerFast(snake_case      ,	do_lower_case=snake_case      ,	split_by_punct=snake_case    )
							_lowercase       :	List[Any]           =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )
							self.assertListEqual(snake_case      ,	snake_case    )






	def       _a(self				: Optional[Any]    ) ->			Tuple:
							# fmt: off
							_lowercase       :	Optional[int]           =   "I was born in 92000, and this is falsé."
							_lowercase       :	Tuple           =   ["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
							# fmt: on

							_lowercase       :	Optional[int]           =   DebertaVaTokenizer(snake_case      ,	do_lower_case=snake_case      ,	split_by_punct=snake_case    )
							_lowercase       :	Dict           =   tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )

							self.assertListEqual(snake_case      ,	snake_case    )

							_lowercase       :	List[Any]           =   DebertaVaTokenizerFast(snake_case      ,	do_lower_case=snake_case      ,	split_by_punct=snake_case    )
							_lowercase       :	Any           =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )

							self.assertListEqual(snake_case      ,	snake_case    )






	def       _a(self				: str    ) ->			Dict:
							# fmt: off
							_lowercase       :	Tuple           =   "I was born in 92000, and this is falsé."
							_lowercase       :	Any           =   ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
							# fmt: on

							_lowercase       :	Union[str, Any]           =   DebertaVaTokenizer(snake_case      ,	do_lower_case=snake_case      ,	split_by_punct=snake_case    )
							_lowercase       :	List[Any]           =   tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )

							self.assertListEqual(snake_case      ,	snake_case    )

							_lowercase       :	List[Any]           =   DebertaVaTokenizerFast(snake_case      ,	do_lower_case=snake_case      ,	split_by_punct=snake_case    )
							_lowercase       :	int           =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )

							self.assertListEqual(snake_case      ,	snake_case    )






	def       _a(self				: List[str]    ) ->			Tuple:
							# fmt: off
							_lowercase       :	Any           =   " \tHeLLo!how  \n Are yoU?  "
							_lowercase       :	str           =   ["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
							# fmt: on

							_lowercase       :	List[str]           =   DebertaVaTokenizer(snake_case      ,	do_lower_case=snake_case      ,	split_by_punct=snake_case    )
							_lowercase       :	int           =   tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )

							self.assertListEqual(snake_case      ,	snake_case    )

							_lowercase       :	int           =   DebertaVaTokenizerFast(snake_case      ,	do_lower_case=snake_case      ,	split_by_punct=snake_case    )
							_lowercase       :	List[Any]           =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )

							self.assertListEqual(snake_case      ,	snake_case    )






	def       _a(self				: Any    ) ->			Any:
							_lowercase       :	Union[str, Any]           =   self.get_tokenizer()
							_lowercase       :	Tuple           =   self.get_rust_tokenizer()

							_lowercase       :	Any           =   "I was born in 92000, and this is falsé."

							_lowercase       :	List[Any]           =   tokenizer.convert_ids_to_tokens(tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )
							_lowercase       :	Dict           =   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )    )
							self.assertListEqual(snake_case      ,	snake_case    )

							_lowercase       :	Any           =   tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )
							_lowercase       :	List[str]           =   rust_tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )

							_lowercase       :	Optional[int]           =   self.get_rust_tokenizer()
							_lowercase       :	List[Any]           =   tokenizer.encode(snake_case    )
							_lowercase       :	Any           =   rust_tokenizer.encode(snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )






	def       _a(self				: Union[str, Any]    ) ->			int:
							_lowercase       :	List[Any]           =   "This is a test"
							_lowercase       :	Union[str, Any]           =   [13, 1, 4398, 25, 21, 1289]
							_lowercase       :	Union[str, Any]           =   ["▁", "T", "his", "▁is", "▁a", "▁test"]
							_lowercase       :	List[str]           =   ["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

							_lowercase       :	Tuple           =   DebertaVaTokenizer(snake_case      ,	keep_accents=snake_case    )
							_lowercase       :	str           =   DebertaVaTokenizerFast(snake_case      ,	keep_accents=snake_case    )

							_lowercase       :	List[str]           =   tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )
							_lowercase       :	Optional[int]           =   tokenizer.tokenize(snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )
							_lowercase       :	List[str]           =   tokenizer.convert_ids_to_tokens(snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )

							_lowercase       :	Union[str, Any]           =   rust_tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )
							_lowercase       :	Optional[Any]           =   rust_tokenizer.tokenize(snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )
							_lowercase       :	List[str]           =   rust_tokenizer.convert_ids_to_tokens(snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )

							# fmt: off
							_lowercase       :	List[Any]           =   "I was born in 92000, and this is falsé."
							_lowercase       :	Tuple           =   [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
							_lowercase       :	Optional[int]           =   ["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
							_lowercase       :	Union[str, Any]           =   ["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
							# fmt: on

							_lowercase       :	Any           =   tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )
							_lowercase       :	List[str]           =   tokenizer.tokenize(snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )
							_lowercase       :	List[Any]           =   tokenizer.convert_ids_to_tokens(snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )

							_lowercase       :	Tuple           =   rust_tokenizer.encode(snake_case      ,	add_special_tokens=snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )
							_lowercase       :	List[Any]           =   rust_tokenizer.tokenize(snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )
							_lowercase       :	int           =   rust_tokenizer.convert_ids_to_tokens(snake_case    )
							self.assertListEqual(snake_case      ,	snake_case    )






	def       _a(self				: List[Any]    ) ->			Optional[Any]:
							_lowercase       :	List[str]           =   DebertaVaTokenizer(snake_case    )

							_lowercase       :	Optional[Any]           =   tokenizer.encode("sequence builders"    )
							_lowercase       :	Optional[int]           =   tokenizer.encode("multi-sequence build"    )

							_lowercase       :	int           =   tokenizer.build_inputs_with_special_tokens(snake_case    )
							_lowercase       :	List[str]           =   tokenizer.build_inputs_with_special_tokens(snake_case      ,	snake_case    )

							self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]      ,	snake_case    )
							self.assertEqual(
							    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]      ,	snake_case      ,	)






	@slow
	def       _a(self				: Any    ) ->			Dict:
							# fmt: off
							_lowercase       :	Optional[int]           =   {"input_ids": [[1, 3_9867, 36, 1_9390, 486, 27, 3_5052, 8_1436, 18, 6_0685, 1225, 7, 3_5052, 8_1436, 18, 9367, 1_6899, 18, 1_5937, 53, 594, 773, 18, 1_6287, 3_0465, 36, 1_5937, 6, 4_1139, 38, 3_6979, 6_0763, 191, 6, 3_4132, 99, 6, 5_0538, 390, 4_3230, 6, 3_4132, 2779, 2_0850, 14, 699, 1072, 1194, 36, 382, 1_0901, 53, 7, 699, 1072, 2084, 36, 2_0422, 630, 53, 19, 105, 3049, 1896, 1053, 1_6899, 1506, 11, 3_7978, 4243, 7, 1237, 3_1869, 200, 1_6566, 654, 6, 3_5052, 8_1436, 7, 5_5630, 1_3593, 4, 2], [1, 26, 1_5011, 13, 667, 8, 1053, 18, 2_3611, 1237, 7_2356, 1_2820, 34, 10_4134, 1209, 35, 1_3313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 1_5785, 1_4951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=snake_case      ,	model_name="microsoft/deberta-v2-xlarge"      ,	revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"      ,	)






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase_						:     Optional[int]	       =							logging.get_logger(__name__)




class   __lowercase						(						__snake_case       ):
	_A       =   "timm_backbone"






	def __init__(self				: Any      ,	snake_case				: List[Any]=None      ,	snake_case				: int=3      ,	snake_case				: Dict=True      ,	snake_case				: Union[str, Any]=True      ,	snake_case				: Any=None      ,	**snake_case				: int      ,	) ->			Optional[int]:
							super().__init__(**snake_case    )
							_lowercase       :	Dict           =   backbone
							_lowercase       :	Optional[Any]           =   num_channels
							_lowercase       :	Union[str, Any]           =   features_only
							_lowercase       :	Tuple           =   use_pretrained_backbone
							_lowercase       :	List[str]           =   True
							_lowercase       :	Tuple           =   out_indices if out_indices is not None else (-1,)




