
import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin

class   lowerCAmelCase__	(      lowercase__    ,      unittest.TestCase     ):
		a__			:		Dict     =						DebertaTokenizer
		a__			:		Optional[int]     =						True
		a__			:		List[str]     =						DebertaTokenizerFast






		def 					__A			(						self						:				Any    )	->				str:
						super().setUp()

						# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
						__lowerCamelCase          =      [
						    '''l''',
						    '''o''',
						    '''w''',
						    '''e''',
						    '''r''',
						    '''s''',
						    '''t''',
						    '''i''',
						    '''d''',
						    '''n''',
						    '''\u0120''',
						    '''\u0120l''',
						    '''\u0120n''',
						    '''\u0120lo''',
						    '''\u0120low''',
						    '''er''',
						    '''\u0120lowest''',
						    '''\u0120newer''',
						    '''\u0120wider''',
						    '''[UNK]''',
						]
						__lowerCamelCase          =      dict(zip(SCREAMING_SNAKE_CASE_				,     range(len(SCREAMING_SNAKE_CASE_    )    )    )    )
						__lowerCamelCase          =      ['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
						__lowerCamelCase          =      {'''unk_token''': '''[UNK]'''}

						__lowerCamelCase          =      os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES['''vocab_file''']    )
						__lowerCamelCase          =      os.path.join(self.tmpdirname				,     VOCAB_FILES_NAMES['''merges_file''']    )
						with open(self.vocab_file				,     '''w'''				,     encoding='''utf-8'''    ) as fp:
										fp.write(json.dumps(SCREAMING_SNAKE_CASE_    ) + '''\n'''    )
						with open(self.merges_file				,     '''w'''				,     encoding='''utf-8'''    ) as fp:
										fp.write('''\n'''.join(SCREAMING_SNAKE_CASE_    )    )






		def 					__A			(						self						:				Tuple				,     **SCREAMING_SNAKE_CASE__						:				str    )	->				List[str]:
						kwargs.update(self.special_tokens_map    )
						return self.tokenizer_class.from_pretrained(self.tmpdirname				,     **SCREAMING_SNAKE_CASE_    )






		def 					__A			(						self						:				Tuple				,     SCREAMING_SNAKE_CASE__						:				Optional[Any]    )	->				Optional[int]:
						__lowerCamelCase          =      '''lower newer'''
						__lowerCamelCase          =      '''lower newer'''
						return input_text, output_text






		def 					__A			(						self						:				str    )	->				Any:
						__lowerCamelCase          =      self.get_tokenizer()
						__lowerCamelCase          =      '''lower newer'''
						__lowerCamelCase          =      ['''l''', '''o''', '''w''', '''er''', '''\u0120''', '''n''', '''e''', '''w''', '''er''']
						__lowerCamelCase          =      tokenizer.tokenize(SCREAMING_SNAKE_CASE_    )
						self.assertListEqual(SCREAMING_SNAKE_CASE_				,     SCREAMING_SNAKE_CASE_    )

						__lowerCamelCase          =      tokens + [tokenizer.unk_token]
						__lowerCamelCase          =      [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
						self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_    )				,     SCREAMING_SNAKE_CASE_    )






		def 					__A			(						self						:				Dict    )	->				Dict:
						__lowerCamelCase          =      self.get_tokenizer()
						__lowerCamelCase          =      tokenizer('''Hello'''				,     '''World'''    )
						__lowerCamelCase          =      [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
						self.assertListEqual(tokd['''token_type_ids''']				,     SCREAMING_SNAKE_CASE_    )






		@slow
		def 					__A			(						self						:				List[str]    )	->				Tuple:
						__lowerCamelCase          =      self.tokenizer_class.from_pretrained('''microsoft/deberta-base'''    )

						__lowerCamelCase          =      tokenizer.encode('''sequence builders'''				,     add_special_tokens=SCREAMING_SNAKE_CASE_    )
						__lowerCamelCase          =      tokenizer.encode('''multi-sequence build'''				,     add_special_tokens=SCREAMING_SNAKE_CASE_    )

						__lowerCamelCase          =      tokenizer.encode(
						    '''sequence builders'''				,     add_special_tokens=SCREAMING_SNAKE_CASE_				,     add_prefix_space=SCREAMING_SNAKE_CASE_    )
						__lowerCamelCase          =      tokenizer.encode(
						    '''sequence builders'''				,     '''multi-sequence build'''				,     add_special_tokens=SCREAMING_SNAKE_CASE_				,     add_prefix_space=SCREAMING_SNAKE_CASE_    )

						__lowerCamelCase          =      tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_    )
						__lowerCamelCase          =      tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE_				,     SCREAMING_SNAKE_CASE_    )

						assert encoded_sentence == encoded_text_from_decode
						assert encoded_pair == encoded_pair_from_decode







		@slow
		def 					__A			(						self						:				List[str]    )	->				List[str]:
						__lowerCamelCase          =      [self.tokenizer_class]
						if self.test_rust_tokenizer:
										tokenizer_classes.append(self.rust_tokenizer_class    )

						for tokenizer_class in tokenizer_classes:
										__lowerCamelCase          =      tokenizer_class.from_pretrained('''microsoft/deberta-base'''    )

										__lowerCamelCase          =      [
										    '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
										    '''ALBERT incorporates two parameter reduction techniques''',
										    '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
										    ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
										    ''' vocabulary embedding.''',
										]

										__lowerCamelCase          =      tokenizer(SCREAMING_SNAKE_CASE_				,     padding=SCREAMING_SNAKE_CASE_    )
										__lowerCamelCase          =      [tokenizer.decode(SCREAMING_SNAKE_CASE_				,     skip_special_tokens=SCREAMING_SNAKE_CASE_    ) for seq in encoding['''input_ids''']]

										# fmt: off
										__lowerCamelCase          =      {
										    '''input_ids''': [
										        [1, 21_18, 1_11_26, 5_65, 35, 83, 2_51_91, 1_63, 1_88_54, 13, 1_21_56, 12, 1_61_01, 2_53_76, 1_38_07, 9, 2_22_05, 2_78_93, 16_35, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
										        [1, 21_18, 1_11_26, 5_65, 2_45_36, 80, 4_37_97, 48_78, 73_73, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
										        [1, 1_33, 78, 65, 16, 10, 37_24, 15_38, 3_31_83, 1_13_03, 4_37_97, 19_38, 4, 8_70, 2_41_65, 2_91_05, 5, 7_39, 3_26_44, 3_31_83, 1_13_03, 3_61_73, 88, 80, 6_50, 78_21, 4_59_40, 6, 52, 25_59, 5, 18_36, 9, 5, 73_97, 1_31_71, 31, 5, 18_36, 9, 3_26_44, 3_31_83, 1_13_03, 4, 2]
										    ],
										    '''token_type_ids''': [
										        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
										        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
										        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
										    ],
										    '''attention_mask''': [
										        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
										        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
										        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
										    ]
										}
										# fmt: on

										__lowerCamelCase          =      [
										    '''ALBERT: A Lite BERT for Self-supervised Learning of Language Representations''',
										    '''ALBERT incorporates two parameter reduction techniques''',
										    '''The first one is a factorized embedding parameterization. By decomposing the large vocabulary'''
										    ''' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'''
										    ''' vocabulary embedding.''',
										]

										self.assertDictEqual(encoding.data				,     SCREAMING_SNAKE_CASE_    )

										for expected, decoded in zip(SCREAMING_SNAKE_CASE_				,     SCREAMING_SNAKE_CASE_    ):
														self.assertEqual(SCREAMING_SNAKE_CASE_				,     SCREAMING_SNAKE_CASE_    )





"""simple docstring"""






from ..utils import DummyObject, requires_backends



class 			lowercase					( metaclass=lowercase__		):
			lowercase        =     ['''flax''', '''transformers''']






			def __init__(self  :	List[Any]      ,*SCREAMING_SNAKE_CASE_  :	Union[str, Any]      ,**SCREAMING_SNAKE_CASE_  :	Union[str, Any]			)       ->	str:





				"""simple docstring"""


				requires_backends(self      ,['''flax''', '''transformers''']			)






			@classmethod
			def 					UpperCAmelCase							(cls  :	List[Any]      ,*SCREAMING_SNAKE_CASE_  :	List[Any]      ,**SCREAMING_SNAKE_CASE_  :	int			)       ->	Optional[int]:





				"""simple docstring"""


				requires_backends(cls      ,['''flax''', '''transformers''']			)






			@classmethod
			def 					UpperCAmelCase							(cls  :	List[Any]      ,*SCREAMING_SNAKE_CASE_  :	str      ,**SCREAMING_SNAKE_CASE_  :	Union[str, Any]			)       ->	Union[str, Any]:





				"""simple docstring"""


				requires_backends(cls      ,['''flax''', '''transformers''']			)





class 			lowercase					( metaclass=lowercase__		):
			lowercase        =     ['''flax''', '''transformers''']






			def __init__(self  :	int      ,*SCREAMING_SNAKE_CASE_  :	List[Any]      ,**SCREAMING_SNAKE_CASE_  :	Any			)       ->	Union[str, Any]:





				"""simple docstring"""


				requires_backends(self      ,['''flax''', '''transformers''']			)






			@classmethod
			def 					UpperCAmelCase							(cls  :	int      ,*SCREAMING_SNAKE_CASE_  :	str      ,**SCREAMING_SNAKE_CASE_  :	int			)       ->	Optional[Any]:





				"""simple docstring"""


				requires_backends(cls      ,['''flax''', '''transformers''']			)






			@classmethod
			def 					UpperCAmelCase							(cls  :	List[str]      ,*SCREAMING_SNAKE_CASE_  :	List[str]      ,**SCREAMING_SNAKE_CASE_  :	Optional[Any]			)       ->	Dict:





				"""simple docstring"""


				requires_backends(cls      ,['''flax''', '''transformers''']			)





class 			lowercase					( metaclass=lowercase__		):
			lowercase        =     ['''flax''', '''transformers''']






			def __init__(self  :	Tuple      ,*SCREAMING_SNAKE_CASE_  :	Optional[int]      ,**SCREAMING_SNAKE_CASE_  :	Dict			)       ->	List[str]:





				"""simple docstring"""


				requires_backends(self      ,['''flax''', '''transformers''']			)






			@classmethod
			def 					UpperCAmelCase							(cls  :	Tuple      ,*SCREAMING_SNAKE_CASE_  :	Dict      ,**SCREAMING_SNAKE_CASE_  :	Optional[Any]			)       ->	List[Any]:





				"""simple docstring"""


				requires_backends(cls      ,['''flax''', '''transformers''']			)






			@classmethod
			def 					UpperCAmelCase							(cls  :	Tuple      ,*SCREAMING_SNAKE_CASE_  :	Tuple      ,**SCREAMING_SNAKE_CASE_  :	Tuple			)       ->	Union[str, Any]:





				"""simple docstring"""


				requires_backends(cls      ,['''flax''', '''transformers''']			)





class 			lowercase					( metaclass=lowercase__		):
			lowercase        =     ['''flax''', '''transformers''']






			def __init__(self  :	Optional[Any]      ,*SCREAMING_SNAKE_CASE_  :	Any      ,**SCREAMING_SNAKE_CASE_  :	str			)       ->	Union[str, Any]:





				"""simple docstring"""


				requires_backends(self      ,['''flax''', '''transformers''']			)






			@classmethod
			def 					UpperCAmelCase							(cls  :	List[Any]      ,*SCREAMING_SNAKE_CASE_  :	Optional[Any]      ,**SCREAMING_SNAKE_CASE_  :	Optional[int]			)       ->	Union[str, Any]:





				"""simple docstring"""


				requires_backends(cls      ,['''flax''', '''transformers''']			)






			@classmethod
			def 					UpperCAmelCase							(cls  :	List[Any]      ,*SCREAMING_SNAKE_CASE_  :	List[Any]      ,**SCREAMING_SNAKE_CASE_  :	List[Any]			)       ->	Optional[Any]:





				"""simple docstring"""


				requires_backends(cls      ,['''flax''', '''transformers''']			)


