
from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
	from transformers import (
	    TFAutoModel,
	    TFAutoModelForCausalLM,
	    TFAutoModelForMaskedLM,
	    TFAutoModelForPreTraining,
	    TFAutoModelForQuestionAnswering,
	    TFAutoModelForSeqaSeqLM,
	    TFAutoModelForSequenceClassification,
	    TFAutoModelForTableQuestionAnswering,
	    TFAutoModelForTokenClassification,
	    TFAutoModelWithLMHead,
	    TFBertForMaskedLM,
	    TFBertForPreTraining,
	    TFBertForQuestionAnswering,
	    TFBertForSequenceClassification,
	    TFBertModel,
	    TFFunnelBaseModel,
	    TFFunnelModel,
	    TFGPTaLMHeadModel,
	    TFRobertaForMaskedLM,
	    TFTaForConditionalGeneration,
	    TFTapasForQuestionAnswering,
	)
	from transformers.models.auto.modeling_tf_auto import (
	    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
	    TF_MODEL_FOR_MASKED_LM_MAPPING,
	    TF_MODEL_FOR_PRETRAINING_MAPPING,
	    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
	    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
	    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
	    TF_MODEL_MAPPING,
	)
	from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
	from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
	from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
	from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST







class 			A_  (     __lowerCamelCase  ):




	'''simple docstring'''


	_UpperCamelCase	:	Dict									=			"""new-model"""


if is_tf_available():







	class 			A_  (     __lowerCamelCase  ):




		'''simple docstring'''


		_UpperCamelCase	:	Tuple									=			NewModelConfig







@require_tf
class 			A_  (     unittest.TestCase  ):




	'''simple docstring'''


	@slow
	def       SCREAMING_SNAKE_CASE__		(			self			):
		lowercase	    =       'bert-base-cased'
		lowercase	    =       AutoConfig.from_pretrained(snake_case			)
		self.assertIsNotNone(snake_case			)
		self.assertIsInstance(snake_case						,       snake_case			)

		lowercase	    =       TFAutoModel.from_pretrained(snake_case			)
		self.assertIsNotNone(snake_case			)
		self.assertIsInstance(snake_case						,       snake_case			)





	@slow
	def       SCREAMING_SNAKE_CASE__		(			self			):
		lowercase	    =       'bert-base-cased'
		lowercase	    =       AutoConfig.from_pretrained(snake_case			)
		self.assertIsNotNone(snake_case			)
		self.assertIsInstance(snake_case						,       snake_case			)

		lowercase	    =       TFAutoModelForPreTraining.from_pretrained(snake_case			)
		self.assertIsNotNone(snake_case			)
		self.assertIsInstance(snake_case						,       snake_case			)





	@slow
	def       SCREAMING_SNAKE_CASE__		(			self			):
		for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
			lowercase	    =       AutoConfig.from_pretrained(snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)

			lowercase	    =       TFAutoModelForCausalLM.from_pretrained(snake_case			)
			lowercase    ,   lowercase	    =       TFAutoModelForCausalLM.from_pretrained(snake_case						,       output_loading_info=snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)





	@slow
	def       SCREAMING_SNAKE_CASE__		(			self			):
		for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
			lowercase	    =       AutoConfig.from_pretrained(snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)

			lowercase	    =       TFAutoModelWithLMHead.from_pretrained(snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)





	@slow
	def       SCREAMING_SNAKE_CASE__		(			self			):
		for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
			lowercase	    =       AutoConfig.from_pretrained(snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)

			lowercase	    =       TFAutoModelForMaskedLM.from_pretrained(snake_case			)
			lowercase    ,   lowercase	    =       TFAutoModelForMaskedLM.from_pretrained(snake_case						,       output_loading_info=snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)





	@slow
	def       SCREAMING_SNAKE_CASE__		(			self			):
		for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
			lowercase	    =       AutoConfig.from_pretrained(snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)

			lowercase	    =       TFAutoModelForSeqaSeqLM.from_pretrained(snake_case			)
			lowercase    ,   lowercase	    =       TFAutoModelForSeqaSeqLM.from_pretrained(snake_case						,       output_loading_info=snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)





	@slow
	def       SCREAMING_SNAKE_CASE__		(			self			):
		# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
		for model_name in ["bert-base-uncased"]:
			lowercase	    =       AutoConfig.from_pretrained(snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)

			lowercase	    =       TFAutoModelForSequenceClassification.from_pretrained(snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)





	@slow
	def       SCREAMING_SNAKE_CASE__		(			self			):
		# for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
		for model_name in ["bert-base-uncased"]:
			lowercase	    =       AutoConfig.from_pretrained(snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)

			lowercase	    =       TFAutoModelForQuestionAnswering.from_pretrained(snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)





	@slow
	@require_tensorflow_probability
	def       SCREAMING_SNAKE_CASE__		(			self			):
		for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
			lowercase	    =       AutoConfig.from_pretrained(snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)

			lowercase	    =       TFAutoModelForTableQuestionAnswering.from_pretrained(snake_case			)
			lowercase    ,   lowercase	    =       TFAutoModelForTableQuestionAnswering.from_pretrained(
			    snake_case						,       output_loading_info=snake_case			)
			self.assertIsNotNone(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)





	def       SCREAMING_SNAKE_CASE__		(			self			):
		lowercase	    =       TFAutoModelWithLMHead.from_pretrained(snake_case			)
		self.assertIsInstance(snake_case						,       snake_case			)
		self.assertEqual(model.num_parameters()						,       1_4410			)
		self.assertEqual(model.num_parameters(only_trainable=snake_case			)						,       1_4410			)





	def       SCREAMING_SNAKE_CASE__		(			self			):
		lowercase	    =       TFAutoModelWithLMHead.from_pretrained(snake_case			)
		self.assertIsInstance(snake_case						,       snake_case			)
		self.assertEqual(model.num_parameters()						,       1_4410			)
		self.assertEqual(model.num_parameters(only_trainable=snake_case			)						,       1_4410			)





	def       SCREAMING_SNAKE_CASE__		(			self			):
		# For the auto model mapping, FunnelConfig has two models: FunnelModel and FunnelBaseModel
		lowercase	    =       TFAutoModel.from_pretrained('sgugger/funnel-random-tiny'			)
		self.assertIsInstance(snake_case						,       snake_case			)

		lowercase	    =       copy.deepcopy(model.config			)
		lowercase	    =       ['FunnelBaseModel']
		lowercase	    =       TFAutoModel.from_config(snake_case			)
		self.assertIsInstance(snake_case						,       snake_case			)

		with tempfile.TemporaryDirectory() as tmp_dir:
			model.save_pretrained(snake_case			)
			lowercase	    =       TFAutoModel.from_pretrained(snake_case			)
			self.assertIsInstance(snake_case						,       snake_case			)





	def       SCREAMING_SNAKE_CASE__		(			self			):
		try:
			AutoConfig.register('new-model'						,       snake_case			)

			lowercase	    =       [
			    TFAutoModel,
			    TFAutoModelForCausalLM,
			    TFAutoModelForMaskedLM,
			    TFAutoModelForPreTraining,
			    TFAutoModelForQuestionAnswering,
			    TFAutoModelForSequenceClassification,
			    TFAutoModelForTokenClassification,
			]

			for auto_class in auto_classes:
				with self.subTest(auto_class.__name__			):
					# Wrong config class will raise an error
					with self.assertRaises(snake_case			):
						auto_class.register(snake_case						,       snake_case			)
					auto_class.register(snake_case						,       snake_case			)
					# Trying to register something existing in the Transformers library will raise an error
					with self.assertRaises(snake_case			):
						auto_class.register(snake_case						,       snake_case			)

					# Now that the config is registered, it can be used as any other config with the auto-API
					lowercase	    =       BertModelTester(self			).get_config()
					lowercase	    =       NewModelConfig(**tiny_config.to_dict()			)
					lowercase	    =       auto_class.from_config(snake_case			)
					self.assertIsInstance(snake_case						,       snake_case			)

					with tempfile.TemporaryDirectory() as tmp_dir:
						model.save_pretrained(snake_case			)
						lowercase	    =       auto_class.from_pretrained(snake_case			)
						self.assertIsInstance(snake_case						,       snake_case			)

		finally:
			if "new-model" in CONFIG_MAPPING._extra_content:
				del CONFIG_MAPPING._extra_content["new-model"]
			for mapping in (
			    TF_MODEL_MAPPING,
			    TF_MODEL_FOR_PRETRAINING_MAPPING,
			    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
			    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
			    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
			    TF_MODEL_FOR_CAUSAL_LM_MAPPING,
			    TF_MODEL_FOR_MASKED_LM_MAPPING,
			):
				if NewModelConfig in mapping._extra_content:
					del mapping._extra_content[NewModelConfig]





	def       SCREAMING_SNAKE_CASE__		(			self			):
		with self.assertRaisesRegex(
		    snake_case						,       'bert-base is not a local folder and is not a valid model identifier'			):
			lowercase	    =       TFAutoModel.from_pretrained('bert-base'			)





	def       SCREAMING_SNAKE_CASE__		(			self			):
		with self.assertRaisesRegex(
		    snake_case						,       r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'			):
			lowercase	    =       TFAutoModel.from_pretrained(snake_case						,       revision='aaaaaa'			)





	def       SCREAMING_SNAKE_CASE__		(			self			):
		with self.assertRaisesRegex(
		    snake_case						,       'hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'						,       ):
			lowercase	    =       TFAutoModel.from_pretrained('hf-internal-testing/config-no-model'			)





	def       SCREAMING_SNAKE_CASE__		(			self			):
		with self.assertRaisesRegex(snake_case						,       'Use `from_pt=True` to load this model'			):
			lowercase	    =       TFAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only'			)





	def       SCREAMING_SNAKE_CASE__		(			self			):
		# Make sure we have cached the model.
		lowercase	    =       TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'			)
		with RequestCounter() as counter:
			lowercase	    =       TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert'			)
			self.assertEqual(counter.get_request_count						,       0			)
			self.assertEqual(counter.head_request_count						,       1			)
			self.assertEqual(counter.other_request_count						,       0			)

		# With a sharded checkpoint
		lowercase	    =       TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'			)
		with RequestCounter() as counter:
			lowercase	    =       TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded'			)
			self.assertEqual(counter.get_request_count						,       0			)
			self.assertEqual(counter.head_request_count						,       1			)
			self.assertEqual(counter.other_request_count						,       0			)


import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCAmelCase       =		get_tests_dir("fixtures/spiece.model")
@require_sentencepiece
@require_tokenizers
class     lowerCAmelCase_     (       __lowercase,  unittest.TestCase ):
							UpperCAmelCase				   =     DebertaVaTokenizer
							UpperCAmelCase				   =     DebertaVaTokenizerFast
							UpperCAmelCase				   =     True
							UpperCAmelCase				   =     True







							def  UpperCamelCase_						(   self      :     List[Any]		):
												super().setUp()

												# We have a SentencePiece fixture for testing
												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			unk_token='''<unk>'''		)
												tokenizer.save_pretrained(self.tmpdirname		)







							def  UpperCamelCase_						(   self      :     Dict ,			_A      :     Union[str, Any]		):
												_UpperCamelCase			     =     '''this is a test'''
												_UpperCamelCase			     =     '''this is a test'''
												return input_text, output_text







							def  UpperCamelCase_						(   self      :     Optional[Any]		):
												_UpperCamelCase			     =     '''<pad>'''
												_UpperCamelCase			     =     0

												self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A		) ,			_A		)
												self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A		) ,			_A		)







							def  UpperCamelCase_						(   self      :     Any		):
												_UpperCamelCase			     =     list(self.get_tokenizer().get_vocab().keys()		)
												self.assertEqual(vocab_keys[0] ,			'''<pad>'''		)
												self.assertEqual(vocab_keys[1] ,			'''<unk>'''		)
												self.assertEqual(vocab_keys[-1] ,			'''[PAD]'''		)
												self.assertEqual(len(_A		) ,			3_0001		)







							def  UpperCamelCase_						(   self      :     List[Any]		):
												self.assertEqual(self.get_tokenizer().vocab_size ,			3_0000		)







							def  UpperCamelCase_						(   self      :     List[str]		):
												# fmt: off
												_UpperCamelCase			     =     ''' \tHeLLo!how  \n Are yoU?  '''
												_UpperCamelCase			     =     ['''▁hello''', '''!''', '''how''', '''▁are''', '''▁you''', '''?''']
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''		)
							def  UpperCamelCase_						(   self      :     Dict		):
												pass







							@unittest.skip('''There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'''		)
							def  UpperCamelCase_						(   self      :     Optional[Any]		):
												pass







							def  UpperCamelCase_						(   self      :     Dict		):
												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     List[Any]		):
												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)
												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Dict		):
												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     int		):
												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', '''▁''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''▁''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Tuple		):
												# fmt: off
												_UpperCamelCase			     =     ''' \tHeLLo!how  \n Are yoU?  '''
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''e''', '''<unk>''', '''o''', '''!''', '''how''', '''▁''', '''<unk>''', '''re''', '''▁yo''', '''<unk>''', '''?''']
												# fmt: on

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			do_lower_case=_A ,			split_by_punct=_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)

												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     List[str]		):
												_UpperCamelCase			     =     self.get_tokenizer()
												_UpperCamelCase			     =     self.get_rust_tokenizer()

												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''

												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(tokenizer.encode(_A ,			add_special_tokens=_A		)		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_A ,			add_special_tokens=_A		)		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     tokenizer.encode(_A ,			add_special_tokens=_A		)
												_UpperCamelCase			     =     rust_tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     self.get_rust_tokenizer()
												_UpperCamelCase			     =     tokenizer.encode(_A		)
												_UpperCamelCase			     =     rust_tokenizer.encode(_A		)
												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Dict		):
												_UpperCamelCase			     =     '''This is a test'''
												_UpperCamelCase			     =     [13, 1, 4398, 25, 21, 1289]
												_UpperCamelCase			     =     ['''▁''', '''T''', '''his''', '''▁is''', '''▁a''', '''▁test''']
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''his''', '''▁is''', '''▁a''', '''▁test''']

												_UpperCamelCase			     =     DebertaVaTokenizer(_A ,			keep_accents=_A		)
												_UpperCamelCase			     =     DebertaVaTokenizerFast(_A ,			keep_accents=_A		)

												_UpperCamelCase			     =     tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     tokenizer.tokenize(_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(_A		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     rust_tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     rust_tokenizer.tokenize(_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(_A		)
												self.assertListEqual(_A ,			_A		)

												# fmt: off
												_UpperCamelCase			     =     '''I was born in 92000, and this is falsé.'''
												_UpperCamelCase			     =     [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
												_UpperCamelCase			     =     ['''▁''', '''I''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''', ]
												_UpperCamelCase			     =     ['''▁''', '''<unk>''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''', ]
												# fmt: on

												_UpperCamelCase			     =     tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     tokenizer.tokenize(_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     tokenizer.convert_ids_to_tokens(_A		)
												self.assertListEqual(_A ,			_A		)

												_UpperCamelCase			     =     rust_tokenizer.encode(_A ,			add_special_tokens=_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     rust_tokenizer.tokenize(_A		)
												self.assertListEqual(_A ,			_A		)
												_UpperCamelCase			     =     rust_tokenizer.convert_ids_to_tokens(_A		)
												self.assertListEqual(_A ,			_A		)







							def  UpperCamelCase_						(   self      :     Any		):
												_UpperCamelCase			     =     DebertaVaTokenizer(_A		)

												_UpperCamelCase			     =     tokenizer.encode('''sequence builders'''		)
												_UpperCamelCase			     =     tokenizer.encode('''multi-sequence build'''		)

												_UpperCamelCase			     =     tokenizer.build_inputs_with_special_tokens(_A		)
												_UpperCamelCase			     =     tokenizer.build_inputs_with_special_tokens(_A ,			_A		)

												self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] ,			_A		)
												self.assertEqual(
												    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id] ,			_A ,			)







							@slow
							def  UpperCamelCase_						(   self      :     Optional[Any]		):
												# fmt: off
												_UpperCamelCase			     =     {'''input_ids''': [[1, 3_9867, 36, 1_9390, 486, 27, 3_5052, 8_1436, 18, 6_0685, 1225, 7, 3_5052, 8_1436, 18, 9367, 1_6899, 18, 1_5937, 53, 594, 773, 18, 1_6287, 3_0465, 36, 1_5937, 6, 4_1139, 38, 3_6979, 6_0763, 191, 6, 3_4132, 99, 6, 5_0538, 390, 4_3230, 6, 3_4132, 2779, 2_0850, 14, 699, 1072, 1194, 36, 382, 1_0901, 53, 7, 699, 1072, 2084, 36, 2_0422, 630, 53, 19, 105, 3049, 1896, 1053, 1_6899, 1506, 11, 3_7978, 4243, 7, 1237, 3_1869, 200, 1_6566, 654, 6, 3_5052, 8_1436, 7, 5_5630, 1_3593, 4, 2], [1, 26, 1_5011, 13, 667, 8, 1053, 18, 2_3611, 1237, 7_2356, 1_2820, 34, 10_4134, 1209, 35, 1_3313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 1_5785, 1_4951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
												# fmt: on

												self.tokenizer_integration_test_util(
												    expected_encoding=_A ,			model_name='''microsoft/deberta-v2-xlarge''' ,			revision='''ad6e42c1532ddf3a15c39246b63f5559d558b670''' ,			)
