"""simple docstring"""







from __future__ import annotations

import collections
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import is_tf_available, is_vision_available

from ...test_modeling_tf_common import floats_tensor, ids_tensor, random_attention_mask
from ..bert.test_modeling_tf_bert import TFBertModelTester
from ..clip.test_modeling_tf_clip import TFCLIPVisionModelTester
from ..deit.test_modeling_tf_deit import TFDeiTModelTester
from ..roberta.test_modeling_tf_roberta import TFRobertaModelTester
from ..vit.test_modeling_tf_vit import TFViTModelTester


if is_tf_available():
						from transformers import (
						    TFBertModel,
						    TFCLIPVisionModel,
						    TFDeiTModel,
						    TFRobertaModel,
						    TFVisionTextDualEncoderModel,
						    TFViTModel,
						    VisionTextDualEncoderConfig,
						)

if is_vision_available():
						from PIL import Image

						from transformers import VisionTextDualEncoderProcessor


def      __lowerCAmelCase(		__UpperCAmelCase			):






	"""simple docstring"""





	if isinstance(__UpperCAmelCase     ,collections.abc.Iterable			):
		return x
	return (x, x)



@require_tf
class  _lowerCamelCase			:







				def 							__UpperCAmelCase					( self    :       Dict			,					lowerCamelCase_    :       Union[str, Any]			,					lowerCamelCase_    :       List[str]						):

					"""simple docstring"""






					pass







				def 							__UpperCAmelCase					( self    :       Optional[Any]						):

					"""simple docstring"""






					pass







				def 							__UpperCAmelCase					( self    :       List[str]						):

					"""simple docstring"""






					pass







				def 							__UpperCAmelCase					( self    :       List[str]			,					lowerCamelCase_    :       Any			,					lowerCamelCase_    :       int			,					lowerCamelCase_    :       Tuple			,					lowerCamelCase_    :       List[str]			,					lowerCamelCase_    :       str=None			,					**lowerCamelCase_    :       str						):

					"""simple docstring"""






					_lowercase :    Union[str, Any]							=							VisionTextDualEncoderConfig.from_vision_text_configs(lowerCamelCase_			,					lowerCamelCase_						)

					_lowercase :    Optional[Any]							=							TFVisionTextDualEncoderModel(lowerCamelCase_						)

					_lowercase :    Optional[int]							=							model(input_ids=lowerCamelCase_			,					pixel_values=lowerCamelCase_			,					attention_mask=lowerCamelCase_						)

					self.assertEqual(output['text_embeds'].shape			,					(input_ids.shape[0], config.projection_dim)						)
					self.assertEqual(output['image_embeds'].shape			,					(pixel_values.shape[0], config.projection_dim)						)







				def 							__UpperCAmelCase					( self    :       Optional[int]			,					lowerCamelCase_    :       int			,					lowerCamelCase_    :       int			,					lowerCamelCase_    :       Any			,					lowerCamelCase_    :       List[Any]			,					lowerCamelCase_    :       Optional[int]=None			,					**lowerCamelCase_    :       Union[str, Any]						):

					"""simple docstring"""






					_lowercase				,      _lowercase :    Any							=							self.get_vision_text_model(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    Optional[int]							=							TFVisionTextDualEncoderModel(vision_model=lowerCamelCase_			,					text_model=lowerCamelCase_						)

					_lowercase :    Union[str, Any]							=							model(input_ids=lowerCamelCase_			,					pixel_values=lowerCamelCase_			,					attention_mask=lowerCamelCase_						)

					self.assertEqual(output['text_embeds'].shape			,					(input_ids.shape[0], model.config.projection_dim)						)
					self.assertEqual(output['image_embeds'].shape			,					(pixel_values.shape[0], model.config.projection_dim)						)







				def 							__UpperCAmelCase					( self    :       Union[str, Any]			,					lowerCamelCase_    :       Any			,					lowerCamelCase_    :       List[Any]			,					lowerCamelCase_    :       Optional[int]			,					lowerCamelCase_    :       List[str]			,					lowerCamelCase_    :       List[Any]=None			,					**lowerCamelCase_    :       Tuple						):

					"""simple docstring"""






					_lowercase				,      _lowercase :    List[str]							=							self.get_vision_text_model(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    str							=							{'vision_model': vision_model, 'text_model': text_model}
					_lowercase :    Optional[Any]							=							TFVisionTextDualEncoderModel.from_vision_text_pretrained(**lowerCamelCase_						)

					_lowercase :    Dict							=							model(input_ids=lowerCamelCase_			,					pixel_values=lowerCamelCase_			,					attention_mask=lowerCamelCase_						)

					self.assertEqual(output['text_embeds'].shape			,					(input_ids.shape[0], model.config.projection_dim)						)
					self.assertEqual(output['image_embeds'].shape			,					(pixel_values.shape[0], model.config.projection_dim)						)







				def 							__UpperCAmelCase					( self    :       List[str]			,					lowerCamelCase_    :       Union[str, Any]			,					lowerCamelCase_    :       Union[str, Any]			,					lowerCamelCase_    :       Union[str, Any]			,					lowerCamelCase_    :       str			,					lowerCamelCase_    :       Union[str, Any]=None			,					**lowerCamelCase_    :       Union[str, Any]						):

					"""simple docstring"""






					_lowercase				,      _lowercase :    str							=							self.get_vision_text_model(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    List[Any]							=							TFVisionTextDualEncoderModel(vision_model=lowerCamelCase_			,					text_model=lowerCamelCase_						)

					_lowercase :    Optional[Any]							=							model(input_ids=lowerCamelCase_			,					pixel_values=lowerCamelCase_			,					attention_mask=lowerCamelCase_						)
					_lowercase :    Optional[Any]							=							output[0].numpy()

					with tempfile.TemporaryDirectory() as tmpdirname:
						model.save_pretrained(lowerCamelCase_						)
						_lowercase :    Tuple							=							TFVisionTextDualEncoderModel.from_pretrained(lowerCamelCase_						)

						_lowercase :    str							=							model(input_ids=lowerCamelCase_			,					pixel_values=lowerCamelCase_			,					attention_mask=lowerCamelCase_						)
						_lowercase :    int							=							after_output[0].numpy()
						_lowercase :    List[Any]							=							np.amax(np.abs(out_a - out_a						)						)
						self.assertLessEqual(lowerCamelCase_			,					1E-5						)







				def 							__UpperCAmelCase					( self    :       Optional[int]			,					lowerCamelCase_    :       Any			,					lowerCamelCase_    :       str			,					lowerCamelCase_    :       List[Any]			,					lowerCamelCase_    :       Tuple			,					lowerCamelCase_    :       Optional[Any]=None			,					**lowerCamelCase_    :       Dict						):

					"""simple docstring"""






					_lowercase				,      _lowercase :    Dict							=							self.get_vision_text_model(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    Any							=							TFVisionTextDualEncoderModel(vision_model=lowerCamelCase_			,					text_model=lowerCamelCase_						)

					_lowercase :    Optional[int]							=							model(
					    input_ids=lowerCamelCase_			,					pixel_values=lowerCamelCase_			,					attention_mask=lowerCamelCase_			,					output_attentions=lowerCamelCase_						)

					_lowercase :    Any							=							output.vision_model_output.attentions
					self.assertEqual(len(lowerCamelCase_						)			,					vision_config.num_hidden_layers						)

					# in ViT, the seq_len equals the number of patches + 1 (we add 1 for the [CLS] token)
					_lowercase :    Dict							=							to_atuple(vision_model.config.image_size						)
					_lowercase :    Tuple							=							to_atuple(vision_model.config.patch_size						)
					_lowercase :    Union[str, Any]							=							(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
					_lowercase :    int							=							num_patches + 1
					self.assertEqual(vision_attentions[0].shape[-3:]			,					(vision_config.num_attention_heads, seq_len, seq_len)						)

					_lowercase :    List[str]							=							output.text_model_output.attentions
					self.assertEqual(len(lowerCamelCase_						)			,					text_config.num_hidden_layers						)

					self.assertEqual(
					    text_attentions[0].shape[-3:]			,					(text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])			,					)







				def 							__UpperCAmelCase					( self    :       Any			,					lowerCamelCase_    :       np.ndarray			,					lowerCamelCase_    :       np.ndarray			,					lowerCamelCase_    :       float						):

					"""simple docstring"""






					_lowercase :    Tuple							=							np.abs((a - b)						).max()
					self.assertLessEqual(lowerCamelCase_			,					lowerCamelCase_			,					F'''Difference between torch and flax is {diff} (>= {tol}).'''						)







				def 							__UpperCAmelCase					( self    :       Optional[int]						):

					"""simple docstring"""






					_lowercase :    Optional[int]							=							self.prepare_config_and_inputs()
					self.check_vision_text_dual_encoder_model(**lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       int						):

					"""simple docstring"""






					_lowercase :    Optional[Any]							=							self.prepare_config_and_inputs()
					self.check_model_from_pretrained_configs(**lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       Optional[Any]						):

					"""simple docstring"""






					_lowercase :    Tuple							=							self.prepare_config_and_inputs()
					self.check_vision_text_dual_encoder_from_pretrained(**lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       List[Any]						):

					"""simple docstring"""






					_lowercase :    List[str]							=							self.prepare_config_and_inputs()
					self.check_save_load(**lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       Dict						):

					"""simple docstring"""






					_lowercase :    Dict							=							self.prepare_config_and_inputs()
					self.check_vision_text_output_attention(**lowerCamelCase_						)







				@slow
				def 							__UpperCAmelCase					( self    :       List[str]						):

					"""simple docstring"""






					_lowercase				,      _lowercase :    List[str]							=							self.get_pretrained_model_and_inputs()

					_lowercase :    Any							=							model_a(**lowerCamelCase_						)
					_lowercase :    Optional[int]							=							outputs[0].numpy()

					with tempfile.TemporaryDirectory() as tmp_dirname:
						model_a.save_pretrained(lowerCamelCase_						)
						_lowercase :    int							=							TFVisionTextDualEncoderModel.from_pretrained(lowerCamelCase_						)

						_lowercase :    Optional[Any]							=							model_a(**lowerCamelCase_						)
						_lowercase :    Optional[int]							=							after_outputs[0].numpy()
						_lowercase :    List[str]							=							np.amax(np.abs(out_a - out_a						)						)
						self.assertLessEqual(lowerCamelCase_			,					1E-5						)



@require_tf
class  _lowerCamelCase			(__lowerCamelCase						,							unittest.TestCase     ):







				def 							__UpperCAmelCase					( self    :       Dict						):

					"""simple docstring"""






					_lowercase :    Any							=							TFVisionTextDualEncoderModel.from_vision_text_pretrained(
					    'hf-internal-testing/tiny-random-vit'			,					'hf-internal-testing/tiny-random-bert'						)
					_lowercase :    Union[str, Any]							=							1_3
					_lowercase :    Optional[Any]							=							floats_tensor(
					    [
					        batch_size,
					        model.vision_model.config.num_channels,
					        model.vision_model.config.image_size,
					        model.vision_model.config.image_size,
					    ]						)
					_lowercase :    List[str]							=							ids_tensor([batch_size, 4]			,					model.text_model.config.vocab_size						)
					_lowercase :    str							=							random_attention_mask([batch_size, 4]						)
					_lowercase :    Optional[Any]							=							{'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

					return model, inputs







				def 							__UpperCAmelCase					( self    :       List[str]			,					lowerCamelCase_    :       str			,					lowerCamelCase_    :       List[str]						):

					"""simple docstring"""






					_lowercase :    Dict							=							TFViTModel(lowerCamelCase_			,					name='vision_model'						)
					_lowercase :    List[Any]							=							TFBertModel(lowerCamelCase_			,					name='text_model'						)
					return vision_model, text_model







				def 							__UpperCAmelCase					( self    :       List[str]						):

					"""simple docstring"""






					_lowercase :    List[Any]							=							TFViTModelTester(self						)
					_lowercase :    List[str]							=							TFBertModelTester(self						)
					_lowercase :    Tuple							=							vit_model_tester.prepare_config_and_inputs()
					_lowercase :    int							=							bert_model_tester.prepare_config_and_inputs()

					_lowercase				,      _lowercase				,      _lowercase :    int							=							vision_config_and_inputs

					(
					    (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      
					) :    List[str]							=							text_config_and_inputs

					return {
					    "text_config": text_config,
					    "vision_config": vision_config,
					    "pixel_values": pixel_values,
					    "attention_mask": input_mask,
					    "input_ids": input_ids,
					    "text_token_type_ids": token_type_ids,
					    "text_sequence_labels": sequence_labels,
					    "text_token_labels": token_labels,
					    "text_choice_labels": choice_labels,
					}



@require_tf
class  _lowerCamelCase			(__lowerCamelCase						,							unittest.TestCase     ):







				def 							__UpperCAmelCase					( self    :       Tuple						):

					"""simple docstring"""






					_lowercase :    int							=							TFVisionTextDualEncoderModel.from_vision_text_pretrained(
					    'Rocketknight1/tiny-random-deit-tf'			,					'hf-internal-testing/tiny-random-roberta'						)
					_lowercase :    int							=							1_3
					_lowercase :    Union[str, Any]							=							floats_tensor(
					    [
					        batch_size,
					        model.vision_model.config.num_channels,
					        model.vision_model.config.image_size,
					        model.vision_model.config.image_size,
					    ]						)
					_lowercase :    Optional[int]							=							ids_tensor([batch_size, 4]			,					model.text_model.config.vocab_size						)
					_lowercase :    str							=							random_attention_mask([batch_size, 4]						)
					_lowercase :    Union[str, Any]							=							{'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

					return model, inputs







				def 							__UpperCAmelCase					( self    :       List[Any]			,					lowerCamelCase_    :       Any			,					lowerCamelCase_    :       List[str]			,					lowerCamelCase_    :       int			,					lowerCamelCase_    :       Union[str, Any]			,					lowerCamelCase_    :       List[str]=None			,					**lowerCamelCase_    :       str						):

					"""simple docstring"""






					_lowercase				,      _lowercase :    Dict							=							self.get_vision_text_model(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    Tuple							=							TFVisionTextDualEncoderModel(vision_model=lowerCamelCase_			,					text_model=lowerCamelCase_						)

					_lowercase :    Any							=							model(
					    input_ids=lowerCamelCase_			,					pixel_values=lowerCamelCase_			,					attention_mask=lowerCamelCase_			,					output_attentions=lowerCamelCase_						)

					_lowercase :    List[str]							=							output.vision_model_output.attentions
					self.assertEqual(len(lowerCamelCase_						)			,					vision_config.num_hidden_layers						)

					# in DEiT, the seq_len equals the number of patches + 2 (we add 2 for the [CLS] and distillation tokens)
					_lowercase :    Dict							=							to_atuple(vision_model.config.image_size						)
					_lowercase :    str							=							to_atuple(vision_model.config.patch_size						)
					_lowercase :    Dict							=							(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
					_lowercase :    int							=							num_patches + 2
					self.assertEqual(vision_attentions[0].shape[-3:]			,					(vision_config.num_attention_heads, seq_len, seq_len)						)

					_lowercase :    str							=							output.text_model_output.attentions
					self.assertEqual(len(lowerCamelCase_						)			,					text_config.num_hidden_layers						)

					self.assertEqual(
					    text_attentions[0].shape[-3:]			,					(text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])			,					)







				def 							__UpperCAmelCase					( self    :       str			,					lowerCamelCase_    :       str			,					lowerCamelCase_    :       List[str]						):

					"""simple docstring"""






					_lowercase :    Dict							=							TFDeiTModel(lowerCamelCase_			,					name='vision_model'						)
					_lowercase :    Union[str, Any]							=							TFRobertaModel(lowerCamelCase_			,					name='text_model'						)
					return vision_model, text_model







				def 							__UpperCAmelCase					( self    :       List[str]						):

					"""simple docstring"""






					_lowercase :    List[Any]							=							TFDeiTModelTester(self						)
					_lowercase :    Union[str, Any]							=							TFRobertaModelTester(self						)
					_lowercase :    List[str]							=							vit_model_tester.prepare_config_and_inputs()
					_lowercase :    Union[str, Any]							=							bert_model_tester.prepare_config_and_inputs()

					_lowercase				,      _lowercase				,      _lowercase :    Union[str, Any]							=							vision_config_and_inputs

					(
					    (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      
					) :    Optional[Any]							=							text_config_and_inputs

					return {
					    "text_config": text_config,
					    "vision_config": vision_config,
					    "pixel_values": pixel_values,
					    "attention_mask": input_mask,
					    "input_ids": input_ids,
					    "text_token_type_ids": token_type_ids,
					    "text_sequence_labels": sequence_labels,
					    "text_token_labels": token_labels,
					    "text_choice_labels": choice_labels,
					}



@require_tf
class  _lowerCamelCase			(__lowerCamelCase						,							unittest.TestCase     ):







				def 							__UpperCAmelCase					( self    :       Dict						):

					"""simple docstring"""






					_lowercase :    Union[str, Any]							=							TFVisionTextDualEncoderModel.from_vision_text_pretrained(
					    'Rocketknight1/tiny-random-clip-tf'			,					'hf-internal-testing/tiny-random-bert'						)
					_lowercase :    int							=							1_3
					_lowercase :    str							=							floats_tensor(
					    [
					        batch_size,
					        model.vision_model.config.num_channels,
					        model.vision_model.config.image_size,
					        model.vision_model.config.image_size,
					    ]						)
					_lowercase :    List[str]							=							ids_tensor([batch_size, 4]			,					model.text_model.config.vocab_size						)
					_lowercase :    Any							=							random_attention_mask([batch_size, 4]						)
					_lowercase :    Optional[Any]							=							{'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

					return model, inputs







				def 							__UpperCAmelCase					( self    :       List[str]			,					lowerCamelCase_    :       Dict			,					lowerCamelCase_    :       Union[str, Any]						):

					"""simple docstring"""






					_lowercase :    Optional[Any]							=							TFCLIPVisionModel(lowerCamelCase_			,					name='vision_model'						)
					_lowercase :    Optional[Any]							=							TFBertModel(lowerCamelCase_			,					name='text_model'						)
					return vision_model, text_model







				def 							__UpperCAmelCase					( self    :       str						):

					"""simple docstring"""






					_lowercase :    str							=							TFCLIPVisionModelTester(self						)
					_lowercase :    Tuple							=							TFBertModelTester(self						)
					_lowercase :    str							=							clip_model_tester.prepare_config_and_inputs()
					_lowercase :    str							=							bert_model_tester.prepare_config_and_inputs()

					_lowercase				,      _lowercase :    Any							=							vision_config_and_inputs

					(
					    (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      (
					    _lowercase
					)				,      
					) :    int							=							text_config_and_inputs

					return {
					    "text_config": text_config,
					    "vision_config": vision_config,
					    "pixel_values": pixel_values,
					    "attention_mask": input_mask,
					    "input_ids": input_ids,
					    "text_token_type_ids": token_type_ids,
					    "text_sequence_labels": sequence_labels,
					    "text_token_labels": token_labels,
					    "text_choice_labels": choice_labels,
					}



@require_vision
@require_tf
class  _lowerCamelCase			(unittest.TestCase     ):







				@slow
				def 							__UpperCAmelCase					( self    :       Union[str, Any]						):

					"""simple docstring"""






					_lowercase :    int							=							TFVisionTextDualEncoderModel.from_pretrained(
					    'clip-italian/clip-italian'			,					logit_scale_init_value=1.0			,					from_pt=lowerCamelCase_						)
					_lowercase :    Any							=							VisionTextDualEncoderProcessor.from_pretrained('clip-italian/clip-italian'						)

					_lowercase :    int							=							Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'						)
					_lowercase :    int							=							processor(
					    text=['una foto di un gatto', 'una foto di un cane']			,					images=lowerCamelCase_			,					padding=lowerCamelCase_			,					return_tensors='np'						)

					_lowercase :    Optional[int]							=							model(**lowerCamelCase_						)

					# verify the logits
					self.assertEqual(outputs.logits_per_image.shape			,					(inputs.pixel_values.shape[0], inputs.input_ids.shape[0])						)
					self.assertEqual(
					    outputs.logits_per_text.shape			,					(inputs.input_ids.shape[0], inputs.pixel_values.shape[0])			,					)

					_lowercase :    Optional[Any]							=							np.array([[1.228_4727, 0.310_4122]]						)

					self.assertTrue(np.allclose(outputs.logits_per_image.numpy()			,					lowerCamelCase_			,					atol=1E-3						)						)




"""simple docstring"""







import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE									=						get_tests_dir('fixtures/spiece.model')



@require_sentencepiece
@require_tokenizers
class  _lowerCamelCase			(__lowerCamelCase						,							unittest.TestCase     ):
				_snake_case     =  DebertaVaTokenizer
				_snake_case     =  DebertaVaTokenizerFast
				_snake_case     =  True
				_snake_case     =  True







				def 							__UpperCAmelCase					( self    :       List[Any]						):

					"""simple docstring"""






					super().setUp()

					# We have a SentencePiece fixture for testing
					_lowercase :    Dict							=							DebertaVaTokenizer(lowerCamelCase_			,					unk_token='<unk>'						)
					tokenizer.save_pretrained(self.tmpdirname						)







				def 							__UpperCAmelCase					( self    :       Dict			,					lowerCamelCase_    :       int						):

					"""simple docstring"""






					_lowercase :    Optional[Any]							=							'this is a test'
					_lowercase :    int							=							'this is a test'
					return input_text, output_text







				def 							__UpperCAmelCase					( self    :       List[Any]						):

					"""simple docstring"""






					_lowercase :    Optional[int]							=							'<pad>'
					_lowercase :    List[Any]							=							0

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase_						)			,					lowerCamelCase_						)
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase_						)			,					lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       int						):

					"""simple docstring"""






					_lowercase :    Union[str, Any]							=							list(self.get_tokenizer().get_vocab().keys()						)
					self.assertEqual(vocab_keys[0]			,					'<pad>'						)
					self.assertEqual(vocab_keys[1]			,					'<unk>'						)
					self.assertEqual(vocab_keys[-1]			,					'[PAD]'						)
					self.assertEqual(len(lowerCamelCase_						)			,					3_0_0_0_1						)







				def 							__UpperCAmelCase					( self    :       str						):

					"""simple docstring"""






					self.assertEqual(self.get_tokenizer().vocab_size			,					3_0_0_0_0						)







				def 							__UpperCAmelCase					( self    :       Any						):

					"""simple docstring"""






					_lowercase :    Optional[int]							=							' \tHeLLo!how  \n Are yoU?  '
					_lowercase :    Optional[int]							=							['▁hello', '!', 'how', '▁are', '▁you', '?']
					# fmt: on

					_lowercase :    Optional[int]							=							DebertaVaTokenizer(lowerCamelCase_			,					do_lower_case=lowerCamelCase_						)
					_lowercase :    Dict							=							tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)

					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

					_lowercase :    int							=							DebertaVaTokenizerFast(lowerCamelCase_			,					do_lower_case=lowerCamelCase_						)
					_lowercase :    Dict							=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)

					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)







				@unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'						)
				def 							__UpperCAmelCase					( self    :       int						):

					"""simple docstring"""






					pass







				@unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.'						)
				def 							__UpperCAmelCase					( self    :       Optional[Any]						):

					"""simple docstring"""






					pass







				def 							__UpperCAmelCase					( self    :       Dict						):

					"""simple docstring"""






					_lowercase :    List[Any]							=							'I was born in 92000, and this is falsé.'
					_lowercase :    int							=							['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
					# fmt: on

					_lowercase :    Union[str, Any]							=							DebertaVaTokenizer(lowerCamelCase_			,					split_by_punct=lowerCamelCase_						)
					_lowercase :    Optional[int]							=							tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)

					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

					_lowercase :    Optional[int]							=							DebertaVaTokenizerFast(lowerCamelCase_			,					split_by_punct=lowerCamelCase_						)
					_lowercase :    Optional[int]							=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)

					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       Optional[Any]						):

					"""simple docstring"""






					_lowercase :    List[Any]							=							'I was born in 92000, and this is falsé.'
					_lowercase :    Dict							=							['▁i', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
					# fmt: on

					_lowercase :    Union[str, Any]							=							DebertaVaTokenizer(lowerCamelCase_			,					do_lower_case=lowerCamelCase_			,					split_by_punct=lowerCamelCase_						)
					_lowercase :    int							=							tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

					_lowercase :    Dict							=							DebertaVaTokenizerFast(lowerCamelCase_			,					do_lower_case=lowerCamelCase_			,					split_by_punct=lowerCamelCase_						)
					_lowercase :    List[Any]							=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       Union[str, Any]						):

					"""simple docstring"""






					_lowercase :    Optional[Any]							=							'I was born in 92000, and this is falsé.'
					_lowercase :    List[str]							=							['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
					# fmt: on

					_lowercase :    Union[str, Any]							=							DebertaVaTokenizer(lowerCamelCase_			,					do_lower_case=lowerCamelCase_			,					split_by_punct=lowerCamelCase_						)
					_lowercase :    Optional[int]							=							tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)

					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

					_lowercase :    Union[str, Any]							=							DebertaVaTokenizerFast(lowerCamelCase_			,					do_lower_case=lowerCamelCase_			,					split_by_punct=lowerCamelCase_						)
					_lowercase :    Dict							=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)

					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       Tuple						):

					"""simple docstring"""






					_lowercase :    Dict							=							'I was born in 92000, and this is falsé.'
					_lowercase :    Union[str, Any]							=							['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
					# fmt: on

					_lowercase :    Optional[int]							=							DebertaVaTokenizer(lowerCamelCase_			,					do_lower_case=lowerCamelCase_			,					split_by_punct=lowerCamelCase_						)
					_lowercase :    Dict							=							tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)

					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

					_lowercase :    Tuple							=							DebertaVaTokenizerFast(lowerCamelCase_			,					do_lower_case=lowerCamelCase_			,					split_by_punct=lowerCamelCase_						)
					_lowercase :    Any							=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)

					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       List[str]						):

					"""simple docstring"""






					_lowercase :    Optional[Any]							=							' \tHeLLo!how  \n Are yoU?  '
					_lowercase :    int							=							['▁', '<unk>', 'e', '<unk>', 'o', '!', 'how', '▁', '<unk>', 're', '▁yo', '<unk>', '?']
					# fmt: on

					_lowercase :    Dict							=							DebertaVaTokenizer(lowerCamelCase_			,					do_lower_case=lowerCamelCase_			,					split_by_punct=lowerCamelCase_						)
					_lowercase :    Dict							=							tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)

					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

					_lowercase :    str							=							DebertaVaTokenizerFast(lowerCamelCase_			,					do_lower_case=lowerCamelCase_			,					split_by_punct=lowerCamelCase_						)
					_lowercase :    Dict							=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)

					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       Union[str, Any]						):

					"""simple docstring"""






					_lowercase :    int							=							self.get_tokenizer()
					_lowercase :    str							=							self.get_rust_tokenizer()

					_lowercase :    Optional[int]							=							'I was born in 92000, and this is falsé.'

					_lowercase :    List[str]							=							tokenizer.convert_ids_to_tokens(tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)
					_lowercase :    Optional[int]							=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

					_lowercase :    List[str]							=							tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)
					_lowercase :    List[Any]							=							rust_tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

					_lowercase :    List[Any]							=							self.get_rust_tokenizer()
					_lowercase :    Tuple							=							tokenizer.encode(lowerCamelCase_						)
					_lowercase :    int							=							rust_tokenizer.encode(lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       int						):

					"""simple docstring"""






					_lowercase :    Union[str, Any]							=							'This is a test'
					_lowercase :    Tuple							=							[1_3, 1, 4_3_9_8, 2_5, 2_1, 1_2_8_9]
					_lowercase :    Optional[Any]							=							['▁', 'T', 'his', '▁is', '▁a', '▁test']
					_lowercase :    str							=							['▁', '<unk>', 'his', '▁is', '▁a', '▁test']

					_lowercase :    List[Any]							=							DebertaVaTokenizer(lowerCamelCase_			,					keep_accents=lowerCamelCase_						)
					_lowercase :    Optional[Any]							=							DebertaVaTokenizerFast(lowerCamelCase_			,					keep_accents=lowerCamelCase_						)

					_lowercase :    Union[str, Any]							=							tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    Optional[Any]							=							tokenizer.tokenize(lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    str							=							tokenizer.convert_ids_to_tokens(lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

					_lowercase :    int							=							rust_tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    int							=							rust_tokenizer.tokenize(lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    int							=							rust_tokenizer.convert_ids_to_tokens(lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

					# fmt: off
					_lowercase :    str							=							'I was born in 92000, and this is falsé.'
					_lowercase :    str							=							[1_3, 1, 2_3, 3_8_6, 1_9, 5_6_1, 3_0_5_0, 1_5, 1_7, 4_8, 2_5, 8_2_5_6, 1_8, 1, 9]
					_lowercase :    str							=							['▁', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.', ]
					_lowercase :    Optional[int]							=							['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
					# fmt: on

					_lowercase :    Tuple							=							tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    List[str]							=							tokenizer.tokenize(lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    Union[str, Any]							=							tokenizer.convert_ids_to_tokens(lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)

					_lowercase :    List[str]							=							rust_tokenizer.encode(lowerCamelCase_			,					add_special_tokens=lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    Any							=							rust_tokenizer.tokenize(lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)
					_lowercase :    str							=							rust_tokenizer.convert_ids_to_tokens(lowerCamelCase_						)
					self.assertListEqual(lowerCamelCase_			,					lowerCamelCase_						)







				def 							__UpperCAmelCase					( self    :       Union[str, Any]						):

					"""simple docstring"""






					_lowercase :    Optional[Any]							=							DebertaVaTokenizer(lowerCamelCase_						)

					_lowercase :    str							=							tokenizer.encode('sequence builders'						)
					_lowercase :    str							=							tokenizer.encode('multi-sequence build'						)

					_lowercase :    Tuple							=							tokenizer.build_inputs_with_special_tokens(lowerCamelCase_						)
					_lowercase :    Union[str, Any]							=							tokenizer.build_inputs_with_special_tokens(lowerCamelCase_			,					lowerCamelCase_						)

					self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]			,					lowerCamelCase_						)
					self.assertEqual(
					    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]			,					lowerCamelCase_			,					)







				@slow
				def 							__UpperCAmelCase					( self    :       Union[str, Any]						):

					"""simple docstring"""






					_lowercase :    Any							=							{'input_ids': [[1, 3_9_8_6_7, 3_6, 1_9_3_9_0, 4_8_6, 2_7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 6_0_6_8_5, 1_2_2_5, 7, 3_5_0_5_2, 8_1_4_3_6, 1_8, 9_3_6_7, 1_6_8_9_9, 1_8, 1_5_9_3_7, 5_3, 5_9_4, 7_7_3, 1_8, 1_6_2_8_7, 3_0_4_6_5, 3_6, 1_5_9_3_7, 6, 4_1_1_3_9, 3_8, 3_6_9_7_9, 6_0_7_6_3, 1_9_1, 6, 3_4_1_3_2, 9_9, 6, 5_0_5_3_8, 3_9_0, 4_3_2_3_0, 6, 3_4_1_3_2, 2_7_7_9, 2_0_8_5_0, 1_4, 6_9_9, 1_0_7_2, 1_1_9_4, 3_6, 3_8_2, 1_0_9_0_1, 5_3, 7, 6_9_9, 1_0_7_2, 2_0_8_4, 3_6, 2_0_4_2_2, 6_3_0, 5_3, 1_9, 1_0_5, 3_0_4_9, 1_8_9_6, 1_0_5_3, 1_6_8_9_9, 1_5_0_6, 1_1, 3_7_9_7_8, 4_2_4_3, 7, 1_2_3_7, 3_1_8_6_9, 2_0_0, 1_6_5_6_6, 6_5_4, 6, 3_5_0_5_2, 8_1_4_3_6, 7, 5_5_6_3_0, 1_3_5_9_3, 4, 2], [1, 2_6, 1_5_0_1_1, 1_3, 6_6_7, 8, 1_0_5_3, 1_8, 2_3_6_1_1, 1_2_3_7, 7_2_3_5_6, 1_2_8_2_0, 3_4, 1_0_4_1_3_4, 1_2_0_9, 3_5, 1_3_3_1_3, 6_6_2_7, 2_1, 2_0_2, 3_4_7, 7, 1_6_4, 2_3_9_9, 1_1, 4_6, 4_4_8_5, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_2_3_2, 2_8_6_4, 1_5_7_8_5, 1_4_9_5_1, 1_0_5, 5, 8_5_8_1, 1_2_5_0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
					# fmt: on

					self.tokenizer_integration_test_util(
					    expected_encoding=lowerCamelCase_			,					model_name='microsoft/deberta-v2-xlarge'			,					revision='ad6e42c1532ddf3a15c39246b63f5559d558b670'			,					)



