





'''simple docstring'''






from __future__ import annotations

import collections
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import is_tf_available, is_vision_available

from ...test_modeling_tf_common import floats_tensor, ids_tensor, random_attention_mask
from ..bert.test_modeling_tf_bert import TFBertModelTester
from ..clip.test_modeling_tf_clip import TFCLIPVisionModelTester
from ..deit.test_modeling_tf_deit import TFDeiTModelTester
from ..roberta.test_modeling_tf_roberta import TFRobertaModelTester
from ..vit.test_modeling_tf_vit import TFViTModelTester


if is_tf_available():
		from transformers import (
		    TFBertModel,
		    TFCLIPVisionModel,
		    TFDeiTModel,
		    TFRobertaModel,
		    TFVisionTextDualEncoderModel,
		    TFViTModel,
		    VisionTextDualEncoderConfig,
		)

if is_vision_available():
		from PIL import Image

		from transformers import VisionTextDualEncoderProcessor





def       lowercase_      (		lowercase__						)   ->Optional[int]:







						if isinstance(lowercase__				,  collections.abc.Iterable						):
												return x
						return (x, x)







@require_tf
class 			lowerCamelCase					:


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[Any]       ,     __snake_case	:  str       ,     __snake_case	:  Optional[Any]       ):





											'''simple docstring'''





											pass


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ):





											'''simple docstring'''





											pass


					def    SCREAMING_SNAKE_CASE_		(					self	:  Any       ):





											'''simple docstring'''





											pass


					def    SCREAMING_SNAKE_CASE_		(					self	:  int       ,     __snake_case	:  Optional[int]       ,     __snake_case	:  Any       ,     __snake_case	:  Any       ,     __snake_case	:  Optional[Any]       ,     __snake_case	:  Dict=None       ,     **__snake_case	:  int       ):





											'''simple docstring'''





											_snake_case:   Optional[int]  					=		VisionTextDualEncoderConfig.from_vision_text_configs(_lowerCAmelCase       ,     _lowerCAmelCase       )

											_snake_case:   List[str]  					=		TFVisionTextDualEncoderModel(_lowerCAmelCase       )

											_snake_case:   str  					=		model(input_ids=_lowerCAmelCase       ,     pixel_values=_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       )

											self.assertEqual(output['text_embeds'].shape       ,     (input_ids.shape[0], config.projection_dim)       )
											self.assertEqual(output['image_embeds'].shape       ,     (pixel_values.shape[0], config.projection_dim)       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  str       ,     __snake_case	:  Tuple       ,     __snake_case	:  str       ,     __snake_case	:  List[Any]       ,     __snake_case	:  List[str]       ,     __snake_case	:  int=None       ,     **__snake_case	:  Dict       ):





											'''simple docstring'''





											_snake_case					,						_snake_case:   List[Any]  					=		self.get_vision_text_model(_lowerCAmelCase       ,     _lowerCAmelCase       )
											_snake_case:   Union[str, Any]  					=		TFVisionTextDualEncoderModel(vision_model=_lowerCAmelCase       ,     text_model=_lowerCAmelCase       )

											_snake_case:   Optional[Any]  					=		model(input_ids=_lowerCAmelCase       ,     pixel_values=_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       )

											self.assertEqual(output['text_embeds'].shape       ,     (input_ids.shape[0], model.config.projection_dim)       )
											self.assertEqual(output['image_embeds'].shape       ,     (pixel_values.shape[0], model.config.projection_dim)       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[Any]       ,     __snake_case	:  List[Any]       ,     __snake_case	:  Any       ,     __snake_case	:  Dict       ,     __snake_case	:  str       ,     __snake_case	:  Any=None       ,     **__snake_case	:  str       ):





											'''simple docstring'''





											_snake_case					,						_snake_case:   Dict  					=		self.get_vision_text_model(_lowerCAmelCase       ,     _lowerCAmelCase       )
											_snake_case:   Any  					=		{'vision_model': vision_model, 'text_model': text_model}
											_snake_case:   Dict  					=		TFVisionTextDualEncoderModel.from_vision_text_pretrained(**_lowerCAmelCase       )

											_snake_case:   int  					=		model(input_ids=_lowerCAmelCase       ,     pixel_values=_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       )

											self.assertEqual(output['text_embeds'].shape       ,     (input_ids.shape[0], model.config.projection_dim)       )
											self.assertEqual(output['image_embeds'].shape       ,     (pixel_values.shape[0], model.config.projection_dim)       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ,     __snake_case	:  Union[str, Any]       ,     __snake_case	:  List[str]       ,     __snake_case	:  Tuple       ,     __snake_case	:  Optional[int]       ,     __snake_case	:  List[Any]=None       ,     **__snake_case	:  Any       ):





											'''simple docstring'''





											_snake_case					,						_snake_case:   Optional[Any]  					=		self.get_vision_text_model(_lowerCAmelCase       ,     _lowerCAmelCase       )
											_snake_case:   Union[str, Any]  					=		TFVisionTextDualEncoderModel(vision_model=_lowerCAmelCase       ,     text_model=_lowerCAmelCase       )

											_snake_case:   Tuple  					=		model(input_ids=_lowerCAmelCase       ,     pixel_values=_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       )
											_snake_case:   Tuple  					=		output[0].numpy()

											with tempfile.TemporaryDirectory() as tmpdirname:
																	model.save_pretrained(_lowerCAmelCase       )
																	_snake_case:   Optional[int]  					=		TFVisionTextDualEncoderModel.from_pretrained(_lowerCAmelCase       )

																	_snake_case:   Optional[Any]  					=		model(input_ids=_lowerCAmelCase       ,     pixel_values=_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       )
																	_snake_case:   Any  					=		after_output[0].numpy()
																	_snake_case:   int  					=		np.amax(np.abs(out_a - out_a       )       )
																	self.assertLessEqual(_lowerCAmelCase       ,     1e-5       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  int       ,     __snake_case	:  Any       ,     __snake_case	:  Optional[int]       ,     __snake_case	:  str       ,     __snake_case	:  Optional[int]       ,     __snake_case	:  Optional[int]=None       ,     **__snake_case	:  Optional[int]       ):





											'''simple docstring'''





											_snake_case					,						_snake_case:   Any  					=		self.get_vision_text_model(_lowerCAmelCase       ,     _lowerCAmelCase       )
											_snake_case:   Dict  					=		TFVisionTextDualEncoderModel(vision_model=_lowerCAmelCase       ,     text_model=_lowerCAmelCase       )

											_snake_case:   int  					=		model(
											    input_ids=_lowerCAmelCase       ,     pixel_values=_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     output_attentions=_lowerCAmelCase       )

											_snake_case:   Any  					=		output.vision_model_output.attentions
											self.assertEqual(len(_lowerCAmelCase       )       ,     vision_config.num_hidden_layers       )

											# in ViT, the seq_len equals the number of patches + 1 (we add 1 for the [CLS] token)
											_snake_case:   str  					=		to_atuple(vision_model.config.image_size       )
											_snake_case:   Tuple  					=		to_atuple(vision_model.config.patch_size       )
											_snake_case:   List[Any]  					=		(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
											_snake_case:   str  					=		num_patches + 1
											self.assertEqual(vision_attentions[0].shape[-3:]       ,     (vision_config.num_attention_heads, seq_len, seq_len)       )

											_snake_case:   int  					=		output.text_model_output.attentions
											self.assertEqual(len(_lowerCAmelCase       )       ,     text_config.num_hidden_layers       )

											self.assertEqual(
											    text_attentions[0].shape[-3:]       ,     (text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])       ,     )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[Any]       ,     __snake_case	:  np.ndarray       ,     __snake_case	:  np.ndarray       ,     __snake_case	:  float       ):





											'''simple docstring'''





											_snake_case:   str  					=		np.abs((a - b)       ).max()
											self.assertLessEqual(_lowerCAmelCase       ,     _lowerCAmelCase       ,     f'''Difference between torch and flax is {diff} (>= {tol}).'''       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[Any]       ):





											'''simple docstring'''





											_snake_case:   Optional[Any]  					=		self.prepare_config_and_inputs()
											self.check_vision_text_dual_encoder_model(**_lowerCAmelCase       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  int       ):





											'''simple docstring'''





											_snake_case:   Optional[Any]  					=		self.prepare_config_and_inputs()
											self.check_model_from_pretrained_configs(**_lowerCAmelCase       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ):





											'''simple docstring'''





											_snake_case:   Optional[int]  					=		self.prepare_config_and_inputs()
											self.check_vision_text_dual_encoder_from_pretrained(**_lowerCAmelCase       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[str]       ):





											'''simple docstring'''





											_snake_case:   Optional[int]  					=		self.prepare_config_and_inputs()
											self.check_save_load(**_lowerCAmelCase       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ):





											'''simple docstring'''





											_snake_case:   int  					=		self.prepare_config_and_inputs()
											self.check_vision_text_output_attention(**_lowerCAmelCase       )




					@slow
					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ):





											'''simple docstring'''





											_snake_case					,						_snake_case:   Tuple  					=		self.get_pretrained_model_and_inputs()

											_snake_case:   Dict  					=		model_a(**_lowerCAmelCase       )
											_snake_case:   Union[str, Any]  					=		outputs[0].numpy()

											with tempfile.TemporaryDirectory() as tmp_dirname:
																	model_a.save_pretrained(_lowerCAmelCase       )
																	_snake_case:   List[Any]  					=		TFVisionTextDualEncoderModel.from_pretrained(_lowerCAmelCase       )

																	_snake_case:   str  					=		model_a(**_lowerCAmelCase       )
																	_snake_case:   Dict  					=		after_outputs[0].numpy()
																	_snake_case:   Optional[Any]  					=		np.amax(np.abs(out_a - out_a       )       )
																	self.assertLessEqual(_lowerCAmelCase       ,     1e-5       )







@require_tf
class 			lowerCamelCase					(						_lowerCAmelCase     ,							unittest.TestCase  ):


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ):





											'''simple docstring'''





											_snake_case:   Optional[Any]  					=		TFVisionTextDualEncoderModel.from_vision_text_pretrained(
											    'hf-internal-testing/tiny-random-vit'       ,     'hf-internal-testing/tiny-random-bert'       )
											_snake_case:   Dict  					=		13
											_snake_case:   Optional[int]  					=		floats_tensor(
											    [
											        batch_size,
											        model.vision_model.config.num_channels,
											        model.vision_model.config.image_size,
											        model.vision_model.config.image_size,
											    ]       )
											_snake_case:   Dict  					=		ids_tensor([batch_size, 4]       ,     model.text_model.config.vocab_size       )
											_snake_case:   Union[str, Any]  					=		random_attention_mask([batch_size, 4]       )
											_snake_case:   Union[str, Any]  					=		{'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

											return model, inputs


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ,     __snake_case	:  Tuple       ,     __snake_case	:  List[str]       ):





											'''simple docstring'''





											_snake_case:   Any  					=		TFViTModel(_lowerCAmelCase       ,     name='vision_model'       )
											_snake_case:   Union[str, Any]  					=		TFBertModel(_lowerCAmelCase       ,     name='text_model'       )
											return vision_model, text_model




					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ):





											'''simple docstring'''





											_snake_case:   Tuple  					=		TFViTModelTester(self       )
											_snake_case:   Dict  					=		TFBertModelTester(self       )
											_snake_case:   Optional[Any]  					=		vit_model_tester.prepare_config_and_inputs()
											_snake_case:   str  					=		bert_model_tester.prepare_config_and_inputs()

											_snake_case					,						_snake_case					,						_snake_case:   Tuple  					=		vision_config_and_inputs

											(
											    (
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,												
											):   int  					=		text_config_and_inputs

											return {
											    "text_config": text_config,
											    "vision_config": vision_config,
											    "pixel_values": pixel_values,
											    "attention_mask": input_mask,
											    "input_ids": input_ids,
											    "text_token_type_ids": token_type_ids,
											    "text_sequence_labels": sequence_labels,
											    "text_token_labels": token_labels,
											    "text_choice_labels": choice_labels,
											}







@require_tf
class 			lowerCamelCase					(						_lowerCAmelCase     ,							unittest.TestCase  ):


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ):





											'''simple docstring'''





											_snake_case:   Tuple  					=		TFVisionTextDualEncoderModel.from_vision_text_pretrained(
											    'Rocketknight1/tiny-random-deit-tf'       ,     'hf-internal-testing/tiny-random-roberta'       )
											_snake_case:   Any  					=		13
											_snake_case:   str  					=		floats_tensor(
											    [
											        batch_size,
											        model.vision_model.config.num_channels,
											        model.vision_model.config.image_size,
											        model.vision_model.config.image_size,
											    ]       )
											_snake_case:   Union[str, Any]  					=		ids_tensor([batch_size, 4]       ,     model.text_model.config.vocab_size       )
											_snake_case:   List[str]  					=		random_attention_mask([batch_size, 4]       )
											_snake_case:   List[str]  					=		{'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

											return model, inputs


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[Any]       ,     __snake_case	:  Union[str, Any]       ,     __snake_case	:  str       ,     __snake_case	:  int       ,     __snake_case	:  Tuple       ,     __snake_case	:  int=None       ,     **__snake_case	:  Tuple       ):





											'''simple docstring'''





											_snake_case					,						_snake_case:   Union[str, Any]  					=		self.get_vision_text_model(_lowerCAmelCase       ,     _lowerCAmelCase       )
											_snake_case:   List[Any]  					=		TFVisionTextDualEncoderModel(vision_model=_lowerCAmelCase       ,     text_model=_lowerCAmelCase       )

											_snake_case:   List[str]  					=		model(
											    input_ids=_lowerCAmelCase       ,     pixel_values=_lowerCAmelCase       ,     attention_mask=_lowerCAmelCase       ,     output_attentions=_lowerCAmelCase       )

											_snake_case:   Any  					=		output.vision_model_output.attentions
											self.assertEqual(len(_lowerCAmelCase       )       ,     vision_config.num_hidden_layers       )

											# in DEiT, the seq_len equals the number of patches + 2 (we add 2 for the [CLS] and distillation tokens)
											_snake_case:   Optional[int]  					=		to_atuple(vision_model.config.image_size       )
											_snake_case:   List[str]  					=		to_atuple(vision_model.config.patch_size       )
											_snake_case:   List[Any]  					=		(image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
											_snake_case:   int  					=		num_patches + 2
											self.assertEqual(vision_attentions[0].shape[-3:]       ,     (vision_config.num_attention_heads, seq_len, seq_len)       )

											_snake_case:   Optional[Any]  					=		output.text_model_output.attentions
											self.assertEqual(len(_lowerCAmelCase       )       ,     text_config.num_hidden_layers       )

											self.assertEqual(
											    text_attentions[0].shape[-3:]       ,     (text_config.num_attention_heads, input_ids.shape[-1], input_ids.shape[-1])       ,     )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ,     __snake_case	:  List[str]       ,     __snake_case	:  Dict       ):





											'''simple docstring'''





											_snake_case:   List[Any]  					=		TFDeiTModel(_lowerCAmelCase       ,     name='vision_model'       )
											_snake_case:   str  					=		TFRobertaModel(_lowerCAmelCase       ,     name='text_model'       )
											return vision_model, text_model




					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ):





											'''simple docstring'''





											_snake_case:   Union[str, Any]  					=		TFDeiTModelTester(self       )
											_snake_case:   Optional[int]  					=		TFRobertaModelTester(self       )
											_snake_case:   List[Any]  					=		vit_model_tester.prepare_config_and_inputs()
											_snake_case:   Union[str, Any]  					=		bert_model_tester.prepare_config_and_inputs()

											_snake_case					,						_snake_case					,						_snake_case:   Tuple  					=		vision_config_and_inputs

											(
											    (
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,												
											):   int  					=		text_config_and_inputs

											return {
											    "text_config": text_config,
											    "vision_config": vision_config,
											    "pixel_values": pixel_values,
											    "attention_mask": input_mask,
											    "input_ids": input_ids,
											    "text_token_type_ids": token_type_ids,
											    "text_sequence_labels": sequence_labels,
											    "text_token_labels": token_labels,
											    "text_choice_labels": choice_labels,
											}







@require_tf
class 			lowerCamelCase					(						_lowerCAmelCase     ,							unittest.TestCase  ):


					def    SCREAMING_SNAKE_CASE_		(					self	:  int       ):





											'''simple docstring'''





											_snake_case:   List[str]  					=		TFVisionTextDualEncoderModel.from_vision_text_pretrained(
											    'Rocketknight1/tiny-random-clip-tf'       ,     'hf-internal-testing/tiny-random-bert'       )
											_snake_case:   Dict  					=		13
											_snake_case:   List[str]  					=		floats_tensor(
											    [
											        batch_size,
											        model.vision_model.config.num_channels,
											        model.vision_model.config.image_size,
											        model.vision_model.config.image_size,
											    ]       )
											_snake_case:   List[Any]  					=		ids_tensor([batch_size, 4]       ,     model.text_model.config.vocab_size       )
											_snake_case:   Optional[Any]  					=		random_attention_mask([batch_size, 4]       )
											_snake_case:   str  					=		{'pixel_values': pixel_values, 'input_ids': input_ids, 'attention_mask': attention_mask}

											return model, inputs


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ,     __snake_case	:  Dict       ,     __snake_case	:  Union[str, Any]       ):





											'''simple docstring'''





											_snake_case:   List[str]  					=		TFCLIPVisionModel(_lowerCAmelCase       ,     name='vision_model'       )
											_snake_case:   int  					=		TFBertModel(_lowerCAmelCase       ,     name='text_model'       )
											return vision_model, text_model




					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ):





											'''simple docstring'''





											_snake_case:   List[Any]  					=		TFCLIPVisionModelTester(self       )
											_snake_case:   Optional[Any]  					=		TFBertModelTester(self       )
											_snake_case:   List[str]  					=		clip_model_tester.prepare_config_and_inputs()
											_snake_case:   Any  					=		bert_model_tester.prepare_config_and_inputs()

											_snake_case					,						_snake_case:   List[Any]  					=		vision_config_and_inputs

											(
											    (
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,						(
											    _snake_case						
											)					,												
											):   Optional[int]  					=		text_config_and_inputs

											return {
											    "text_config": text_config,
											    "vision_config": vision_config,
											    "pixel_values": pixel_values,
											    "attention_mask": input_mask,
											    "input_ids": input_ids,
											    "text_token_type_ids": token_type_ids,
											    "text_sequence_labels": sequence_labels,
											    "text_token_labels": token_labels,
											    "text_choice_labels": choice_labels,
											}







@require_vision
@require_tf
class 			lowerCamelCase					(						unittest.TestCase  ):


					@slow
					def    SCREAMING_SNAKE_CASE_		(					self	:  int       ):





											'''simple docstring'''





											_snake_case:   Tuple  					=		TFVisionTextDualEncoderModel.from_pretrained(
											    'clip-italian/clip-italian'       ,     logit_scale_init_value=1.0       ,     from_pt=_lowerCAmelCase       )
											_snake_case:   Optional[Any]  					=		VisionTextDualEncoderProcessor.from_pretrained('clip-italian/clip-italian'       )

											_snake_case:   Dict  					=		Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'       )
											_snake_case:   int  					=		processor(
											    text=['una foto di un gatto', 'una foto di un cane']       ,     images=_lowerCAmelCase       ,     padding=_lowerCAmelCase       ,     return_tensors='np'       )

											_snake_case:   int  					=		model(**_lowerCAmelCase       )

											# verify the logits
											self.assertEqual(outputs.logits_per_image.shape       ,     (inputs.pixel_values.shape[0], inputs.input_ids.shape[0])       )
											self.assertEqual(
											    outputs.logits_per_text.shape       ,     (inputs.input_ids.shape[0], inputs.pixel_values.shape[0])       ,     )

											_snake_case:   Any  					=		np.array([[1.2_284_727, 0.3_104_122]]       )

											self.assertTrue(np.allclose(outputs.logits_per_image.numpy()       ,     _lowerCAmelCase       ,     atol=1e-3       )       )












'''simple docstring'''






import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


A   : Tuple             =			'▁'

A   : Optional[Any]             =			get_tests_dir('fixtures/test_sentencepiece.model')







@require_sentencepiece
@require_tokenizers
class 			lowerCamelCase					(						__UpperCAmelCase     ,							unittest.TestCase  ):
					_SCREAMING_SNAKE_CASE							   =    BigBirdTokenizer
					_SCREAMING_SNAKE_CASE							   =    BigBirdTokenizerFast
					_SCREAMING_SNAKE_CASE							   =    True
					_SCREAMING_SNAKE_CASE							   =    True


					def    SCREAMING_SNAKE_CASE_		(					self	:  str       ):





											'''simple docstring'''





											super().setUp()

											_snake_case:   Dict  					=		self.tokenizer_class(__snake_case       ,     keep_accents=__snake_case       )
											tokenizer.save_pretrained(self.tmpdirname       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ):





											'''simple docstring'''





											_snake_case:   List[str]  					=		'<s>'
											_snake_case:   Tuple  					=		1

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(__snake_case       )       ,     __snake_case       )
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(__snake_case       )       ,     __snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  int       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		list(self.get_tokenizer().get_vocab().keys()       )

											self.assertEqual(vocab_keys[0]       ,     '<unk>'       )
											self.assertEqual(vocab_keys[1]       ,     '<s>'       )
											self.assertEqual(vocab_keys[-1]       ,     '[MASK]'       )
											self.assertEqual(len(__snake_case       )       ,     10_04       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  int       ):





											'''simple docstring'''





											self.assertEqual(self.get_tokenizer().vocab_size       ,     10_00       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[str]       ):





											'''simple docstring'''





											if not self.test_rust_tokenizer:
																	return

											_snake_case:   Optional[int]  					=		self.get_tokenizer()
											_snake_case:   Union[str, Any]  					=		self.get_rust_tokenizer()

											_snake_case:   List[str]  					=		'I was born in 92000, and this is falsé.'

											_snake_case:   str  					=		tokenizer.tokenize(__snake_case       )
											_snake_case:   Dict  					=		rust_tokenizer.tokenize(__snake_case       )
											self.assertListEqual(__snake_case       ,     __snake_case       )

											_snake_case:   Union[str, Any]  					=		tokenizer.encode(__snake_case       ,     add_special_tokens=__snake_case       )
											_snake_case:   List[Any]  					=		rust_tokenizer.encode(__snake_case       ,     add_special_tokens=__snake_case       )
											self.assertListEqual(__snake_case       ,     __snake_case       )

											_snake_case:   Optional[int]  					=		self.get_rust_tokenizer()
											_snake_case:   int  					=		tokenizer.encode(__snake_case       )
											_snake_case:   Tuple  					=		rust_tokenizer.encode(__snake_case       )
											self.assertListEqual(__snake_case       ,     __snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  str       ):





											'''simple docstring'''





											_snake_case:   Tuple  					=		BigBirdTokenizer(__snake_case       ,     keep_accents=__snake_case       )

											_snake_case:   Optional[int]  					=		tokenizer.tokenize('This is a test'       )
											self.assertListEqual(__snake_case       ,     ['▁This', '▁is', '▁a', '▁t', 'est']       )

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(__snake_case       )       ,     [2_85, 46, 10, 1_70, 3_82]       ,     )

											_snake_case:   Any  					=		tokenizer.tokenize('I was born in 92000, and this is falsé.'       )
											self.assertListEqual(
											    __snake_case       ,     [
											        SPIECE_UNDERLINE + 'I',
											        SPIECE_UNDERLINE + 'was',
											        SPIECE_UNDERLINE + 'b',
											        'or',
											        'n',
											        SPIECE_UNDERLINE + 'in',
											        SPIECE_UNDERLINE + '',
											        '9',
											        '2',
											        '0',
											        '0',
											        '0',
											        ',',
											        SPIECE_UNDERLINE + 'and',
											        SPIECE_UNDERLINE + 'this',
											        SPIECE_UNDERLINE + 'is',
											        SPIECE_UNDERLINE + 'f',
											        'al',
											        's',
											        'é',
											        '.',
											    ]       ,     )
											_snake_case:   List[Any]  					=		tokenizer.convert_tokens_to_ids(__snake_case       )
											self.assertListEqual(
											    __snake_case       ,     [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]       ,     )

											_snake_case:   Optional[Any]  					=		tokenizer.convert_ids_to_tokens(__snake_case       )
											self.assertListEqual(
											    __snake_case       ,     [
											        SPIECE_UNDERLINE + 'I',
											        SPIECE_UNDERLINE + 'was',
											        SPIECE_UNDERLINE + 'b',
											        'or',
											        'n',
											        SPIECE_UNDERLINE + 'in',
											        SPIECE_UNDERLINE + '',
											        '<unk>',
											        '2',
											        '0',
											        '0',
											        '0',
											        ',',
											        SPIECE_UNDERLINE + 'and',
											        SPIECE_UNDERLINE + 'this',
											        SPIECE_UNDERLINE + 'is',
											        SPIECE_UNDERLINE + 'f',
											        'al',
											        's',
											        '<unk>',
											        '.',
											    ]       ,     )


					@cached_property
					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ):





											'''simple docstring'''





											return BigBirdTokenizer.from_pretrained('google/bigbird-roberta-base'       )


					@slow
					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		'Hello World!'
											_snake_case:   Optional[int]  					=		[65, 1_85_36, 22_60, 1_01, 66]

											self.assertListEqual(__snake_case       ,     self.big_tokenizer.encode(__snake_case       )       )


					@slow
					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ):





											'''simple docstring'''





											_snake_case:   str  					=		(
											    'This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) " [ ] ! : - . Also we will'
											    ' add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth'
											)
											# fmt: off
											_snake_case:   str  					=		[65, 8_71, 4_19, 3_58, 9_46, 9_91, 25_21, 4_52, 3_58, 13_57, 3_87, 77_51, 35_36, 1_12, 9_85, 4_56, 1_26, 8_65, 9_38, 54_00, 57_34, 4_58, 13_68, 4_67, 7_86, 24_62, 52_46, 11_59, 6_33, 8_65, 45_19, 4_57, 5_82, 8_52, 25_57, 4_27, 9_16, 5_08, 4_05, 3_43_24, 4_97, 3_91, 4_08, 1_13_42, 12_44, 3_85, 1_00, 9_38, 9_85, 4_56, 5_74, 3_62, 1_25_97, 32_00, 31_29, 11_72, 66]  # noqa: E231
											# fmt: on
											self.assertListEqual(__snake_case       ,     self.big_tokenizer.encode(__snake_case       )       )


					@require_torch
					@slow
					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ):





											'''simple docstring'''





											import torch

											from transformers import BigBirdConfig, BigBirdModel

											# Build sequence
											_snake_case:   Tuple  					=		list(self.big_tokenizer.get_vocab().keys()       )[:10]
											_snake_case:   Union[str, Any]  					=		' '.join(__snake_case       )
											_snake_case:   Optional[Any]  					=		self.big_tokenizer.encode_plus(__snake_case       ,     return_tensors='pt'       ,     return_token_type_ids=__snake_case       )
											_snake_case:   int  					=		self.big_tokenizer.batch_encode_plus(
											    [sequence + ' ' + sequence]       ,     return_tensors='pt'       ,     return_token_type_ids=__snake_case       )

											_snake_case:   int  					=		BigBirdConfig(attention_type='original_full'       )
											_snake_case:   int  					=		BigBirdModel(__snake_case       )

											assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

											with torch.no_grad():
																	model(**__snake_case       )
																	model(**__snake_case       )


					@slow
					def    SCREAMING_SNAKE_CASE_		(					self	:  List[Any]       ):





											'''simple docstring'''





											_snake_case:   Tuple  					=		BigBirdTokenizer.from_pretrained('google/bigbird-roberta-base'       )
											_snake_case:   Optional[Any]  					=		tokenizer.decode(tokenizer('Paris is the [MASK].'       ).input_ids       )

											self.assertTrue(decoded_text == '[CLS] Paris is the[MASK].[SEP]'       )




					@slow
					def    SCREAMING_SNAKE_CASE_		(					self	:  int       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		{'input_ids': [[65, 3_92_86, 4_58, 3_63_35, 20_01, 4_56, 1_30_73, 1_32_66, 4_55, 1_13, 77_46, 17_41, 1_11_57, 3_91, 1_30_73, 1_32_66, 4_55, 1_13, 39_67, 3_54_12, 1_13, 49_36, 1_09, 38_70, 23_77, 1_13, 3_00_84, 4_57_20, 4_58, 1_34, 1_74_96, 1_12, 5_03, 1_16_72, 1_13, 1_18, 1_12, 56_65, 1_33_47, 3_86_87, 1_12, 14_96, 3_13_89, 1_12, 32_68, 4_72_64, 1_34, 9_62, 1_12, 1_63_77, 80_35, 2_31_30, 4_30, 1_21_69, 1_55_18, 2_85_92, 4_58, 1_46, 4_16_97, 1_09, 3_91, 1_21_69, 1_55_18, 1_66_89, 4_58, 1_46, 4_13_58, 1_09, 4_52, 7_26, 40_34, 1_11, 7_63, 3_54_12, 50_82, 3_88, 19_03, 1_11, 90_51, 3_91, 28_70, 4_89_18, 19_00, 11_23, 5_50, 9_98, 1_12, 95_86, 1_59_85, 4_55, 3_91, 4_10, 2_29_55, 3_76_36, 1_14, 66], [65, 4_48, 1_74_96, 4_19, 36_63, 3_85, 7_63, 1_13, 2_75_33, 28_70, 32_83, 1_30_43, 16_39, 2_47_13, 5_23, 6_56, 2_40_13, 1_85_50, 25_21, 5_17, 2_70_14, 2_12_44, 4_20, 12_12, 14_65, 3_91, 9_27, 48_33, 3_88, 5_78, 1_17_86, 1_14, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 4_84, 21_69, 76_87, 2_19_32, 1_81_46, 7_26, 3_63, 1_70_32, 33_91, 1_14, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=__snake_case       ,     model_name='google/bigbird-roberta-base'       ,     revision='215c99f1600e06f83acce68422f2035b2b5c3510'       ,     )





