

'''simple docstring'''





import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers.testing_utils import require_vision
from transformers.utils import is_vision_available


if is_vision_available():
							from PIL import Image

							from transformers import (
							    AutoProcessor,
							    BertTokenizerFast,
							    BlipImageProcessor,
							    GPTaTokenizer,
							    InstructBlipProcessor,
							    PreTrainedTokenizerFast,
							)





@require_vision
class 		__SCREAMING_SNAKE_CASE       ( unittest.TestCase			):







				def    lowerCamelCase_  (					self    :		str      ):
										'''simple docstring'''
										lowercase   :       List[Any]								=tempfile.mkdtemp()

										lowercase   :       Optional[Any]								=BlipImageProcessor()
										lowercase   :       List[str]								=GPTaTokenizer.from_pretrained('''hf-internal-testing/tiny-random-GPT2Model'''      )
										lowercase   :       Optional[int]								=BertTokenizerFast.from_pretrained('''hf-internal-testing/tiny-random-bert'''      )

										lowercase   :       Tuple								=InstructBlipProcessor(UpperCAmelCase__		,				UpperCAmelCase__		,				UpperCAmelCase__      )

										processor.save_pretrained(self.tmpdirname      )







				def    lowerCamelCase_  (					self    :		List[Any]		,				**UpperCAmelCase__    :		Optional[int]      ):
										'''simple docstring'''
										return AutoProcessor.from_pretrained(self.tmpdirname		,				**UpperCAmelCase__      ).tokenizer







				def    lowerCamelCase_  (					self    :		List[str]		,				**UpperCAmelCase__    :		Any      ):
										'''simple docstring'''
										return AutoProcessor.from_pretrained(self.tmpdirname		,				**UpperCAmelCase__      ).image_processor







				def    lowerCamelCase_  (					self    :		str		,				**UpperCAmelCase__    :		List[Any]      ):
										'''simple docstring'''
										return AutoProcessor.from_pretrained(self.tmpdirname		,				**UpperCAmelCase__      ).qformer_tokenizer







				def    lowerCamelCase_  (					self    :		List[str]      ):
										'''simple docstring'''
										shutil.rmtree(self.tmpdirname      )







				def    lowerCamelCase_  (					self    :		Any      ):
										'''simple docstring'''
										lowercase   :       int								=[np.random.randint(255		,				size=(3, 30, 400)		,				dtype=np.uinta      )]

										lowercase   :       Optional[Any]								=[Image.fromarray(np.moveaxis(UpperCAmelCase__		,				0		,				-1      )      ) for x in image_inputs]

										return image_inputs







				def    lowerCamelCase_  (					self    :		str      ):
										'''simple docstring'''
										lowercase   :       Tuple								=InstructBlipProcessor(
										    tokenizer=self.get_tokenizer()		,				image_processor=self.get_image_processor()		,				qformer_tokenizer=self.get_qformer_tokenizer()		,				)
										processor.save_pretrained(self.tmpdirname      )

										lowercase   :       Dict								=self.get_tokenizer(bos_token='''(BOS)'''		,				eos_token='''(EOS)'''      )
										lowercase   :       Tuple								=self.get_image_processor(do_normalize=UpperCAmelCase__		,				padding_value=1.0      )

										lowercase   :       Dict								=InstructBlipProcessor.from_pretrained(
										    self.tmpdirname		,				bos_token='''(BOS)'''		,				eos_token='''(EOS)'''		,				do_normalize=UpperCAmelCase__		,				padding_value=1.0      )

										self.assertEqual(processor.tokenizer.get_vocab()		,				tokenizer_add_kwargs.get_vocab()      )
										self.assertIsInstance(processor.tokenizer		,				UpperCAmelCase__      )

										self.assertEqual(processor.image_processor.to_json_string()		,				image_processor_add_kwargs.to_json_string()      )
										self.assertIsInstance(processor.image_processor		,				UpperCAmelCase__      )
										self.assertIsInstance(processor.qformer_tokenizer		,				UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		Union[str, Any]      ):
										'''simple docstring'''
										lowercase   :       Tuple								=self.get_image_processor()
										lowercase   :       Optional[int]								=self.get_tokenizer()
										lowercase   :       Union[str, Any]								=self.get_qformer_tokenizer()

										lowercase   :       int								=InstructBlipProcessor(
										    tokenizer=UpperCAmelCase__		,				image_processor=UpperCAmelCase__		,				qformer_tokenizer=UpperCAmelCase__      )

										lowercase   :       Optional[Any]								=self.prepare_image_inputs()

										lowercase   :       Any								=image_processor(UpperCAmelCase__		,				return_tensors='''np'''      )
										lowercase   :       Any								=processor(images=UpperCAmelCase__		,				return_tensors='''np'''      )

										for key in input_feat_extract.keys():
																self.assertAlmostEqual(input_feat_extract[key].sum()		,				input_processor[key].sum()		,				delta=1E-2      )







				def    lowerCamelCase_  (					self    :		Any      ):
										'''simple docstring'''
										lowercase   :       Optional[Any]								=self.get_image_processor()
										lowercase   :       str								=self.get_tokenizer()
										lowercase   :       Any								=self.get_qformer_tokenizer()

										lowercase   :       int								=InstructBlipProcessor(
										    tokenizer=UpperCAmelCase__		,				image_processor=UpperCAmelCase__		,				qformer_tokenizer=UpperCAmelCase__      )

										lowercase   :       Any								='''lower newer'''

										lowercase   :       List[Any]								=processor(text=UpperCAmelCase__      )

										lowercase   :       int								=tokenizer(UpperCAmelCase__		,				return_token_type_ids=UpperCAmelCase__      )
										lowercase   :       Optional[Any]								=qformer_tokenizer(UpperCAmelCase__		,				return_token_type_ids=UpperCAmelCase__      )

										for key in encoded_tokens.keys():
																self.assertListEqual(encoded_tokens[key]		,				encoded_processor[key]      )

										for key in encoded_tokens_qformer.keys():
																self.assertListEqual(encoded_tokens_qformer[key]		,				encoded_processor['''qformer_''' + key]      )







				def    lowerCamelCase_  (					self    :		Any      ):
										'''simple docstring'''
										lowercase   :       Optional[int]								=self.get_image_processor()
										lowercase   :       Union[str, Any]								=self.get_tokenizer()
										lowercase   :       int								=self.get_qformer_tokenizer()

										lowercase   :       List[Any]								=InstructBlipProcessor(
										    tokenizer=UpperCAmelCase__		,				image_processor=UpperCAmelCase__		,				qformer_tokenizer=UpperCAmelCase__      )

										lowercase   :       int								='''lower newer'''
										lowercase   :       Optional[Any]								=self.prepare_image_inputs()

										lowercase   :       Union[str, Any]								=processor(text=UpperCAmelCase__		,				images=UpperCAmelCase__      )

										self.assertListEqual(
										    list(inputs.keys()      )		,				['''input_ids''', '''attention_mask''', '''qformer_input_ids''', '''qformer_attention_mask''', '''pixel_values''']		,				)

										# test if it raises when no input is passed
										with pytest.raises(UpperCAmelCase__      ):
																processor()







				def    lowerCamelCase_  (					self    :		str      ):
										'''simple docstring'''
										lowercase   :       int								=self.get_image_processor()
										lowercase   :       int								=self.get_tokenizer()
										lowercase   :       List[Any]								=self.get_qformer_tokenizer()

										lowercase   :       List[str]								=InstructBlipProcessor(
										    tokenizer=UpperCAmelCase__		,				image_processor=UpperCAmelCase__		,				qformer_tokenizer=UpperCAmelCase__      )

										lowercase   :       Any								=[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

										lowercase   :       List[Any]								=processor.batch_decode(UpperCAmelCase__      )
										lowercase   :       int								=tokenizer.batch_decode(UpperCAmelCase__      )

										self.assertListEqual(UpperCAmelCase__		,				UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		Optional[int]      ):
										'''simple docstring'''
										lowercase   :       Any								=self.get_image_processor()
										lowercase   :       Any								=self.get_tokenizer()
										lowercase   :       List[str]								=self.get_qformer_tokenizer()

										lowercase   :       Union[str, Any]								=InstructBlipProcessor(
										    tokenizer=UpperCAmelCase__		,				image_processor=UpperCAmelCase__		,				qformer_tokenizer=UpperCAmelCase__      )

										lowercase   :       str								='''lower newer'''
										lowercase   :       List[Any]								=self.prepare_image_inputs()

										lowercase   :       Optional[int]								=processor(text=UpperCAmelCase__		,				images=UpperCAmelCase__      )

										self.assertListEqual(
										    list(inputs.keys()      )		,				['''input_ids''', '''attention_mask''', '''qformer_input_ids''', '''qformer_attention_mask''', '''pixel_values''']		,				)








'''simple docstring'''





import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 		__SCREAMING_SNAKE_CASE       ( lowercase__	,       unittest.TestCase			):
				lowerCamelCase_							=						None
				lowerCamelCase_							=						BloomTokenizerFast
				lowerCamelCase_							=						BloomTokenizerFast
				lowerCamelCase_							=						True
				lowerCamelCase_							=						False
				lowerCamelCase_							=						'tokenizer_file'
				lowerCamelCase_							=						{'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '<pad>'}







				def    lowerCamelCase_  (					self    :		List[str]      ):
										'''simple docstring'''
										super().setUp()
										lowercase   :       Union[str, Any]								=BloomTokenizerFast.from_pretrained('''bigscience/tokenizer'''      )
										tokenizer.save_pretrained(self.tmpdirname      )







				def    lowerCamelCase_  (					self    :		Union[str, Any]		,				**UpperCAmelCase__    :		Any      ):
										'''simple docstring'''
										kwargs.update(self.special_tokens_map      )
										return BloomTokenizerFast.from_pretrained(self.tmpdirname		,				**UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		Any      ):
										'''simple docstring'''
										lowercase   :       str								=self.get_rust_tokenizer()

										lowercase   :       List[str]								=['''The quick brown fox</s>''', '''jumps over the lazy dog</s>''']
										lowercase   :       Any								=[[2175, 23714, 73173, 144252, 2], [77, 132619, 3478, 368, 109586, 35433, 2]]

										lowercase   :       Any								=tokenizer.batch_encode_plus(UpperCAmelCase__      )['''input_ids''']
										self.assertListEqual(UpperCAmelCase__		,				UpperCAmelCase__      )

										lowercase   :       int								=tokenizer.batch_decode(UpperCAmelCase__      )
										self.assertListEqual(UpperCAmelCase__		,				UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		Tuple		,				UpperCAmelCase__    :		Any=6      ):
										'''simple docstring'''
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
																						lowercase   :       Optional[int]								=self.rust_tokenizer_class.from_pretrained(UpperCAmelCase__		,				**UpperCAmelCase__      )
																						# tokenizer_r.pad_token = None # Hotfixing padding = None
																						# Simple input
																						lowercase   :       Tuple								='''This is a simple input'''
																						lowercase   :       int								=['''This is a simple input 1''', '''This is a simple input 2''']
																						lowercase   :       Optional[Any]								=('''This is a simple input''', '''This is a pair''')
																						lowercase   :       int								=[
																						    ('''This is a simple input 1''', '''This is a simple input 2'''),
																						    ('''This is a simple pair 1''', '''This is a simple pair 2'''),
																						]

																						# Simple input tests
																						try:
																												tokenizer_r.encode(UpperCAmelCase__		,				max_length=UpperCAmelCase__      )
																												tokenizer_r.encode_plus(UpperCAmelCase__		,				max_length=UpperCAmelCase__      )

																												tokenizer_r.batch_encode_plus(UpperCAmelCase__		,				max_length=UpperCAmelCase__      )
																												tokenizer_r.encode(UpperCAmelCase__		,				max_length=UpperCAmelCase__      )
																												tokenizer_r.batch_encode_plus(UpperCAmelCase__		,				max_length=UpperCAmelCase__      )
																						except ValueError:
																												self.fail('''Bloom Tokenizer should be able to deal with padding'''      )

																						lowercase   :       Optional[int]								=None  # Hotfixing padding = None
																						self.assertRaises(UpperCAmelCase__		,				tokenizer_r.encode		,				UpperCAmelCase__		,				max_length=UpperCAmelCase__		,				padding='''max_length'''      )

																						# Simple input
																						self.assertRaises(UpperCAmelCase__		,				tokenizer_r.encode_plus		,				UpperCAmelCase__		,				max_length=UpperCAmelCase__		,				padding='''max_length'''      )

																						# Simple input
																						self.assertRaises(
																						    UpperCAmelCase__		,				tokenizer_r.batch_encode_plus		,				UpperCAmelCase__		,				max_length=UpperCAmelCase__		,				padding='''max_length'''		,				)

																						# Pair input
																						self.assertRaises(UpperCAmelCase__		,				tokenizer_r.encode		,				UpperCAmelCase__		,				max_length=UpperCAmelCase__		,				padding='''max_length'''      )

																						# Pair input
																						self.assertRaises(UpperCAmelCase__		,				tokenizer_r.encode_plus		,				UpperCAmelCase__		,				max_length=UpperCAmelCase__		,				padding='''max_length'''      )

																						# Pair input
																						self.assertRaises(
																						    UpperCAmelCase__		,				tokenizer_r.batch_encode_plus		,				UpperCAmelCase__		,				max_length=UpperCAmelCase__		,				padding='''max_length'''		,				)







				def    lowerCamelCase_  (					self    :		Union[str, Any]      ):
										'''simple docstring'''
										lowercase   :       Union[str, Any]								=self.get_rust_tokenizer()
										lowercase   :       Dict								=load_dataset('''xnli'''		,				'''all_languages'''		,				split='''test'''		,				streaming=UpperCAmelCase__      )

										lowercase   :       Union[str, Any]								=next(iter(UpperCAmelCase__      )      )['''premise''']  # pick up one data
										lowercase   :       int								=list(sample_data.values()      )

										lowercase   :       Any								=list(map(tokenizer.encode		,				UpperCAmelCase__      )      )
										lowercase   :       List[str]								=[tokenizer.decode(UpperCAmelCase__		,				clean_up_tokenization_spaces=UpperCAmelCase__      ) for x in output_tokens]
										self.assertListEqual(UpperCAmelCase__		,				UpperCAmelCase__      )







				def    lowerCamelCase_  (					self    :		List[Any]      ):
										'''simple docstring'''
										# The test has to be overriden because BLOOM uses ALiBi positional embeddings that does not have
										# any sequence length constraints. This test of the parent class will fail since it relies on the
										# maximum sequence length of the positoonal embeddings.
										self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map      )		,				1      )
										self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()      )[0]      )		,				1      )





