





import unittest

from datasets import load_dataset

from transformers import BloomTokenizerFast
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 			lowerCamelCase					(						__UpperCAmelCase     ,							unittest.TestCase  ):
					_SCREAMING_SNAKE_CASE							   =    None
					_SCREAMING_SNAKE_CASE							   =    BloomTokenizerFast
					_SCREAMING_SNAKE_CASE							   =    BloomTokenizerFast
					_SCREAMING_SNAKE_CASE							   =    True
					_SCREAMING_SNAKE_CASE							   =    False
					_SCREAMING_SNAKE_CASE							   =    "tokenizer_file"
					_SCREAMING_SNAKE_CASE							   =    {"bos_token": "<s>", "eos_token": "</s>", "unk_token": "<unk>", "pad_token": "<pad>"}


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ):





											'''simple docstring'''





											super().setUp()
											_snake_case:   Optional[int]  					=		BloomTokenizerFast.from_pretrained('bigscience/tokenizer'       )
											tokenizer.save_pretrained(self.tmpdirname       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ,     **__snake_case	:  Optional[int]       ):





											'''simple docstring'''





											kwargs.update(self.special_tokens_map       )
											return BloomTokenizerFast.from_pretrained(self.tmpdirname       ,     **__snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  str       ):





											'''simple docstring'''





											_snake_case:   Tuple  					=		self.get_rust_tokenizer()

											_snake_case:   List[str]  					=		['The quick brown fox</s>', 'jumps over the lazy dog</s>']
											_snake_case:   List[str]  					=		[[21_75, 2_37_14, 7_31_73, 14_42_52, 2], [77, 13_26_19, 34_78, 3_68, 10_95_86, 3_54_33, 2]]

											_snake_case:   int  					=		tokenizer.batch_encode_plus(__snake_case       )['input_ids']
											self.assertListEqual(__snake_case       ,     __snake_case       )

											_snake_case:   Any  					=		tokenizer.batch_decode(__snake_case       )
											self.assertListEqual(__snake_case       ,     __snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ,     __snake_case	:  List[Any]=6       ):





											'''simple docstring'''





											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																	with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''       ):
																							_snake_case:   List[str]  					=		self.rust_tokenizer_class.from_pretrained(__snake_case       ,     **__snake_case       )
																							# tokenizer_r.pad_token = None # Hotfixing padding = None
																							# Simple input
																							_snake_case:   List[Any]  					=		'This is a simple input'
																							_snake_case:   int  					=		['This is a simple input 1', 'This is a simple input 2']
																							_snake_case:   Dict  					=		('This is a simple input', 'This is a pair')
																							_snake_case:   Union[str, Any]  					=		[
																							    ('This is a simple input 1', 'This is a simple input 2'),
																							    ('This is a simple pair 1', 'This is a simple pair 2'),
																							]

																							# Simple input tests
																							try:
																													tokenizer_r.encode(__snake_case       ,     max_length=__snake_case       )
																													tokenizer_r.encode_plus(__snake_case       ,     max_length=__snake_case       )

																													tokenizer_r.batch_encode_plus(__snake_case       ,     max_length=__snake_case       )
																													tokenizer_r.encode(__snake_case       ,     max_length=__snake_case       )
																													tokenizer_r.batch_encode_plus(__snake_case       ,     max_length=__snake_case       )
																							except ValueError:
																													self.fail('Bloom Tokenizer should be able to deal with padding'       )

																							_snake_case:   Union[str, Any]  					=		None  # Hotfixing padding = None
																							self.assertRaises(__snake_case       ,     tokenizer_r.encode       ,     __snake_case       ,     max_length=__snake_case       ,     padding='max_length'       )

																							# Simple input
																							self.assertRaises(__snake_case       ,     tokenizer_r.encode_plus       ,     __snake_case       ,     max_length=__snake_case       ,     padding='max_length'       )

																							# Simple input
																							self.assertRaises(
																							    __snake_case       ,     tokenizer_r.batch_encode_plus       ,     __snake_case       ,     max_length=__snake_case       ,     padding='max_length'       ,     )

																							# Pair input
																							self.assertRaises(__snake_case       ,     tokenizer_r.encode       ,     __snake_case       ,     max_length=__snake_case       ,     padding='max_length'       )

																							# Pair input
																							self.assertRaises(__snake_case       ,     tokenizer_r.encode_plus       ,     __snake_case       ,     max_length=__snake_case       ,     padding='max_length'       )

																							# Pair input
																							self.assertRaises(
																							    __snake_case       ,     tokenizer_r.batch_encode_plus       ,     __snake_case       ,     max_length=__snake_case       ,     padding='max_length'       ,     )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[Any]       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		self.get_rust_tokenizer()
											_snake_case:   Optional[int]  					=		load_dataset('xnli'       ,     'all_languages'       ,     split='test'       ,     streaming=__snake_case       )

											_snake_case:   List[Any]  					=		next(iter(__snake_case       )       )['premise']  # pick up one data
											_snake_case:   str  					=		list(sample_data.values()       )

											_snake_case:   Tuple  					=		list(map(tokenizer.encode       ,     __snake_case       )       )
											_snake_case:   str  					=		[tokenizer.decode(__snake_case       ,     clean_up_tokenization_spaces=__snake_case       ) for x in output_tokens]
											self.assertListEqual(__snake_case       ,     __snake_case       )




					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ):





											'''simple docstring'''





											self.assertGreaterEqual(len(self.tokenizer_class.pretrained_vocab_files_map       )       ,     1       )
											self.assertGreaterEqual(len(list(self.tokenizer_class.pretrained_vocab_files_map.values()       )[0]       )       ,     1       )












'''simple docstring'''






import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
		from PIL import Image

		from transformers import CLIPImageProcessor, CLIPProcessor







@require_vision
class 			lowerCamelCase					(						unittest.TestCase  ):


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ):





											'''simple docstring'''





											_snake_case:   Optional[Any]  					=		tempfile.mkdtemp()

											# fmt: off
											_snake_case:   Union[str, Any]  					=		['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
											# fmt: on
											_snake_case:   str  					=		dict(zip(__snake_case       ,     range(len(__snake_case       )       )       )       )
											_snake_case:   str  					=		['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>', '']
											_snake_case:   Optional[int]  					=		{'unk_token': '<unk>'}

											_snake_case:   int  					=		os.path.join(self.tmpdirname       ,     VOCAB_FILES_NAMES['vocab_file']       )
											_snake_case:   Optional[int]  					=		os.path.join(self.tmpdirname       ,     VOCAB_FILES_NAMES['merges_file']       )
											with open(self.vocab_file       ,     'w'       ,     encoding='utf-8'       ) as fp:
																	fp.write(json.dumps(__snake_case       ) + '\n'       )
											with open(self.merges_file       ,     'w'       ,     encoding='utf-8'       ) as fp:
																	fp.write('\n'.join(__snake_case       )       )

											_snake_case:   Dict  					=		{
											    'do_resize': True,
											    'size': 20,
											    'do_center_crop': True,
											    'crop_size': 18,
											    'do_normalize': True,
											    'image_mean': [0.48_145_466, 0.4_578_275, 0.40_821_073],
											    'image_std': [0.26_862_954, 0.26_130_258, 0.27_577_711],
											}
											_snake_case:   Union[str, Any]  					=		os.path.join(self.tmpdirname       ,     __snake_case       )
											with open(self.image_processor_file       ,     'w'       ,     encoding='utf-8'       ) as fp:
																	json.dump(__snake_case       ,     __snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[Any]       ,     **__snake_case	:  Tuple       ):





											'''simple docstring'''





											return CLIPTokenizer.from_pretrained(self.tmpdirname       ,     **__snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ,     **__snake_case	:  Any       ):





											'''simple docstring'''





											return CLIPTokenizerFast.from_pretrained(self.tmpdirname       ,     **__snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  str       ,     **__snake_case	:  List[Any]       ):





											'''simple docstring'''





											return CLIPImageProcessor.from_pretrained(self.tmpdirname       ,     **__snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ):





											'''simple docstring'''





											shutil.rmtree(self.tmpdirname       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[Any]       ):





											'''simple docstring'''





											_snake_case:   str  					=		[np.random.randint(2_55       ,     size=(3, 30, 4_00)       ,     dtype=np.uinta       )]

											_snake_case:   Any  					=		[Image.fromarray(np.moveaxis(__snake_case       ,     0       ,     -1       )       ) for x in image_inputs]

											return image_inputs


					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		self.get_tokenizer()
											_snake_case:   List[str]  					=		self.get_rust_tokenizer()
											_snake_case:   List[str]  					=		self.get_image_processor()

											_snake_case:   Union[str, Any]  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )
											processor_slow.save_pretrained(self.tmpdirname       )
											_snake_case:   Any  					=		CLIPProcessor.from_pretrained(self.tmpdirname       ,     use_fast=__snake_case       )

											_snake_case:   Any  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )
											processor_fast.save_pretrained(self.tmpdirname       )
											_snake_case:   int  					=		CLIPProcessor.from_pretrained(self.tmpdirname       )

											self.assertEqual(processor_slow.tokenizer.get_vocab()       ,     tokenizer_slow.get_vocab()       )
											self.assertEqual(processor_fast.tokenizer.get_vocab()       ,     tokenizer_fast.get_vocab()       )
											self.assertEqual(tokenizer_slow.get_vocab()       ,     tokenizer_fast.get_vocab()       )
											self.assertIsInstance(processor_slow.tokenizer       ,     __snake_case       )
											self.assertIsInstance(processor_fast.tokenizer       ,     __snake_case       )

											self.assertEqual(processor_slow.image_processor.to_json_string()       ,     image_processor.to_json_string()       )
											self.assertEqual(processor_fast.image_processor.to_json_string()       ,     image_processor.to_json_string()       )
											self.assertIsInstance(processor_slow.image_processor       ,     __snake_case       )
											self.assertIsInstance(processor_fast.image_processor       ,     __snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[Any]       ):





											'''simple docstring'''





											_snake_case:   Any  					=		CLIPProcessor(tokenizer=self.get_tokenizer()       ,     image_processor=self.get_image_processor()       )
											processor.save_pretrained(self.tmpdirname       )

											_snake_case:   List[Any]  					=		self.get_tokenizer(bos_token='(BOS)'       ,     eos_token='(EOS)'       )
											_snake_case:   List[Any]  					=		self.get_image_processor(do_normalize=__snake_case       ,     padding_value=1.0       )

											_snake_case:   Any  					=		CLIPProcessor.from_pretrained(
											    self.tmpdirname       ,     bos_token='(BOS)'       ,     eos_token='(EOS)'       ,     do_normalize=__snake_case       ,     padding_value=1.0       )

											self.assertEqual(processor.tokenizer.get_vocab()       ,     tokenizer_add_kwargs.get_vocab()       )
											self.assertIsInstance(processor.tokenizer       ,     __snake_case       )

											self.assertEqual(processor.image_processor.to_json_string()       ,     image_processor_add_kwargs.to_json_string()       )
											self.assertIsInstance(processor.image_processor       ,     __snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ):





											'''simple docstring'''





											_snake_case:   List[Any]  					=		self.get_image_processor()
											_snake_case:   List[str]  					=		self.get_tokenizer()

											_snake_case:   Any  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )

											_snake_case:   List[str]  					=		self.prepare_image_inputs()

											_snake_case:   List[str]  					=		image_processor(__snake_case       ,     return_tensors='np'       )
											_snake_case:   Dict  					=		processor(images=__snake_case       ,     return_tensors='np'       )

											for key in input_image_proc.keys():
																	self.assertAlmostEqual(input_image_proc[key].sum()       ,     input_processor[key].sum()       ,     delta=1e-2       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[str]       ):





											'''simple docstring'''





											_snake_case:   List[Any]  					=		self.get_image_processor()
											_snake_case:   Any  					=		self.get_tokenizer()

											_snake_case:   List[str]  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )

											_snake_case:   int  					=		'lower newer'

											_snake_case:   str  					=		processor(text=__snake_case       )

											_snake_case:   List[Any]  					=		tokenizer(__snake_case       )

											for key in encoded_tok.keys():
																	self.assertListEqual(encoded_tok[key]       ,     encoded_processor[key]       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ):





											'''simple docstring'''





											_snake_case:   List[str]  					=		self.get_image_processor()
											_snake_case:   List[Any]  					=		self.get_tokenizer()

											_snake_case:   Tuple  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )

											_snake_case:   Optional[int]  					=		'lower newer'
											_snake_case:   Union[str, Any]  					=		self.prepare_image_inputs()

											_snake_case:   int  					=		processor(text=__snake_case       ,     images=__snake_case       )

											self.assertListEqual(list(inputs.keys()       )       ,     ['input_ids', 'attention_mask', 'pixel_values']       )

											# test if it raises when no input is passed
											with pytest.raises(__snake_case       ):
																	processor()


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		self.get_image_processor()
											_snake_case:   Dict  					=		self.get_tokenizer()

											_snake_case:   str  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )

											_snake_case:   str  					=		[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

											_snake_case:   List[Any]  					=		processor.batch_decode(__snake_case       )
											_snake_case:   Any  					=		tokenizer.batch_decode(__snake_case       )

											self.assertListEqual(__snake_case       ,     __snake_case       )




					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ):





											'''simple docstring'''





											_snake_case:   Union[str, Any]  					=		self.get_image_processor()
											_snake_case:   Any  					=		self.get_tokenizer()

											_snake_case:   Any  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )

											_snake_case:   Any  					=		'lower newer'
											_snake_case:   Optional[Any]  					=		self.prepare_image_inputs()

											_snake_case:   int  					=		processor(text=__snake_case       ,     images=__snake_case       )

											self.assertListEqual(list(inputs.keys()       )       ,     processor.model_input_names       )





