





import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
		A   : List[str]             =			'pt'
elif is_tf_available():
		A   : Any             =			'tf'
else:
		A   : Dict             =			'jax'







class 			lowerCamelCase					(						__UpperCAmelCase     ,							unittest.TestCase  ):
					_SCREAMING_SNAKE_CASE							   =    ByTaTokenizer
					_SCREAMING_SNAKE_CASE							   =    False


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ):





											'''simple docstring'''





											super().setUp()
											_snake_case:   Any  					=		ByTaTokenizer()
											tokenizer.save_pretrained(self.tmpdirname       )


					@cached_property
					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ):





											'''simple docstring'''





											return ByTaTokenizer.from_pretrained('google/byt5-small'       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ,     **__snake_case	:  Optional[Any]       ):





											'''simple docstring'''





											return self.tokenizer_class.from_pretrained(self.tmpdirname       ,     **__snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  str       ,     __snake_case	:  Any       ,     __snake_case	:  Dict=False       ,     __snake_case	:  Dict=20       ,     __snake_case	:  Tuple=5       ):





											'''simple docstring'''





											_snake_case:   int  					=		[]
											for i in range(len(__snake_case       )       ):
																	try:
																							_snake_case:   Any  					=		tokenizer.decode([i]       ,     clean_up_tokenization_spaces=__snake_case       )
																	except UnicodeDecodeError:
																							pass
																	toks.append((i, tok)       )

											_snake_case:   Union[str, Any]  					=		list(filter(lambda __snake_case       : re.match(R'^[ a-zA-Z]+$'       ,     t[1]       )       ,     __snake_case       )       )
											_snake_case:   List[str]  					=		list(filter(lambda __snake_case       : [t[0]] == tokenizer.encode(t[1]       ,     add_special_tokens=__snake_case       )       ,     __snake_case       )       )
											if max_length is not None and len(__snake_case       ) > max_length:
																	_snake_case:   List[Any]  					=		toks[:max_length]
											if min_length is not None and len(__snake_case       ) < min_length and len(__snake_case       ) > 0:
																	while len(__snake_case       ) < min_length:
																							_snake_case:   List[str]  					=		toks + toks
        # toks_str = [t[1] for t in toks]
											_snake_case:   Tuple  					=		[t[0] for t in toks]

											# Ensure consistency
											_snake_case:   int  					=		tokenizer.decode(__snake_case       ,     clean_up_tokenization_spaces=__snake_case       )
											if " " not in output_txt and len(__snake_case       ) > 1:
																	_snake_case:   Tuple  					=		(
																	    tokenizer.decode([toks_ids[0]]       ,     clean_up_tokenization_spaces=__snake_case       )
																	    + ' '
																	    + tokenizer.decode(toks_ids[1:]       ,     clean_up_tokenization_spaces=__snake_case       )
																	)
											if with_prefix_space:
																	_snake_case:   Dict  					=		' ' + output_txt
											_snake_case:   Any  					=		tokenizer.encode(__snake_case       ,     add_special_tokens=__snake_case       )
											return output_txt, output_ids


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[Any]       ):





											'''simple docstring'''





											_snake_case:   Any  					=		self.ta_base_tokenizer
											_snake_case:   Any  					=		tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']       )
											_snake_case:   Union[str, Any]  					=		tokenizer(['hi', 'I went to the gym', '']       )
											self.assertListEqual(batch_with_eos_added['input_ids']       ,     batch_without_eos_added['input_ids']       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ):





											'''simple docstring'''





											_snake_case:   Union[str, Any]  					=		self.ta_base_tokenizer
											_snake_case:   Any  					=		'Unicode €.'
											_snake_case:   Optional[int]  					=		tokenizer(__snake_case       )
											_snake_case:   str  					=		[88, 1_13, 1_08, 1_02, 1_14, 1_03, 1_04, 35, 2_29, 1_33, 1_75, 49, 1]
											self.assertEqual(encoded['input_ids']       ,     __snake_case       )

											# decoding
											_snake_case:   Union[str, Any]  					=		tokenizer.decode(__snake_case       )
											self.assertEqual(__snake_case       ,     'Unicode €.</s>'       )

											_snake_case:   Tuple  					=		tokenizer('e è é ê ë'       )
											_snake_case:   Dict  					=		[1_04, 35, 1_98, 1_71, 35, 1_98, 1_72, 35, 1_98, 1_73, 35, 1_98, 1_74, 1]
											self.assertEqual(encoded['input_ids']       ,     __snake_case       )
											# decoding
											_snake_case:   List[Any]  					=		tokenizer.decode(__snake_case       )
											self.assertEqual(__snake_case       ,     'e è é ê ë</s>'       )

											# encode/decode, but with `encode` instead of `__call__`
											self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'       )       )       ,     'e è é ê ë</s>'       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ):





											'''simple docstring'''





											_snake_case:   str  					=		self.ta_base_tokenizer
											_snake_case:   Optional[Any]  					=		['A long paragraph for summarization.', 'Another paragraph for summarization.']
											# fmt: off
											_snake_case:   List[Any]  					=		[68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 1, 0]
											# fmt: on
											_snake_case:   str  					=		tokenizer(__snake_case       ,     padding=__snake_case       ,     return_tensors=__snake_case       )
											self.assertIsInstance(__snake_case       ,     __snake_case       )

											if FRAMEWORK != "jax":
																	_snake_case:   Optional[int]  					=		list(batch.input_ids.numpy()[0]       )
											else:
																	_snake_case:   Union[str, Any]  					=		list(batch.input_ids.tolist()[0]       )

											self.assertListEqual(__snake_case       ,     __snake_case       )

											self.assertEqual((2, 37)       ,     batch.input_ids.shape       )
											self.assertEqual((2, 37)       ,     batch.attention_mask.shape       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		self.ta_base_tokenizer
											_snake_case:   Tuple  					=		['A long paragraph for summarization.', 'Another paragraph for summarization.']
											_snake_case:   Union[str, Any]  					=		tokenizer(__snake_case       ,     padding=__snake_case       ,     return_tensors=__snake_case       )
											# check if input_ids are returned and no decoder_input_ids
											self.assertIn('input_ids'       ,     __snake_case       )
											self.assertIn('attention_mask'       ,     __snake_case       )
											self.assertNotIn('decoder_input_ids'       ,     __snake_case       )
											self.assertNotIn('decoder_attention_mask'       ,     __snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  str       ):





											'''simple docstring'''





											_snake_case:   List[str]  					=		self.ta_base_tokenizer
											_snake_case:   Union[str, Any]  					=		[
											    'Summary of the text.',
											    'Another summary.',
											]
											_snake_case:   int  					=		tokenizer(
											    text_target=__snake_case       ,     max_length=32       ,     padding='max_length'       ,     truncation=__snake_case       ,     return_tensors=__snake_case       )
											self.assertEqual(32       ,     targets['input_ids'].shape[1]       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ):





											'''simple docstring'''





											_snake_case:   Optional[Any]  					=		self.ta_base_tokenizer
											_snake_case:   Any  					=		['A long paragraph for summarization. </s>']
											_snake_case:   Dict  					=		['Summary of the text. </s>']
											# fmt: off
											_snake_case:   Union[str, Any]  					=		[68, 35, 1_11, 1_14, 1_13, 1_06, 35, 1_15, 1_00, 1_17, 1_00, 1_06, 1_17, 1_00, 1_15, 1_07, 35, 1_05, 1_14, 1_17, 35, 1_18, 1_20, 1_12, 1_12, 1_00, 1_17, 1_08, 1_25, 1_00, 1_19, 1_08, 1_14, 1_13, 49, 35, 1]
											_snake_case:   int  					=		[86, 1_20, 1_12, 1_12, 1_00, 1_17, 1_24, 35, 1_14, 1_05, 35, 1_19, 1_07, 1_04, 35, 1_19, 1_04, 1_23, 1_19, 49, 35, 1]
											# fmt: on

											_snake_case:   int  					=		tokenizer(__snake_case       ,     text_target=__snake_case       )

											self.assertEqual(__snake_case       ,     batch['input_ids'][0]       )
											self.assertEqual(__snake_case       ,     batch['labels'][0]       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  int       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		self.get_tokenizers()
											for tokenizer in tokenizers:
																	with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
																							self.assertNotEqual(tokenizer.model_max_length       ,     42       )

        # Now let's start the test
											_snake_case:   List[str]  					=		self.get_tokenizers()
											for tokenizer in tokenizers:
																	with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
																							# Isolate this from the other tests because we save additional tokens/etc
																							_snake_case:   List[Any]  					=		tempfile.mkdtemp()

																							_snake_case:   Optional[Any]  					=		' He is very happy, UNwant\u00E9d,running'
																							_snake_case:   List[Any]  					=		tokenizer.encode(__snake_case       ,     add_special_tokens=__snake_case       )
																							tokenizer.save_pretrained(__snake_case       )

																							_snake_case:   int  					=		tokenizer.__class__.from_pretrained(__snake_case       )
																							_snake_case:   Optional[int]  					=		after_tokenizer.encode(__snake_case       ,     add_special_tokens=__snake_case       )
																							self.assertListEqual(__snake_case       ,     __snake_case       )

																							shutil.rmtree(__snake_case       )

											_snake_case:   List[Any]  					=		self.get_tokenizers(model_max_length=42       )
											for tokenizer in tokenizers:
																	with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
																							# Isolate this from the other tests because we save additional tokens/etc
																							_snake_case:   Optional[int]  					=		tempfile.mkdtemp()

																							_snake_case:   Dict  					=		' He is very happy, UNwant\u00E9d,running'
																							tokenizer.add_tokens(['bim', 'bambam']       )
																							_snake_case:   int  					=		tokenizer.additional_special_tokens
																							additional_special_tokens.append('new_additional_special_token'       )
																							tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}       )
																							_snake_case:   int  					=		tokenizer.encode(__snake_case       ,     add_special_tokens=__snake_case       )
																							tokenizer.save_pretrained(__snake_case       )

																							_snake_case:   Any  					=		tokenizer.__class__.from_pretrained(__snake_case       )
																							_snake_case:   Optional[int]  					=		after_tokenizer.encode(__snake_case       ,     add_special_tokens=__snake_case       )
																							self.assertListEqual(__snake_case       ,     __snake_case       )
																							self.assertIn('new_additional_special_token'       ,     after_tokenizer.additional_special_tokens       )
																							self.assertEqual(after_tokenizer.model_max_length       ,     42       )

																							_snake_case:   Union[str, Any]  					=		tokenizer.__class__.from_pretrained(__snake_case       ,     model_max_length=43       )
																							self.assertEqual(tokenizer.model_max_length       ,     43       )

																							shutil.rmtree(__snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Any       ):





											'''simple docstring'''





											_snake_case:   Any  					=		[]
											if self.test_slow_tokenizer:
																	tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

											if self.test_rust_tokenizer:
																	tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

											for tokenizer_class, tokenizer_utils in tokenizer_list:
																	with tempfile.TemporaryDirectory() as tmp_dir:
																							tokenizer_utils.save_pretrained(__snake_case       )

																							with open(os.path.join(__snake_case       ,     'special_tokens_map.json'       )       ,     encoding='utf-8'       ) as json_file:
																													_snake_case:   int  					=		json.load(__snake_case       )

																							with open(os.path.join(__snake_case       ,     'tokenizer_config.json'       )       ,     encoding='utf-8'       ) as json_file:
																													_snake_case:   Union[str, Any]  					=		json.load(__snake_case       )

																							_snake_case:   Tuple  					=		[f'''<extra_id_{i}>''' for i in range(1_25       )]

																							_snake_case:   Optional[int]  					=		added_tokens_extra_ids + [
																							    'an_additional_special_token'
																							]
																							_snake_case:   List[str]  					=		added_tokens_extra_ids + [
																							    'an_additional_special_token'
																							]

																							with open(os.path.join(__snake_case       ,     'special_tokens_map.json'       )       ,     'w'       ,     encoding='utf-8'       ) as outfile:
																													json.dump(__snake_case       ,     __snake_case       )
																							with open(os.path.join(__snake_case       ,     'tokenizer_config.json'       )       ,     'w'       ,     encoding='utf-8'       ) as outfile:
																													json.dump(__snake_case       ,     __snake_case       )

																							# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																							# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																							# "special_tokens_map.json" files
																							_snake_case:   str  					=		tokenizer_class.from_pretrained(
																							    __snake_case       ,     )
																							self.assertIn(
																							    'an_additional_special_token'       ,     tokenizer_without_change_in_init.additional_special_tokens       )
																							# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																							self.assertEqual(
																							    ['an_additional_special_token']       ,     tokenizer_without_change_in_init.convert_ids_to_tokens(
																							        tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']       )       )       ,     )

																							# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																							_snake_case:   List[Any]  					=		added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'       ,     lstrip=__snake_case       )]
																							_snake_case:   Tuple  					=		tokenizer_class.from_pretrained(
																							    __snake_case       ,     additional_special_tokens=__snake_case       ,     )

																							self.assertIn('a_new_additional_special_token'       ,     tokenizer.additional_special_tokens       )
																							self.assertEqual(
																							    ['a_new_additional_special_token']       ,     tokenizer.convert_ids_to_tokens(
																							        tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']       )       )       ,     )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		[]
											if self.test_slow_tokenizer:
																	tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

											if self.test_rust_tokenizer:
																	tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

											for tokenizer_class, tokenizer_utils in tokenizer_list:
																	with tempfile.TemporaryDirectory() as tmp_dir:
																							tokenizer_utils.save_pretrained(__snake_case       )

																							_snake_case:   str  					=		tokenizer_class.from_pretrained(__snake_case       )

																							self.assertTrue(tokenizer.decode([2_55]       ) == ''       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ):





											'''simple docstring'''





											pass


					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ):





											'''simple docstring'''





											pass


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ):





											'''simple docstring'''





											pass


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ):





											'''simple docstring'''





											pass


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[Any]       ):





											'''simple docstring'''





											_snake_case:   Any  					=		self.get_tokenizers(fast=__snake_case       ,     do_lower_case=__snake_case       )
											for tokenizer in tokenizers:
																	with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
																							_snake_case:   List[str]  					=		['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
																							_snake_case:   Optional[int]  					=		tokenizer.convert_tokens_to_string(__snake_case       )

																							self.assertIsInstance(__snake_case       ,     __snake_case       )




					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ):





											'''simple docstring'''





											_snake_case:   int  					=		self.get_tokenizers()
											for tokenizer in tokenizers:
																	with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
																							_snake_case:   Tuple  					=		[
																							    'bos_token',
																							    'eos_token',
																							    'unk_token',
																							    'sep_token',
																							    'pad_token',
																							    'cls_token',
																							    'mask_token',
																							]

																							_snake_case:   Tuple  					=		0
																							_snake_case:   Tuple  					=		tokenizer.convert_ids_to_tokens(
																							    __snake_case       ,     skip_special_tokens=__snake_case       )

																							for attr in attributes_list:
																													setattr(__snake_case       ,     attr + '_id'       ,     __snake_case       )
																													self.assertEqual(getattr(__snake_case       ,     __snake_case       )       ,     __snake_case       )
																													self.assertEqual(getattr(__snake_case       ,     attr + '_id'       )       ,     __snake_case       )

																													setattr(__snake_case       ,     attr + '_id'       ,     __snake_case       )
																													self.assertEqual(getattr(__snake_case       ,     __snake_case       )       ,     __snake_case       )
																													self.assertEqual(getattr(__snake_case       ,     attr + '_id'       )       ,     __snake_case       )

																							setattr(__snake_case       ,     'additional_special_tokens_ids'       ,     []       )
																							self.assertListEqual(getattr(__snake_case       ,     'additional_special_tokens'       )       ,     []       )
																							self.assertListEqual(getattr(__snake_case       ,     'additional_special_tokens_ids'       )       ,     []       )

																							setattr(__snake_case       ,     'additional_special_tokens_ids'       ,     [token_id_to_test_setters]       )
																							self.assertListEqual(getattr(__snake_case       ,     'additional_special_tokens'       )       ,     [token_to_test_setters]       )
																							self.assertListEqual(getattr(__snake_case       ,     'additional_special_tokens_ids'       )       ,     [token_id_to_test_setters]       )












'''simple docstring'''






import json
import os
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
		from PIL import Image

		from transformers import CLIPImageProcessor, CLIPProcessor







@require_vision
class 			lowerCamelCase					(						unittest.TestCase  ):


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ):





											'''simple docstring'''





											_snake_case:   Optional[Any]  					=		tempfile.mkdtemp()

											# fmt: off
											_snake_case:   Union[str, Any]  					=		['l', 'o', 'w', 'e', 'r', 's', 't', 'i', 'd', 'n', 'lo', 'l</w>', 'w</w>', 'r</w>', 't</w>', 'low</w>', 'er</w>', 'lowest</w>', 'newer</w>', 'wider', '<unk>', '<|startoftext|>', '<|endoftext|>']
											# fmt: on
											_snake_case:   str  					=		dict(zip(__snake_case       ,     range(len(__snake_case       )       )       )       )
											_snake_case:   str  					=		['#version: 0.2', 'l o', 'lo w</w>', 'e r</w>', '']
											_snake_case:   Optional[int]  					=		{'unk_token': '<unk>'}

											_snake_case:   int  					=		os.path.join(self.tmpdirname       ,     VOCAB_FILES_NAMES['vocab_file']       )
											_snake_case:   Optional[int]  					=		os.path.join(self.tmpdirname       ,     VOCAB_FILES_NAMES['merges_file']       )
											with open(self.vocab_file       ,     'w'       ,     encoding='utf-8'       ) as fp:
																	fp.write(json.dumps(__snake_case       ) + '\n'       )
											with open(self.merges_file       ,     'w'       ,     encoding='utf-8'       ) as fp:
																	fp.write('\n'.join(__snake_case       )       )

											_snake_case:   Dict  					=		{
											    'do_resize': True,
											    'size': 20,
											    'do_center_crop': True,
											    'crop_size': 18,
											    'do_normalize': True,
											    'image_mean': [0.48_145_466, 0.4_578_275, 0.40_821_073],
											    'image_std': [0.26_862_954, 0.26_130_258, 0.27_577_711],
											}
											_snake_case:   Union[str, Any]  					=		os.path.join(self.tmpdirname       ,     __snake_case       )
											with open(self.image_processor_file       ,     'w'       ,     encoding='utf-8'       ) as fp:
																	json.dump(__snake_case       ,     __snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[Any]       ,     **__snake_case	:  Tuple       ):





											'''simple docstring'''





											return CLIPTokenizer.from_pretrained(self.tmpdirname       ,     **__snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ,     **__snake_case	:  Any       ):





											'''simple docstring'''





											return CLIPTokenizerFast.from_pretrained(self.tmpdirname       ,     **__snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  str       ,     **__snake_case	:  List[Any]       ):





											'''simple docstring'''





											return CLIPImageProcessor.from_pretrained(self.tmpdirname       ,     **__snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ):





											'''simple docstring'''





											shutil.rmtree(self.tmpdirname       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[Any]       ):





											'''simple docstring'''





											_snake_case:   str  					=		[np.random.randint(2_55       ,     size=(3, 30, 4_00)       ,     dtype=np.uinta       )]

											_snake_case:   Any  					=		[Image.fromarray(np.moveaxis(__snake_case       ,     0       ,     -1       )       ) for x in image_inputs]

											return image_inputs


					def    SCREAMING_SNAKE_CASE_		(					self	:  Union[str, Any]       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		self.get_tokenizer()
											_snake_case:   List[str]  					=		self.get_rust_tokenizer()
											_snake_case:   List[str]  					=		self.get_image_processor()

											_snake_case:   Union[str, Any]  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )
											processor_slow.save_pretrained(self.tmpdirname       )
											_snake_case:   Any  					=		CLIPProcessor.from_pretrained(self.tmpdirname       ,     use_fast=__snake_case       )

											_snake_case:   Any  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )
											processor_fast.save_pretrained(self.tmpdirname       )
											_snake_case:   int  					=		CLIPProcessor.from_pretrained(self.tmpdirname       )

											self.assertEqual(processor_slow.tokenizer.get_vocab()       ,     tokenizer_slow.get_vocab()       )
											self.assertEqual(processor_fast.tokenizer.get_vocab()       ,     tokenizer_fast.get_vocab()       )
											self.assertEqual(tokenizer_slow.get_vocab()       ,     tokenizer_fast.get_vocab()       )
											self.assertIsInstance(processor_slow.tokenizer       ,     __snake_case       )
											self.assertIsInstance(processor_fast.tokenizer       ,     __snake_case       )

											self.assertEqual(processor_slow.image_processor.to_json_string()       ,     image_processor.to_json_string()       )
											self.assertEqual(processor_fast.image_processor.to_json_string()       ,     image_processor.to_json_string()       )
											self.assertIsInstance(processor_slow.image_processor       ,     __snake_case       )
											self.assertIsInstance(processor_fast.image_processor       ,     __snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[Any]       ):





											'''simple docstring'''





											_snake_case:   Any  					=		CLIPProcessor(tokenizer=self.get_tokenizer()       ,     image_processor=self.get_image_processor()       )
											processor.save_pretrained(self.tmpdirname       )

											_snake_case:   List[Any]  					=		self.get_tokenizer(bos_token='(BOS)'       ,     eos_token='(EOS)'       )
											_snake_case:   List[Any]  					=		self.get_image_processor(do_normalize=__snake_case       ,     padding_value=1.0       )

											_snake_case:   Any  					=		CLIPProcessor.from_pretrained(
											    self.tmpdirname       ,     bos_token='(BOS)'       ,     eos_token='(EOS)'       ,     do_normalize=__snake_case       ,     padding_value=1.0       )

											self.assertEqual(processor.tokenizer.get_vocab()       ,     tokenizer_add_kwargs.get_vocab()       )
											self.assertIsInstance(processor.tokenizer       ,     __snake_case       )

											self.assertEqual(processor.image_processor.to_json_string()       ,     image_processor_add_kwargs.to_json_string()       )
											self.assertIsInstance(processor.image_processor       ,     __snake_case       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Dict       ):





											'''simple docstring'''





											_snake_case:   List[Any]  					=		self.get_image_processor()
											_snake_case:   List[str]  					=		self.get_tokenizer()

											_snake_case:   Any  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )

											_snake_case:   List[str]  					=		self.prepare_image_inputs()

											_snake_case:   List[str]  					=		image_processor(__snake_case       ,     return_tensors='np'       )
											_snake_case:   Dict  					=		processor(images=__snake_case       ,     return_tensors='np'       )

											for key in input_image_proc.keys():
																	self.assertAlmostEqual(input_image_proc[key].sum()       ,     input_processor[key].sum()       ,     delta=1e-2       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  List[str]       ):





											'''simple docstring'''





											_snake_case:   List[Any]  					=		self.get_image_processor()
											_snake_case:   Any  					=		self.get_tokenizer()

											_snake_case:   List[str]  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )

											_snake_case:   int  					=		'lower newer'

											_snake_case:   str  					=		processor(text=__snake_case       )

											_snake_case:   List[Any]  					=		tokenizer(__snake_case       )

											for key in encoded_tok.keys():
																	self.assertListEqual(encoded_tok[key]       ,     encoded_processor[key]       )


					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ):





											'''simple docstring'''





											_snake_case:   List[str]  					=		self.get_image_processor()
											_snake_case:   List[Any]  					=		self.get_tokenizer()

											_snake_case:   Tuple  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )

											_snake_case:   Optional[int]  					=		'lower newer'
											_snake_case:   Union[str, Any]  					=		self.prepare_image_inputs()

											_snake_case:   int  					=		processor(text=__snake_case       ,     images=__snake_case       )

											self.assertListEqual(list(inputs.keys()       )       ,     ['input_ids', 'attention_mask', 'pixel_values']       )

											# test if it raises when no input is passed
											with pytest.raises(__snake_case       ):
																	processor()


					def    SCREAMING_SNAKE_CASE_		(					self	:  Tuple       ):





											'''simple docstring'''





											_snake_case:   Dict  					=		self.get_image_processor()
											_snake_case:   Dict  					=		self.get_tokenizer()

											_snake_case:   str  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )

											_snake_case:   str  					=		[[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

											_snake_case:   List[Any]  					=		processor.batch_decode(__snake_case       )
											_snake_case:   Any  					=		tokenizer.batch_decode(__snake_case       )

											self.assertListEqual(__snake_case       ,     __snake_case       )




					def    SCREAMING_SNAKE_CASE_		(					self	:  Optional[int]       ):





											'''simple docstring'''





											_snake_case:   Union[str, Any]  					=		self.get_image_processor()
											_snake_case:   Any  					=		self.get_tokenizer()

											_snake_case:   Any  					=		CLIPProcessor(tokenizer=__snake_case       ,     image_processor=__snake_case       )

											_snake_case:   Any  					=		'lower newer'
											_snake_case:   Optional[Any]  					=		self.prepare_image_inputs()

											_snake_case:   int  					=		processor(text=__snake_case       ,     images=__snake_case       )

											self.assertListEqual(list(inputs.keys()       )       ,     processor.model_input_names       )





