



import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
				__snake_case     :str			       ='pt'
elif is_tf_available():
				__snake_case     :List[Any]			       ='tf'
else:
				__snake_case     :Any			       ='jax'



class        lowerCAmelCase__					(  _lowerCamelCase ,				unittest.TestCase       ):
				A_   :     Tuple								=     ByTaTokenizer
				A_   :     str								=     False


				def __UpperCamelCase						(							self      :  List[str]       )		->     Optional[int]:
											super().setUp()
											A      						=    ByTaTokenizer()
											tokenizer.save_pretrained(self.tmpdirname       )


				@cached_property
				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     Optional[Any]:
											return ByTaTokenizer.from_pretrained('google/byt5-small'       )


				def __UpperCamelCase						(							self      :  Optional[Any]						, **__UpperCamelCase      :  Union[str, Any]       )		->     ByTaTokenizer:
											return self.tokenizer_class.from_pretrained(self.tmpdirname						, **__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Optional[int]						, __UpperCamelCase      :  Tuple						, __UpperCamelCase      :  str=False						, __UpperCamelCase      :  Optional[Any]=20						, __UpperCamelCase      :  str=5       )		->     Tuple[str, list]:
											# XXX The default common tokenizer tests assume that every ID is decodable on its own.
											# This assumption is invalid for ByT5 because single bytes might not be
											# valid utf-8 (byte 128 for instance).
											# Here we're overriding the smallest possible method to provide
											# a clean sequence without making the same assumption.

											A      						=    []
											for i in range(len(__UpperCamelCase       )       ):
																		try:
																									A      						=    tokenizer.decode([i]						, clean_up_tokenization_spaces=__UpperCamelCase       )
																		except UnicodeDecodeError:
																									pass
																		toks.append((i, tok)       )

											A      						=    list(filter(lambda __UpperCamelCase       : re.match(R'^[ a-zA-Z]+$'						, t[1]       )						, __UpperCamelCase       )       )
											A      						=    list(filter(lambda __UpperCamelCase       : [t[0]] == tokenizer.encode(t[1]						, add_special_tokens=__UpperCamelCase       )						, __UpperCamelCase       )       )
											if max_length is not None and len(__UpperCamelCase       ) > max_length:
																		A      						=    toks[:max_length]
											if min_length is not None and len(__UpperCamelCase       ) < min_length and len(__UpperCamelCase       ) > 0:
																		while len(__UpperCamelCase       ) < min_length:
																									A      						=    toks + toks
        # toks_str = [t[1] for t in toks]
											A      						=    [t[0] for t in toks]

											# Ensure consistency
											A      						=    tokenizer.decode(__UpperCamelCase						, clean_up_tokenization_spaces=__UpperCamelCase       )
											if " " not in output_txt and len(__UpperCamelCase       ) > 1:
																		A      						=    (
																		    tokenizer.decode([toks_ids[0]]						, clean_up_tokenization_spaces=__UpperCamelCase       )
																		    + ' '
																		    + tokenizer.decode(toks_ids[1:]						, clean_up_tokenization_spaces=__UpperCamelCase       )
																		)
											if with_prefix_space:
																		A      						=    ' ' + output_txt
											A      						=    tokenizer.encode(__UpperCamelCase						, add_special_tokens=__UpperCamelCase       )
											return output_txt, output_ids


				def __UpperCamelCase						(							self      :  List[str]       )		->     Any:
											A      						=    self.ta_base_tokenizer
											A      						=    tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']       )
											A      						=    tokenizer(['hi', 'I went to the gym', '']       )
											self.assertListEqual(batch_with_eos_added['input_ids']						, batch_without_eos_added['input_ids']       )


				def __UpperCamelCase						(							self      :  Any       )		->     Optional[Any]:
											A      						=    self.ta_base_tokenizer
											A      						=    'Unicode €.'
											A      						=    tokenizer(__UpperCamelCase       )
											A      						=    [88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
											self.assertEqual(encoded['input_ids']						, __UpperCamelCase       )

											# decoding
											A      						=    tokenizer.decode(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, 'Unicode €.</s>'       )

											A      						=    tokenizer('e è é ê ë'       )
											A      						=    [104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
											self.assertEqual(encoded['input_ids']						, __UpperCamelCase       )
											# decoding
											A      						=    tokenizer.decode(__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, 'e è é ê ë</s>'       )

											# encode/decode, but with `encode` instead of `__call__`
											self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'       )       )						, 'e è é ê ë</s>'       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[str]:
											A      						=    self.ta_base_tokenizer
											A      						=    ['A long paragraph for summarization.', 'Another paragraph for summarization.']
											# fmt: off
											A      						=    [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
											# fmt: on
											A      						=    tokenizer(__UpperCamelCase						, padding=__UpperCamelCase						, return_tensors=__UpperCamelCase       )
											self.assertIsInstance(__UpperCamelCase						, __UpperCamelCase       )

											if FRAMEWORK != "jax":
																		A      						=    list(batch.input_ids.numpy()[0]       )
											else:
																		A      						=    list(batch.input_ids.tolist()[0]       )

											self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )

											self.assertEqual((2, 37)						, batch.input_ids.shape       )
											self.assertEqual((2, 37)						, batch.attention_mask.shape       )


				def __UpperCamelCase						(							self      :  List[str]       )		->     List[Any]:
											A      						=    self.ta_base_tokenizer
											A      						=    ['A long paragraph for summarization.', 'Another paragraph for summarization.']
											A      						=    tokenizer(__UpperCamelCase						, padding=__UpperCamelCase						, return_tensors=__UpperCamelCase       )
											# check if input_ids are returned and no decoder_input_ids
											self.assertIn('input_ids'						, __UpperCamelCase       )
											self.assertIn('attention_mask'						, __UpperCamelCase       )
											self.assertNotIn('decoder_input_ids'						, __UpperCamelCase       )
											self.assertNotIn('decoder_attention_mask'						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Dict       )		->     Optional[Any]:
											A      						=    self.ta_base_tokenizer
											A      						=    [
											    'Summary of the text.',
											    'Another summary.',
											]
											A      						=    tokenizer(
											    text_target=__UpperCamelCase						, max_length=32						, padding='max_length'						, truncation=__UpperCamelCase						, return_tensors=__UpperCamelCase       )
											self.assertEqual(32						, targets['input_ids'].shape[1]       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     Any:
											A      						=    self.ta_base_tokenizer
											A      						=    ['A long paragraph for summarization. </s>']
											A      						=    ['Summary of the text. </s>']
											# fmt: off
											A      						=    [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
											A      						=    [86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
											# fmt: on

											A      						=    tokenizer(__UpperCamelCase						, text_target=__UpperCamelCase       )

											self.assertEqual(__UpperCamelCase						, batch['input_ids'][0]       )
											self.assertEqual(__UpperCamelCase						, batch['labels'][0]       )


				def __UpperCamelCase						(							self      :  Any       )		->     str:
											# safety check on max_len default value so we are sure the test works
											A      						=    self.get_tokenizers()
											for tokenizer in tokenizers:
																		with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
																									self.assertNotEqual(tokenizer.model_max_length						, 42       )

        # Now let's start the test
											A      						=    self.get_tokenizers()
											for tokenizer in tokenizers:
																		with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
																									# Isolate this from the other tests because we save additional tokens/etc
																									A      						=    tempfile.mkdtemp()

																									A      						=    ' He is very happy, UNwant\u00E9d,running'
																									A      						=    tokenizer.encode(__UpperCamelCase						, add_special_tokens=__UpperCamelCase       )
																									tokenizer.save_pretrained(__UpperCamelCase       )

																									A      						=    tokenizer.__class__.from_pretrained(__UpperCamelCase       )
																									A      						=    after_tokenizer.encode(__UpperCamelCase						, add_special_tokens=__UpperCamelCase       )
																									self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )

																									shutil.rmtree(__UpperCamelCase       )

											A      						=    self.get_tokenizers(model_max_length=42       )
											for tokenizer in tokenizers:
																		with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
																									# Isolate this from the other tests because we save additional tokens/etc
																									A      						=    tempfile.mkdtemp()

																									A      						=    ' He is very happy, UNwant\u00E9d,running'
																									tokenizer.add_tokens(['bim', 'bambam']       )
																									A      						=    tokenizer.additional_special_tokens
																									additional_special_tokens.append('new_additional_special_token'       )
																									tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}       )
																									A      						=    tokenizer.encode(__UpperCamelCase						, add_special_tokens=__UpperCamelCase       )
																									tokenizer.save_pretrained(__UpperCamelCase       )

																									A      						=    tokenizer.__class__.from_pretrained(__UpperCamelCase       )
																									A      						=    after_tokenizer.encode(__UpperCamelCase						, add_special_tokens=__UpperCamelCase       )
																									self.assertListEqual(__UpperCamelCase						, __UpperCamelCase       )
																									self.assertIn('new_additional_special_token'						, after_tokenizer.additional_special_tokens       )
																									self.assertEqual(after_tokenizer.model_max_length						, 42       )

																									A      						=    tokenizer.__class__.from_pretrained(__UpperCamelCase						, model_max_length=43       )
																									self.assertEqual(tokenizer.model_max_length						, 43       )

																									shutil.rmtree(__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Tuple       )		->     str:
											A      						=    []
											if self.test_slow_tokenizer:
																		tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

											if self.test_rust_tokenizer:
																		tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

											for tokenizer_class, tokenizer_utils in tokenizer_list:
																		with tempfile.TemporaryDirectory() as tmp_dir:
																									tokenizer_utils.save_pretrained(__UpperCamelCase       )

																									with open(os.path.join(__UpperCamelCase						, 'special_tokens_map.json'       )						, encoding='utf-8'       ) as json_file:
																																A      						=    json.load(__UpperCamelCase       )

																									with open(os.path.join(__UpperCamelCase						, 'tokenizer_config.json'       )						, encoding='utf-8'       ) as json_file:
																																A      						=    json.load(__UpperCamelCase       )

																									A      						=    [f'''<extra_id_{i}>''' for i in range(125       )]

																									A      						=    added_tokens_extra_ids + [
																									    'an_additional_special_token'
																									]
																									A      						=    added_tokens_extra_ids + [
																									    'an_additional_special_token'
																									]

																									with open(os.path.join(__UpperCamelCase						, 'special_tokens_map.json'       )						, 'w'						, encoding='utf-8'       ) as outfile:
																																json.dump(__UpperCamelCase						, __UpperCamelCase       )
																									with open(os.path.join(__UpperCamelCase						, 'tokenizer_config.json'       )						, 'w'						, encoding='utf-8'       ) as outfile:
																																json.dump(__UpperCamelCase						, __UpperCamelCase       )

																									# the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
																									# into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
																									# "special_tokens_map.json" files
																									A      						=    tokenizer_class.from_pretrained(
																									    __UpperCamelCase						, )
																									self.assertIn(
																									    'an_additional_special_token'						, tokenizer_without_change_in_init.additional_special_tokens       )
																									# self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
																									self.assertEqual(
																									    ['an_additional_special_token']						, tokenizer_without_change_in_init.convert_ids_to_tokens(
																									        tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']       )       )						, )

																									# Now we test that we can change the value of additional_special_tokens in the from_pretrained
																									A      						=    added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'						, lstrip=__UpperCamelCase       )]
																									A      						=    tokenizer_class.from_pretrained(
																									    __UpperCamelCase						, additional_special_tokens=__UpperCamelCase						, )

																									self.assertIn('a_new_additional_special_token'						, tokenizer.additional_special_tokens       )
																									self.assertEqual(
																									    ['a_new_additional_special_token']						, tokenizer.convert_ids_to_tokens(
																									        tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']       )       )						, )


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     List[str]:
											A      						=    []
											if self.test_slow_tokenizer:
																		tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())       )

											if self.test_rust_tokenizer:
																		tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())       )

											for tokenizer_class, tokenizer_utils in tokenizer_list:
																		with tempfile.TemporaryDirectory() as tmp_dir:
																									tokenizer_utils.save_pretrained(__UpperCamelCase       )

																									A      						=    tokenizer_class.from_pretrained(__UpperCamelCase       )

																									self.assertTrue(tokenizer.decode([255]       ) == ''       )


				def __UpperCamelCase						(							self      :  List[str]       )		->     Any:
											pass


				def __UpperCamelCase						(							self      :  Dict       )		->     Optional[Any]:
											pass


				def __UpperCamelCase						(							self      :  Any       )		->     List[Any]:
											pass


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     Optional[int]:
											pass


				def __UpperCamelCase						(							self      :  int       )		->     Dict:
											# The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
											# and special added tokens as tokens
											A      						=    self.get_tokenizers(fast=__UpperCamelCase						, do_lower_case=__UpperCamelCase       )
											for tokenizer in tokenizers:
																		with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
																									A      						=    ['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
																									A      						=    tokenizer.convert_tokens_to_string(__UpperCamelCase       )

																									self.assertIsInstance(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     Union[str, Any]:
											A      						=    self.get_tokenizers()
											for tokenizer in tokenizers:
																		with self.subTest(f'''{tokenizer.__class__.__name__}'''       ):
																									A      						=    [
																									    'bos_token',
																									    'eos_token',
																									    'unk_token',
																									    'sep_token',
																									    'pad_token',
																									    'cls_token',
																									    'mask_token',
																									]

																									A      						=    0
																									A      						=    tokenizer.convert_ids_to_tokens(
																									    __UpperCamelCase						, skip_special_tokens=__UpperCamelCase       )

																									for attr in attributes_list:
																																setattr(__UpperCamelCase						, attr + '_id'						, __UpperCamelCase       )
																																self.assertEqual(getattr(__UpperCamelCase						, __UpperCamelCase       )						, __UpperCamelCase       )
																																self.assertEqual(getattr(__UpperCamelCase						, attr + '_id'       )						, __UpperCamelCase       )

																																setattr(__UpperCamelCase						, attr + '_id'						, __UpperCamelCase       )
																																self.assertEqual(getattr(__UpperCamelCase						, __UpperCamelCase       )						, __UpperCamelCase       )
																																self.assertEqual(getattr(__UpperCamelCase						, attr + '_id'       )						, __UpperCamelCase       )

																									setattr(__UpperCamelCase						, 'additional_special_tokens_ids'						, []       )
																									self.assertListEqual(getattr(__UpperCamelCase						, 'additional_special_tokens'       )						, []       )
																									self.assertListEqual(getattr(__UpperCamelCase						, 'additional_special_tokens_ids'       )						, []       )

																									setattr(__UpperCamelCase						, 'additional_special_tokens_ids'						, [token_id_to_test_setters]       )
																									self.assertListEqual(getattr(__UpperCamelCase						, 'additional_special_tokens'       )						, [token_to_test_setters]       )
																									self.assertListEqual(getattr(__UpperCamelCase						, 'additional_special_tokens_ids'       )						, [token_id_to_test_setters]       )




import gc
import tempfile
import unittest

import numpy as np
import torch

from diffusers import VersatileDiffusionPipeline
from diffusers.utils.testing_utils import load_image, nightly, require_torch_gpu, torch_device


__snake_case     :str			       =False



class        lowerCAmelCase__					(  unittest.TestCase       ):
				pass





@nightly
@require_torch_gpu
class        lowerCAmelCase__					(  unittest.TestCase       ):


				def __UpperCamelCase						(							self      :  Dict       )		->     List[Any]:
											# clean up the VRAM after each test
											super().tearDown()
											gc.collect()
											torch.cuda.empty_cache()


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     List[Any]:
											A      						=    VersatileDiffusionPipeline.from_pretrained('shi-labs/versatile-diffusion'						, torch_dtype=torch.floataa       )
											pipe.to(__UpperCamelCase       )
											pipe.set_progress_bar_config(disable=__UpperCamelCase       )

											A      						=    load_image(
											    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg'       )

											A      						=    torch.manual_seed(0       )
											A      						=    pipe.dual_guided(
											    prompt='first prompt'						, image=__UpperCamelCase						, text_to_image_strength=0.7_5						, generator=__UpperCamelCase						, guidance_scale=7.5						, num_inference_steps=2						, output_type='numpy'						, ).images

											with tempfile.TemporaryDirectory() as tmpdirname:
																		pipe.save_pretrained(__UpperCamelCase       )
																		A      						=    VersatileDiffusionPipeline.from_pretrained(__UpperCamelCase						, torch_dtype=torch.floataa       )
											pipe.to(__UpperCamelCase       )
											pipe.set_progress_bar_config(disable=__UpperCamelCase       )

											A      						=    generator.manual_seed(0       )
											A      						=    pipe.dual_guided(
											    prompt='first prompt'						, image=__UpperCamelCase						, text_to_image_strength=0.7_5						, generator=__UpperCamelCase						, guidance_scale=7.5						, num_inference_steps=2						, output_type='numpy'						, ).images

											assert np.abs(image - new_image       ).sum() < 1e-5, "Models don't have the same forward pass"


				def __UpperCamelCase						(							self      :  Tuple       )		->     List[str]:
											A      						=    VersatileDiffusionPipeline.from_pretrained('shi-labs/versatile-diffusion'						, torch_dtype=torch.floataa       )
											pipe.to(__UpperCamelCase       )
											pipe.set_progress_bar_config(disable=__UpperCamelCase       )

											A      						=    'cyberpunk 2077'
											A      						=    load_image(
											    'https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/versatile_diffusion/benz.jpg'       )
											A      						=    torch.manual_seed(0       )
											A      						=    pipe.dual_guided(
											    prompt=__UpperCamelCase						, image=__UpperCamelCase						, text_to_image_strength=0.7_5						, generator=__UpperCamelCase						, guidance_scale=7.5						, num_inference_steps=50						, output_type='numpy'						, ).images

											A      						=    image[0, 253:256, 253:256, -1]

											assert image.shape == (1, 512, 512, 3)
											A      						=    np.array([0.1_4_4_8, 0.1_6_1_9, 0.1_7_4_1, 0.1_0_8_6, 0.1_1_4_7, 0.1_1_2_8, 0.1_1_9_9, 0.1_1_6_5, 0.1_0_0_1]       )

											assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1e-1

											A      						=    'A painting of a squirrel eating a burger '
											A      						=    torch.manual_seed(0       )
											A      						=    pipe.text_to_image(
											    prompt=__UpperCamelCase						, generator=__UpperCamelCase						, guidance_scale=7.5						, num_inference_steps=50						, output_type='numpy'       ).images

											A      						=    image[0, 253:256, 253:256, -1]

											assert image.shape == (1, 512, 512, 3)
											A      						=    np.array([0.3_3_6_7, 0.3_1_6_9, 0.2_6_5_6, 0.3_8_7_0, 0.4_7_9_0, 0.3_7_9_6, 0.4_0_0_9, 0.4_8_7_8, 0.4_7_7_8]       )

											assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1e-1

											A      						=    pipe.image_variation(__UpperCamelCase						, generator=__UpperCamelCase						, output_type='numpy'       ).images

											A      						=    image[0, 253:256, 253:256, -1]

											assert image.shape == (1, 512, 512, 3)
											A      						=    np.array([0.3_0_7_6, 0.3_1_2_3, 0.3_2_8_4, 0.3_7_8_2, 0.3_7_7_0, 0.3_8_9_4, 0.4_2_9_7, 0.4_3_3_1, 0.4_4_5_6]       )

											assert np.abs(image_slice.flatten() - expected_slice       ).max() < 1e-1