


import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase			=				get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
			from transformers.models.plbart.modeling_plbart import shift_tokens_right

lowerCAmelCase			=				5_0003
lowerCAmelCase			=				5_0002


@require_sentencepiece
@require_tokenizers
class 			_a				(     UpperCamelCase__	,      unittest.TestCase     ):
							_lowercase      :     int     =					PLBartTokenizer
							_lowercase      :     Optional[Any]     =					None
							_lowercase      :     int     =					False

							def  lowerCamelCase_					(				self:      Tuple				)     ->  Any:





									"""simple docstring"""







									super().setUp()

									# We have a SentencePiece fixture for testing
									lowercase__				= PLBartTokenizer(UpperCamelCase_       ,		language_codes='''base'''       ,		keep_accents=UpperCamelCase_				)
									tokenizer.save_pretrained(self.tmpdirname				)

							def  lowerCamelCase_					(				self:      int				)     ->  Tuple:





									"""simple docstring"""







									lowercase__				= PLBartTokenizer(UpperCamelCase_       ,		language_codes='''base'''       ,		keep_accents=UpperCamelCase_				)

									lowercase__				= tokenizer.tokenize('''This is a test'''				)
									self.assertListEqual(UpperCamelCase_       ,		['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(UpperCamelCase_				)       ,		[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,		)

									lowercase__				= tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
									self.assertListEqual(
									    UpperCamelCase_       ,		[
									        SPIECE_UNDERLINE + '''I''',
									        SPIECE_UNDERLINE + '''was''',
									        SPIECE_UNDERLINE + '''b''',
									        '''or''',
									        '''n''',
									        SPIECE_UNDERLINE + '''in''',
									        SPIECE_UNDERLINE + '''''',
									        '''9''',
									        '''2''',
									        '''0''',
									        '''0''',
									        '''0''',
									        ''',''',
									        SPIECE_UNDERLINE + '''and''',
									        SPIECE_UNDERLINE + '''this''',
									        SPIECE_UNDERLINE + '''is''',
									        SPIECE_UNDERLINE + '''f''',
									        '''al''',
									        '''s''',
									        '''é''',
									        '''.''',
									    ]       ,		)
									lowercase__				= tokenizer.convert_tokens_to_ids(UpperCamelCase_				)
									self.assertListEqual(
									    UpperCamelCase_       ,		[
									        value + tokenizer.fairseq_offset
									        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
									    ]       ,		)

									lowercase__				= tokenizer.convert_ids_to_tokens(UpperCamelCase_				)
									self.assertListEqual(
									    UpperCamelCase_       ,		[
									        SPIECE_UNDERLINE + '''I''',
									        SPIECE_UNDERLINE + '''was''',
									        SPIECE_UNDERLINE + '''b''',
									        '''or''',
									        '''n''',
									        SPIECE_UNDERLINE + '''in''',
									        SPIECE_UNDERLINE + '''''',
									        '''<unk>''',
									        '''2''',
									        '''0''',
									        '''0''',
									        '''0''',
									        ''',''',
									        SPIECE_UNDERLINE + '''and''',
									        SPIECE_UNDERLINE + '''this''',
									        SPIECE_UNDERLINE + '''is''',
									        SPIECE_UNDERLINE + '''f''',
									        '''al''',
									        '''s''',
									        '''<unk>''',
									        '''.''',
									    ]       ,		)

									lowercase__				= tokenizer.vocab_size
									lowercase__				= [tokenizer.convert_ids_to_tokens(UpperCamelCase_				) for x in range(end - 4       ,		UpperCamelCase_				)]

									self.assertListEqual(UpperCamelCase_       ,		['''__java__''', '''__python__''', '''__en_XX__''', '''<mask>''']				)

									lowercase__				= '''java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'''
									lowercase__				= tokenizer(UpperCamelCase_				).input_ids
									self.assertEqual(
									    tokenizer.decode(UpperCamelCase_       ,		skip_special_tokens=UpperCamelCase_       ,		clean_up_tokenization_spaces=UpperCamelCase_				)       ,		UpperCamelCase_       ,		)


							def  lowerCamelCase_					(				self:      int				)     ->  str:





									"""simple docstring"""







									lowercase__				= PLBartTokenizer(UpperCamelCase_       ,		language_codes='''multi'''       ,		keep_accents=UpperCamelCase_				)

									lowercase__				= tokenizer.tokenize('''This is a test'''				)
									self.assertListEqual(UpperCamelCase_       ,		['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']				)

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(UpperCamelCase_				)       ,		[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,		)

									lowercase__				= tokenizer.tokenize('''I was born in 92000, and this is falsé.'''				)
									self.assertListEqual(
									    UpperCamelCase_       ,		[
									        SPIECE_UNDERLINE + '''I''',
									        SPIECE_UNDERLINE + '''was''',
									        SPIECE_UNDERLINE + '''b''',
									        '''or''',
									        '''n''',
									        SPIECE_UNDERLINE + '''in''',
									        SPIECE_UNDERLINE + '''''',
									        '''9''',
									        '''2''',
									        '''0''',
									        '''0''',
									        '''0''',
									        ''',''',
									        SPIECE_UNDERLINE + '''and''',
									        SPIECE_UNDERLINE + '''this''',
									        SPIECE_UNDERLINE + '''is''',
									        SPIECE_UNDERLINE + '''f''',
									        '''al''',
									        '''s''',
									        '''é''',
									        '''.''',
									    ]       ,		)
									lowercase__				= tokenizer.convert_tokens_to_ids(UpperCamelCase_				)
									self.assertListEqual(
									    UpperCamelCase_       ,		[
									        value + tokenizer.fairseq_offset
									        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
									    ]       ,		)

									lowercase__				= tokenizer.convert_ids_to_tokens(UpperCamelCase_				)
									self.assertListEqual(
									    UpperCamelCase_       ,		[
									        SPIECE_UNDERLINE + '''I''',
									        SPIECE_UNDERLINE + '''was''',
									        SPIECE_UNDERLINE + '''b''',
									        '''or''',
									        '''n''',
									        SPIECE_UNDERLINE + '''in''',
									        SPIECE_UNDERLINE + '''''',
									        '''<unk>''',
									        '''2''',
									        '''0''',
									        '''0''',
									        '''0''',
									        ''',''',
									        SPIECE_UNDERLINE + '''and''',
									        SPIECE_UNDERLINE + '''this''',
									        SPIECE_UNDERLINE + '''is''',
									        SPIECE_UNDERLINE + '''f''',
									        '''al''',
									        '''s''',
									        '''<unk>''',
									        '''.''',
									    ]       ,		)
									lowercase__				= tokenizer.vocab_size
									lowercase__				= [tokenizer.convert_ids_to_tokens(UpperCamelCase_				) for x in range(end - 7       ,		UpperCamelCase_				)]

									self.assertListEqual(
									    UpperCamelCase_       ,		['''__java__''', '''__python__''', '''__en_XX__''', '''__javascript__''', '''__php__''', '''__ruby__''', '''__go__''']				)
									lowercase__				= '''java.lang.Exception, python.lang.Exception, javascript, php, ruby, go'''
									lowercase__				= tokenizer(UpperCamelCase_				).input_ids
									self.assertEqual(
									    tokenizer.decode(UpperCamelCase_       ,		skip_special_tokens=UpperCamelCase_       ,		clean_up_tokenization_spaces=UpperCamelCase_				)       ,		UpperCamelCase_       ,		)


@require_torch
@require_sentencepiece
@require_tokenizers
class 			_a				(     unittest.TestCase     ):
							_lowercase      :     Union[str, Any]     =					'''uclanlp/plbart-python-en_XX'''
							_lowercase      :     Any     =					[
							    '''def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])''',
							    '''def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])''',
							]
							_lowercase      :     Union[str, Any]     =					[
							    '''Returns the maximum value of a b c.''',
							    '''Sums the values of a b c.''',
							]
							_lowercase      :     Tuple     =					[
							    134,
							    5452,
							    33460,
							    33441,
							    33463,
							    33465,
							    33463,
							    33449,
							    988,
							    20,
							    33456,
							    19,
							    33456,
							    771,
							    39,
							    4258,
							    889,
							    3318,
							    33441,
							    33463,
							    33465,
							    33463,
							    33449,
							    2471,
							    2,
							    PYTHON_CODE,
							]

							@classmethod
							def  lowerCamelCase_					(				cls:      List[str]				)     ->  Optional[int]:





									"""simple docstring"""







									lowercase__				= PLBartTokenizer.from_pretrained(
									    cls.checkpoint_name       ,		language_codes='''base'''       ,		src_lang='''python'''       ,		tgt_lang='''en_XX'''				)
									lowercase__				= 1
									return cls

							def  lowerCamelCase_					(				self:      Optional[int]				)     ->  Optional[Any]:





									"""simple docstring"""







									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''__java__''']       ,		50_001				)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''__python__''']       ,		50_002				)
									self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''__en_XX__''']       ,		50_003				)

							def  lowerCamelCase_					(				self:      Any				)     ->  Optional[Any]:





									"""simple docstring"""







									lowercase__				= self.tokenizer.batch_encode_plus(self.src_text				).input_ids[0]
									self.assertListEqual(self.expected_src_tokens       ,		UpperCamelCase_				)

							def  lowerCamelCase_					(				self:      Union[str, Any]				)     ->  Dict:





									"""simple docstring"""







									self.assertIn(UpperCamelCase_       ,		self.tokenizer.all_special_ids				)
									lowercase__				= [EN_CODE, 9_037, 33_442, 57, 752, 153, 14, 56, 18, 9, 2]
									lowercase__				= self.tokenizer.decode(UpperCamelCase_       ,		skip_special_tokens=UpperCamelCase_				)
									lowercase__				= self.tokenizer.decode(generated_ids[1:]       ,		skip_special_tokens=UpperCamelCase_				)
									self.assertEqual(UpperCamelCase_       ,		UpperCamelCase_				)
									self.assertNotIn(self.tokenizer.eos_token       ,		UpperCamelCase_				)

							def  lowerCamelCase_					(				self:      Tuple				)     ->  Any:





									"""simple docstring"""







									lowercase__				= ['''def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])''' * 20]
									self.assertIsInstance(src_text[0]       ,		UpperCamelCase_				)
									lowercase__				= 10
									lowercase__				= self.tokenizer(UpperCamelCase_       ,		max_length=UpperCamelCase_       ,		truncation=UpperCamelCase_				).input_ids[0]
									self.assertEqual(ids[-2]       ,		2				)
									self.assertEqual(ids[-1]       ,		UpperCamelCase_				)
									self.assertEqual(len(UpperCamelCase_				)       ,		UpperCamelCase_				)

							def  lowerCamelCase_					(				self:      int				)     ->  List[Any]:





									"""simple docstring"""







									self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''__java__''']				)       ,		[50_004, 50_001]				)

							def  lowerCamelCase_					(				self:      Union[str, Any]				)     ->  List[Any]:





									"""simple docstring"""







									lowercase__				= tempfile.mkdtemp()
									lowercase__				= self.tokenizer.fairseq_tokens_to_ids
									self.tokenizer.save_pretrained(UpperCamelCase_				)
									lowercase__				= PLBartTokenizer.from_pretrained(UpperCamelCase_				)
									self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,		UpperCamelCase_				)

							@require_torch
							def  lowerCamelCase_					(				self:      str				)     ->  Optional[int]:





									"""simple docstring"""







									lowercase__				= self.tokenizer(self.src_text       ,		text_target=self.tgt_text       ,		padding=UpperCamelCase_       ,		return_tensors='''pt'''				)
									lowercase__				= shift_tokens_right(batch['''labels''']       ,		self.tokenizer.pad_token_id				)

									# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
									self.assertEqual(batch.input_ids[1][-2:].tolist()       ,		[2, PYTHON_CODE]				)
									self.assertEqual(batch.decoder_input_ids[1][0]       ,		UpperCamelCase_				)
									self.assertEqual(batch.decoder_input_ids[1][-1]       ,		2				)
									self.assertEqual(batch.labels[1][-2:].tolist()       ,		[2, EN_CODE]				)

							@require_torch
							def  lowerCamelCase_					(				self:      List[Any]				)     ->  Dict:





									"""simple docstring"""







									lowercase__				= self.tokenizer(
									    self.src_text       ,		text_target=self.tgt_text       ,		padding=UpperCamelCase_       ,		truncation=UpperCamelCase_       ,		max_length=len(self.expected_src_tokens				)       ,		return_tensors='''pt'''       ,		)
									lowercase__				= shift_tokens_right(batch['''labels''']       ,		self.tokenizer.pad_token_id				)

									self.assertIsInstance(UpperCamelCase_       ,		UpperCamelCase_				)

									self.assertEqual((2, 26)       ,		batch.input_ids.shape				)
									self.assertEqual((2, 26)       ,		batch.attention_mask.shape				)
									lowercase__				= batch.input_ids.tolist()[0]
									self.assertListEqual(self.expected_src_tokens       ,		UpperCamelCase_				)
									self.assertEqual(2       ,		batch.decoder_input_ids[0, -1]				)  # EOS
									# Test that special tokens are reset
									self.assertEqual(self.tokenizer.prefix_tokens       ,		[]				)
									self.assertEqual(self.tokenizer.suffix_tokens       ,		[self.tokenizer.eos_token_id, PYTHON_CODE]				)

							def  lowerCamelCase_					(				self:      Optional[int]				)     ->  Optional[int]:





									"""simple docstring"""







									lowercase__				= self.tokenizer(self.src_text       ,		padding=UpperCamelCase_       ,		truncation=UpperCamelCase_       ,		max_length=3       ,		return_tensors='''pt'''				)
									lowercase__				= self.tokenizer(
									    text_target=self.tgt_text       ,		padding=UpperCamelCase_       ,		truncation=UpperCamelCase_       ,		max_length=10       ,		return_tensors='''pt'''				)
									lowercase__				= targets['''input_ids''']
									lowercase__				= shift_tokens_right(UpperCamelCase_       ,		self.tokenizer.pad_token_id				)

									self.assertEqual(batch.input_ids.shape[1]       ,		3				)
									self.assertEqual(batch.decoder_input_ids.shape[1]       ,		10				)


							@require_torch
							def  lowerCamelCase_					(				self:      Union[str, Any]				)     ->  int:





									"""simple docstring"""







									lowercase__				= self.tokenizer._build_translation_inputs(
									    '''A test'''       ,		return_tensors='''pt'''       ,		src_lang='''en_XX'''       ,		tgt_lang='''java'''				)

									self.assertEqual(
									    nested_simplify(UpperCamelCase_				)       ,		{
									        # A, test, EOS, en_XX
									        '''input_ids''': [[150, 242, 2, 50_003]],
									        '''attention_mask''': [[1, 1, 1, 1]],
									        # java
									        '''forced_bos_token_id''': 50_001,
									    }       ,		)






import unittest

import numpy as np
import torch
from torch import nn
from transformers import (
    CLIPImageProcessor,
    CLIPTextConfig,
    CLIPTextModelWithProjection,
    CLIPTokenizer,
    CLIPVisionConfig,
    CLIPVisionModelWithProjection,
)

from diffusers import KandinskyVaaPriorPipeline, PriorTransformer, UnCLIPScheduler
from diffusers.utils import torch_device
from diffusers.utils.testing_utils import enable_full_determinism, skip_mps

from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()




class       lowercase_					(lowercase__      ,					unittest.TestCase   ):
				snake_case	    =KandinskyVaaPriorPipeline
				snake_case	    =['prompt']
				snake_case	    =['prompt', 'negative_prompt']
				snake_case	    =[
				    'num_images_per_prompt',
				    'generator',
				    'num_inference_steps',
				    'latents',
				    'negative_prompt',
				    'guidance_scale',
				    'output_type',
				    'return_dict',
				]
				snake_case	    =False





				@property
				def 						__UpperCamelCase    (				self)  -> Optional[int]:
											return 32





				@property
				def 						__UpperCamelCase    (				self)  -> Tuple:
											return 32





				@property
				def 						__UpperCamelCase    (				self)  -> int:
											return self.time_input_dim





				@property
				def 						__UpperCamelCase    (				self)  -> str:
											return self.time_input_dim * 4





				@property
				def 						__UpperCamelCase    (				self)  -> Optional[int]:
											return 100





				@property
				def 						__UpperCamelCase    (				self)  -> Union[str, Any]:
											a__      =CLIPTokenizer.from_pretrained('hf-internal-testing/tiny-random-clip')
											return tokenizer





				@property
				def 						__UpperCamelCase    (				self)  -> Union[str, Any]:
											torch.manual_seed(0)
											a__      =CLIPTextConfig(
											    bos_token_id=0       ,  eos_token_id=2       ,  hidden_size=self.text_embedder_hidden_size       ,  projection_dim=self.text_embedder_hidden_size       ,  intermediate_size=37       ,  layer_norm_eps=1e-05       ,  num_attention_heads=4       ,  num_hidden_layers=5       ,  pad_token_id=1       ,  vocab_size=1000       ,  )
											return CLIPTextModelWithProjection(lowercase_)





				@property
				def 						__UpperCamelCase    (				self)  -> Tuple:
											torch.manual_seed(0)

											a__      ={
											    'num_attention_heads': 2,
											    'attention_head_dim': 12,
											    'embedding_dim': self.text_embedder_hidden_size,
											    'num_layers': 1,
											}

											a__      =PriorTransformer(**lowercase_)
											# clip_std and clip_mean is initialized to be 0 so PriorTransformer.post_process_latents will always return 0 - set clip_std to be 1 so it won't return 0
											a__      =nn.Parameter(torch.ones(model.clip_std.shape))
											return model





				@property
				def 						__UpperCamelCase    (				self)  -> Any:
											torch.manual_seed(0)
											a__      =CLIPVisionConfig(
											    hidden_size=self.text_embedder_hidden_size       ,  image_size=224       ,  projection_dim=self.text_embedder_hidden_size       ,  intermediate_size=37       ,  num_attention_heads=4       ,  num_channels=3       ,  num_hidden_layers=5       ,  patch_size=14       ,  )

											a__      =CLIPVisionModelWithProjection(lowercase_)
											return model





				@property
				def 						__UpperCamelCase    (				self)  -> Optional[int]:
											a__      =CLIPImageProcessor(
											    crop_size=224       ,  do_center_crop=lowercase_       ,  do_normalize=lowercase_       ,  do_resize=lowercase_       ,  image_mean=[0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73]       ,  image_std=[0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11]       ,  resample=3       ,  size=224       ,  )

											return image_processor





				def 						__UpperCamelCase    (				self)  -> Any:
											a__      =self.dummy_prior
											a__      =self.dummy_image_encoder
											a__      =self.dummy_text_encoder
											a__      =self.dummy_tokenizer
											a__      =self.dummy_image_processor

											a__      =UnCLIPScheduler(
											    variance_type='fixed_small_log'       ,  prediction_type='sample'       ,  num_train_timesteps=1000       ,  clip_sample=lowercase_       ,  clip_sample_range=10.0       ,  )

											a__      ={
											    'prior': prior,
											    'image_encoder': image_encoder,
											    'text_encoder': text_encoder,
											    'tokenizer': tokenizer,
											    'scheduler': scheduler,
											    'image_processor': image_processor,
											}

											return components





				def 						__UpperCamelCase    (				self       ,  lowercase_       ,  lowercase_=0)  -> Tuple:
											if str(lowercase_).startswith('mps'):
																		a__      =torch.manual_seed(lowercase_)
											else:
																		a__      =torch.Generator(device=lowercase_).manual_seed(lowercase_)
											a__      ={
											    'prompt': 'horse',
											    'generator': generator,
											    'guidance_scale': 4.0,
											    'num_inference_steps': 2,
											    'output_type': 'np',
											}
											return inputs





				def 						__UpperCamelCase    (				self)  -> int:
											a__      ='cpu'

											a__      =self.get_dummy_components()

											a__      =self.pipeline_class(**lowercase_)
											a__      =pipe.to(lowercase_)

											pipe.set_progress_bar_config(disable=lowercase_)

											a__      =pipe(**self.get_dummy_inputs(lowercase_))
											a__      =output.image_embeds

											a__      =pipe(
											    **self.get_dummy_inputs(lowercase_)       ,  return_dict=lowercase_       ,  )[0]

											a__      =image[0, -10:]
											a__      =image_from_tuple[0, -10:]

											assert image.shape == (1, 32)

											a__      =np.array(
											    [-0.05_32, 1.71_20, 0.36_56, -1.08_52, -0.89_46, -1.17_56, 0.43_48, 0.24_82, 0.51_46, -0.11_56])

											assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2
											assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2





				@skip_mps
				def 						__UpperCamelCase    (				self)  -> List[Any]:
											a__      =torch_device == 'cpu'
											a__      =True
											a__      =False

											self._test_inference_batch_single_identical(
											    test_max_difference=lowercase_       ,  relax_max_difference=lowercase_       ,  test_mean_pixel_difference=lowercase_       ,  )





				@skip_mps
				def 						__UpperCamelCase    (				self)  -> Optional[int]:
											a__      =torch_device == 'cpu'
											a__      =False

											self._test_attention_slicing_forward_pass(
											    test_max_difference=lowercase_       ,  test_mean_pixel_difference=lowercase_       ,  )

