





import json
import os
import unittest

from transformers import AutoTokenizer, GPTaTokenizer, GPTaTokenizerFast
from transformers.models.gpta.tokenization_gpta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 					__snake_case	(  lowercase_			,     unittest.TestCase    ):


    '''simple docstring'''







    lowerCamelCase__    :		List[str]				=	GPTaTokenizer
    lowerCamelCase__    :		Optional[int]				=	GPTaTokenizerFast
    lowerCamelCase__    :		List[Any]				=	True
    lowerCamelCase__    :		List[str]				=	{"add_prefix_space": True}
    lowerCamelCase__    :		Optional[int]				=	False



    def  lowercase_					(       self     ):





        '''simple docstring'''

        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        SCREAMING_SNAKE_CASE__											=			[
            """l""",
            """o""",
            """w""",
            """e""",
            """r""",
            """s""",
            """t""",
            """i""",
            """d""",
            """n""",
            """\u0120""",
            """\u0120l""",
            """\u0120n""",
            """\u0120lo""",
            """\u0120low""",
            """er""",
            """\u0120lowest""",
            """\u0120newer""",
            """\u0120wider""",
            """<unk>""",
            """<|endoftext|>""",
        ]
        SCREAMING_SNAKE_CASE__											=			dict(zip(a__   ,				range(len(a__     )     )     )     )
        SCREAMING_SNAKE_CASE__											=			["""#version: 0.2""", """\u0120 l""", """\u0120l o""", """\u0120lo w""", """e r""", """"""]
        SCREAMING_SNAKE_CASE__											=			{"""unk_token""": """<unk>"""}

        SCREAMING_SNAKE_CASE__											=			os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''vocab_file''']     )
        SCREAMING_SNAKE_CASE__											=			os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES['''merges_file''']     )
        with open(self.vocab_file   ,				'''w'''   ,				encoding='''utf-8'''     ) as fp:
            fp.write(json.dumps(a__     ) + '''\n'''     )
        with open(self.merges_file   ,				'''w'''   ,				encoding='''utf-8'''     ) as fp:
            fp.write('''\n'''.join(a__     )     )



    def  lowercase_					(       self   ,				**A_     ):





        '''simple docstring'''

        kwargs.update(self.special_tokens_map     )
        return GPTaTokenizer.from_pretrained(self.tmpdirname   ,				**a__     )



    def  lowercase_					(       self   ,				**A_     ):





        '''simple docstring'''

        kwargs.update(self.special_tokens_map     )
        return GPTaTokenizerFast.from_pretrained(self.tmpdirname   ,				**a__     )



    def  lowercase_					(       self   ,				A_     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			"""lower newer"""
        SCREAMING_SNAKE_CASE__											=			"""lower newer"""
        return input_text, output_text



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			GPTaTokenizer(self.vocab_file   ,				self.merges_file   ,				**self.special_tokens_map     )
        SCREAMING_SNAKE_CASE__											=			"""lower newer"""
        SCREAMING_SNAKE_CASE__											=			["""\u0120low""", """er""", """\u0120""", """n""", """e""", """w""", """er"""]
        SCREAMING_SNAKE_CASE__											=			tokenizer.tokenize(a__   ,				add_prefix_space=a__     )
        self.assertListEqual(a__   ,				a__     )

        SCREAMING_SNAKE_CASE__											=			tokens + [tokenizer.unk_token]
        SCREAMING_SNAKE_CASE__											=			[14, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(a__     )   ,				a__     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        if not self.test_rust_tokenizer:
            return

        SCREAMING_SNAKE_CASE__											=			self.get_tokenizer()
        SCREAMING_SNAKE_CASE__											=			self.get_rust_tokenizer(add_prefix_space=a__     )

        SCREAMING_SNAKE_CASE__											=			"""lower newer"""

        # Testing tokenization
        SCREAMING_SNAKE_CASE__											=			tokenizer.tokenize(a__   ,				add_prefix_space=a__     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.tokenize(a__     )
        self.assertListEqual(a__   ,				a__     )

        # Testing conversion to ids without special tokens
        SCREAMING_SNAKE_CASE__											=			tokenizer.encode(a__   ,				add_special_tokens=a__   ,				add_prefix_space=a__     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.encode(a__   ,				add_special_tokens=a__     )
        self.assertListEqual(a__   ,				a__     )

        # Testing conversion to ids with special tokens
        SCREAMING_SNAKE_CASE__											=			self.get_rust_tokenizer(add_prefix_space=a__     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.encode(a__   ,				add_prefix_space=a__     )
        SCREAMING_SNAKE_CASE__											=			rust_tokenizer.encode(a__     )
        self.assertListEqual(a__   ,				a__     )

        # Testing the unknown token
        SCREAMING_SNAKE_CASE__											=			tokens + [rust_tokenizer.unk_token]
        SCREAMING_SNAKE_CASE__											=			[14, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(a__     )   ,				a__     )



    def  lowercase_					(       self   ,				*A_   ,				**A_     ):





        '''simple docstring'''

        pass



    def  lowercase_					(       self   ,				A_=15     ):





        '''simple docstring'''

        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
            with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''     ):
                SCREAMING_SNAKE_CASE__											=			self.rust_tokenizer_class.from_pretrained(a__   ,				**a__     )

                # Simple input
                SCREAMING_SNAKE_CASE__											=			"""This is a simple input"""
                SCREAMING_SNAKE_CASE__											=			["""This is a simple input 1""", """This is a simple input 2"""]
                SCREAMING_SNAKE_CASE__											=			("""This is a simple input""", """This is a pair""")
                SCREAMING_SNAKE_CASE__											=			[
                    ("""This is a simple input 1""", """This is a simple input 2"""),
                    ("""This is a simple pair 1""", """This is a simple pair 2"""),
                ]

                # Simple input tests
                self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length'''     )

                # Simple input
                self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''     )

                # Simple input
                self.assertRaises(
                    a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)

                # Pair input
                self.assertRaises(a__   ,				tokenizer_r.encode   ,				a__   ,				max_length=a__   ,				padding='''max_length'''     )

                # Pair input
                self.assertRaises(a__   ,				tokenizer_r.encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''     )

                # Pair input
                self.assertRaises(
                    a__   ,				tokenizer_r.batch_encode_plus   ,				a__   ,				max_length=a__   ,				padding='''max_length'''   ,				)



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			GPTaTokenizer.from_pretrained(self.tmpdirname   ,				pad_token='''<pad>'''     )

        # Simple input
        SCREAMING_SNAKE_CASE__											=			"""This is a simple input"""
        SCREAMING_SNAKE_CASE__											=			["""This is a simple input looooooooong""", """This is a simple input"""]
        SCREAMING_SNAKE_CASE__											=			("""This is a simple input""", """This is a pair""")
        SCREAMING_SNAKE_CASE__											=			[
            ("""This is a simple input loooooong""", """This is a simple input"""),
            ("""This is a simple pair loooooong""", """This is a simple pair"""),
        ]

        SCREAMING_SNAKE_CASE__											=			tokenizer.pad_token_id

        SCREAMING_SNAKE_CASE__											=			tokenizer(a__   ,				padding='''max_length'''   ,				max_length=30   ,				return_tensors='''np'''     )
        SCREAMING_SNAKE_CASE__											=			tokenizer(a__   ,				padding=a__   ,				truncate=a__   ,				return_tensors='''np'''     )
        SCREAMING_SNAKE_CASE__											=			tokenizer(*a__   ,				padding='''max_length'''   ,				max_length=60   ,				return_tensors='''np'''     )
        SCREAMING_SNAKE_CASE__											=			tokenizer(a__   ,				padding=a__   ,				truncate=a__   ,				return_tensors='''np'''     )

        # s
        # test single string max_length padding
        self.assertEqual(out_s['''input_ids'''].shape[-1]   ,				30     )
        self.assertTrue(pad_token_id in out_s['''input_ids''']     )
        self.assertTrue(0 in out_s['''attention_mask''']     )

        # s2
        # test automatic padding
        self.assertEqual(out_sa['''input_ids'''].shape[-1]   ,				33     )
        # long slice doesn't have padding
        self.assertFalse(pad_token_id in out_sa['''input_ids'''][0]     )
        self.assertFalse(0 in out_sa['''attention_mask'''][0]     )
        # short slice does have padding
        self.assertTrue(pad_token_id in out_sa['''input_ids'''][1]     )
        self.assertTrue(0 in out_sa['''attention_mask'''][1]     )

        # p
        # test single pair max_length padding
        self.assertEqual(out_p['''input_ids'''].shape[-1]   ,				60     )
        self.assertTrue(pad_token_id in out_p['''input_ids''']     )
        self.assertTrue(0 in out_p['''attention_mask''']     )

        # p2
        # test automatic padding pair
        self.assertEqual(out_pa['''input_ids'''].shape[-1]   ,				52     )
        # long slice pair doesn't have padding
        self.assertFalse(pad_token_id in out_pa['''input_ids'''][0]     )
        self.assertFalse(0 in out_pa['''attention_mask'''][0]     )
        # short slice pair does have padding
        self.assertTrue(pad_token_id in out_pa['''input_ids'''][1]     )
        self.assertTrue(0 in out_pa['''attention_mask'''][1]     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			"""$$$"""
        SCREAMING_SNAKE_CASE__											=			GPTaTokenizer.from_pretrained(self.tmpdirname   ,				bos_token=a__   ,				add_bos_token=a__     )

        SCREAMING_SNAKE_CASE__											=			"""This is a simple input"""
        SCREAMING_SNAKE_CASE__											=			["""This is a simple input 1""", """This is a simple input 2"""]

        SCREAMING_SNAKE_CASE__											=			tokenizer.bos_token_id

        SCREAMING_SNAKE_CASE__											=			tokenizer(a__     )
        SCREAMING_SNAKE_CASE__											=			tokenizer(a__     )

        self.assertEqual(out_s.input_ids[0]   ,				a__     )
        self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids     )     )

        SCREAMING_SNAKE_CASE__											=			tokenizer.decode(out_s.input_ids     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.batch_decode(out_sa.input_ids     )

        self.assertEqual(decode_s.split()[0]   ,				a__     )
        self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa     )     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        pass




    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			[self.get_tokenizer(do_lower_case=a__   ,				add_bos_token=a__     )]
        for tokenizer in tokenizers:
            with self.subTest(f'''{tokenizer.__class__.__name__}'''     ):
                SCREAMING_SNAKE_CASE__											=			"""Encode this."""
                SCREAMING_SNAKE_CASE__											=			"""This one too please."""
                SCREAMING_SNAKE_CASE__											=			tokenizer.encode(a__   ,				add_special_tokens=a__     )
                encoded_sequence += tokenizer.encode(a__   ,				add_special_tokens=a__     )
                SCREAMING_SNAKE_CASE__											=			tokenizer.encode_plus(
                    a__   ,				a__   ,				add_special_tokens=a__   ,				return_special_tokens_mask=a__   ,				)
                SCREAMING_SNAKE_CASE__											=			encoded_sequence_dict["""input_ids"""]
                SCREAMING_SNAKE_CASE__											=			encoded_sequence_dict["""special_tokens_mask"""]
                self.assertEqual(len(a__     )   ,				len(a__     )     )

                SCREAMING_SNAKE_CASE__											=			[
                    (x if not special_tokens_mask[i] else None) for i, x in enumerate(a__     )
                ]
                SCREAMING_SNAKE_CASE__											=			[x for x in filtered_sequence if x is not None]
                self.assertEqual(a__   ,				a__     )





@require_tokenizers
class 					__snake_case	(  unittest.TestCase    ):


    '''simple docstring'''







    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			AutoTokenizer.from_pretrained('''facebook/opt-350m'''   ,				from_slow=a__     )
        SCREAMING_SNAKE_CASE__											=			"""A photo of a cat"""

        SCREAMING_SNAKE_CASE__											=			tokenizer.encode(
            a__   ,				)
        self.assertEqual(a__   ,				[2, 2_50, 13_45, 9, 10, 47_58]     )
        tokenizer.save_pretrained('''test_opt'''     )

        SCREAMING_SNAKE_CASE__											=			AutoTokenizer.from_pretrained('''./test_opt'''     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.encode(
            a__   ,				)
        self.assertEqual(a__   ,				[2, 2_50, 13_45, 9, 10, 47_58]     )



    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			AutoTokenizer.from_pretrained('''facebook/opt-350m'''   ,				use_slow=a__     )
        SCREAMING_SNAKE_CASE__											=			"""A photo of a cat"""

        SCREAMING_SNAKE_CASE__											=			tokenizer.encode(
            a__   ,				)
        # Same as above
        self.assertEqual(a__   ,				[2, 2_50, 13_45, 9, 10, 47_58]     )




    @unittest.skip('''This test is failing because of a bug in the fast tokenizer'''     )
    def  lowercase_					(       self     ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__											=			AutoTokenizer.from_pretrained('''facebook/opt-350m'''   ,				from_slow=a__     )

        SCREAMING_SNAKE_CASE__											=			"""bos"""
        SCREAMING_SNAKE_CASE__											=			tokenizer.get_vocab()["""bos"""]

        SCREAMING_SNAKE_CASE__											=			"""A photo of a cat"""
        SCREAMING_SNAKE_CASE__											=			tokenizer.encode(
            a__   ,				)
        # We changed the bos token
        self.assertEqual(a__   ,				[3_19_57, 2_50, 13_45, 9, 10, 47_58]     )
        tokenizer.save_pretrained('''./tok'''     )
        SCREAMING_SNAKE_CASE__											=			AutoTokenizer.from_pretrained('''./tok'''     )
        self.assertTrue(tokenizer.is_fast     )
        SCREAMING_SNAKE_CASE__											=			tokenizer.encode(
            a__   ,				)
        self.assertEqual(a__   ,				[3_19_57, 2_50, 13_45, 9, 10, 47_58]     )








import unittest

import numpy as np
import torch
from torch import nn
from transformers import (
    CLIPImageProcessor,
    CLIPTextConfig,
    CLIPTextModelWithProjection,
    CLIPTokenizer,
    CLIPVisionConfig,
    CLIPVisionModelWithProjection,
)

from diffusers import KandinskyVaaPriorPipeline, PriorTransformer, UnCLIPScheduler
from diffusers.utils import torch_device
from diffusers.utils.testing_utils import enable_full_determinism, skip_mps

from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()




class       snake_case__ (				lowercase_						,     unittest.TestCase):

       '''simple docstring'''
       lowerCamelCase				:					Tuple								=			KandinskyVaaPriorPipeline
       lowerCamelCase				:					Optional[Any]								=			["prompt"]
       lowerCamelCase				:					Any								=			["prompt", "negative_prompt"]
       lowerCamelCase				:					Tuple								=			[
           "num_images_per_prompt",
           "generator",
           "num_inference_steps",
           "latents",
           "negative_prompt",
           "guidance_scale",
           "output_type",
           "return_dict",
       ]
       lowerCamelCase				:					Any								=			False



       @property
       def        __lowercase    (				self     )			->   Union[str, Any]:






            '''simple docstring'''





            return 32



       @property
       def        __lowercase    (				self     )			->   Any:






            '''simple docstring'''





            return 32



       @property
       def        __lowercase    (				self     )			->   List[Any]:






            '''simple docstring'''





            return self.time_input_dim



       @property
       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            return self.time_input_dim * 4



       @property
       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            return 1_00



       @property
       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            __snake_case      :Tuple 							=    CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""     )
            return tokenizer



       @property
       def        __lowercase    (				self     )			->   str:






            '''simple docstring'''





            torch.manual_seed(0     )
            __snake_case      :Dict 							=    CLIPTextConfig(
                bos_token_id=0      ,    eos_token_id=2      ,    hidden_size=self.text_embedder_hidden_size      ,    projection_dim=self.text_embedder_hidden_size      ,    intermediate_size=37      ,    layer_norm_eps=1e-05      ,    num_attention_heads=4      ,    num_hidden_layers=5      ,    pad_token_id=1      ,    vocab_size=10_00      ,    )
            return CLIPTextModelWithProjection(a__     )



       @property
       def        __lowercase    (				self     )			->   str:






            '''simple docstring'''





            torch.manual_seed(0     )

            __snake_case      :List[Any] 							=    {
                """num_attention_heads""": 2,
                """attention_head_dim""": 12,
                """embedding_dim""": self.text_embedder_hidden_size,
                """num_layers""": 1,
            }

            __snake_case      :Union[str, Any] 							=    PriorTransformer(**a__     )
            # clip_std and clip_mean is initialized to be 0 so PriorTransformer.post_process_latents will always return 0 - set clip_std to be 1 so it won't return 0
            __snake_case      :int 							=    nn.Parameter(torch.ones(model.clip_std.shape     )     )
            return model



       @property
       def        __lowercase    (				self     )			->   List[str]:






            '''simple docstring'''





            torch.manual_seed(0     )
            __snake_case      :List[Any] 							=    CLIPVisionConfig(
                hidden_size=self.text_embedder_hidden_size      ,    image_size=2_24      ,    projection_dim=self.text_embedder_hidden_size      ,    intermediate_size=37      ,    num_attention_heads=4      ,    num_channels=3      ,    num_hidden_layers=5      ,    patch_size=14      ,    )

            __snake_case      :Tuple 							=    CLIPVisionModelWithProjection(a__     )
            return model



       @property
       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            __snake_case      :Optional[int] 							=    CLIPImageProcessor(
                crop_size=2_24      ,    do_center_crop=a__      ,    do_normalize=a__      ,    do_resize=a__      ,    image_mean=[0.48_14_54_66, 0.4_57_82_75, 0.40_82_10_73]      ,    image_std=[0.26_86_29_54, 0.26_13_02_58, 0.27_57_77_11]      ,    resample=3      ,    size=2_24      ,    )

            return image_processor



       def        __lowercase    (				self     )			->   Optional[Any]:






            '''simple docstring'''





            __snake_case      :Optional[Any] 							=    self.dummy_prior
            __snake_case      :Tuple 							=    self.dummy_image_encoder
            __snake_case      :List[Any] 							=    self.dummy_text_encoder
            __snake_case      :List[str] 							=    self.dummy_tokenizer
            __snake_case      :int 							=    self.dummy_image_processor

            __snake_case      :Union[str, Any] 							=    UnCLIPScheduler(
                variance_type="""fixed_small_log"""      ,    prediction_type="""sample"""      ,    num_train_timesteps=10_00      ,    clip_sample=a__      ,    clip_sample_range=10.0      ,    )

            __snake_case      :Union[str, Any] 							=    {
                """prior""": prior,
                """image_encoder""": image_encoder,
                """text_encoder""": text_encoder,
                """tokenizer""": tokenizer,
                """scheduler""": scheduler,
                """image_processor""": image_processor,
            }

            return components



       def        __lowercase    (				self      ,    a__      ,    a__=0     )			->   Optional[Any]:






            '''simple docstring'''





            if str(a__     ).startswith("""mps"""     ):
                 __snake_case      :Any 							=    torch.manual_seed(a__     )
            else:
                 __snake_case      :Any 							=    torch.Generator(device=a__     ).manual_seed(a__     )
            __snake_case      :Union[str, Any] 							=    {
                """prompt""": """horse""",
                """generator""": generator,
                """guidance_scale""": 4.0,
                """num_inference_steps""": 2,
                """output_type""": """np""",
            }
            return inputs



       def        __lowercase    (				self     )			->   List[str]:






            '''simple docstring'''





            __snake_case      :str 							=    """cpu"""

            __snake_case      :Optional[Any] 							=    self.get_dummy_components()

            __snake_case      :List[Any] 							=    self.pipeline_class(**a__     )
            __snake_case      :Union[str, Any] 							=    pipe.to(a__     )

            pipe.set_progress_bar_config(disable=a__     )

            __snake_case      :List[Any] 							=    pipe(**self.get_dummy_inputs(a__     )     )
            __snake_case      :Any 							=    output.image_embeds

            __snake_case      :Any 							=    pipe(
                **self.get_dummy_inputs(a__     )      ,    return_dict=a__      ,    )[0]

            __snake_case      :Any 							=    image[0, -10:]
            __snake_case      :Any 							=    image_from_tuple[0, -10:]

            assert image.shape == (1, 32)

            __snake_case      :List[str] 							=    np.array(
                [-0.05_32, 1.71_20, 0.36_56, -1.08_52, -0.89_46, -1.17_56, 0.43_48, 0.24_82, 0.51_46, -0.11_56]     )

            assert np.abs(image_slice.flatten() - expected_slice     ).max() < 1e-2
            assert np.abs(image_from_tuple_slice.flatten() - expected_slice     ).max() < 1e-2



       @skip_mps
       def        __lowercase    (				self     )			->   str:






            '''simple docstring'''





            __snake_case      :List[Any] 							=    torch_device == """cpu"""
            __snake_case      :List[str] 							=    True
            __snake_case      :Optional[Any] 							=    False

            self._test_inference_batch_single_identical(
                test_max_difference=a__      ,    relax_max_difference=a__      ,    test_mean_pixel_difference=a__      ,    )



       @skip_mps
       def        __lowercase    (				self     )			->   List[Any]:






            '''simple docstring'''





            __snake_case      :Any 							=    torch_device == """cpu"""
            __snake_case      :str 							=    False

            self._test_attention_slicing_forward_pass(
                test_max_difference=a__      ,    test_mean_pixel_difference=a__      ,    )


