
"""simple docstring"""



import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a       :						Union[str, Any]       		=		get_tests_dir('''fixtures/test_sentencepiece.model''')




@require_sentencepiece
@require_tokenizers
class 		a_    (			_UpperCAmelCase  ,    unittest.TestCase						):
    a   :					Dict			     =					XLNetTokenizer
    a   :					Dict			     =					XLNetTokenizerFast
    a   :					Any			     =					True
    a   :					List[str]			     =					True







    def   _snake_case       (      self      :							Optional[int]  )		->Optional[Any]:






        '''simple docstring'''






        super().setUp()

        # We have a SentencePiece fixture for testing
        _UpperCAmelCase        = XLNetTokenizer(__UpperCamelCase  ,       keep_accents=__UpperCamelCase  )
        tokenizer.sanitize_special_tokens()
        tokenizer.save_pretrained(self.tmpdirname  )







    def   _snake_case       (      self      :							Optional[Any]  )		->str:






        '''simple docstring'''






        _UpperCAmelCase        = """<s>"""
        _UpperCAmelCase        = 1

        self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCamelCase  )  ,       __UpperCamelCase  )
        self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCamelCase  )  ,       __UpperCamelCase  )







    def   _snake_case       (      self      :							int  )		->Any:






        '''simple docstring'''






        _UpperCAmelCase        = list(self.get_tokenizer().get_vocab().keys()  )

        self.assertEqual(vocab_keys[0]  ,       """<unk>"""  )
        self.assertEqual(vocab_keys[1]  ,       """<s>"""  )
        self.assertEqual(vocab_keys[-1]  ,       """<eod>"""  )
        self.assertEqual(len(__UpperCamelCase  )  ,       10_06  )







    def   _snake_case       (      self      :							Optional[int]  )		->Tuple:






        '''simple docstring'''






        self.assertEqual(self.get_tokenizer().vocab_size  ,       10_00  )







    def   _snake_case       (      self      :							Any  )		->Dict:






        '''simple docstring'''






        _UpperCAmelCase        = XLNetTokenizer(__UpperCamelCase  ,       keep_accents=__UpperCamelCase  )

        _UpperCAmelCase        = tokenizer.tokenize("""This is a test"""  )
        self.assertListEqual(__UpperCamelCase  ,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]  )

        self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase  )  ,       [2_85, 46, 10, 1_70, 3_82]  )

        _UpperCAmelCase        = tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
        self.assertListEqual(
            __UpperCamelCase  ,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """é""",
                """.""",
            ]  ,       )
        _UpperCAmelCase        = tokenizer.convert_tokens_to_ids(__UpperCamelCase  )
        self.assertListEqual(__UpperCamelCase  ,       [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]  )

        _UpperCAmelCase        = tokenizer.convert_ids_to_tokens(__UpperCamelCase  )
        self.assertListEqual(
            __UpperCamelCase  ,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """<unk>""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """s""",
                """<unk>""",
                """.""",
            ]  ,       )







    def   _snake_case       (      self      :							str  )		->List[Any]:






        '''simple docstring'''






        _UpperCAmelCase        = XLNetTokenizer(__UpperCamelCase  ,       do_lower_case=__UpperCamelCase  )
        _UpperCAmelCase        = tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
        self.assertListEqual(
            __UpperCamelCase  ,       [
                SPIECE_UNDERLINE + """""",
                """i""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """se""",
                """.""",
            ]  ,       )
        self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""  )  ,       ["""▁he""", """ll""", """o"""]  )







    def   _snake_case       (      self      :							Union[str, Any]  )		->int:






        '''simple docstring'''






        _UpperCAmelCase        = XLNetTokenizer(__UpperCamelCase  ,       do_lower_case=__UpperCamelCase  )
        _UpperCAmelCase        = tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
        self.assertListEqual(
            __UpperCamelCase  ,       [
                SPIECE_UNDERLINE + """I""",
                SPIECE_UNDERLINE + """was""",
                SPIECE_UNDERLINE + """b""",
                """or""",
                """n""",
                SPIECE_UNDERLINE + """in""",
                SPIECE_UNDERLINE + """""",
                """9""",
                """2""",
                """0""",
                """0""",
                """0""",
                """,""",
                SPIECE_UNDERLINE + """and""",
                SPIECE_UNDERLINE + """this""",
                SPIECE_UNDERLINE + """is""",
                SPIECE_UNDERLINE + """f""",
                """al""",
                """se""",
                """.""",
            ]  ,       )







    @slow
    def   _snake_case       (      self      :							Optional[int]  )		->List[str]:






        '''simple docstring'''






        _UpperCAmelCase        = XLNetTokenizer.from_pretrained("""xlnet-base-cased"""  )

        _UpperCAmelCase        = tokenizer.encode("""sequence builders"""  ,       add_special_tokens=__UpperCamelCase  )
        _UpperCAmelCase        = tokenizer.encode("""multi-sequence build"""  ,       add_special_tokens=__UpperCamelCase  )

        _UpperCAmelCase        = tokenizer.build_inputs_with_special_tokens(__UpperCamelCase  )
        _UpperCAmelCase        = tokenizer.build_inputs_with_special_tokens(__UpperCamelCase  ,       __UpperCamelCase  )

        assert encoded_sentence == text + [4, 3]
        assert encoded_pair == text + [4] + text_a + [4, 3]







    @slow
    def   _snake_case       (      self      :							Dict  )		->int:






        '''simple docstring'''






        _UpperCAmelCase        = {"""input_ids""": [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
        # fmt: on

        self.tokenizer_integration_test_util(
            expected_encoding=__UpperCamelCase  ,       model_name="""xlnet-base-cased"""  ,       revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""  ,       )

"""simple docstring"""



import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    StableDiffusionSAGPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()




class 		a_    (			_UpperCAmelCase  ,    _UpperCAmelCase  ,    unittest.TestCase						):
    a   :					Dict			     =					StableDiffusionSAGPipeline
    a   :					Any			     =					TEXT_TO_IMAGE_PARAMS
    a   :					Optional[Any]			     =					TEXT_TO_IMAGE_BATCH_PARAMS
    a   :					Optional[int]			     =					TEXT_TO_IMAGE_IMAGE_PARAMS
    a   :					Optional[int]			     =					TEXT_TO_IMAGE_IMAGE_PARAMS
    a   :					Tuple			     =					False







    def   _snake_case       (      self      :							List[Any]  )		->Tuple:






        '''simple docstring'''






        torch.manual_seed(0  )
        _UpperCAmelCase        = UNetaDConditionModel(
            block_out_channels=(32, 64)  ,       layers_per_block=2  ,       sample_size=32  ,       in_channels=4  ,       out_channels=4  ,       down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")  ,       up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")  ,       cross_attention_dim=32  ,       )
        _UpperCAmelCase        = DDIMScheduler(
            beta_start=0.0_0_0_8_5  ,       beta_end=0.0_1_2  ,       beta_schedule="""scaled_linear"""  ,       clip_sample=__UpperCamelCase  ,       set_alpha_to_one=__UpperCamelCase  ,       )
        torch.manual_seed(0  )
        _UpperCAmelCase        = AutoencoderKL(
            block_out_channels=[32, 64]  ,       in_channels=3  ,       out_channels=3  ,       down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]  ,       up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]  ,       latent_channels=4  ,       )
        torch.manual_seed(0  )
        _UpperCAmelCase        = CLIPTextConfig(
            bos_token_id=0  ,       eos_token_id=2  ,       hidden_size=32  ,       intermediate_size=37  ,       layer_norm_eps=1e-05  ,       num_attention_heads=4  ,       num_hidden_layers=5  ,       pad_token_id=1  ,       vocab_size=10_00  ,       )
        _UpperCAmelCase        = CLIPTextModel(__UpperCamelCase  )
        _UpperCAmelCase        = CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""  )

        _UpperCAmelCase        = {
            """unet""": unet,
            """scheduler""": scheduler,
            """vae""": vae,
            """text_encoder""": text_encoder,
            """tokenizer""": tokenizer,
            """safety_checker""": None,
            """feature_extractor""": None,
        }
        return components







    def   _snake_case       (      self      :							Union[str, Any]  ,       __UpperCamelCase      :							Union[str, Any]  ,       __UpperCamelCase      :							Any=0  )		->Any:






        '''simple docstring'''






        if str(__UpperCamelCase  ).startswith("""mps"""  ):
            _UpperCAmelCase        = torch.manual_seed(__UpperCamelCase  )
        else:
            _UpperCAmelCase        = torch.Generator(device=__UpperCamelCase  ).manual_seed(__UpperCamelCase  )
        _UpperCAmelCase        = {
            """prompt""": """.""",
            """generator""": generator,
            """num_inference_steps""": 2,
            """guidance_scale""": 1.0,
            """sag_scale""": 1.0,
            """output_type""": """numpy""",
        }
        return inputs







    def   _snake_case       (      self      :							Optional[Any]  )		->Optional[int]:






        '''simple docstring'''






        super().test_inference_batch_single_identical(expected_max_diff=3e-3  )







@slow
@require_torch_gpu
class 		a_    (			unittest.TestCase						):







    def   _snake_case       (      self      :							Union[str, Any]  )		->str:






        '''simple docstring'''






        super().tearDown()
        gc.collect()
        torch.cuda.empty_cache()







    def   _snake_case       (      self      :							Tuple  )		->Union[str, Any]:






        '''simple docstring'''






        _UpperCAmelCase        = StableDiffusionSAGPipeline.from_pretrained("""CompVis/stable-diffusion-v1-4"""  )
        _UpperCAmelCase        = sag_pipe.to(__UpperCamelCase  )
        sag_pipe.set_progress_bar_config(disable=__UpperCamelCase  )

        _UpperCAmelCase        = """."""
        _UpperCAmelCase        = torch.manual_seed(0  )
        _UpperCAmelCase        = sag_pipe(
            [prompt]  ,       generator=__UpperCamelCase  ,       guidance_scale=7.5  ,       sag_scale=1.0  ,       num_inference_steps=20  ,       output_type="""np"""  )

        _UpperCAmelCase        = output.images

        _UpperCAmelCase        = image[0, -3:, -3:, -1]

        assert image.shape == (1, 5_12, 5_12, 3)
        _UpperCAmelCase        = np.array([0.1_5_6_8, 0.1_7_3_8, 0.1_6_9_5, 0.1_6_9_3, 0.1_5_0_7, 0.1_7_0_5, 0.1_5_4_7, 0.1_7_5_1, 0.1_9_4_9]  )

        assert np.abs(image_slice.flatten() - expected_slice  ).max() < 5e-2







    def   _snake_case       (      self      :							Union[str, Any]  )		->Tuple:






        '''simple docstring'''






        _UpperCAmelCase        = StableDiffusionSAGPipeline.from_pretrained("""stabilityai/stable-diffusion-2-1-base"""  )
        _UpperCAmelCase        = sag_pipe.to(__UpperCamelCase  )
        sag_pipe.set_progress_bar_config(disable=__UpperCamelCase  )

        _UpperCAmelCase        = """."""
        _UpperCAmelCase        = torch.manual_seed(0  )
        _UpperCAmelCase        = sag_pipe(
            [prompt]  ,       generator=__UpperCamelCase  ,       guidance_scale=7.5  ,       sag_scale=1.0  ,       num_inference_steps=20  ,       output_type="""np"""  )

        _UpperCAmelCase        = output.images

        _UpperCAmelCase        = image[0, -3:, -3:, -1]

        assert image.shape == (1, 5_12, 5_12, 3)
        _UpperCAmelCase        = np.array([0.3_4_5_9, 0.2_8_7_6, 0.2_5_3_7, 0.3_0_0_2, 0.2_6_7_1, 0.2_1_6_0, 0.3_0_2_6, 0.2_2_6_2, 0.2_3_7_1]  )

        assert np.abs(image_slice.flatten() - expected_slice  ).max() < 5e-2







    def   _snake_case       (      self      :							str  )		->str:






        '''simple docstring'''






        _UpperCAmelCase        = StableDiffusionSAGPipeline.from_pretrained("""stabilityai/stable-diffusion-2-1-base"""  )
        _UpperCAmelCase        = sag_pipe.to(__UpperCamelCase  )
        sag_pipe.set_progress_bar_config(disable=__UpperCamelCase  )

        _UpperCAmelCase        = """."""
        _UpperCAmelCase        = torch.manual_seed(0  )
        _UpperCAmelCase        = sag_pipe(
            [prompt]  ,       width=7_68  ,       height=5_12  ,       generator=__UpperCamelCase  ,       guidance_scale=7.5  ,       sag_scale=1.0  ,       num_inference_steps=20  ,       output_type="""np"""  ,       )

        _UpperCAmelCase        = output.images

        assert image.shape == (1, 5_12, 7_68, 3)