
'''simple docstring'''
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase												=	logging.get_logger(__name__)

__lowerCAmelCase												=	{'vocab_file': 'sentencepiece.bpe.model'}

__lowerCAmelCase												=	{
    'vocab_file': {
        'camembert-base': 'https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model',
    }
}

__lowerCAmelCase												=	{
    'camembert-base': 512,
}

__lowerCAmelCase												=	'▁'


class       _lowerCAmelCase   (   __snake_case				):







							'''simple docstring'''

							lowerCAmelCase_ 						=						VOCAB_FILES_NAMES
							lowerCAmelCase_ 						=						PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase_ 						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCAmelCase_ 						=						["input_ids", "attention_mask"]

							def __init__(self  ,  UpperCAmelCase  ,  UpperCAmelCase="<s>"  ,  UpperCAmelCase="</s>"  ,  UpperCAmelCase="</s>"  ,  UpperCAmelCase="<s>"  ,  UpperCAmelCase="<unk>"  ,  UpperCAmelCase="<pad>"  ,  UpperCAmelCase="<mask>"  ,  UpperCAmelCase=["<s>NOTUSED", "</s>NOTUSED"]  ,  UpperCAmelCase = None  ,  **UpperCAmelCase  ,  )					->       None:
														# Mask token behave like a normal word, i.e. include the space before it
														_snake_case       =  AddedToken(UpperCAmelCase  ,  lstrip=UpperCAmelCase  ,  rstrip=UpperCAmelCase  ) if isinstance(UpperCAmelCase  ,  UpperCAmelCase  ) else mask_token

														_snake_case       =  {} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    bos_token=UpperCAmelCase  ,  eos_token=UpperCAmelCase  ,  unk_token=UpperCAmelCase  ,  sep_token=UpperCAmelCase  ,  cls_token=UpperCAmelCase  ,  pad_token=UpperCAmelCase  ,  mask_token=UpperCAmelCase  ,  additional_special_tokens=UpperCAmelCase  ,  sp_model_kwargs=self.sp_model_kwargs  ,  **UpperCAmelCase  ,  )
														_snake_case       =  spm.SentencePieceProcessor(**self.sp_model_kwargs  )
														self.sp_model.Load(str(UpperCAmelCase  )  )
														_snake_case       =  vocab_file
														# HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
														# sentencepiece vocabulary (this is the case for <s> and </s>
														_snake_case       =  {"""<s>NOTUSED""": 0, """<pad>""": 1, """</s>NOTUSED""": 2, """<unk>""": 3}
														_snake_case       =  len(self.fairseq_tokens_to_ids  )
														_snake_case       =  len(self.sp_model  ) + len(self.fairseq_tokens_to_ids  )
														_snake_case       =  {v: k for k, v in self.fairseq_tokens_to_ids.items()}

							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase = None  )					->       List[int]:

														if token_ids_a is None:
																					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
														_snake_case       =  [self.cls_token_id]
														_snake_case       =  [self.sep_token_id]
														return cls + token_ids_a + sep + sep + token_ids_a + sep

							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase = None  ,  UpperCAmelCase = False  )					->       List[int]:
														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=UpperCAmelCase  ,  token_ids_a=UpperCAmelCase  ,  already_has_special_tokens=UpperCAmelCase  )

														if token_ids_a is None:
																					return [1] + ([0] * len(UpperCAmelCase  )) + [1]
														return [1] + ([0] * len(UpperCAmelCase  )) + [1, 1] + ([0] * len(UpperCAmelCase  )) + [1]

							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase = None  )					->       List[int]:
														_snake_case       =  [self.sep_token_id]
														_snake_case       =  [self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep  ) * [0]
														return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]

							@property
							def 	lowercase							(self  )					->       Tuple:
														return len(self.fairseq_tokens_to_ids  ) + len(self.sp_model  )

							def 	lowercase							(self  )					->       Tuple:
														_snake_case       =  {self.convert_ids_to_tokens(UpperCAmelCase  ): i for i in range(self.vocab_size  )}
														vocab.update(self.added_tokens_encoder  )
														return vocab

							def 	lowercase							(self  ,  UpperCAmelCase  )					->       List[str]:
														return self.sp_model.encode(UpperCAmelCase  ,  out_type=UpperCAmelCase  )

							def 	lowercase							(self  ,  UpperCAmelCase  )					->       Tuple:
														if token in self.fairseq_tokens_to_ids:
																					return self.fairseq_tokens_to_ids[token]
														elif self.sp_model.PieceToId(UpperCAmelCase  ) == 0:
																					# Convert sentence piece unk token to fairseq unk token index
																					return self.unk_token_id
														return self.fairseq_offset + self.sp_model.PieceToId(UpperCAmelCase  )

							def 	lowercase							(self  ,  UpperCAmelCase  )					->       Optional[int]:
														if index in self.fairseq_ids_to_tokens:
																					return self.fairseq_ids_to_tokens[index]
														return self.sp_model.IdToPiece(index - self.fairseq_offset  )

							def 	lowercase							(self  ,  UpperCAmelCase  )					->       Tuple:
														_snake_case       =  []
														_snake_case       =  """"""
														_snake_case       =  False
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												if not prev_is_special:
																																			out_string += " "
																												out_string += self.sp_model.decode(UpperCAmelCase  ) + token
																												_snake_case       =  True
																												_snake_case       =  []
																					else:
																												current_sub_tokens.append(UpperCAmelCase  )
																												_snake_case       =  False
														out_string += self.sp_model.decode(UpperCAmelCase  )
														return out_string.strip()

							def __getstate__(self  )					->       Any:
														_snake_case       =  self.__dict__.copy()
														_snake_case       =  None
														return state

							def __setstate__(self  ,  UpperCAmelCase  )					->       List[Any]:
														_snake_case       =  d

														# for backward compatibility
														if not hasattr(self  ,  """sp_model_kwargs"""  ):
																					_snake_case       =  {}

														_snake_case       =  spm.SentencePieceProcessor(**self.sp_model_kwargs  )
														self.sp_model.Load(self.vocab_file  )





							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase = None  )					->       Tuple[str]:
														if not os.path.isdir(UpperCAmelCase  ):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
																					return
														_snake_case       =  os.path.join(
														    UpperCAmelCase  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )

														if os.path.abspath(self.vocab_file  ) != os.path.abspath(UpperCAmelCase  ) and os.path.isfile(self.vocab_file  ):
																					copyfile(self.vocab_file  ,  UpperCAmelCase  )
														elif not os.path.isfile(self.vocab_file  ):
																					with open(UpperCAmelCase  ,  """wb"""  ) as fi:
																												_snake_case       =  self.sp_model.serialized_model_proto()
																												fi.write(UpperCAmelCase  )

														return (out_vocab_file,)

'''simple docstring'''
import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMScheduler,
    EulerAncestralDiscreteScheduler,
    LMSDiscreteScheduler,
    PNDMScheduler,
    StableDiffusionPanoramaPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, skip_mps

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()


@skip_mps
class       _lowerCAmelCase   (   __snake_case					,							__snake_case					,							unittest.TestCase				):







							'''simple docstring'''

							lowerCAmelCase_ 						=						StableDiffusionPanoramaPipeline
							lowerCAmelCase_ 						=						TEXT_TO_IMAGE_PARAMS
							lowerCAmelCase_ 						=						TEXT_TO_IMAGE_BATCH_PARAMS
							lowerCAmelCase_ 						=						TEXT_TO_IMAGE_IMAGE_PARAMS
							lowerCAmelCase_ 						=						TEXT_TO_IMAGE_IMAGE_PARAMS

							def 	lowercase							(self  )					->       List[Any]:
														torch.manual_seed(0  )
														_snake_case       =  UNetaDConditionModel(
														    block_out_channels=(32, 64)  ,  layers_per_block=1  ,  sample_size=32  ,  in_channels=4  ,  out_channels=4  ,  down_block_types=("""DownBlock2D""", """CrossAttnDownBlock2D""")  ,  up_block_types=("""CrossAttnUpBlock2D""", """UpBlock2D""")  ,  cross_attention_dim=32  ,  )
														_snake_case       =  DDIMScheduler()
														torch.manual_seed(0  )
														_snake_case       =  AutoencoderKL(
														    block_out_channels=[32, 64]  ,  in_channels=3  ,  out_channels=3  ,  down_block_types=["""DownEncoderBlock2D""", """DownEncoderBlock2D"""]  ,  up_block_types=["""UpDecoderBlock2D""", """UpDecoderBlock2D"""]  ,  latent_channels=4  ,  )
														torch.manual_seed(0  )
														_snake_case       =  CLIPTextConfig(
														    bos_token_id=0  ,  eos_token_id=2  ,  hidden_size=32  ,  intermediate_size=37  ,  layer_norm_eps=1e-0_5  ,  num_attention_heads=4  ,  num_hidden_layers=5  ,  pad_token_id=1  ,  vocab_size=1000  ,  )
														_snake_case       =  CLIPTextModel(UpperCAmelCase  )
														_snake_case       =  CLIPTokenizer.from_pretrained("""hf-internal-testing/tiny-random-clip"""  )

														_snake_case       =  {
														    """unet""": unet,
														    """scheduler""": scheduler,
														    """vae""": vae,
														    """text_encoder""": text_encoder,
														    """tokenizer""": tokenizer,
														    """safety_checker""": None,
														    """feature_extractor""": None,
														}
														return components

							def 	lowercase							(self  ,  UpperCAmelCase  ,  UpperCAmelCase=0  )					->       Tuple:
														_snake_case       =  torch.manual_seed(UpperCAmelCase  )
														_snake_case       =  {
														    """prompt""": """a photo of the dolomites""",
														    """generator""": generator,
														    # Setting height and width to None to prevent OOMs on CPU.
														    """height""": None,
														    """width""": None,
														    """num_inference_steps""": 1,
														    """guidance_scale""": 6.0,
														    """output_type""": """numpy""",
														}
														return inputs

							def 	lowercase							(self  )					->       Tuple:
														_snake_case       =  """cpu"""  # ensure determinism for the device-dependent torch.Generator
														_snake_case       =  self.get_dummy_components()
														_snake_case       =  StableDiffusionPanoramaPipeline(**UpperCAmelCase  )
														_snake_case       =  sd_pipe.to(UpperCAmelCase  )
														sd_pipe.set_progress_bar_config(disable=UpperCAmelCase  )

														_snake_case       =  self.get_dummy_inputs(UpperCAmelCase  )
														_snake_case       =  sd_pipe(**UpperCAmelCase  ).images
														_snake_case       =  image[0, -3:, -3:, -1]
														assert image.shape == (1, 64, 64, 3)

														_snake_case       =  np.array([0.6186, 0.5374, 0.4915, 0.4135, 0.4114, 0.4563, 0.5128, 0.4977, 0.4757]  )

														assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2

							def 	lowercase							(self  )					->       Tuple:
														super().test_inference_batch_consistent(batch_sizes=[1, 2]  )

							def 	lowercase							(self  )					->       Any:
														super().test_inference_batch_single_identical(batch_size=2  ,  expected_max_diff=3.2_5e-3  )

							def 	lowercase							(self  )					->       Any:
														_snake_case       =  """cpu"""  # ensure determinism for the device-dependent torch.Generator
														_snake_case       =  self.get_dummy_components()
														_snake_case       =  StableDiffusionPanoramaPipeline(**UpperCAmelCase  )
														_snake_case       =  sd_pipe.to(UpperCAmelCase  )
														sd_pipe.set_progress_bar_config(disable=UpperCAmelCase  )

														_snake_case       =  self.get_dummy_inputs(UpperCAmelCase  )
														_snake_case       =  """french fries"""
														_snake_case       =  sd_pipe(**UpperCAmelCase  ,  negative_prompt=UpperCAmelCase  )
														_snake_case       =  output.images
														_snake_case       =  image[0, -3:, -3:, -1]

														assert image.shape == (1, 64, 64, 3)

														_snake_case       =  np.array([0.6187, 0.5375, 0.4915, 0.4136, 0.4114, 0.4563, 0.5128, 0.4976, 0.4757]  )

														assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2

							def 	lowercase							(self  )					->       str:
														_snake_case       =  """cpu"""  # ensure determinism for the device-dependent torch.Generator
														_snake_case       =  self.get_dummy_components()
														_snake_case       =  StableDiffusionPanoramaPipeline(**UpperCAmelCase  )
														_snake_case       =  sd_pipe.to(UpperCAmelCase  )
														sd_pipe.set_progress_bar_config(disable=UpperCAmelCase  )

														_snake_case       =  self.get_dummy_inputs(UpperCAmelCase  )
														_snake_case       =  sd_pipe(**UpperCAmelCase  ,  view_batch_size=2  )
														_snake_case       =  output.images
														_snake_case       =  image[0, -3:, -3:, -1]

														assert image.shape == (1, 64, 64, 3)

														_snake_case       =  np.array([0.6187, 0.5375, 0.4915, 0.4136, 0.4114, 0.4563, 0.5128, 0.4976, 0.4757]  )

														assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2

							def 	lowercase							(self  )					->       Tuple:
														_snake_case       =  """cpu"""  # ensure determinism for the device-dependent torch.Generator
														_snake_case       =  self.get_dummy_components()
														_snake_case       =  EulerAncestralDiscreteScheduler(
														    beta_start=0.0_0085  ,  beta_end=0.012  ,  beta_schedule="""scaled_linear"""  )
														_snake_case       =  StableDiffusionPanoramaPipeline(**UpperCAmelCase  )
														_snake_case       =  sd_pipe.to(UpperCAmelCase  )
														sd_pipe.set_progress_bar_config(disable=UpperCAmelCase  )

														_snake_case       =  self.get_dummy_inputs(UpperCAmelCase  )
														_snake_case       =  sd_pipe(**UpperCAmelCase  ).images
														_snake_case       =  image[0, -3:, -3:, -1]

														assert image.shape == (1, 64, 64, 3)

														_snake_case       =  np.array([0.4024, 0.6510, 0.4901, 0.5378, 0.5813, 0.5622, 0.4795, 0.4467, 0.4952]  )

														assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2





							def 	lowercase							(self  )					->       str:
														_snake_case       =  """cpu"""  # ensure determinism for the device-dependent torch.Generator
														_snake_case       =  self.get_dummy_components()
														_snake_case       =  PNDMScheduler(
														    beta_start=0.0_0085  ,  beta_end=0.012  ,  beta_schedule="""scaled_linear"""  ,  skip_prk_steps=UpperCAmelCase  )
														_snake_case       =  StableDiffusionPanoramaPipeline(**UpperCAmelCase  )
														_snake_case       =  sd_pipe.to(UpperCAmelCase  )
														sd_pipe.set_progress_bar_config(disable=UpperCAmelCase  )

														_snake_case       =  self.get_dummy_inputs(UpperCAmelCase  )
														_snake_case       =  sd_pipe(**UpperCAmelCase  ).images
														_snake_case       =  image[0, -3:, -3:, -1]

														assert image.shape == (1, 64, 64, 3)

														_snake_case       =  np.array([0.6391, 0.6291, 0.4861, 0.5134, 0.5552, 0.4578, 0.5032, 0.5023, 0.4539]  )

														assert np.abs(image_slice.flatten() - expected_slice  ).max() < 1e-2


@slow
@require_torch_gpu
class       _lowerCAmelCase   (   unittest.TestCase				):







							'''simple docstring'''

							def 	lowercase							(self  )					->       Optional[Any]:
														super().tearDown()
														gc.collect()
														torch.cuda.empty_cache()

							def 	lowercase							(self  ,  UpperCAmelCase=0  )					->       List[str]:
														_snake_case       =  torch.manual_seed(UpperCAmelCase  )
														_snake_case       =  {
														    """prompt""": """a photo of the dolomites""",
														    """generator""": generator,
														    """num_inference_steps""": 3,
														    """guidance_scale""": 7.5,
														    """output_type""": """numpy""",
														}
														return inputs

							def 	lowercase							(self  )					->       List[Any]:
														_snake_case       =  """stabilityai/stable-diffusion-2-base"""
														_snake_case       =  DDIMScheduler.from_pretrained(UpperCAmelCase  ,  subfolder="""scheduler"""  )
														_snake_case       =  StableDiffusionPanoramaPipeline.from_pretrained(UpperCAmelCase  ,  scheduler=UpperCAmelCase  ,  safety_checker=UpperCAmelCase  )
														pipe.to(UpperCAmelCase  )
														pipe.set_progress_bar_config(disable=UpperCAmelCase  )
														pipe.enable_attention_slicing()

														_snake_case       =  self.get_inputs()
														_snake_case       =  pipe(**UpperCAmelCase  ).images
														_snake_case       =  image[0, -3:, -3:, -1].flatten()

														assert image.shape == (1, 512, 2048, 3)

														_snake_case       =  np.array(
														    [
														        0.3696_8392,
														        0.2702_5372,
														        0.3244_6766,
														        0.2837_9387,
														        0.3636_3274,
														        0.3073_3347,
														        0.2710_0027,
														        0.2705_4125,
														        0.2553_6096,
														    ]  )

														assert np.abs(expected_slice - image_slice  ).max() < 1e-2

							def 	lowercase							(self  )					->       str:
														_snake_case       =  StableDiffusionPanoramaPipeline.from_pretrained(
														    """stabilityai/stable-diffusion-2-base"""  ,  safety_checker=UpperCAmelCase  )
														_snake_case       =  LMSDiscreteScheduler.from_config(pipe.scheduler.config  )
														pipe.to(UpperCAmelCase  )
														pipe.set_progress_bar_config(disable=UpperCAmelCase  )
														pipe.enable_attention_slicing()

														_snake_case       =  self.get_inputs()
														_snake_case       =  pipe(**UpperCAmelCase  ).images
														_snake_case       =  image[0, -3:, -3:, -1].flatten()

														assert image.shape == (1, 512, 2048, 3)

														_snake_case       =  np.array(
														    [
														        [
														            0.0,
														            0.0,
														            0.0,
														            0.0,
														            0.0,
														            0.0,
														            0.0,
														            0.0,
														            0.0,
														        ]
														    ]  )

														assert np.abs(expected_slice - image_slice  ).max() < 1e-3

							def 	lowercase							(self  )					->       Optional[int]:
														_snake_case       =  0

														def callback_fn(UpperCAmelCase  ,  UpperCAmelCase  ,  UpperCAmelCase  ) -> None:
																					_snake_case       =  True
																					nonlocal number_of_steps
																					number_of_steps += 1
																					if step == 1:
																												_snake_case       =  latents.detach().cpu().numpy()
																												assert latents.shape == (1, 4, 64, 256)
																												_snake_case       =  latents[0, -3:, -3:, -1]

																												_snake_case       =  np.array(
																												    [
																												        0.1868_1869,
																												        0.3390_7816,
																												        0.536_1276,
																												        0.1443_2865,
																												        -0.0285_6611,
																												        -0.7394_1123,
																												        0.2339_7987,
																												        0.4732_2682,
																												        -0.3782_3164,
																												    ]  )
																												assert np.abs(latents_slice.flatten() - expected_slice  ).max() < 5e-2
																					elif step == 2:
																												_snake_case       =  latents.detach().cpu().numpy()
																												assert latents.shape == (1, 4, 64, 256)
																												_snake_case       =  latents[0, -3:, -3:, -1]

																												_snake_case       =  np.array(
																												    [
																												        0.1853_9645,
																												        0.3398_7248,
																												        0.537_8559,
																												        0.1443_7142,
																												        -0.0245_5261,
																												        -0.733_8317,
																												        0.2399_0755,
																												        0.4735_6272,
																												        -0.378_6505,
																												    ]  )

																												assert np.abs(latents_slice.flatten() - expected_slice  ).max() < 5e-2

														_snake_case       =  False

														_snake_case       =  """stabilityai/stable-diffusion-2-base"""
														_snake_case       =  DDIMScheduler.from_pretrained(UpperCAmelCase  ,  subfolder="""scheduler"""  )
														_snake_case       =  StableDiffusionPanoramaPipeline.from_pretrained(UpperCAmelCase  ,  scheduler=UpperCAmelCase  ,  safety_checker=UpperCAmelCase  )
														_snake_case       =  pipe.to(UpperCAmelCase  )
														pipe.set_progress_bar_config(disable=UpperCAmelCase  )
														pipe.enable_attention_slicing()

														_snake_case       =  self.get_inputs()
														pipe(**UpperCAmelCase  ,  callback=UpperCAmelCase  ,  callback_steps=1  )
														assert callback_fn.has_been_called
														assert number_of_steps == 3





							def 	lowercase							(self  )					->       List[Any]:
														torch.cuda.empty_cache()
														torch.cuda.reset_max_memory_allocated()
														torch.cuda.reset_peak_memory_stats()

														_snake_case       =  """stabilityai/stable-diffusion-2-base"""
														_snake_case       =  DDIMScheduler.from_pretrained(UpperCAmelCase  ,  subfolder="""scheduler"""  )
														_snake_case       =  StableDiffusionPanoramaPipeline.from_pretrained(UpperCAmelCase  ,  scheduler=UpperCAmelCase  ,  safety_checker=UpperCAmelCase  )
														_snake_case       =  pipe.to(UpperCAmelCase  )
														pipe.set_progress_bar_config(disable=UpperCAmelCase  )
														pipe.enable_attention_slicing(1  )
														pipe.enable_sequential_cpu_offload()

														_snake_case       =  self.get_inputs()
														_snake_case       =  pipe(**UpperCAmelCase  )

														_snake_case       =  torch.cuda.max_memory_allocated()
														# make sure that less than 5.2 GB is allocated
														assert mem_bytes < 5.5 * 10**9