class_docstr: CLIP guided stable diffusion based on the amazing repo by @crowsonkb and @Jack000
  - https://github.com/Jack000/glid-3-xl
  - https://github.dev/crowsonkb/k-diffusion
class_docstr_tok: ["CLIP","guided","stable","diffusion","based","on","the","amazing","repo","by","@","crowsonkb","and","@","Jack000","-","https",":","\/\/github.com\/Jack000\/glid-3-xl","-","https",":","\/\/github.dev\/crowsonkb\/k-diffusion"]
code: def __call__(
      self,
      prompt: Union[str, List[str]],
      height: Optional[int] = 512,
      width: Optional[int] = 512,
      image: Union[torch.FloatTensor, PIL.Image.Image] = None,
      strength: float = 0.8,
      num_inference_steps: Optional[int] = 50,
      guidance_scale: Optional[float] = 7.5,
      num_images_per_prompt: Optional[int] = 1,
      eta: float = 0.0,
      clip_guidance_scale: Optional[float] = 100,
      clip_prompt: Optional[Union[str, List[str]]] = None,
      num_cutouts: Optional[int] = 4,
      use_cutouts: Optional[bool] = True,
      generator: Optional[torch.Generator] = None,
      latents: Optional[torch.FloatTensor] = None,
      output_type: Optional[str] = "pil",
      return_dict: bool = True,
  ):
      if isinstance(prompt, str):
          batch_size = 1
      elif isinstance(prompt, list):
          batch_size = len(prompt)
      else:
          raise ValueError(
              f"`prompt` has to be of type `str` or `list` but is {type(prompt)}"
          )
  
      if height % 8 != 0 or width % 8 != 0:
          raise ValueError(
              f"`height` and `width` have to be divisible by 8 but are {height} and {width}."
          )
  
      # get prompt text embeddings
      text_input = self.tokenizer(
          prompt,
          padding="max_length",
          max_length=self.tokenizer.model_max_length,
          truncation=True,
          return_tensors="pt",
      )
      text_embeddings = self.text_encoder(
          text_input.input_ids.to(self.device)
      )[0]
      # duplicate text embeddings for each generation per prompt
      text_embeddings = text_embeddings.repeat_interleave(
          num_images_per_prompt, dim=0
      )
  
      # set timesteps
      accepts_offset = "offset" in set(
          inspect.signature(
              self.scheduler.set_timesteps
          ).parameters.keys()
      )
      extra_set_kwargs = {}
      if accepts_offset:
          extra_set_kwargs["offset"] = 1
  
      self.scheduler.set_timesteps(
          num_inference_steps, **extra_set_kwargs
      )
      # Some schedulers like PNDM have timesteps as arrays
      # It's more optimized to move all timesteps to correct device beforehand
      self.scheduler.timesteps.to(self.device)
  
      timesteps, num_inference_steps = self.get_timesteps(
          num_inference_steps, strength, self.device
      )
      latent_timestep = timesteps[:1].repeat(
          batch_size * num_images_per_prompt
      )
  
      # Preprocess image
      image = preprocess(image, width, height)
      latents = self.prepare_latents(
          image,
          latent_timestep,
          batch_size,
          num_images_per_prompt,
          text_embeddings.dtype,
          self.device,
          generator,
      )
  
      if clip_guidance_scale > 0:
          if clip_prompt is not None:
              clip_text_input = self.tokenizer(
                  clip_prompt,
                  padding="max_length",
                  max_length=self.tokenizer.model_max_length,
                  truncation=True,
                  return_tensors="pt",
              ).input_ids.to(self.device)
          else:
              clip_text_input = text_input.input_ids.to(self.device)
          text_embeddings_clip = self.clip_model.get_text_features(
              clip_text_input
          )
          text_embeddings_clip = (
              text_embeddings_clip
              / text_embeddings_clip.norm(p=2, dim=-1, keepdim=True)
          )
          # duplicate text embeddings clip for each generation per prompt
          text_embeddings_clip = text_embeddings_clip.repeat_interleave(
              num_images_per_prompt, dim=0
          )
  
      # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
      # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
      # corresponds to doing no classifier free guidance.
      do_classifier_free_guidance = guidance_scale > 1.0
      # get unconditional embeddings for classifier free guidance
      if do_classifier_free_guidance:
          max_length = text_input.input_ids.shape[-1]
          uncond_input = self.tokenizer(
              [""],
              padding="max_length",
              max_length=max_length,
              return_tensors="pt",
          )
          uncond_embeddings = self.text_encoder(
              uncond_input.input_ids.to(self.device)
          )[0]
          # duplicate unconditional embeddings for each generation per prompt
          uncond_embeddings = uncond_embeddings.repeat_interleave(
              num_images_per_prompt, dim=0
          )
  
          # For classifier free guidance, we need to do two forward passes.
          # Here we concatenate the unconditional and text embeddings into a single batch
          # to avoid doing two forward passes
          text_embeddings = torch.cat(
              [uncond_embeddings, text_embeddings]
          )
  
      # get the initial random noise unless the user supplied it
  
      # Unlike in other pipelines, latents need to be generated in the target device
      # for 1-to-1 results reproducibility with the CompVis implementation.
      # However this currently doesn't work in `mps`.
      latents_shape = (
          batch_size * num_images_per_prompt,
          self.unet.config.in_channels,
          height // 8,
          width // 8,
      )
      latents_dtype = text_embeddings.dtype
      if latents is None:
          if self.device.type == "mps":
              # randn does not work reproducibly on mps
              latents = torch.randn(
                  latents_shape,
                  generator=generator,
                  device="cpu",
                  dtype=latents_dtype,
              ).to(self.device)
          else:
              latents = torch.randn(
                  latents_shape,
                  generator=generator,
                  device=self.device,
                  dtype=latents_dtype,
              )
      else:
          if latents.shape != latents_shape:
              raise ValueError(
                  f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}"
              )
          latents = latents.to(self.device)
  
      # scale the initial noise by the standard deviation required by the scheduler
      latents = latents * self.scheduler.init_noise_sigma
  
      # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
      # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
      # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
      # and should be between [0, 1]
      accepts_eta = "eta" in set(
          inspect.signature(self.scheduler.step).parameters.keys()
      )
      extra_step_kwargs = {}
      if accepts_eta:
          extra_step_kwargs["eta"] = eta
  
      # check if the scheduler accepts generator
      accepts_generator = "generator" in set(
          inspect.signature(self.scheduler.step).parameters.keys()
      )
      if accepts_generator:
          extra_step_kwargs["generator"] = generator
  
      with self.progress_bar(total=num_inference_steps):
          for i, t in enumerate(timesteps):
              # expand the latents if we are doing classifier free guidance
              latent_model_input = (
                  torch.cat([latents] * 2)
                  if do_classifier_free_guidance
                  else latents
              )
              latent_model_input = self.scheduler.scale_model_input(
                  latent_model_input, t
              )
  
              # predict the noise residual
              noise_pred = self.unet(
                  latent_model_input,
                  t,
                  encoder_hidden_states=text_embeddings,
              ).sample
  
              # perform classifier free guidance
              if do_classifier_free_guidance:
                  noise_pred_uncond, noise_pred_text = noise_pred.chunk(
                      2
                  )
                  noise_pred = noise_pred_uncond + guidance_scale * (
                      noise_pred_text - noise_pred_uncond
                  )
  
              # perform clip guidance
              if clip_guidance_scale > 0:
                  text_embeddings_for_guidance = (
                      text_embeddings.chunk(2)[1]
                      if do_classifier_free_guidance
                      else text_embeddings
                  )
                  noise_pred, latents = self.cond_fn(
                      latents,
                      t,
                      i,
                      text_embeddings_for_guidance,
                      noise_pred,
                      text_embeddings_clip,
                      clip_guidance_scale,
                      num_cutouts,
                      use_cutouts,
                  )
  
              # compute the previous noisy sample x_t -> x_t-1
              latents = self.scheduler.step(
                  noise_pred, t, latents, **extra_step_kwargs
              ).prev_sample
  
      # scale and decode the image latents with vae
      latents = 1 / self.vae.config.scaling_factor * latents
      image = self.vae.decode(latents).sample
  
      image = (image / 2 + 0.5).clamp(0, 1)
      image = image.cpu().permute(0, 2, 3, 1).numpy()
  
      if output_type == "pil":
          image = self.numpy_to_pil(image)
  
      if not return_dict:
          return (image, None)
  
      return StableDiffusionPipelineOutput(
          images=image, nsfw_content_detected=None
      )
  
imports: import inspect
  from typing import List, Optional, Union
  import numpy
  import PIL.Image
  import torch
  from torch import nn
  from torch.nn import functional
  from torchvision import transforms
  from transformers import CLIPFeatureExtractor, CLIPModel, CLIPTextModel, CLIPTokenizer
  from diffusers import AutoencoderKL, DDIMScheduler, DiffusionPipeline, DPMSolverMultistepScheduler, LMSDiscreteScheduler, PNDMScheduler, UNet2DConditionModel
  from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import StableDiffusionPipelineOutput
  from diffusers.utils import PIL_INTERPOLATION, deprecate
  from diffusers.utils.torch_utils import randn_tensor
label_desc: Use image node_id 11 for calling the CLIPGuidedStableDiffusion obj's underlying member method code with example usage: obj.__call__(prompt, height, width, image, strength, num_inference_steps, guidance_scale, num_images_per_prompt, eta, clip_guidance_scale, clip_prompt, num_cutouts, use_cutouts, generator, latents, output_type, return_dict) and returns: StableDiffusionPipelineOutput, image,  None