diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..c734b1cfbab72d854464e6d8c45684446b9b35f2 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,18 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
*.zip filter=lfs diff=lfs merge=lfs -text
*.zst filter=lfs diff=lfs merge=lfs -text
*tfevents* filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_053639/output.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_055109/output.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_173140/output.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_181436/input.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_181633/input.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_214810/output.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_214908/output.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_215043/output.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_221300/output.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_222257/output.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_222442/output.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_222522/output.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_223634/output.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241126_223719/output.png filter=lfs diff=lfs merge=lfs -text
+saved_results/20241127_025429/output.png filter=lfs diff=lfs merge=lfs -text
diff --git a/app.py b/app.py
new file mode 100644
index 0000000000000000000000000000000000000000..029a67a1e2d6e9df4fe9d6697bc5b7ad5c30bfcf
--- /dev/null
+++ b/app.py
@@ -0,0 +1,427 @@
+import spaces
+import gradio as gr
+import torch
+from diffusers import StableDiffusionInpaintPipeline, StableDiffusionImg2ImgPipeline
+from PIL import Image
+import random
+import numpy as np
+import torch
+import os
+import json
+from datetime import datetime
+
+from fluxcombined import FluxPipeline
+from scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler
+
+# Load the Stable Diffusion Inpainting model
+scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained("black-forest-labs/FLUX.1-dev", subfolder="scheduler")
+pipe = FluxPipeline.from_pretrained("black-forest-labs/FLUX.1-dev", torch_dtype=torch.float16, scheduler=scheduler)
+pipe.to("cuda") # Comment this line if GPU is not available
+
+# Function to process the image
+@spaces.GPU(duration=120)
+def process_image(
+ mode, image_layers, prompt, edit_prompt, seed, randomize_seed, num_inference_steps,
+ max_steps, learning_rate, max_source_steps, optimization_steps, true_cfg, mask_input
+):
+ image_with_mask = {
+ "image": image_layers["background"],
+ "mask": image_layers["layers"][0] if mask_input is None else mask_input
+ }
+
+ # Set seed
+ if randomize_seed or seed is None:
+ seed = random.randint(0, 2**32 - 1)
+ generator = torch.Generator("cuda").manual_seed(int(seed))
+
+ # Unpack image and mask
+ if image_with_mask is None:
+ return None, f"❌ Please upload an image and create a mask."
+ image = image_with_mask["image"]
+ mask = image_with_mask["mask"]
+
+ if image is None or mask is None:
+ return None, f"❌ Please ensure both image and mask are provided."
+
+ # Convert images to RGB
+ image = image.convert("RGB")
+ mask = mask.split()[-1] # Convert mask to grayscale
+
+ if mode == "Inpainting":
+ if not prompt:
+ return None, f"❌ Please provide a prompt for inpainting."
+ with torch.autocast("cuda"):
+ # Placeholder for using advanced parameters in the future
+ # Adjust parameters according to advanced settings if applicable
+ result = pipe.inpaint(
+ prompt=prompt,
+ input_image=image.resize((1024, 1024)),
+ mask_image=mask.resize((1024, 1024)),
+ num_inference_steps=num_inference_steps,
+ guidance_scale=0.5,
+ generator=generator,
+ save_masked_image=False,
+ output_path="",
+ learning_rate=learning_rate,
+ max_steps=max_steps
+ ).images[0]
+ pipe.vae = pipe.vae.to(torch.float16)
+ return result, f"✅ Inpainting completed with seed {seed}."
+ elif mode == "Editing":
+ if not edit_prompt:
+ return None, f"❌ Please provide a prompt for editing."
+ if not prompt:
+ prompt = ""
+ # Resize the mask to match the image
+ # mask = mask.resize(image.size)
+ with torch.autocast("cuda"):
+ # Placeholder for using advanced parameters in the future
+ # Adjust parameters according to advanced settings if applicable
+ result = pipe.edit2(
+ prompt=edit_prompt,
+ input_image=image.resize((1024, 1024)),
+ mask_image=mask.resize((1024, 1024)),
+ num_inference_steps=num_inference_steps,
+ guidance_scale=0.0,
+ generator=generator,
+ save_masked_image=False,
+ output_path="",
+ learning_rate=learning_rate,
+ max_steps=max_steps,
+ optimization_steps=optimization_steps,
+ true_cfg=true_cfg,
+ negative_prompt=prompt,
+ source_steps=max_source_steps,
+ ).images[0]
+ return result, f"✅ Editing completed with seed {seed}."
+ else:
+ return None, f"❌ Invalid mode selected."
+
+# Design the Gradio interface
+with gr.Blocks() as demo:
+ gr.Markdown(
+ """
+
+ """
+ )
+ gr.Markdown("🍲 FlowChef 🍲
")
+ gr.Markdown("Inversion/Gradient/Training-free Steering of Flux.1[Dev]
")
+ gr.Markdown("Project Page | Paper
(Steering Rectified Flow Models in the Vector Field for Controlled Image Generation)")
+ gr.Markdown("💡 We recommend going through our tutorial introduction before getting started!
")
+
+ # Store current state
+ current_input_image = None
+ current_mask = None
+ current_output_image = None
+ current_params = {}
+
+ # Images at the top
+ with gr.Row():
+ with gr.Column():
+ image_input = gr.ImageMask(
+ # source="upload",
+ # tool="sketch",
+ type="pil",
+ label="Input Image and Mask",
+ image_mode="RGBA",
+ height=512,
+ width=512,
+ )
+ with gr.Column():
+ output_image = gr.Image(label="Output Image")
+
+ # All options below
+ with gr.Column():
+ mode = gr.Radio(
+ choices=["Inpainting", "Editing"], label="Select Mode", value="Inpainting"
+ )
+ prompt = gr.Textbox(
+ label="Prompt",
+ placeholder="Describe what should appear in the masked area..."
+ )
+ edit_prompt = gr.Textbox(
+ label="Editing Prompt",
+ placeholder="Describe how you want to edit the image..."
+ )
+ with gr.Row():
+ seed = gr.Number(label="Seed (Optional)", value=None)
+ randomize_seed = gr.Checkbox(label="Randomize Seed", value=True)
+ num_inference_steps = gr.Slider(
+ label="Inference Steps", minimum=1, maximum=50, value=30
+ )
+ # Advanced settings in an accordion
+ with gr.Accordion("Advanced Settings", open=False):
+ max_steps = gr.Slider(label="Max Steps", minimum=1, maximum=30, value=30)
+ learning_rate = gr.Slider(label="Learning Rate", minimum=0.1, maximum=1.0, value=0.5)
+ true_cfg = gr.Slider(label="Guidance Scale (only for editing)", minimum=1, maximum=20, value=2)
+ max_source_steps = gr.Slider(label="Max Source Steps (only for editing)", minimum=1, maximum=30, value=20)
+ optimization_steps = gr.Slider(label="Optimization Steps", minimum=1, maximum=10, value=1)
+ mask_input = gr.Image(
+ type="pil",
+ label="Optional Mask",
+ image_mode="RGBA",
+ )
+ with gr.Row():
+ run_button = gr.Button("Run", variant="primary")
+ save_button = gr.Button("Save Data", variant="secondary")
+
+ def update_visibility(selected_mode):
+ if selected_mode == "Inpainting":
+ return gr.update(visible=True), gr.update(visible=False)
+ else:
+ return gr.update(visible=True), gr.update(visible=True)
+
+ mode.change(
+ update_visibility,
+ inputs=mode,
+ outputs=[prompt, edit_prompt],
+ )
+
+ def run_and_update_status(
+ mode, image_with_mask, prompt, edit_prompt, seed, randomize_seed, num_inference_steps,
+ max_steps, learning_rate, max_source_steps, optimization_steps, true_cfg, mask_input
+ ):
+ result_image, result_status = process_image(
+ mode, image_with_mask, prompt, edit_prompt, seed, randomize_seed, num_inference_steps,
+ max_steps, learning_rate, max_source_steps, optimization_steps, true_cfg, mask_input
+ )
+
+ # Store current state
+ global current_input_image, current_mask, current_output_image, current_params
+
+ current_input_image = image_with_mask["background"] if image_with_mask else None
+ current_mask = mask_input if mask_input is not None else (image_with_mask["layers"][0] if image_with_mask else None)
+ current_output_image = result_image
+ current_params = {
+ "mode": mode,
+ "prompt": prompt,
+ "edit_prompt": edit_prompt,
+ "seed": seed,
+ "randomize_seed": randomize_seed,
+ "num_inference_steps": num_inference_steps,
+ "max_steps": max_steps,
+ "learning_rate": learning_rate,
+ "max_source_steps": max_source_steps,
+ "optimization_steps": optimization_steps,
+ "true_cfg": true_cfg
+ }
+
+ return result_image
+
+ def save_data():
+ if not os.path.exists("saved_results"):
+ os.makedirs("saved_results")
+
+ timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+ save_dir = os.path.join("saved_results", timestamp)
+ os.makedirs(save_dir)
+
+ # Save images
+ if current_input_image:
+ current_input_image.save(os.path.join(save_dir, "input.png"))
+ if current_mask:
+ current_mask.save(os.path.join(save_dir, "mask.png"))
+ if current_output_image:
+ current_output_image.save(os.path.join(save_dir, "output.png"))
+
+ # Save parameters
+ with open(os.path.join(save_dir, "parameters.json"), "w") as f:
+ json.dump(current_params, f, indent=4)
+
+ return f"✅ Data saved in {save_dir}"
+
+ run_button.click(
+ fn=run_and_update_status,
+ inputs=[
+ mode,
+ image_input,
+ prompt,
+ edit_prompt,
+ seed,
+ randomize_seed,
+ num_inference_steps,
+ max_steps,
+ learning_rate,
+ max_source_steps,
+ optimization_steps,
+ true_cfg,
+ mask_input
+ ],
+ outputs=output_image,
+ )
+
+ save_button.click(fn=save_data)
+
+ gr.Markdown(
+ ""
+ )
+
+ def load_example_image_with_mask(image_path):
+ # Load the image
+ image = Image.open(image_path)
+ # Create an empty mask of the same size
+ mask = Image.new('L', image.size, 0)
+ return {"background": image, "layers": [mask], "composite": image}
+
+ examples_dir = "assets"
+ volcano_dict = load_example_image_with_mask(os.path.join(examples_dir, "vulcano.jpg"))
+ dog_dict = load_example_image_with_mask(os.path.join(examples_dir, "dog.webp"))
+
+ gr.Examples(
+ examples=[
+ [
+ "Inpainting", # mode
+ "./saved_results/20241126_053639/input.png", # image with mask
+ "./saved_results/20241126_053639/mask.png",
+ "./saved_results/20241126_053639/output.png",
+ "a dog", # prompt
+ " ", # edit_prompt
+ 0, # seed
+ True, # randomize_seed
+ 30, # num_inference_steps
+ 30, # max_steps
+ 1.0, # learning_rate
+ 20, # max_source_steps
+ 10, # optimization_steps
+ 2, # true_cfg
+ ],
+ [
+ "Inpainting", # mode
+ "./saved_results/20241126_173140/input.png", # image with mask
+ "./saved_results/20241126_173140/mask.png",
+ "./saved_results/20241126_173140/output.png",
+ "a cat with blue eyes", # prompt
+ " ", # edit_prompt
+ 0, # seed
+ True, # randomize_seed
+ 30, # num_inference_steps
+ 20, # max_steps
+ 1.0, # learning_rate
+ 20, # max_source_steps
+ 10, # optimization_steps
+ 2, # true_cfg
+ ],
+ [
+ "Editing", # mode
+ "./saved_results/20241126_181633/input.png", # image with mask
+ "./saved_results/20241126_181633/mask.png",
+ "./saved_results/20241126_181633/output.png",
+ " ", # prompt
+ "volcano eruption", # edit_prompt
+ 0, # seed
+ True, # randomize_seed
+ 30, # num_inference_steps
+ 20, # max_steps
+ 0.5, # learning_rate
+ 2, # max_source_steps
+ 3, # optimization_steps
+ 4.5, # true_cfg
+ ],
+ [
+ "Editing", # mode
+ "./saved_results/20241126_214810/input.png", # image with mask
+ "./saved_results/20241126_214810/mask.png",
+ "./saved_results/20241126_214810/output.png",
+ " ", # prompt
+ "a dog with flowers in the mouth", # edit_prompt
+ 0, # seed
+ True, # randomize_seed
+ 30, # num_inference_steps
+ 30, # max_steps
+ 1, # learning_rate
+ 5, # max_source_steps
+ 3, # optimization_steps
+ 4.5, # true_cfg
+ ],
+ [
+ "Inpainting", # mode
+ "./saved_results/20241127_025429/input.png", # image with mask
+ "./saved_results/20241127_025429/mask.png",
+ "./saved_results/20241127_025429/output.png",
+ "A building with \"ASU\" written on it.", # prompt
+ "", # edit_prompt
+ 52, # seed
+ False, # randomize_seed
+ 30, # num_inference_steps
+ 30, # max_steps
+ 1, # learning_rate
+ 20, # max_source_steps
+ 10, # optimization_steps
+ 2, # true_cfg
+ ],
+ [
+ "Inpainting", # mode
+ "./saved_results/20241126_222257/input.png", # image with mask
+ "./saved_results/20241126_222257/mask.png",
+ "./saved_results/20241126_222257/output.png",
+ "A cute pig with big eyes", # prompt
+ "", # edit_prompt
+ 0, # seed
+ True, # randomize_seed
+ 30, # num_inference_steps
+ 20, # max_steps
+ 1, # learning_rate
+ 20, # max_source_steps
+ 5, # optimization_steps
+ 2, # true_cfg
+ ],
+ [
+ "Editing", # mode
+ "./saved_results/20241126_222522/input.png", # image with mask
+ "./saved_results/20241126_222522/mask.png",
+ "./saved_results/20241126_222522/output.png",
+ "A cute rabbit with big eyes", # prompt
+ "A cute pig with big eyes", # edit_prompt
+ 0, # seed
+ True, # randomize_seed
+ 30, # num_inference_steps
+ 20, # max_steps
+ 0.4, # learning_rate
+ 5, # max_source_steps
+ 5, # optimization_steps
+ 4.5, # true_cfg
+ ],
+ [
+ "Editing", # mode
+ "./saved_results/20241126_223719/input.png", # image with mask
+ "./saved_results/20241126_223719/mask.png",
+ "./saved_results/20241126_223719/output.png",
+ "a cat", # prompt
+ "a tiger", # edit_prompt
+ 0, # seed
+ True, # randomize_seed
+ 30, # num_inference_steps
+ 30, # max_steps
+ 0.6, # learning_rate
+ 10, # max_source_steps
+ 5, # optimization_steps
+ 4.5, # true_cfg
+ ],
+ ],
+ inputs=[
+ mode,
+ image_input,
+ mask_input,
+ output_image,
+ prompt,
+ edit_prompt,
+ seed,
+ randomize_seed,
+ num_inference_steps,
+ max_steps,
+ learning_rate,
+ max_source_steps,
+ optimization_steps,
+ true_cfg,
+ ],
+ # outputs=[output_image],
+ # fn=run_and_update_status,
+ # cache_examples=True,
+ )
+demo.launch()
diff --git a/assets/dog.webp b/assets/dog.webp
new file mode 100644
index 0000000000000000000000000000000000000000..a6a06d82b3005294c420d83f59f3a8c1bf9ec13a
Binary files /dev/null and b/assets/dog.webp differ
diff --git a/assets/vulcano.jpg b/assets/vulcano.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..2433ee61e0a507a77cf61595f6c4d15bffc9f543
Binary files /dev/null and b/assets/vulcano.jpg differ
diff --git a/assets/vulcano_mask.webp b/assets/vulcano_mask.webp
new file mode 100644
index 0000000000000000000000000000000000000000..3293b6c76c6a3fbe05fd0b70031d19a2059f9079
Binary files /dev/null and b/assets/vulcano_mask.webp differ
diff --git a/fluxcombined.py b/fluxcombined.py
new file mode 100644
index 0000000000000000000000000000000000000000..50ad5b16965b5bb63c506635688b6e27d1bfbc3a
--- /dev/null
+++ b/fluxcombined.py
@@ -0,0 +1,1607 @@
+# Copyright 2024 Black Forest Labs and The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import inspect
+from typing import Any, Callable, Dict, List, Optional, Union
+
+import numpy as np
+import torch
+from transformers import CLIPTextModel, CLIPTokenizer, T5EncoderModel, T5TokenizerFast
+
+from diffusers.image_processor import VaeImageProcessor
+from diffusers.loaders import FluxLoraLoaderMixin, FromSingleFileMixin
+from diffusers.models.autoencoders import AutoencoderKL
+from diffusers.models.transformers import FluxTransformer2DModel
+from scheduling_flow_match_euler_discrete import FlowMatchEulerDiscreteScheduler
+from diffusers.utils import (
+ USE_PEFT_BACKEND,
+ is_torch_xla_available,
+ logging,
+ replace_example_docstring,
+ scale_lora_layers,
+ unscale_lora_layers,
+)
+from diffusers.utils.torch_utils import randn_tensor
+from diffusers.pipelines.pipeline_utils import DiffusionPipeline
+from diffusers.pipelines.flux.pipeline_output import FluxPipelineOutput
+
+import os
+import torch
+import torch.nn as nn
+from os.path import expanduser # pylint: disable=import-outside-toplevel
+from urllib.request import urlretrieve # pylint: disable=import-outside-toplevel
+from torchvision import transforms as TF
+
+if is_torch_xla_available():
+ import torch_xla.core.xla_model as xm
+
+ XLA_AVAILABLE = True
+else:
+ XLA_AVAILABLE = False
+
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+EXAMPLE_DOC_STRING = """
+ Examples:
+ ```py
+ >>> import torch
+ >>> from diffusers import FluxPipeline
+
+ >>> pipe = FluxPipeline.from_pretrained("black-forest-labs/FLUX.1-schnell", torch_dtype=torch.bfloat16)
+ >>> pipe.to("cuda")
+ >>> prompt = "A cat holding a sign that says hello world"
+ >>> # Depending on the variant being used, the pipeline call will slightly vary.
+ >>> # Refer to the pipeline documentation for more details.
+ >>> image = pipe(prompt, num_inference_steps=4, guidance_scale=0.0).images[0]
+ >>> image.save("flux.png")
+ ```
+"""
+
+import sys
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..')))
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+def retrieve_latents(
+ encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample"
+):
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
+ return encoder_output.latent_dist.sample(generator)
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
+ return encoder_output.latent_dist.mode()
+ elif hasattr(encoder_output, "latents"):
+ return encoder_output.latents
+ else:
+ raise AttributeError("Could not access latents of provided encoder_output")
+
+
+def calculate_shift(
+ image_seq_len,
+ base_seq_len: int = 256,
+ max_seq_len: int = 4096,
+ base_shift: float = 0.5,
+ max_shift: float = 1.16,
+):
+ m = (max_shift - base_shift) / (max_seq_len - base_seq_len)
+ b = base_shift - m * base_seq_len
+ mu = image_seq_len * m + b
+ return mu
+
+
+# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps
+def retrieve_timesteps(
+ scheduler,
+ num_inference_steps: Optional[int] = None,
+ device: Optional[Union[str, torch.device]] = None,
+ timesteps: Optional[List[int]] = None,
+ sigmas: Optional[List[float]] = None,
+ **kwargs,
+):
+ """
+ Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles
+ custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`.
+
+ Args:
+ scheduler (`SchedulerMixin`):
+ The scheduler to get timesteps from.
+ num_inference_steps (`int`):
+ The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps`
+ must be `None`.
+ device (`str` or `torch.device`, *optional*):
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
+ timesteps (`List[int]`, *optional*):
+ Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed,
+ `num_inference_steps` and `sigmas` must be `None`.
+ sigmas (`List[float]`, *optional*):
+ Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed,
+ `num_inference_steps` and `timesteps` must be `None`.
+
+ Returns:
+ `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the
+ second element is the number of inference steps.
+ """
+ if timesteps is not None and sigmas is not None:
+ raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values")
+ if timesteps is not None:
+ accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
+ if not accepts_timesteps:
+ raise ValueError(
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
+ f" timestep schedules. Please check whether you are using the correct scheduler."
+ )
+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)
+ timesteps = scheduler.timesteps
+ num_inference_steps = len(timesteps)
+ elif sigmas is not None:
+ accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys())
+ if not accept_sigmas:
+ raise ValueError(
+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"
+ f" sigmas schedules. Please check whether you are using the correct scheduler."
+ )
+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)
+ timesteps = scheduler.timesteps
+ num_inference_steps = len(timesteps)
+ else:
+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)
+ timesteps = scheduler.timesteps
+ return timesteps, num_inference_steps
+
+
+class FluxPipeline(DiffusionPipeline, FluxLoraLoaderMixin):
+ r"""
+ The Flux pipeline for text-to-image generation.
+
+ Reference: https://blackforestlabs.ai/announcing-black-forest-labs/
+
+ Args:
+ transformer ([`FluxTransformer2DModel`]):
+ Conditional Transformer (MMDiT) architecture to denoise the encoded image latents.
+ scheduler ([`FlowMatchEulerDiscreteScheduler`]):
+ A scheduler to be used in combination with `transformer` to denoise the encoded image latents.
+ vae ([`AutoencoderKL`]):
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations.
+ text_encoder ([`CLIPTextModel`]):
+ [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically
+ the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant.
+ text_encoder_2 ([`T5EncoderModel`]):
+ [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically
+ the [google/t5-v1_1-xxl](https://huggingface.co/google/t5-v1_1-xxl) variant.
+ tokenizer (`CLIPTokenizer`):
+ Tokenizer of class
+ [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer).
+ tokenizer_2 (`T5TokenizerFast`):
+ Second Tokenizer of class
+ [T5TokenizerFast](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5TokenizerFast).
+ """
+
+ model_cpu_offload_seq = "text_encoder->text_encoder_2->transformer->vae"
+ _optional_components = []
+ _callback_tensor_inputs = ["latents", "prompt_embeds"]
+
+ def __init__(
+ self,
+ scheduler: FlowMatchEulerDiscreteScheduler,
+ vae: AutoencoderKL,
+ text_encoder: CLIPTextModel,
+ tokenizer: CLIPTokenizer,
+ text_encoder_2: T5EncoderModel,
+ tokenizer_2: T5TokenizerFast,
+ transformer: FluxTransformer2DModel,
+ ):
+ super().__init__()
+
+ self.register_modules(
+ vae=vae,
+ text_encoder=text_encoder,
+ text_encoder_2=text_encoder_2,
+ tokenizer=tokenizer,
+ tokenizer_2=tokenizer_2,
+ transformer=transformer,
+ scheduler=scheduler,
+ )
+ self.vae_scale_factor = (
+ 2 ** (len(self.vae.config.block_out_channels)) if hasattr(self, "vae") and self.vae is not None else 16
+ )
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
+ self.tokenizer_max_length = (
+ self.tokenizer.model_max_length if hasattr(self, "tokenizer") and self.tokenizer is not None else 77
+ )
+ self.default_sample_size = 64
+
+ def _get_t5_prompt_embeds(
+ self,
+ prompt: Union[str, List[str]] = None,
+ num_images_per_prompt: int = 1,
+ max_sequence_length: int = 512,
+ device: Optional[torch.device] = None,
+ dtype: Optional[torch.dtype] = None,
+ ):
+ device = device or self._execution_device
+ dtype = dtype or self.text_encoder.dtype
+
+ prompt = [prompt] if isinstance(prompt, str) else prompt
+ batch_size = len(prompt)
+
+ text_inputs = self.tokenizer_2(
+ prompt,
+ padding="max_length",
+ max_length=max_sequence_length,
+ truncation=True,
+ return_length=False,
+ return_overflowing_tokens=False,
+ return_tensors="pt",
+ )
+ text_input_ids = text_inputs.input_ids
+ untruncated_ids = self.tokenizer_2(prompt, padding="longest", return_tensors="pt").input_ids
+
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
+ removed_text = self.tokenizer_2.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
+ logger.warning(
+ "The following part of your input was truncated because `max_sequence_length` is set to "
+ f" {max_sequence_length} tokens: {removed_text}"
+ )
+
+ prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0]
+
+ dtype = self.text_encoder_2.dtype
+ prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
+
+ _, seq_len, _ = prompt_embeds.shape
+
+ # duplicate text embeddings and attention mask for each generation per prompt, using mps friendly method
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
+ prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
+
+ return prompt_embeds
+
+ def _get_clip_prompt_embeds(
+ self,
+ prompt: Union[str, List[str]],
+ num_images_per_prompt: int = 1,
+ device: Optional[torch.device] = None,
+ ):
+ device = device or self._execution_device
+
+ prompt = [prompt] if isinstance(prompt, str) else prompt
+ batch_size = len(prompt)
+
+ text_inputs = self.tokenizer(
+ prompt,
+ padding="max_length",
+ max_length=self.tokenizer_max_length,
+ truncation=True,
+ return_overflowing_tokens=False,
+ return_length=False,
+ return_tensors="pt",
+ )
+
+ text_input_ids = text_inputs.input_ids
+ untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids):
+ removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1])
+ logger.warning(
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
+ f" {self.tokenizer_max_length} tokens: {removed_text}"
+ )
+ prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False)
+
+ # Use pooled output of CLIPTextModel
+ prompt_embeds = prompt_embeds.pooler_output
+ prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device)
+
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt)
+ prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, -1)
+
+ return prompt_embeds
+
+ def encode_prompt(
+ self,
+ prompt: Union[str, List[str]],
+ prompt_2: Union[str, List[str]],
+ device: Optional[torch.device] = None,
+ num_images_per_prompt: int = 1,
+ prompt_embeds: Optional[torch.FloatTensor] = None,
+ pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+ max_sequence_length: int = 512,
+ lora_scale: Optional[float] = None,
+ ):
+ r"""
+
+ Args:
+ prompt (`str` or `List[str]`, *optional*):
+ prompt to be encoded
+ prompt_2 (`str` or `List[str]`, *optional*):
+ The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
+ used in all text-encoders
+ device: (`torch.device`):
+ torch device
+ num_images_per_prompt (`int`):
+ number of images that should be generated per prompt
+ prompt_embeds (`torch.FloatTensor`, *optional*):
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
+ provided, text embeddings will be generated from `prompt` input argument.
+ pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
+ lora_scale (`float`, *optional*):
+ A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded.
+ """
+ device = device or self._execution_device
+
+ # set lora scale so that monkey patched LoRA
+ # function of text encoder can correctly access it
+ if lora_scale is not None and isinstance(self, FluxLoraLoaderMixin):
+ self._lora_scale = lora_scale
+
+ # dynamically adjust the LoRA scale
+ if self.text_encoder is not None and USE_PEFT_BACKEND:
+ scale_lora_layers(self.text_encoder, lora_scale)
+ if self.text_encoder_2 is not None and USE_PEFT_BACKEND:
+ scale_lora_layers(self.text_encoder_2, lora_scale)
+
+ prompt = [prompt] if isinstance(prompt, str) else prompt
+ if prompt is not None:
+ batch_size = len(prompt)
+ else:
+ batch_size = prompt_embeds.shape[0]
+
+ if prompt_embeds is None:
+ prompt_2 = prompt_2 or prompt
+ prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
+
+ # We only use the pooled prompt output from the CLIPTextModel
+ pooled_prompt_embeds = self._get_clip_prompt_embeds(
+ prompt=prompt,
+ device=device,
+ num_images_per_prompt=num_images_per_prompt,
+ )
+ prompt_embeds = self._get_t5_prompt_embeds(
+ prompt=prompt_2,
+ num_images_per_prompt=num_images_per_prompt,
+ max_sequence_length=max_sequence_length,
+ device=device,
+ )
+
+ if self.text_encoder is not None:
+ if isinstance(self, FluxLoraLoaderMixin) and USE_PEFT_BACKEND:
+ # Retrieve the original scale by scaling back the LoRA layers
+ unscale_lora_layers(self.text_encoder, lora_scale)
+
+ if self.text_encoder_2 is not None:
+ if isinstance(self, FluxLoraLoaderMixin) and USE_PEFT_BACKEND:
+ # Retrieve the original scale by scaling back the LoRA layers
+ unscale_lora_layers(self.text_encoder_2, lora_scale)
+
+ dtype = self.text_encoder.dtype if self.text_encoder is not None else self.transformer.dtype
+ text_ids = torch.zeros(prompt_embeds.shape[1], 3).to(device=device, dtype=dtype)
+ # text_ids = text_ids.repeat(num_images_per_prompt, 1, 1)
+
+ return prompt_embeds, pooled_prompt_embeds, text_ids
+
+ def encode_prompt_edit(
+ self,
+ prompt: Union[str, List[str]],
+ prompt_2: Union[str, List[str]],
+ negative_prompt: Union[str, List[str]] = None,
+ negative_prompt_2: Union[str, List[str]] = None,
+ device: Optional[torch.device] = None,
+ num_images_per_prompt: int = 1,
+ prompt_embeds: Optional[torch.FloatTensor] = None,
+ pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+ negative_prompt_embeds: Optional[torch.FloatTensor] = None,
+ negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+ max_sequence_length: int = 512,
+ lora_scale: Optional[float] = None,
+ do_true_cfg: bool = False,
+ ):
+ device = device or self._execution_device
+
+ # Set LoRA scale if applicable
+ if lora_scale is not None and isinstance(self, FluxLoraLoaderMixin):
+ self._lora_scale = lora_scale
+
+ if self.text_encoder is not None and USE_PEFT_BACKEND:
+ scale_lora_layers(self.text_encoder, lora_scale)
+ if self.text_encoder_2 is not None and USE_PEFT_BACKEND:
+ scale_lora_layers(self.text_encoder_2, lora_scale)
+
+ prompt = [prompt] if isinstance(prompt, str) else prompt
+ batch_size = len(prompt)
+
+ if do_true_cfg and negative_prompt is not None:
+ negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt
+ negative_batch_size = len(negative_prompt)
+
+ if negative_batch_size != batch_size:
+ raise ValueError(
+ f"Negative prompt batch size ({negative_batch_size}) does not match prompt batch size ({batch_size})"
+ )
+
+ # Concatenate prompts
+ prompts = prompt + negative_prompt
+ prompts_2 = (
+ prompt_2 + negative_prompt_2 if prompt_2 is not None and negative_prompt_2 is not None else None
+ )
+ else:
+ prompts = prompt
+ prompts_2 = prompt_2
+
+ if prompt_embeds is None:
+ if prompts_2 is None:
+ prompts_2 = prompts
+
+ # Get pooled prompt embeddings from CLIPTextModel
+ pooled_prompt_embeds = self._get_clip_prompt_embeds(
+ prompt=prompts,
+ device=device,
+ num_images_per_prompt=num_images_per_prompt,
+ )
+ prompt_embeds = self._get_t5_prompt_embeds(
+ prompt=prompts_2,
+ num_images_per_prompt=num_images_per_prompt,
+ max_sequence_length=max_sequence_length,
+ device=device,
+ )
+
+ if do_true_cfg and negative_prompt is not None:
+ # Split embeddings back into positive and negative parts
+ total_batch_size = batch_size * num_images_per_prompt
+ positive_indices = slice(0, total_batch_size)
+ negative_indices = slice(total_batch_size, 2 * total_batch_size)
+
+ positive_pooled_prompt_embeds = pooled_prompt_embeds[positive_indices]
+ negative_pooled_prompt_embeds = pooled_prompt_embeds[negative_indices]
+
+ positive_prompt_embeds = prompt_embeds[positive_indices]
+ negative_prompt_embeds = prompt_embeds[negative_indices]
+
+ pooled_prompt_embeds = positive_pooled_prompt_embeds
+ prompt_embeds = positive_prompt_embeds
+
+ # Unscale LoRA layers
+ if self.text_encoder is not None:
+ if isinstance(self, FluxLoraLoaderMixin) and USE_PEFT_BACKEND:
+ unscale_lora_layers(self.text_encoder, lora_scale)
+
+ if self.text_encoder_2 is not None:
+ if isinstance(self, FluxLoraLoaderMixin) and USE_PEFT_BACKEND:
+ unscale_lora_layers(self.text_encoder_2, lora_scale)
+
+ dtype = self.text_encoder.dtype if self.text_encoder is not None else self.transformer.dtype
+ text_ids = torch.zeros(prompt_embeds.shape[1], 3).to(device=device, dtype=dtype)
+
+ if do_true_cfg and negative_prompt is not None:
+ return (
+ prompt_embeds,
+ pooled_prompt_embeds,
+ text_ids,
+ negative_prompt_embeds,
+ negative_pooled_prompt_embeds,
+ )
+ else:
+ return prompt_embeds, pooled_prompt_embeds, text_ids, None, None
+
+
+ def check_inputs(
+ self,
+ prompt,
+ prompt_2,
+ height,
+ width,
+ prompt_embeds=None,
+ pooled_prompt_embeds=None,
+ callback_on_step_end_tensor_inputs=None,
+ max_sequence_length=None,
+ ):
+ if height % 8 != 0 or width % 8 != 0:
+ raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
+
+ if callback_on_step_end_tensor_inputs is not None and not all(
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
+ ):
+ raise ValueError(
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
+ )
+
+ if prompt is not None and prompt_embeds is not None:
+ raise ValueError(
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
+ " only forward one of the two."
+ )
+ elif prompt_2 is not None and prompt_embeds is not None:
+ raise ValueError(
+ f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
+ " only forward one of the two."
+ )
+ elif prompt is None and prompt_embeds is None:
+ raise ValueError(
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
+ )
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
+ elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)):
+ raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}")
+
+ if prompt_embeds is not None and pooled_prompt_embeds is None:
+ raise ValueError(
+ "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`."
+ )
+
+ if max_sequence_length is not None and max_sequence_length > 512:
+ raise ValueError(f"`max_sequence_length` cannot be greater than 512 but is {max_sequence_length}")
+
+ @staticmethod
+ def _prepare_latent_image_ids(batch_size, height, width, device, dtype):
+ latent_image_ids = torch.zeros(height // 2, width // 2, 3)
+ latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height // 2)[:, None]
+ latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width // 2)[None, :]
+
+ latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape
+
+ # latent_image_ids = latent_image_ids[None, :].repeat(batch_size, 1, 1, 1)
+ latent_image_ids = latent_image_ids.reshape(
+ latent_image_id_height * latent_image_id_width, latent_image_id_channels
+ )
+
+ return latent_image_ids.to(device=device, dtype=dtype)
+
+ @staticmethod
+ def _pack_latents(latents, batch_size, num_channels_latents, height, width):
+ latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2)
+ latents = latents.permute(0, 2, 4, 1, 3, 5)
+ latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4)
+
+ return latents
+
+ @staticmethod
+ def _unpack_latents(latents, height, width, vae_scale_factor):
+ batch_size, num_patches, channels = latents.shape
+
+ height = height // vae_scale_factor
+ width = width // vae_scale_factor
+
+ latents = latents.view(batch_size, height, width, channels // 4, 2, 2)
+ latents = latents.permute(0, 3, 1, 4, 2, 5)
+
+ latents = latents.reshape(batch_size, channels // (2 * 2), height * 2, width * 2)
+
+ return latents
+
+ def prepare_latents(
+ self,
+ batch_size,
+ num_channels_latents,
+ height,
+ width,
+ dtype,
+ device,
+ generator,
+ latents=None,
+ ):
+ height = 2 * (int(height) // self.vae_scale_factor)
+ width = 2 * (int(width) // self.vae_scale_factor)
+
+ shape = (batch_size, num_channels_latents, height, width)
+
+ if latents is not None:
+ latent_image_ids = self._prepare_latent_image_ids(batch_size, height, width, device, dtype)
+ return latents.to(device=device, dtype=dtype), latent_image_ids
+
+ if isinstance(generator, list) and len(generator) != batch_size:
+ raise ValueError(
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
+ )
+
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
+ latents = self._pack_latents(latents, batch_size, num_channels_latents, height, width)
+
+ latent_image_ids = self._prepare_latent_image_ids(batch_size, height, width, device, dtype)
+
+ return latents, latent_image_ids
+
+ @property
+ def guidance_scale(self):
+ return self._guidance_scale
+
+ @property
+ def joint_attention_kwargs(self):
+ return self._joint_attention_kwargs
+
+ @property
+ def num_timesteps(self):
+ return self._num_timesteps
+
+ @property
+ def interrupt(self):
+ return self._interrupt
+
+
+ def prepare_mask_latents(
+ self,
+ mask,
+ masked_image,
+ batch_size,
+ num_channels_latents,
+ num_images_per_prompt,
+ height,
+ width,
+ dtype,
+ device,
+ generator,
+ ):
+ height = 2 * (int(height) // self.vae_scale_factor)
+ width = 2 * (int(width) // self.vae_scale_factor)
+ # resize the mask to latents shape as we concatenate the mask to the latents
+ # we do that before converting to dtype to avoid breaking in case we're using cpu_offload
+ # and half precision
+ mask = torch.nn.functional.interpolate(mask, size=(height, width))
+ mask = mask.to(device=device, dtype=dtype)
+
+ batch_size = batch_size * num_images_per_prompt
+
+ masked_image = masked_image.to(device=device, dtype=dtype)
+
+ if masked_image.shape[1] == 16:
+ masked_image_latents = masked_image
+ else:
+ masked_image_latents = retrieve_latents(self.vae.encode(masked_image), generator=generator)
+
+ masked_image_latents = (masked_image_latents - self.vae.config.shift_factor) * self.vae.config.scaling_factor
+
+ # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method
+ if mask.shape[0] < batch_size:
+ if not batch_size % mask.shape[0] == 0:
+ raise ValueError(
+ "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to"
+ f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number"
+ " of masks that you pass is divisible by the total requested batch size."
+ )
+ mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1)
+ if masked_image_latents.shape[0] < batch_size:
+ if not batch_size % masked_image_latents.shape[0] == 0:
+ raise ValueError(
+ "The passed images and the required batch size don't match. Images are supposed to be duplicated"
+ f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed."
+ " Make sure the number of images that you pass is divisible by the total requested batch size."
+ )
+ masked_image_latents = masked_image_latents.repeat(batch_size // masked_image_latents.shape[0], 1, 1, 1)
+
+ # aligning device to prevent device errors when concating it with the latent model input
+ masked_image_latents = masked_image_latents.to(device=device, dtype=dtype)
+
+ masked_image_latents = self._pack_latents(
+ masked_image_latents,
+ batch_size,
+ num_channels_latents,
+ height,
+ width,
+ )
+ mask = self._pack_latents(
+ mask.repeat(1, num_channels_latents, 1, 1),
+ batch_size,
+ num_channels_latents,
+ height,
+ width,
+ )
+
+ return mask, masked_image_latents
+
+ @torch.no_grad()
+ def inpaint(
+ self,
+ prompt: Union[str, List[str]] = None,
+ prompt_2: Optional[Union[str, List[str]]] = None,
+ height: Optional[int] = None,
+ width: Optional[int] = None,
+ num_inference_steps: int = 28,
+ timesteps: List[int] = None,
+ guidance_scale: float = 7.0,
+ num_images_per_prompt: Optional[int] = 1,
+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+ latents: Optional[torch.FloatTensor] = None,
+ prompt_embeds: Optional[torch.FloatTensor] = None,
+ pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+ output_type: Optional[str] = "pil",
+ return_dict: bool = True,
+ joint_attention_kwargs: Optional[Dict[str, Any]] = None,
+ callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
+ callback_on_step_end_tensor_inputs: List[str] = ["latents"],
+ max_sequence_length: int = 512,
+ optimization_steps: int = 3,
+ learning_rate: float = 0.8,
+ max_steps: int = 5,
+ input_image = None,
+ save_masked_image = False,
+ output_path="",
+ mask_image = None,
+ ):
+
+ height = height or self.default_sample_size * self.vae_scale_factor
+ width = width or self.default_sample_size * self.vae_scale_factor
+
+ # 1. Check inputs. Raise error if not correct
+ self.check_inputs(
+ prompt,
+ prompt_2,
+ height,
+ width,
+ prompt_embeds=prompt_embeds,
+ pooled_prompt_embeds=pooled_prompt_embeds,
+ callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
+ max_sequence_length=max_sequence_length,
+ )
+
+ self._guidance_scale = guidance_scale
+ self._joint_attention_kwargs = joint_attention_kwargs
+ self._interrupt = False
+
+ # 2. Define call parameters
+ if prompt is not None and isinstance(prompt, str):
+ batch_size = 1
+ elif prompt is not None and isinstance(prompt, list):
+ batch_size = len(prompt)
+ else:
+ batch_size = prompt_embeds.shape[0]
+
+ device = self._execution_device
+
+ lora_scale = (
+ self.joint_attention_kwargs.get("scale", None) if self.joint_attention_kwargs is not None else None
+ )
+ (
+ prompt_embeds,
+ pooled_prompt_embeds,
+ text_ids,
+ ) = self.encode_prompt(
+ prompt=prompt,
+ prompt_2=prompt_2,
+ prompt_embeds=prompt_embeds,
+ pooled_prompt_embeds=pooled_prompt_embeds,
+ device=device,
+ num_images_per_prompt=num_images_per_prompt,
+ max_sequence_length=max_sequence_length,
+ lora_scale=lora_scale,
+ )
+
+ # 4. Prepare latent variables
+ num_channels_latents = self.transformer.config.in_channels // 4
+ random_latents, latent_image_ids = self.prepare_latents(
+ batch_size * num_images_per_prompt,
+ num_channels_latents,
+ height,
+ width,
+ prompt_embeds.dtype,
+ device,
+ generator,
+ latents,
+ )
+
+ # 5. Prepare timesteps
+ sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps)
+ image_seq_len = random_latents.shape[1]
+ mu = calculate_shift(
+ image_seq_len,
+ self.scheduler.config.base_image_seq_len,
+ self.scheduler.config.max_image_seq_len,
+ self.scheduler.config.base_shift,
+ self.scheduler.config.max_shift,
+ )
+ timesteps, num_inference_steps = retrieve_timesteps(
+ self.scheduler,
+ num_inference_steps,
+ device,
+ timesteps,
+ sigmas,
+ mu=mu,
+ )
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
+ self._num_timesteps = len(timesteps)
+
+ # 4. Preprocess image
+ # Preprocess mask image
+ mask_image = mask_image.convert("L")
+ mask = TF.ToTensor()(mask_image).to(device=device, dtype=self.transformer.dtype)
+ mask = TF.Resize(input_image.size, interpolation=TF.InterpolationMode.NEAREST)(mask)
+ mask = (mask > 0.5)
+ mask = ~mask
+
+ # # Convert input image to tensor and apply mask
+ # input_image = TF.ToTensor()(input_image).to(device=device, dtype=self.transformer.dtype)
+ # input_image = input_image * mask.float().expand_as(input_image)
+ # input_image = TF.ToPILImage()(input_image.cpu())
+
+ image = self.image_processor.preprocess(input_image)
+ image = image.to(device=device, dtype=self.transformer.dtype)
+ latents = retrieve_latents(self.vae.encode(image), generator=generator) * self.vae.config.scaling_factor
+
+
+ h, w = latents.shape[2], latents.shape[3]
+ mask = TF.ToTensor()(mask_image).to(device=device, dtype=self.transformer.dtype)
+ mask = TF.Resize((h, w), interpolation=TF.InterpolationMode.NEAREST)(mask)
+
+ # Slightly dilate the mask to increase coverage
+ kernel_size = 1 # Decreased from 3 to 2
+ kernel = torch.ones((1, 1, kernel_size, kernel_size), device=device)
+ mask = torch.nn.functional.conv2d(
+ mask.unsqueeze(0),
+ kernel,
+ padding=0
+ ).squeeze(0)
+ mask = torch.clamp(mask, 0, 1)
+
+ mask = (mask > 0.1).float()
+
+ # Remove extra channel dimension if present
+ if len(mask.shape) == 3 and mask.shape[0] == 1:
+ mask = mask.squeeze(0)
+
+ bool_mask = mask.bool().unsqueeze(0).unsqueeze(0).expand_as(latents)
+ mask=~bool_mask
+
+ print(mask.shape, latents.shape)
+
+ masked_latents = (latents * mask).clone().detach() # apply the mask and get gt_latents
+ masked_latents = self._pack_latents(masked_latents, batch_size, num_channels_latents, 2 * (int(height) // self.vae_scale_factor), 2 * (int(width) // self.vae_scale_factor))
+
+ mask = self._pack_latents(mask, batch_size, num_channels_latents, 2 * (int(height) // self.vae_scale_factor), 2 * (int(width) // self.vae_scale_factor))
+
+ # Decode and save the masked image
+ if save_masked_image:
+ with torch.no_grad():
+ save_masked_latents = self._unpack_latents(masked_latents, 1024, 1024, self.vae_scale_factor)
+ save_masked_latents = (save_masked_latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
+ mask_image = self.vae.decode(save_masked_latents, return_dict=False)[0]
+ mask_image = self.image_processor.postprocess(mask_image, output_type="pil")
+ mask_image_path = output_path.replace(".png", "_masked.png")
+ mask_image[0].save(mask_image_path)
+
+
+ # initialize the random noise for denoising
+ latents = random_latents.clone().detach()
+
+ self.vae = self.vae.to(torch.float32)
+
+ # 9. Denoising loop
+ self.transformer.eval()
+ self.vae.eval()
+
+ # 6. Denoising loop
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
+ for i, t in enumerate(timesteps):
+ if self.interrupt:
+ continue
+
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
+ timestep = t.expand(latents.shape[0]).to(latents.dtype)
+
+ # handle guidance
+ if self.transformer.config.guidance_embeds:
+ guidance = torch.tensor([guidance_scale], device=device)
+ guidance = guidance.expand(latents.shape[0])
+ else:
+ guidance = None
+
+ noise_pred = self.transformer(
+ hidden_states=latents,
+ # YiYi notes: divide it by 1000 for now because we scale it by 1000 in the transforme rmodel (we should not keep it but I want to keep the inputs same for the model for testing)
+ timestep=timestep / 1000,
+ guidance=guidance,
+ pooled_projections=pooled_prompt_embeds,
+ encoder_hidden_states=prompt_embeds,
+ txt_ids=text_ids,
+ img_ids=latent_image_ids,
+ joint_attention_kwargs=self.joint_attention_kwargs,
+ return_dict=False,
+ )[0]
+
+ # compute the previous noisy sample x_t -> x_t-1
+ latents_dtype = latents.dtype
+
+ # perform CG
+ if i < max_steps:
+ opt_latents = latents.detach().clone()
+ with torch.enable_grad():
+ opt_latents = opt_latents.detach().requires_grad_()
+ opt_latents = torch.autograd.Variable(opt_latents, requires_grad=True)
+ # optimizer = torch.optim.Adam([opt_latents], lr=learning_rate)
+
+ for _ in range(optimization_steps):
+ latents_p = self.scheduler.step_final(noise_pred, t, opt_latents, return_dict=False)[0]
+ loss = (1000*torch.nn.functional.mse_loss(latents_p, masked_latents, reduction='none')*mask).mean()
+
+ grad = torch.autograd.grad(loss, opt_latents)[0]
+ # grad = torch.clamp(grad, -0.5, 0.5)
+ opt_latents = opt_latents - learning_rate * grad
+
+ latents = opt_latents.detach().clone()
+
+ latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
+
+ if latents.dtype != latents_dtype:
+ if torch.backends.mps.is_available():
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
+ latents = latents.to(latents_dtype)
+
+ if callback_on_step_end is not None:
+ callback_kwargs = {}
+ for k in callback_on_step_end_tensor_inputs:
+ callback_kwargs[k] = locals()[k]
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
+
+ latents = callback_outputs.pop("latents", latents)
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
+
+ # call the callback, if provided
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
+ progress_bar.update()
+
+ if XLA_AVAILABLE:
+ xm.mark_step()
+
+ if output_type == "latent":
+ image = latents
+
+ else:
+ latents = self._unpack_latents(latents, height, width, self.vae_scale_factor)
+ latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
+ image = self.vae.decode(latents.to(torch.float32), return_dict=False)[0]
+ image = self.image_processor.postprocess(image, output_type=output_type)
+
+ # Offload all models
+ self.maybe_free_model_hooks()
+
+ if not return_dict:
+ return (image,)
+
+ return FluxPipelineOutput(images=image)
+
+ def get_diff_image(self, latents):
+ latents = self._unpack_latents(latents, 1024, 1024, self.vae_scale_factor)
+ latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
+ image = self.vae.decode(latents.to(torch.float32), return_dict=False)[0]
+ image = self.image_processor.postprocess(image, output_type="pt")
+ return image
+
+ def load_and_preprocess_image(self, image_path, custom_image_processor, device):
+ from diffusers.utils import load_image
+ img = load_image(image_path)
+ img = img.resize((512, 512))
+ return custom_image_processor(img).unsqueeze(0).to(device)
+
+
+ @torch.no_grad()
+ def edit(
+ self,
+ prompt: Union[str, List[str]] = None,
+ prompt_2: Optional[Union[str, List[str]]] = None,
+ negative_prompt: Union[str, List[str]] = None, #
+ negative_prompt_2: Optional[Union[str, List[str]]] = None,
+ true_cfg: float = 1.0, #
+ height: Optional[int] = None,
+ width: Optional[int] = None,
+ num_inference_steps: int = 28,
+ timesteps: List[int] = None,
+ guidance_scale: float = 3.5,
+ num_images_per_prompt: Optional[int] = 1,
+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+ latents: Optional[torch.FloatTensor] = None,
+ prompt_embeds: Optional[torch.FloatTensor] = None,
+ pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+ negative_prompt_embeds: Optional[torch.FloatTensor] = None,
+ negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+ output_type: Optional[str] = "pil",
+ return_dict: bool = True,
+ joint_attention_kwargs: Optional[Dict[str, Any]] = None,
+ callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
+ callback_on_step_end_tensor_inputs: List[str] = ["latents"],
+ max_sequence_length: int = 512,
+ optimization_steps: int = 3,
+ learning_rate: float = 0.8,
+ max_steps: int = 5,
+ input_image = None,
+ save_masked_image = False,
+ output_path="",
+ mask_image=None,
+ source_steps=1,
+ ):
+
+ height = height or self.default_sample_size * self.vae_scale_factor
+ width = width or self.default_sample_size * self.vae_scale_factor
+
+ # 1. Check inputs. Raise error if not correct
+ self.check_inputs(
+ prompt,
+ prompt_2,
+ height,
+ width,
+ # negative_prompt=negative_prompt,
+ # negative_prompt_2=negative_prompt_2,
+ prompt_embeds=prompt_embeds,
+ # negative_prompt_embeds=negative_prompt_embeds,
+ pooled_prompt_embeds=pooled_prompt_embeds,
+ # negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+ callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
+ max_sequence_length=max_sequence_length,
+ )
+
+ self._guidance_scale = guidance_scale
+ self._joint_attention_kwargs = joint_attention_kwargs
+ self._interrupt = False
+
+ # 2. Define call parameters
+ if prompt is not None and isinstance(prompt, str):
+ batch_size = 1
+ elif prompt is not None and isinstance(prompt, list):
+ batch_size = len(prompt)
+ else:
+ batch_size = prompt_embeds.shape[0]
+
+ device = self._execution_device
+
+ lora_scale = (
+ self.joint_attention_kwargs.get("scale", None) if self.joint_attention_kwargs is not None else None
+ )
+ do_true_cfg = true_cfg > 1 and negative_prompt is not None
+ (
+ prompt_embeds,
+ pooled_prompt_embeds,
+ text_ids,
+ negative_prompt_embeds,
+ negative_pooled_prompt_embeds,
+ ) = self.encode_prompt_edit(
+ prompt=prompt,
+ prompt_2=prompt_2,
+ negative_prompt=negative_prompt,
+ negative_prompt_2=negative_prompt_2,
+ prompt_embeds=prompt_embeds,
+ pooled_prompt_embeds=pooled_prompt_embeds,
+ negative_prompt_embeds=negative_prompt_embeds,
+ negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+ device=device,
+ num_images_per_prompt=num_images_per_prompt,
+ max_sequence_length=max_sequence_length,
+ lora_scale=lora_scale,
+ do_true_cfg=do_true_cfg,
+ )
+ # text_ids = text_ids.repeat(batch_size, 1, 1)
+
+ if do_true_cfg:
+ # Concatenate embeddings
+ prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
+ pooled_prompt_embeds = torch.cat([negative_pooled_prompt_embeds, pooled_prompt_embeds], dim=0)
+
+ # 4. Prepare latent variables
+ num_channels_latents = self.transformer.config.in_channels // 4
+ random_latents, latent_image_ids = self.prepare_latents(
+ batch_size * num_images_per_prompt,
+ num_channels_latents,
+ height,
+ width,
+ prompt_embeds.dtype,
+ device,
+ generator,
+ latents,
+ )
+ # latent_image_ids = latent_image_ids.repeat(batch_size, 1, 1)
+
+ # 5. Prepare timesteps
+ sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps)
+ image_seq_len = random_latents.shape[1]
+ mu = calculate_shift(
+ image_seq_len,
+ self.scheduler.config.base_image_seq_len,
+ self.scheduler.config.max_image_seq_len,
+ self.scheduler.config.base_shift,
+ self.scheduler.config.max_shift,
+ )
+ timesteps, num_inference_steps = retrieve_timesteps(
+ self.scheduler,
+ num_inference_steps,
+ device,
+ timesteps,
+ sigmas,
+ mu=mu,
+ )
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
+ self._num_timesteps = len(timesteps)
+
+ # 4. Preprocess image
+ image = self.image_processor.preprocess(input_image)
+ image = image.to(device=device, dtype=self.transformer.dtype)
+ latents = retrieve_latents(self.vae.encode(image), generator=generator) * self.vae.config.scaling_factor
+
+
+ # Convert PIL image to tensor
+ if mask_image:
+ from torchvision import transforms as TF
+
+ h, w = latents.shape[2], latents.shape[3]
+ mask = TF.ToTensor()(mask_image).to(device=device, dtype=self.transformer.dtype)
+ mask = TF.Resize((h, w), interpolation=TF.InterpolationMode.NEAREST)(mask)
+ mask = (mask > 0.5).float()
+ mask = mask.squeeze(0)#.squeeze(0) # Remove the added dimensions
+ else:
+ mask = torch.ones_like(latents).to(device=device)
+
+ print(mask.shape, latents.shape)
+
+ bool_mask = mask.unsqueeze(0).unsqueeze(0).expand_as(latents)
+ mask=(1-bool_mask*1.0).to(latents.dtype)
+
+ masked_latents = (latents * mask).clone().detach() # apply the mask and get gt_latents
+ masked_latents = self._pack_latents(masked_latents, batch_size, num_channels_latents, 2 * (int(height) // self.vae_scale_factor), 2 * (int(width) // self.vae_scale_factor))
+
+ source_latents = (latents).clone().detach() # apply the mask and get gt_latents
+ source_latents = self._pack_latents(source_latents, batch_size, num_channels_latents, 2 * (int(height) // self.vae_scale_factor), 2 * (int(width) // self.vae_scale_factor))
+
+ mask = self._pack_latents(mask, batch_size, num_channels_latents, 2 * (int(height) // self.vae_scale_factor), 2 * (int(width) // self.vae_scale_factor))
+
+ # initialize the random noise for denoising
+ latents = random_latents.clone().detach()
+
+ self.vae = self.vae.to(torch.float32)
+
+ # 9. Denoising loop
+ self.transformer.eval()
+ self.vae.eval()
+
+ # 6. Denoising loop
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
+ for i, t in enumerate(timesteps):
+ if self.interrupt:
+ continue
+
+ latent_model_input = torch.cat([latents] * 2) if do_true_cfg else latents
+
+ # handle guidance
+ if self.transformer.config.guidance_embeds:
+ guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32)
+ guidance = guidance.expand(latent_model_input.shape[0])
+ else:
+ guidance = None
+
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
+ timestep = t.expand(latent_model_input.shape[0]).to(latent_model_input.dtype)
+
+ noise_pred = self.transformer(
+ hidden_states=latent_model_input,
+ timestep=timestep / 1000,
+ guidance=guidance,
+ pooled_projections=pooled_prompt_embeds,
+ encoder_hidden_states=prompt_embeds,
+ txt_ids=text_ids,
+ img_ids=latent_image_ids,
+ joint_attention_kwargs=self.joint_attention_kwargs,
+ return_dict=False,
+ )[0]
+
+ if do_true_cfg:
+ neg_noise_pred, noise_pred = noise_pred.chunk(2)
+ # noise_pred = neg_noise_pred + true_cfg * (noise_pred - neg_noise_pred)
+ noise_pred = noise_pred + (1-mask)*(noise_pred - neg_noise_pred) * true_cfg
+ # else:
+ # neg_noise_pred, noise_pred = noise_pred.chunk(2)
+
+ # perform CG
+ if i < max_steps:
+ opt_latents = latents.detach().clone()
+ with torch.enable_grad():
+ opt_latents = opt_latents.detach().requires_grad_()
+ opt_latents = torch.autograd.Variable(opt_latents, requires_grad=True)
+ # optimizer = torch.optim.Adam([opt_latents], lr=learning_rate)
+
+ for _ in range(optimization_steps):
+ latents_p = self.scheduler.step_final(noise_pred, t, opt_latents, return_dict=False)[0]
+ if i < source_steps:
+ loss = (1000*torch.nn.functional.mse_loss(latents_p, source_latents, reduction='none')).mean()
+ else:
+ loss = (1000*torch.nn.functional.mse_loss(latents_p, masked_latents, reduction='none')*mask).mean()
+
+ grad = torch.autograd.grad(loss, opt_latents)[0]
+ # grad = torch.clamp(grad, -0.5, 0.5)
+ opt_latents = opt_latents - learning_rate * grad
+
+ latents = opt_latents.detach().clone()
+
+
+ # compute the previous noisy sample x_t -> x_t-1
+ latents_dtype = latents.dtype
+ latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
+
+ if latents.dtype != latents_dtype:
+ if torch.backends.mps.is_available():
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
+ latents = latents.to(latents_dtype)
+
+ if callback_on_step_end is not None:
+ callback_kwargs = {}
+ for k in callback_on_step_end_tensor_inputs:
+ callback_kwargs[k] = locals()[k]
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
+
+ latents = callback_outputs.pop("latents", latents)
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
+
+ # call the callback, if provided
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
+ progress_bar.update()
+
+ if XLA_AVAILABLE:
+ xm.mark_step()
+
+ if output_type == "latent":
+ image = latents
+
+ else:
+ latents = self._unpack_latents(latents, height, width, self.vae_scale_factor)
+ latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
+ image = self.vae.decode(latents.to(torch.float32), return_dict=False)[0]
+ image = self.image_processor.postprocess(image, output_type=output_type)
+
+ # Offload all models
+ self.maybe_free_model_hooks()
+
+ if not return_dict:
+ return (image,)
+
+ return FluxPipelineOutput(images=image)
+
+ @torch.no_grad()
+ def edit2(
+ self,
+ prompt: Union[str, List[str]] = None,
+ prompt_2: Optional[Union[str, List[str]]] = None,
+ negative_prompt: Union[str, List[str]] = None, #
+ negative_prompt_2: Optional[Union[str, List[str]]] = None,
+ true_cfg: float = 1.0, #
+ height: Optional[int] = None,
+ width: Optional[int] = None,
+ num_inference_steps: int = 28,
+ timesteps: List[int] = None,
+ guidance_scale: float = 3.5,
+ num_images_per_prompt: Optional[int] = 1,
+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
+ latents: Optional[torch.FloatTensor] = None,
+ prompt_embeds: Optional[torch.FloatTensor] = None,
+ pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+ negative_prompt_embeds: Optional[torch.FloatTensor] = None,
+ negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
+ output_type: Optional[str] = "pil",
+ return_dict: bool = True,
+ joint_attention_kwargs: Optional[Dict[str, Any]] = None,
+ callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
+ callback_on_step_end_tensor_inputs: List[str] = ["latents"],
+ max_sequence_length: int = 512,
+ optimization_steps: int = 3,
+ learning_rate: float = 0.8,
+ max_steps: int = 5,
+ input_image = None,
+ save_masked_image = False,
+ output_path="",
+ mask_image=None,
+ source_steps=1,
+ ):
+ r"""
+ Function invoked when calling the pipeline for generation.
+
+ Args:
+ prompt (`str` or `List[str]`, *optional*):
+ The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`.
+ instead.
+ prompt_2 (`str` or `List[str]`, *optional*):
+ The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is
+ will be used instead
+ height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
+ The height in pixels of the generated image. This is set to 1024 by default for the best results.
+ width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor):
+ The width in pixels of the generated image. This is set to 1024 by default for the best results.
+ num_inference_steps (`int`, *optional*, defaults to 50):
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
+ expense of slower inference.
+ timesteps (`List[int]`, *optional*):
+ Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument
+ in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is
+ passed will be used. Must be in descending order.
+ guidance_scale (`float`, *optional*, defaults to 7.0):
+ Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598).
+ `guidance_scale` is defined as `w` of equation 2. of [Imagen
+ Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale >
+ 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`,
+ usually at the expense of lower image quality.
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
+ The number of images to generate per prompt.
+ generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
+ One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html)
+ to make generation deterministic.
+ latents (`torch.FloatTensor`, *optional*):
+ Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image
+ generation. Can be used to tweak the same generation with different prompts. If not provided, a latents
+ tensor will ge generated by sampling using the supplied random `generator`.
+ prompt_embeds (`torch.FloatTensor`, *optional*):
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
+ provided, text embeddings will be generated from `prompt` input argument.
+ pooled_prompt_embeds (`torch.FloatTensor`, *optional*):
+ Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting.
+ If not provided, pooled text embeddings will be generated from `prompt` input argument.
+ output_type (`str`, *optional*, defaults to `"pil"`):
+ The output format of the generate image. Choose between
+ [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
+ return_dict (`bool`, *optional*, defaults to `True`):
+ Whether or not to return a [`~pipelines.flux.FluxPipelineOutput`] instead of a plain tuple.
+ joint_attention_kwargs (`dict`, *optional*):
+ A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under
+ `self.processor` in
+ [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
+ callback_on_step_end (`Callable`, *optional*):
+ A function that calls at the end of each denoising steps during the inference. The function is called
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
+ `callback_on_step_end_tensor_inputs`.
+ callback_on_step_end_tensor_inputs (`List`, *optional*):
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
+ `._callback_tensor_inputs` attribute of your pipeline class.
+ max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`.
+
+ Examples:
+
+ Returns:
+ [`~pipelines.flux.FluxPipelineOutput`] or `tuple`: [`~pipelines.flux.FluxPipelineOutput`] if `return_dict`
+ is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated
+ images.
+ """
+
+ height = height or self.default_sample_size * self.vae_scale_factor
+ width = width or self.default_sample_size * self.vae_scale_factor
+
+ # 1. Check inputs. Raise error if not correct
+ self.check_inputs(
+ prompt,
+ prompt_2,
+ height,
+ width,
+ # negative_prompt=negative_prompt,
+ # negative_prompt_2=negative_prompt_2,
+ prompt_embeds=prompt_embeds,
+ # negative_prompt_embeds=negative_prompt_embeds,
+ pooled_prompt_embeds=pooled_prompt_embeds,
+ # negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+ callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
+ max_sequence_length=max_sequence_length,
+ )
+
+ self._guidance_scale = guidance_scale
+ self._joint_attention_kwargs = joint_attention_kwargs
+ self._interrupt = False
+
+ # 2. Define call parameters
+ if prompt is not None and isinstance(prompt, str):
+ batch_size = 1
+ elif prompt is not None and isinstance(prompt, list):
+ batch_size = len(prompt)
+ else:
+ batch_size = prompt_embeds.shape[0]
+
+ device = self._execution_device
+
+ lora_scale = (
+ self.joint_attention_kwargs.get("scale", None) if self.joint_attention_kwargs is not None else None
+ )
+ do_true_cfg = true_cfg > 1 and negative_prompt is not None
+ (
+ prompt_embeds,
+ pooled_prompt_embeds,
+ text_ids,
+ negative_prompt_embeds,
+ negative_pooled_prompt_embeds,
+ ) = self.encode_prompt_edit(
+ prompt=prompt,
+ prompt_2=prompt_2,
+ negative_prompt=negative_prompt,
+ negative_prompt_2=negative_prompt_2,
+ prompt_embeds=prompt_embeds,
+ pooled_prompt_embeds=pooled_prompt_embeds,
+ negative_prompt_embeds=negative_prompt_embeds,
+ negative_pooled_prompt_embeds=negative_pooled_prompt_embeds,
+ device=device,
+ num_images_per_prompt=num_images_per_prompt,
+ max_sequence_length=max_sequence_length,
+ lora_scale=lora_scale,
+ do_true_cfg=do_true_cfg,
+ )
+ # text_ids = text_ids.repeat(batch_size, 1, 1)
+
+ if do_true_cfg:
+ # Concatenate embeddings
+ prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
+ pooled_prompt_embeds = torch.cat([negative_pooled_prompt_embeds, pooled_prompt_embeds], dim=0)
+
+ # 4. Prepare latent variables
+ num_channels_latents = self.transformer.config.in_channels // 4
+ random_latents, latent_image_ids = self.prepare_latents(
+ batch_size * num_images_per_prompt,
+ num_channels_latents,
+ height,
+ width,
+ prompt_embeds.dtype,
+ device,
+ generator,
+ latents,
+ )
+ # latent_image_ids = latent_image_ids.repeat(batch_size, 1, 1)
+
+ # 5. Prepare timesteps
+ sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps)
+ image_seq_len = random_latents.shape[1]
+ mu = calculate_shift(
+ image_seq_len,
+ self.scheduler.config.base_image_seq_len,
+ self.scheduler.config.max_image_seq_len,
+ self.scheduler.config.base_shift,
+ self.scheduler.config.max_shift,
+ )
+ timesteps, num_inference_steps = retrieve_timesteps(
+ self.scheduler,
+ num_inference_steps,
+ device,
+ timesteps,
+ sigmas,
+ mu=mu,
+ )
+ num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0)
+ self._num_timesteps = len(timesteps)
+
+ # 4. Preprocess image
+ image = self.image_processor.preprocess(input_image)
+ image = image.to(device=device, dtype=self.transformer.dtype)
+ latents = retrieve_latents(self.vae.encode(image), generator=generator) * self.vae.config.scaling_factor
+
+
+ # Convert PIL image to tensor
+ if mask_image:
+ from torchvision import transforms as TF
+
+ h, w = latents.shape[2], latents.shape[3]
+ mask = TF.ToTensor()(mask_image).to(device=device, dtype=self.transformer.dtype)
+ mask = TF.Resize((h, w), interpolation=TF.InterpolationMode.NEAREST)(mask)
+ mask = (mask > 0.1).float()
+ mask = mask.squeeze(0)#.squeeze(0) # Remove the added dimensions
+ else:
+ mask = torch.ones_like(latents).to(device=device)
+
+ bool_mask = mask.unsqueeze(0).unsqueeze(0).expand_as(latents)
+ mask=(1-bool_mask*1.0).to(latents.dtype)
+
+ masked_latents = (latents * mask).clone().detach() # apply the mask and get gt_latents
+ masked_latents = self._pack_latents(masked_latents, batch_size, num_channels_latents, 2 * (int(height) // self.vae_scale_factor), 2 * (int(width) // self.vae_scale_factor))
+
+ source_latents = (latents).clone().detach() # apply the mask and get gt_latents
+ source_latents = self._pack_latents(source_latents, batch_size, num_channels_latents, 2 * (int(height) // self.vae_scale_factor), 2 * (int(width) // self.vae_scale_factor))
+
+ mask = self._pack_latents(mask, batch_size, num_channels_latents, 2 * (int(height) // self.vae_scale_factor), 2 * (int(width) // self.vae_scale_factor))
+
+ # initialize the random noise for denoising
+ latents = random_latents.clone().detach()
+
+ self.vae = self.vae.to(torch.float32)
+
+ # 9. Denoising loop
+ self.transformer.eval()
+ self.vae.eval()
+
+ # 6. Denoising loop
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
+ for i, t in enumerate(timesteps):
+ if self.interrupt:
+ continue
+
+ latent_model_input = torch.cat([latents] * 2) if do_true_cfg else latents
+
+ # handle guidance
+ if self.transformer.config.guidance_embeds:
+ guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32)
+ guidance = guidance.expand(latent_model_input.shape[0])
+ else:
+ guidance = None
+
+ # broadcast to batch dimension in a way that's compatible with ONNX/Core ML
+ timestep = t.expand(latent_model_input.shape[0]).to(latent_model_input.dtype)
+
+ noise_pred = self.transformer(
+ hidden_states=latent_model_input,
+ timestep=timestep / 1000,
+ guidance=guidance,
+ pooled_projections=pooled_prompt_embeds,
+ encoder_hidden_states=prompt_embeds,
+ txt_ids=text_ids,
+ img_ids=latent_image_ids,
+ joint_attention_kwargs=self.joint_attention_kwargs,
+ return_dict=False,
+ )[0]
+
+ if do_true_cfg and i < max_steps:
+ neg_noise_pred, noise_pred = noise_pred.chunk(2)
+ # noise_pred = neg_noise_pred + true_cfg * (noise_pred - neg_noise_pred)
+ noise_pred = noise_pred + (1-mask)*(noise_pred - neg_noise_pred) * true_cfg
+ else:
+ neg_noise_pred, noise_pred = noise_pred.chunk(2)
+
+ # perform CG
+ if i < max_steps:
+ opt_latents = latents.detach().clone()
+ with torch.enable_grad():
+ opt_latents = opt_latents.detach().requires_grad_()
+ opt_latents = torch.autograd.Variable(opt_latents, requires_grad=True)
+ # optimizer = torch.optim.Adam([opt_latents], lr=learning_rate)
+
+ for _ in range(optimization_steps):
+ latents_p = self.scheduler.step_final(noise_pred, t, opt_latents, return_dict=False)[0]
+ if i < source_steps:
+ loss = (1000*torch.nn.functional.mse_loss(latents_p, source_latents, reduction='none')).mean()
+ else:
+ loss = (1000*torch.nn.functional.mse_loss(latents_p, masked_latents, reduction='none')*mask).mean()
+
+ grad = torch.autograd.grad(loss, opt_latents)[0]
+ # grad = torch.clamp(grad, -0.5, 0.5)
+ opt_latents = opt_latents - learning_rate * grad
+
+ latents = opt_latents.detach().clone()
+
+
+ # compute the previous noisy sample x_t -> x_t-1
+ latents_dtype = latents.dtype
+ latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
+
+ if latents.dtype != latents_dtype:
+ if torch.backends.mps.is_available():
+ # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272
+ latents = latents.to(latents_dtype)
+
+ if callback_on_step_end is not None:
+ callback_kwargs = {}
+ for k in callback_on_step_end_tensor_inputs:
+ callback_kwargs[k] = locals()[k]
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
+
+ latents = callback_outputs.pop("latents", latents)
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
+
+ # call the callback, if provided
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
+ progress_bar.update()
+
+ if XLA_AVAILABLE:
+ xm.mark_step()
+
+ if output_type == "latent":
+ image = latents
+
+ else:
+ latents = self._unpack_latents(latents, height, width, self.vae_scale_factor)
+ latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor
+ image = self.vae.decode(latents.to(torch.float32), return_dict=False)[0]
+ image = self.image_processor.postprocess(image, output_type=output_type)
+
+ # Offload all models
+ self.maybe_free_model_hooks()
+
+ if not return_dict:
+ return (image,)
+
+ return FluxPipelineOutput(images=image)
+
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000000000000000000000000000000000000..77a338c9b8cd6339b4ddc7b394607d5d5414c587
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,9 @@
+spaces
+diffusers==0.31.0
+gradio==5.6.0
+numpy==2.1.3
+Pillow==11.0.0
+torch==2.1.2
+torch_xla==2.5.1
+torchvision==0.16.2
+transformers==4.45.2
diff --git a/saved_results/20241126_053639/input.png b/saved_results/20241126_053639/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..24df90880d636cc8b7e61552fc1a03f282787640
Binary files /dev/null and b/saved_results/20241126_053639/input.png differ
diff --git a/saved_results/20241126_053639/mask.png b/saved_results/20241126_053639/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..13e9d0b9698bad75929b566d43c8640db48e8973
Binary files /dev/null and b/saved_results/20241126_053639/mask.png differ
diff --git a/saved_results/20241126_053639/output.png b/saved_results/20241126_053639/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..bdb4e11dcbe8f82001ef8ea9a0aed499f454072a
--- /dev/null
+++ b/saved_results/20241126_053639/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f1fdeb3a98da1b0cc536e4f97f4d36cfc912b9645225607402b87b7221047ef
+size 1077405
diff --git a/saved_results/20241126_053639/parameters.json b/saved_results/20241126_053639/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..64f0ae49e8bcfad7666a4c8cde69b4339607f020
--- /dev/null
+++ b/saved_results/20241126_053639/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Inpainting",
+ "prompt": "a dog",
+ "edit_prompt": "",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 30,
+ "learning_rate": 1,
+ "max_source_steps": 20,
+ "optimization_steps": 10,
+ "true_cfg": 2
+}
\ No newline at end of file
diff --git a/saved_results/20241126_055109/input.png b/saved_results/20241126_055109/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..24df90880d636cc8b7e61552fc1a03f282787640
Binary files /dev/null and b/saved_results/20241126_055109/input.png differ
diff --git a/saved_results/20241126_055109/mask.png b/saved_results/20241126_055109/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..13e9d0b9698bad75929b566d43c8640db48e8973
Binary files /dev/null and b/saved_results/20241126_055109/mask.png differ
diff --git a/saved_results/20241126_055109/output.png b/saved_results/20241126_055109/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..66f121daf37affe44f1f88d6430341671db74fd3
--- /dev/null
+++ b/saved_results/20241126_055109/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b8111afa5b56049ddf13a1a35c0a8c3be4ea9ab777476c7669e1719b6a4343d1
+size 1057241
diff --git a/saved_results/20241126_055109/parameters.json b/saved_results/20241126_055109/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..64f0ae49e8bcfad7666a4c8cde69b4339607f020
--- /dev/null
+++ b/saved_results/20241126_055109/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Inpainting",
+ "prompt": "a dog",
+ "edit_prompt": "",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 30,
+ "learning_rate": 1,
+ "max_source_steps": 20,
+ "optimization_steps": 10,
+ "true_cfg": 2
+}
\ No newline at end of file
diff --git a/saved_results/20241126_173140/input.png b/saved_results/20241126_173140/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..be95fe96bbaa270dd63caf011576ff05074f4184
Binary files /dev/null and b/saved_results/20241126_173140/input.png differ
diff --git a/saved_results/20241126_173140/mask.png b/saved_results/20241126_173140/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..285fbe49fde92796ed5da1adf99abeb42b44c8fa
Binary files /dev/null and b/saved_results/20241126_173140/mask.png differ
diff --git a/saved_results/20241126_173140/output.png b/saved_results/20241126_173140/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..7e830e0842f38ddf6dfc81cd31567f8bbcc9a997
--- /dev/null
+++ b/saved_results/20241126_173140/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c484065f2bb53cb427858a9137c4f5e2b32426609a9a1bb401883368f2a6f6da
+size 1372203
diff --git a/saved_results/20241126_173140/parameters.json b/saved_results/20241126_173140/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..e8801087f410535dd147674475f993bea3da378c
--- /dev/null
+++ b/saved_results/20241126_173140/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Inpainting",
+ "prompt": "a cat with blue eyes",
+ "edit_prompt": "",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 20,
+ "learning_rate": 1,
+ "max_source_steps": 20,
+ "optimization_steps": 10,
+ "true_cfg": 2
+}
\ No newline at end of file
diff --git a/saved_results/20241126_181436/input.png b/saved_results/20241126_181436/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..f4a7c53b76f1a339575ef5bacd464967f23c7cbc
--- /dev/null
+++ b/saved_results/20241126_181436/input.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e6a77c0cb907998fe78e28658c7f65c674f1be6bf8d6984279f2fdcbc30482e
+size 4840733
diff --git a/saved_results/20241126_181436/mask.png b/saved_results/20241126_181436/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..9a662b5bc71b41b0a3092539d24e602c515e21c6
Binary files /dev/null and b/saved_results/20241126_181436/mask.png differ
diff --git a/saved_results/20241126_181436/output.png b/saved_results/20241126_181436/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..fb3e3e2bc699195f3a37bee3a5304295347c8211
Binary files /dev/null and b/saved_results/20241126_181436/output.png differ
diff --git a/saved_results/20241126_181436/parameters.json b/saved_results/20241126_181436/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..90d36587a8d074c691e2ef9bec9b10fe1046563d
--- /dev/null
+++ b/saved_results/20241126_181436/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Editing",
+ "prompt": " ",
+ "edit_prompt": "volcano eruption",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 20,
+ "learning_rate": 0.5,
+ "max_source_steps": 2,
+ "optimization_steps": 3,
+ "true_cfg": 4.5
+}
\ No newline at end of file
diff --git a/saved_results/20241126_181633/input.png b/saved_results/20241126_181633/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..f4a7c53b76f1a339575ef5bacd464967f23c7cbc
--- /dev/null
+++ b/saved_results/20241126_181633/input.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e6a77c0cb907998fe78e28658c7f65c674f1be6bf8d6984279f2fdcbc30482e
+size 4840733
diff --git a/saved_results/20241126_181633/mask.png b/saved_results/20241126_181633/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..9a662b5bc71b41b0a3092539d24e602c515e21c6
Binary files /dev/null and b/saved_results/20241126_181633/mask.png differ
diff --git a/saved_results/20241126_181633/output.png b/saved_results/20241126_181633/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..64781a7558b29b4ef46730b89321728643341a23
Binary files /dev/null and b/saved_results/20241126_181633/output.png differ
diff --git a/saved_results/20241126_181633/parameters.json b/saved_results/20241126_181633/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..90d36587a8d074c691e2ef9bec9b10fe1046563d
--- /dev/null
+++ b/saved_results/20241126_181633/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Editing",
+ "prompt": " ",
+ "edit_prompt": "volcano eruption",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 20,
+ "learning_rate": 0.5,
+ "max_source_steps": 2,
+ "optimization_steps": 3,
+ "true_cfg": 4.5
+}
\ No newline at end of file
diff --git a/saved_results/20241126_214810/input.png b/saved_results/20241126_214810/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..24df90880d636cc8b7e61552fc1a03f282787640
Binary files /dev/null and b/saved_results/20241126_214810/input.png differ
diff --git a/saved_results/20241126_214810/mask.png b/saved_results/20241126_214810/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..cc704aedc87e982432e573fd958d711529907b6a
Binary files /dev/null and b/saved_results/20241126_214810/mask.png differ
diff --git a/saved_results/20241126_214810/output.png b/saved_results/20241126_214810/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..1338e85d4996a112ddfa0189704098364128697b
--- /dev/null
+++ b/saved_results/20241126_214810/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c89f9f31166f6418be1533d462e74aefadb2b4521dd8015640f1d9403673a6f8
+size 1085186
diff --git a/saved_results/20241126_214810/parameters.json b/saved_results/20241126_214810/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..bb40b1286014f00bfedd1ac54aca45e941736f1c
--- /dev/null
+++ b/saved_results/20241126_214810/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Editing",
+ "prompt": " ",
+ "edit_prompt": "a dog with flowers in the mouth",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 30,
+ "learning_rate": 1,
+ "max_source_steps": 5,
+ "optimization_steps": 3,
+ "true_cfg": 4.5
+}
\ No newline at end of file
diff --git a/saved_results/20241126_214908/input.png b/saved_results/20241126_214908/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..24df90880d636cc8b7e61552fc1a03f282787640
Binary files /dev/null and b/saved_results/20241126_214908/input.png differ
diff --git a/saved_results/20241126_214908/mask.png b/saved_results/20241126_214908/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..cc704aedc87e982432e573fd958d711529907b6a
Binary files /dev/null and b/saved_results/20241126_214908/mask.png differ
diff --git a/saved_results/20241126_214908/output.png b/saved_results/20241126_214908/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..99934c9fd57a12e04d7f359acda6735ae1e60e3f
--- /dev/null
+++ b/saved_results/20241126_214908/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e8b13890624af98097b57ca314f2af4ec98e73f04c26be10ec93466edf7cabf
+size 1083367
diff --git a/saved_results/20241126_214908/parameters.json b/saved_results/20241126_214908/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..8af7aa996a1c3492ee30839d4483b0cd21cd55b5
--- /dev/null
+++ b/saved_results/20241126_214908/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Editing",
+ "prompt": " ",
+ "edit_prompt": "a dog with flowers in the mouth",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 20,
+ "learning_rate": 1,
+ "max_source_steps": 5,
+ "optimization_steps": 3,
+ "true_cfg": 4.5
+}
\ No newline at end of file
diff --git a/saved_results/20241126_215043/input.png b/saved_results/20241126_215043/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..24df90880d636cc8b7e61552fc1a03f282787640
Binary files /dev/null and b/saved_results/20241126_215043/input.png differ
diff --git a/saved_results/20241126_215043/mask.png b/saved_results/20241126_215043/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..cc704aedc87e982432e573fd958d711529907b6a
Binary files /dev/null and b/saved_results/20241126_215043/mask.png differ
diff --git a/saved_results/20241126_215043/output.png b/saved_results/20241126_215043/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..b555497eeae7eb2c888e5cb2c179dedc085782f8
--- /dev/null
+++ b/saved_results/20241126_215043/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d29bd6e39a70b4a9273467af120c0c1a19ae4019ba975a6b88bdefc4e1620271
+size 1085121
diff --git a/saved_results/20241126_215043/parameters.json b/saved_results/20241126_215043/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..6a29fd516e67ae868894aa0b63e587e9d19f6ab6
--- /dev/null
+++ b/saved_results/20241126_215043/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Editing",
+ "prompt": " ",
+ "edit_prompt": "a dog with flowers in the mouth",
+ "seed": 52,
+ "randomize_seed": false,
+ "num_inference_steps": 30,
+ "max_steps": 20,
+ "learning_rate": 1,
+ "max_source_steps": 5,
+ "optimization_steps": 3,
+ "true_cfg": 4.5
+}
\ No newline at end of file
diff --git a/saved_results/20241126_221300/input.png b/saved_results/20241126_221300/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..a9a0d7821c068f2200e906bf2983b9b048abb72f
Binary files /dev/null and b/saved_results/20241126_221300/input.png differ
diff --git a/saved_results/20241126_221300/mask.png b/saved_results/20241126_221300/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..fba17fb8723f9ba09017cb439af32bca69e6f7fa
Binary files /dev/null and b/saved_results/20241126_221300/mask.png differ
diff --git a/saved_results/20241126_221300/output.png b/saved_results/20241126_221300/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..a19d716c8b860f72243f9016a4b4166b0aa55969
--- /dev/null
+++ b/saved_results/20241126_221300/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b09790e41c4f7502684c6682c623029934c9a8774ed5b45ceba5c1be474638b
+size 1392628
diff --git a/saved_results/20241126_221300/parameters.json b/saved_results/20241126_221300/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..2a820ae5f1798eb861bc35d9dc63796c9e3a136c
--- /dev/null
+++ b/saved_results/20241126_221300/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Inpainting",
+ "prompt": "A building with \"ASU\" written on it.",
+ "edit_prompt": "",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 30,
+ "learning_rate": 1,
+ "max_source_steps": 20,
+ "optimization_steps": 5,
+ "true_cfg": 2
+}
\ No newline at end of file
diff --git a/saved_results/20241126_222257/input.png b/saved_results/20241126_222257/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..bb63c2d883c8f7c23b357c224b8714cf4c00428e
Binary files /dev/null and b/saved_results/20241126_222257/input.png differ
diff --git a/saved_results/20241126_222257/mask.png b/saved_results/20241126_222257/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..7ccc5f5e61ddde1786d5b1291883e33237076896
Binary files /dev/null and b/saved_results/20241126_222257/mask.png differ
diff --git a/saved_results/20241126_222257/output.png b/saved_results/20241126_222257/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..059f90cd3fe9b46a9fd457b46c62186e09f2e5e2
--- /dev/null
+++ b/saved_results/20241126_222257/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:825b90e103c5eb976f47c58cd31cd502f0334714edd26675d76b9e7e1fcad7b9
+size 1007279
diff --git a/saved_results/20241126_222257/parameters.json b/saved_results/20241126_222257/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..aae5df0eb285dc280cc5210ce1787b9460690d0c
--- /dev/null
+++ b/saved_results/20241126_222257/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Inpainting",
+ "prompt": "A cute pig with big eyes",
+ "edit_prompt": "",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 19.8,
+ "learning_rate": 1,
+ "max_source_steps": 20,
+ "optimization_steps": 5,
+ "true_cfg": 2
+}
\ No newline at end of file
diff --git a/saved_results/20241126_222442/input.png b/saved_results/20241126_222442/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..bb63c2d883c8f7c23b357c224b8714cf4c00428e
Binary files /dev/null and b/saved_results/20241126_222442/input.png differ
diff --git a/saved_results/20241126_222442/mask.png b/saved_results/20241126_222442/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..7ccc5f5e61ddde1786d5b1291883e33237076896
Binary files /dev/null and b/saved_results/20241126_222442/mask.png differ
diff --git a/saved_results/20241126_222442/output.png b/saved_results/20241126_222442/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..5e603e4aadbd5c2b4eb595befc23da3e65b36315
--- /dev/null
+++ b/saved_results/20241126_222442/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8850c5c54035ebb6d7ab27f91779c942936b6736292b2e488583b06610d6e4e
+size 1000741
diff --git a/saved_results/20241126_222442/parameters.json b/saved_results/20241126_222442/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..541c6ed154aa64f9dc9ce0c7d2474d58dc1ae94c
--- /dev/null
+++ b/saved_results/20241126_222442/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Editing",
+ "prompt": "A cute rabbit with big eyes",
+ "edit_prompt": "A cute pig with big eyes",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 30,
+ "learning_rate": 0.4,
+ "max_source_steps": 5,
+ "optimization_steps": 5,
+ "true_cfg": 4.5
+}
\ No newline at end of file
diff --git a/saved_results/20241126_222522/input.png b/saved_results/20241126_222522/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..bb63c2d883c8f7c23b357c224b8714cf4c00428e
Binary files /dev/null and b/saved_results/20241126_222522/input.png differ
diff --git a/saved_results/20241126_222522/mask.png b/saved_results/20241126_222522/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..7ccc5f5e61ddde1786d5b1291883e33237076896
Binary files /dev/null and b/saved_results/20241126_222522/mask.png differ
diff --git a/saved_results/20241126_222522/output.png b/saved_results/20241126_222522/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..cc0e7695c7c00bbc5a7f0ac36518eed1f5cbf104
--- /dev/null
+++ b/saved_results/20241126_222522/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7eb826ec17b61f0dc186821984736eaa6db995b32b2754d4f442926862893c28
+size 1004960
diff --git a/saved_results/20241126_222522/parameters.json b/saved_results/20241126_222522/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..541c6ed154aa64f9dc9ce0c7d2474d58dc1ae94c
--- /dev/null
+++ b/saved_results/20241126_222522/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Editing",
+ "prompt": "A cute rabbit with big eyes",
+ "edit_prompt": "A cute pig with big eyes",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 30,
+ "learning_rate": 0.4,
+ "max_source_steps": 5,
+ "optimization_steps": 5,
+ "true_cfg": 4.5
+}
\ No newline at end of file
diff --git a/saved_results/20241126_223634/input.png b/saved_results/20241126_223634/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..a29a7898249ae277f1899bd48a283a6f09bdcec1
Binary files /dev/null and b/saved_results/20241126_223634/input.png differ
diff --git a/saved_results/20241126_223634/mask.png b/saved_results/20241126_223634/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..ce8e75fad3c47e13f9bf2cfb68de2dedc9760025
Binary files /dev/null and b/saved_results/20241126_223634/mask.png differ
diff --git a/saved_results/20241126_223634/output.png b/saved_results/20241126_223634/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..a169969549aca141d041a77d3f77b2d02eb931c4
--- /dev/null
+++ b/saved_results/20241126_223634/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:49a55d834a56efe47f5c6a17d0f693af616eaa477132110f33b691692229551a
+size 1146868
diff --git a/saved_results/20241126_223634/parameters.json b/saved_results/20241126_223634/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..e686d9ce3a1c16f784e61a03d4199f34d5f09a73
--- /dev/null
+++ b/saved_results/20241126_223634/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Editing",
+ "prompt": "a cat",
+ "edit_prompt": "a tiger",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 30,
+ "learning_rate": 0.6,
+ "max_source_steps": 10,
+ "optimization_steps": 5,
+ "true_cfg": 4.5
+}
\ No newline at end of file
diff --git a/saved_results/20241126_223719/input.png b/saved_results/20241126_223719/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..a29a7898249ae277f1899bd48a283a6f09bdcec1
Binary files /dev/null and b/saved_results/20241126_223719/input.png differ
diff --git a/saved_results/20241126_223719/mask.png b/saved_results/20241126_223719/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..ce8e75fad3c47e13f9bf2cfb68de2dedc9760025
Binary files /dev/null and b/saved_results/20241126_223719/mask.png differ
diff --git a/saved_results/20241126_223719/output.png b/saved_results/20241126_223719/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..030a7e876d01c797f5513f12d095dfff9d82268e
--- /dev/null
+++ b/saved_results/20241126_223719/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f30135ee9cdf48f45dc82d8262d0fe4f9592c30f901ded2cac91ea89dfe234a
+size 1118660
diff --git a/saved_results/20241126_223719/parameters.json b/saved_results/20241126_223719/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..e686d9ce3a1c16f784e61a03d4199f34d5f09a73
--- /dev/null
+++ b/saved_results/20241126_223719/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Editing",
+ "prompt": "a cat",
+ "edit_prompt": "a tiger",
+ "seed": 0,
+ "randomize_seed": true,
+ "num_inference_steps": 30,
+ "max_steps": 30,
+ "learning_rate": 0.6,
+ "max_source_steps": 10,
+ "optimization_steps": 5,
+ "true_cfg": 4.5
+}
\ No newline at end of file
diff --git a/saved_results/20241127_025429/input.png b/saved_results/20241127_025429/input.png
new file mode 100644
index 0000000000000000000000000000000000000000..a9a0d7821c068f2200e906bf2983b9b048abb72f
Binary files /dev/null and b/saved_results/20241127_025429/input.png differ
diff --git a/saved_results/20241127_025429/mask.png b/saved_results/20241127_025429/mask.png
new file mode 100644
index 0000000000000000000000000000000000000000..fba17fb8723f9ba09017cb439af32bca69e6f7fa
Binary files /dev/null and b/saved_results/20241127_025429/mask.png differ
diff --git a/saved_results/20241127_025429/output.png b/saved_results/20241127_025429/output.png
new file mode 100644
index 0000000000000000000000000000000000000000..627b30d98bfe384597887acc93ba62512d59499c
--- /dev/null
+++ b/saved_results/20241127_025429/output.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a6c73b284eb402d622db24e0d16f9e9fb9f8b8517f184a589f007a37f1ef3cc
+size 1396508
diff --git a/saved_results/20241127_025429/parameters.json b/saved_results/20241127_025429/parameters.json
new file mode 100644
index 0000000000000000000000000000000000000000..8958ce0d8ca87e1a92438530a876df6efb928882
--- /dev/null
+++ b/saved_results/20241127_025429/parameters.json
@@ -0,0 +1,13 @@
+{
+ "mode": "Inpainting",
+ "prompt": "A building with \"ASU\" written on it.",
+ "edit_prompt": "",
+ "seed": 52,
+ "randomize_seed": false,
+ "num_inference_steps": 30,
+ "max_steps": 30,
+ "learning_rate": 1,
+ "max_source_steps": 20,
+ "optimization_steps": 10,
+ "true_cfg": 2
+}
\ No newline at end of file
diff --git a/scheduling_flow_match_euler_discrete.py b/scheduling_flow_match_euler_discrete.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d415dfc5023f22d177302ecf723bc2e7c761000
--- /dev/null
+++ b/scheduling_flow_match_euler_discrete.py
@@ -0,0 +1,464 @@
+# Copyright 2024 Stability AI, Katherine Crowson and The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from dataclasses import dataclass
+from typing import List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+
+from diffusers.configuration_utils import ConfigMixin, register_to_config
+from diffusers.utils import BaseOutput, logging
+from diffusers.schedulers.scheduling_utils import SchedulerMixin
+
+
+logger = logging.get_logger(__name__) # pylint: disable=invalid-name
+
+
+@dataclass
+class FlowMatchEulerDiscreteSchedulerOutput(BaseOutput):
+ """
+ Output class for the scheduler's `step` function output.
+
+ Args:
+ prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images):
+ Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the
+ denoising loop.
+ """
+
+ prev_sample: torch.FloatTensor
+
+
+class FlowMatchEulerDiscreteScheduler(SchedulerMixin, ConfigMixin):
+ """
+ Euler scheduler.
+
+ This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic
+ methods the library implements for all schedulers such as loading and saving.
+
+ Args:
+ num_train_timesteps (`int`, defaults to 1000):
+ The number of diffusion steps to train the model.
+ timestep_spacing (`str`, defaults to `"linspace"`):
+ The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and
+ Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.
+ shift (`float`, defaults to 1.0):
+ The shift value for the timestep schedule.
+ """
+
+ _compatibles = []
+ order = 1
+
+ @register_to_config
+ def __init__(
+ self,
+ num_train_timesteps: int = 1000,
+ shift: float = 1.0,
+ use_dynamic_shifting=False,
+ base_shift: Optional[float] = 0.5,
+ max_shift: Optional[float] = 1.15,
+ base_image_seq_len: Optional[int] = 256,
+ max_image_seq_len: Optional[int] = 4096,
+ ):
+ timesteps = np.linspace(1, num_train_timesteps, num_train_timesteps, dtype=np.float32)[::-1].copy()
+ timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32)
+
+ sigmas = timesteps / num_train_timesteps
+ if not use_dynamic_shifting:
+ # when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution
+ sigmas = shift * sigmas / (1 + (shift - 1) * sigmas)
+
+ self.timesteps = sigmas * num_train_timesteps
+
+ self._step_index = None
+ self._begin_index = None
+
+ self.sigmas = sigmas.to("cpu") # to avoid too much CPU/GPU communication
+ self.sigma_min = self.sigmas[-1].item()
+ self.sigma_max = self.sigmas[0].item()
+
+ @property
+ def step_index(self):
+ """
+ The index counter for current timestep. It will increase 1 after each scheduler step.
+ """
+ return self._step_index
+
+ @property
+ def begin_index(self):
+ """
+ The index for the first timestep. It should be set from pipeline with `set_begin_index` method.
+ """
+ return self._begin_index
+
+ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index
+ def set_begin_index(self, begin_index: int = 0):
+ """
+ Sets the begin index for the scheduler. This function should be run from pipeline before the inference.
+
+ Args:
+ begin_index (`int`):
+ The begin index for the scheduler.
+ """
+ self._begin_index = begin_index
+
+ def scale_noise(
+ self,
+ sample: torch.FloatTensor,
+ timestep: Union[float, torch.FloatTensor],
+ noise: Optional[torch.FloatTensor] = None,
+ ) -> torch.FloatTensor:
+ """
+ Forward process in flow-matching
+
+ Args:
+ sample (`torch.FloatTensor`):
+ The input sample.
+ timestep (`int`, *optional*):
+ The current timestep in the diffusion chain.
+
+ Returns:
+ `torch.FloatTensor`:
+ A scaled input sample.
+ """
+ # Make sure sigmas and timesteps have the same device and dtype as original_samples
+ sigmas = self.sigmas.to(device=sample.device, dtype=sample.dtype)
+
+ if sample.device.type == "mps" and torch.is_floating_point(timestep):
+ # mps does not support float64
+ schedule_timesteps = self.timesteps.to(sample.device, dtype=torch.float32)
+ timestep = timestep.to(sample.device, dtype=torch.float32)
+ else:
+ schedule_timesteps = self.timesteps.to(sample.device)
+ timestep = timestep.to(sample.device)
+
+ # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index
+ if self.begin_index is None:
+ step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timestep]
+ elif self.step_index is not None:
+ # add_noise is called after first denoising step (for inpainting)
+ step_indices = [self.step_index] * timestep.shape[0]
+ else:
+ # add noise is called before first denoising step to create initial latent(img2img)
+ step_indices = [self.begin_index] * timestep.shape[0]
+
+ sigma = sigmas[step_indices].flatten()
+ while len(sigma.shape) < len(sample.shape):
+ sigma = sigma.unsqueeze(-1)
+
+ sample = sigma * noise + (1.0 - sigma) * sample
+
+ return sample
+
+ def _sigma_to_t(self, sigma):
+ return sigma * self.config.num_train_timesteps
+
+ def time_shift(self, mu: float, sigma: float, t: torch.Tensor):
+ return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma)
+
+ def set_timesteps(
+ self,
+ num_inference_steps: int = None,
+ device: Union[str, torch.device] = None,
+ sigmas: Optional[List[float]] = None,
+ mu: Optional[float] = None,
+ ):
+ """
+ Sets the discrete timesteps used for the diffusion chain (to be run before inference).
+
+ Args:
+ num_inference_steps (`int`):
+ The number of diffusion steps used when generating samples with a pre-trained model.
+ device (`str` or `torch.device`, *optional*):
+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.
+ """
+
+ if self.config.use_dynamic_shifting and mu is None:
+ raise ValueError(" you have a pass a value for `mu` when `use_dynamic_shifting` is set to be `True`")
+
+ if sigmas is None:
+ self.num_inference_steps = num_inference_steps
+ timesteps = np.linspace(
+ self._sigma_to_t(self.sigma_max), self._sigma_to_t(self.sigma_min), num_inference_steps
+ )
+
+ sigmas = timesteps / self.config.num_train_timesteps
+
+ if self.config.use_dynamic_shifting:
+ sigmas = self.time_shift(mu, 1.0, sigmas)
+ else:
+ sigmas = self.config.shift * sigmas / (1 + (self.config.shift - 1) * sigmas)
+
+ sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32, device=device)
+ timesteps = sigmas * self.config.num_train_timesteps
+
+ self.timesteps = timesteps.to(device=device)
+ self.sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)])
+
+ self._step_index = None
+ self._begin_index = None
+
+ def index_for_timestep(self, timestep, schedule_timesteps=None):
+ if schedule_timesteps is None:
+ schedule_timesteps = self.timesteps
+
+ indices = (schedule_timesteps == timestep).nonzero()
+
+ # The sigma index that is taken for the **very** first `step`
+ # is always the second index (or the last index if there is only 1)
+ # This way we can ensure we don't accidentally skip a sigma in
+ # case we start in the middle of the denoising schedule (e.g. for image-to-image)
+ pos = 1 if len(indices) > 1 else 0
+
+ return indices[pos].item()
+
+ def _init_step_index(self, timestep):
+ if self.begin_index is None:
+ if isinstance(timestep, torch.Tensor):
+ timestep = timestep.to(self.timesteps.device)
+ self._step_index = self.index_for_timestep(timestep)
+ else:
+ self._step_index = self._begin_index
+
+ def step(
+ self,
+ model_output: torch.FloatTensor,
+ timestep: Union[float, torch.FloatTensor],
+ sample: torch.FloatTensor,
+ s_churn: float = 0.0,
+ s_tmin: float = 0.0,
+ s_tmax: float = float("inf"),
+ s_noise: float = 1.0,
+ generator: Optional[torch.Generator] = None,
+ return_dict: bool = True,
+ ) -> Union[FlowMatchEulerDiscreteSchedulerOutput, Tuple]:
+ """
+ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
+ process from the learned model outputs (most often the predicted noise).
+
+ Args:
+ model_output (`torch.FloatTensor`):
+ The direct output from learned diffusion model.
+ timestep (`float`):
+ The current discrete timestep in the diffusion chain.
+ sample (`torch.FloatTensor`):
+ A current instance of a sample created by the diffusion process.
+ s_churn (`float`):
+ s_tmin (`float`):
+ s_tmax (`float`):
+ s_noise (`float`, defaults to 1.0):
+ Scaling factor for noise added to the sample.
+ generator (`torch.Generator`, *optional*):
+ A random number generator.
+ return_dict (`bool`):
+ Whether or not to return a [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] or
+ tuple.
+
+ Returns:
+ [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] or `tuple`:
+ If return_dict is `True`, [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] is
+ returned, otherwise a tuple is returned where the first element is the sample tensor.
+ """
+
+ if (
+ isinstance(timestep, int)
+ or isinstance(timestep, torch.IntTensor)
+ or isinstance(timestep, torch.LongTensor)
+ ):
+ raise ValueError(
+ (
+ "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to"
+ " `EulerDiscreteScheduler.step()` is not supported. Make sure to pass"
+ " one of the `scheduler.timesteps` as a timestep."
+ ),
+ )
+
+ if self.step_index is None:
+ self._init_step_index(timestep)
+
+ # Upcast to avoid precision issues when computing prev_sample
+ sample = sample.to(torch.float32)
+
+ sigma = self.sigmas[self.step_index]
+ sigma_next = self.sigmas[self.step_index + 1]
+
+ prev_sample = sample + (sigma_next - sigma) * model_output
+
+ # Cast sample back to model compatible dtype
+ prev_sample = prev_sample.to(model_output.dtype)
+
+ # upon completion increase step index by one
+ self._step_index += 1
+
+ if not return_dict:
+ return (prev_sample,)
+
+ return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=prev_sample)
+
+ def step_no_update(
+ self,
+ model_output: torch.FloatTensor,
+ timestep: Union[float, torch.FloatTensor],
+ sample: torch.FloatTensor,
+ s_churn: float = 0.0,
+ s_tmin: float = 0.0,
+ s_tmax: float = float("inf"),
+ s_noise: float = 1.0,
+ generator: Optional[torch.Generator] = None,
+ return_dict: bool = True,
+ ) -> Union[FlowMatchEulerDiscreteSchedulerOutput, Tuple]:
+ """
+ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
+ process from the learned model outputs (most often the predicted noise).
+
+ Args:
+ model_output (`torch.FloatTensor`):
+ The direct output from learned diffusion model.
+ timestep (`float`):
+ The current discrete timestep in the diffusion chain.
+ sample (`torch.FloatTensor`):
+ A current instance of a sample created by the diffusion process.
+ s_churn (`float`):
+ s_tmin (`float`):
+ s_tmax (`float`):
+ s_noise (`float`, defaults to 1.0):
+ Scaling factor for noise added to the sample.
+ generator (`torch.Generator`, *optional*):
+ A random number generator.
+ return_dict (`bool`):
+ Whether or not to return a [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] or
+ tuple.
+
+ Returns:
+ [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] or `tuple`:
+ If return_dict is `True`, [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] is
+ returned, otherwise a tuple is returned where the first element is the sample tensor.
+ """
+
+ if (
+ isinstance(timestep, int)
+ or isinstance(timestep, torch.IntTensor)
+ or isinstance(timestep, torch.LongTensor)
+ ):
+ raise ValueError(
+ (
+ "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to"
+ " `EulerDiscreteScheduler.step()` is not supported. Make sure to pass"
+ " one of the `scheduler.timesteps` as a timestep."
+ ),
+ )
+
+ if self.step_index is None:
+ self._init_step_index(timestep)
+
+ # Upcast to avoid precision issues when computing prev_sample
+ sample = sample.to(torch.float32)
+
+ sigma = self.sigmas[self.step_index]
+ sigma_next = self.sigmas[self.step_index + 1]
+
+ prev_sample = sample + (sigma_next - sigma) * model_output
+
+ # Cast sample back to model compatible dtype
+ prev_sample = prev_sample.to(model_output.dtype)
+
+ # Note: we do not need to do this if we are just testing the model
+ # # upon completion increase step index by one
+ # self._step_index += 1
+
+ if not return_dict:
+ return (prev_sample,)
+
+ return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=prev_sample)
+
+ def step_final(
+ self,
+ model_output: torch.FloatTensor,
+ timestep: Union[float, torch.FloatTensor],
+ sample: torch.FloatTensor,
+ s_churn: float = 0.0,
+ s_tmin: float = 0.0,
+ s_tmax: float = float("inf"),
+ s_noise: float = 1.0,
+ generator: Optional[torch.Generator] = None,
+ return_dict: bool = True,
+ ) -> Union[FlowMatchEulerDiscreteSchedulerOutput, Tuple]:
+ """
+ Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
+ process from the learned model outputs (most often the predicted noise).
+
+ Args:
+ model_output (`torch.FloatTensor`):
+ The direct output from learned diffusion model.
+ timestep (`float`):
+ The current discrete timestep in the diffusion chain.
+ sample (`torch.FloatTensor`):
+ A current instance of a sample created by the diffusion process.
+ s_churn (`float`):
+ s_tmin (`float`):
+ s_tmax (`float`):
+ s_noise (`float`, defaults to 1.0):
+ Scaling factor for noise added to the sample.
+ generator (`torch.Generator`, *optional*):
+ A random number generator.
+ return_dict (`bool`):
+ Whether or not to return a [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] or
+ tuple.
+
+ Returns:
+ [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] or `tuple`:
+ If return_dict is `True`, [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] is
+ returned, otherwise a tuple is returned where the first element is the sample tensor.
+ """
+
+ if (
+ isinstance(timestep, int)
+ or isinstance(timestep, torch.IntTensor)
+ or isinstance(timestep, torch.LongTensor)
+ ):
+ raise ValueError(
+ (
+ "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to"
+ " `EulerDiscreteScheduler.step()` is not supported. Make sure to pass"
+ " one of the `scheduler.timesteps` as a timestep."
+ ),
+ )
+
+ if self.step_index is None:
+ self._init_step_index(timestep)
+
+ # Upcast to avoid precision issues when computing prev_sample
+ sample = sample.to(torch.float32)
+
+ sigma = self.sigmas[self.step_index]
+ sigma_next = self.sigmas[-1]
+
+ prev_sample = sample + (sigma_next - sigma) * model_output
+
+ # Cast sample back to model compatible dtype
+ prev_sample = prev_sample.to(model_output.dtype)
+
+ # Note: we do not need to do this to estimate final image
+ # # upon completion increase step index by one
+ # self._step_index += 1
+
+ if not return_dict:
+ return (prev_sample,)
+
+ return FlowMatchEulerDiscreteSchedulerOutput(prev_sample=prev_sample)
+
+
+ def __len__(self):
+ return self.config.num_train_timesteps
\ No newline at end of file