import io import os import shutil import uuid import torch import random import spaces import gradio as gr import numpy as np from PIL import Image, ImageCms import torch from diffusers import FluxTransformer2DModel from diffusers.utils import load_image from pipeline_flux_control_removal import FluxControlRemovalPipeline pipe = None torch.set_grad_enabled(False) image_examples = [ [ "example/image/3c43156c-2b44-4ebf-9c47-7707ec60b166.png", "example/mask/3c43156c-2b44-4ebf-9c47-7707ec60b166.png" ], [ "example/image/0e5124d8-fe43-4b5c-819f-7212f23a6d2a.png", "example/mask/0e5124d8-fe43-4b5c-819f-7212f23a6d2a.png" ], [ "example/image/0f900fe8-6eab-4f85-8121-29cac9509b94.png", "example/mask/0f900fe8-6eab-4f85-8121-29cac9509b94.png" ], [ "example/image/3ed1ee18-33b0-4964-b679-0e214a0d8848.png", "example/mask/3ed1ee18-33b0-4964-b679-0e214a0d8848.png" ], [ "example/image/87cdf3e2-0fa1-4d80-a228-cbb4aba3f44f.png", "example/mask/87cdf3e2-0fa1-4d80-a228-cbb4aba3f44f.png" ], [ "example/image/55dd199b-d99b-47a2-a691-edfd92233a6b.png", "example/mask/55dd199b-d99b-47a2-a691-edfd92233a6b.png" ] ] base_model_path = 'black-forest-labs/FLUX.1-dev' lora_path = 'theSure/Omnieraser' transformer = FluxTransformer2DModel.from_pretrained(base_model_path, subfolder='transformer', torch_dtype=torch.bfloat16) gr.Info(str(f"Model loading: {int((40 / 100) * 100)}%")) with torch.no_grad(): initial_input_channels = transformer.config.in_channels new_linear = torch.nn.Linear( transformer.x_embedder.in_features*4, transformer.x_embedder.out_features, bias=transformer.x_embedder.bias is not None, dtype=transformer.dtype, device=transformer.device, ) new_linear.weight.zero_() new_linear.weight[:, :initial_input_channels].copy_(transformer.x_embedder.weight) if transformer.x_embedder.bias is not None: new_linear.bias.copy_(transformer.x_embedder.bias) transformer.x_embedder = new_linear transformer.register_to_config(in_channels=initial_input_channels*4) pipe = FluxControlRemovalPipeline.from_pretrained( base_model_path, transformer=transformer, torch_dtype=torch.bfloat16 ).to("cuda") pipe.transformer.to(torch.bfloat16) gr.Info(str(f"Model loading: {int((80 / 100) * 100)}%")) gr.Info(str(f"Inject LoRA: {lora_path}")) pipe.load_lora_weights(lora_path, weight_name="pytorch_lora_weights.safetensors") gr.Info(str(f"Model loading: {int((100 / 100) * 100)}%")) @spaces.GPU def set_seed(seed): torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.cuda.manual_seed_all(seed) np.random.seed(seed) random.seed(seed) @spaces.GPU def predict( input_image, uploaded_mask, prompt, ddim_steps, seed, scale, ): gr.Info(str(f"Set seed = {seed}")) size1, size2 = input_image.convert("RGB").size icc_profile = input_image.info.get('icc_profile') if icc_profile: gr.Info(str(f"Image detected to contain ICC profile, converting color space to sRGB...")) srgb_profile = ImageCms.createProfile("sRGB") io_handle = io.BytesIO(icc_profile) src_profile = ImageCms.ImageCmsProfile(io_handle) input_image = ImageCms.profileToProfile(input_image, src_profile, srgb_profile) input_image.info.pop('icc_profile', None) if size1 < size2: input_image = input_image.convert("RGB").resize((1024, int(size2 / size1 * 1024))) else: input_image = input_image.convert("RGB").resize((int(size1 / size2 * 1024), 1024)) img = np.array(input_image.convert("RGB")) W = int(np.shape(img)[1] - np.shape(img)[1] % 16) H = int(np.shape(img)[0] - np.shape(img)[0] % 16) input_image = input_image.resize((H, W)) uploaded_mask = uploaded_mask.resize((H, W)) if seed == -1: seed = random.randint(1, 2147483647) set_seed(random.randint(1, 2147483647)) else: set_seed(seed) base_model_path = 'black-forest-labs/FLUX.1-dev' lora_path = 'theSure/Omnieraser' result = pipe( prompt=prompt, control_image=input_image.convert("RGB"), control_mask=uploaded_mask.convert("RGB"), width=W, height=H, num_inference_steps=ddim_steps, generator=torch.Generator("cuda").manual_seed(seed), guidance_scale=scale, max_sequence_length=512, ).images[0] mask_np = np.array(uploaded_mask.convert("RGB")) red = np.array(input_image).astype("float") * 1 red[:, :, 0] = 180.0 red[:, :, 2] = 0 red[:, :, 1] = 0 result_m = np.array(input_image) result_m = Image.fromarray( ( result_m.astype("float") * (1 - mask_np.astype("float") / 512.0) + mask_np.astype("float") / 512.0 * red ).astype("uint8") ) dict_res = [input_image, uploaded_mask, result_m, result] dict_out = [result] image_path = None mask_path = None return dict_out, dict_res def infer( input_image, uploaded_mask, ddim_steps, seed, scale, removal_prompt, ): return predict(input_image, uploaded_mask, removal_prompt, ddim_steps, seed, scale, ) def process_example(image_paths, mask_paths): global image_path, mask_path image = Image.open(image_paths).convert("RGB") mask = Image.open(mask_paths).convert("L") black_background = Image.new("RGB", image.size, (0, 0, 0)) masked_image = Image.composite(black_background, image, mask) image_path = image_paths mask_path = mask_paths return masked_image custom_css = """ .contain { max-width: 1200px !important; } .custom-image { border: 2px dashed #7e22ce !important; border-radius: 12px !important; transition: all 0.3s ease !important; } .custom-image:hover { border-color: #9333ea !important; box-shadow: 0 4px 15px rgba(158, 109, 202, 0.2) !important; } .btn-primary { background: linear-gradient(45deg, #7e22ce, #9333ea) !important; border: none !important; color: white !important; border-radius: 8px !important; } #inline-examples { border: 1px solid #e2e8f0 !important; border-radius: 12px !important; padding: 16px !important; margin-top: 8px !important; } #inline-examples .thumbnail { border-radius: 8px !important; transition: transform 0.2s ease !important; } #inline-examples .thumbnail:hover { transform: scale(1.05); box-shadow: 0 4px 6px -1px rgba(0, 0, 0, 0.1); } .example-title h3 { margin: 0 0 12px 0 !important; color: #475569 !important; font-size: 1.1em !important; display: flex !important; align-items: center !important; } .example-title h3::before { content: "📚"; margin-right: 8px; font-size: 1.2em; } .row { align-items: stretch !important; } .panel { height: 100%; } """ with gr.Blocks( css=custom_css, theme=gr.themes.Soft( primary_hue="purple", secondary_hue="purple", font=[gr.themes.GoogleFont('Inter'), 'sans-serif'] ), title="Omnieraser" ) as demo: ddim_steps = gr.Slider(visible=False, value=28) scale = gr.Slider(visible=False, value=3.5) seed = gr.Slider(visible=False, value=-1) removal_prompt = gr.Textbox(visible=False, value="There is nothing here.") gr.Markdown("""

🪄 Omnieraser

""") gr.Markdown(""" This is the demo of the paper "OmniEraser: Remove Objects and Their Effects in Images with Paired Video-Frame Data". To use this application: 1. Upload an image. 2. Upload a pre-defined mask (Unfortunatey, you cannot sketch mask here due to the compatibility issues with zerogpu, if you need draw mask manually, please use our offline gradio script available in our GitHub repository). 3. Set the seed (default is 1234). 4. Click 'Start Processing' to process the image. 5. The result will be displayed. Note: The mask should be a binary image where the object to be removed is white and the background is black. More details can be found at our [GitHub Repository](https://github.com/PRIS-CV/Omnieraser). """) with gr.Row(equal_height=False): with gr.Column(scale=1, variant="panel"): gr.Markdown("## 📥 Input Panel") with gr.Group(): input_image = gr.Image(label="Upload Image", type="pil", image_mode="RGB") uploaded_mask = gr.Image(label="Upload Mask", type="pil", image_mode="L") with gr.Row(variant="compact"): run_button = gr.Button( "🚀 Start Processing", variant="primary", size="lg" ) with gr.Group(): gr.Markdown("### ⚙️ Control Parameters") seed = gr.Slider( label="Random Seed", minimum=-1, maximum=2147483647, value=1234, step=1, info="-1 for random generation" ) with gr.Column(scale=1, variant="panel"): gr.Markdown("## 📤 Output Panel") with gr.Tabs(): with gr.Tab("Final Result"): inpaint_result = gr.Gallery( label="Generated Image", columns=2, height=450, preview=True, object_fit="contain" ) with gr.Tab("Visualization Steps"): gallery = gr.Gallery( label="Workflow Steps", columns=2, height=450, object_fit="contain" ) with gr.Column(scale=1, variant="panel"): with gr.Column(variant="panel"): gr.Markdown("### 🖼️ Example Gallery", elem_classes=["example-title"]) example = gr.Examples( examples=image_examples, inputs=[ input_image, uploaded_mask ], outputs=[inpaint_result, gallery], examples_per_page=10, label="Click any example to load", elem_id="inline-examples" ) run_button.click( fn=infer, inputs=[ input_image, uploaded_mask, ddim_steps, seed, scale, removal_prompt, ], outputs=[inpaint_result, gallery] ) demo.launch()