Update main app

Initial commit to convert from prompt to image input.

app.py

@@ -1,21 +1,41 @@
 import gradio as gr
 import numpy as np
-import random
+
+# import random
 
 # import spaces #[uncomment to use ZeroGPU]
-from diffusers import DiffusionPipeline
+from diffusers import (
+    StableDiffusionControlNetImg2ImgPipeline,
+    ControlNetModel,
+)
 import torch
 
+import requests
+from fastapi import FastAPI, HTTPException
+from PIL import Image
+from controlnet_aux import CannyDetector
+
 device = "cuda" if torch.cuda.is_available() else "cpu"
-model_repo_id = "stabilityai/sdxl-turbo"  # Replace to the model you would like to use
+# model_repo_id = "stabilityai/sdxl-turbo"  # Replace to the model you would like to use
+model_repo_id = "runwayml/stable-diffusion-v1-5"
 
 if torch.cuda.is_available():
     torch_dtype = torch.float16
 else:
     torch_dtype = torch.float32
 
-pipe = DiffusionPipeline.from_pretrained(model_repo_id, torch_dtype=torch_dtype)
+controlnet = ControlNetModel.from_pretrained(
+    "lllyasviel/sd-controlnet-canny", torch_dtype=torch.float32
+)
+
+# pipe = DiffusionPipeline.from_pretrained(model_repo_id, torch_dtype=torch_dtype)
+pipe = StableDiffusionControlNetImg2ImgPipeline.from_pretrained(
+    model_repo_id,
+    controlnet=controlnet,
+    torch_dtype=torch_dtype,
+).to(device)
 pipe = pipe.to(device)
+canny = CannyDetector()
 
 MAX_SEED = np.iinfo(np.int32).max
 MAX_IMAGE_SIZE = 1024
@@ -23,32 +43,67 @@ MAX_IMAGE_SIZE = 1024
 
 # @spaces.GPU #[uncomment to use ZeroGPU]
 def infer(
-    prompt,
-    negative_prompt,
-    seed,
-    randomize_seed,
+    image_url,
+    # negative_prompt,
+    # seed,
+    # randomize_seed,
     width,
     height,
     guidance_scale,
     num_inference_steps,
     progress=gr.Progress(track_tqdm=True),
 ):
-    if randomize_seed:
-        seed = random.randint(0, MAX_SEED)
-
-    generator = torch.Generator().manual_seed(seed)
+    # if randomize_seed:
+    #     seed = random.randint(0, MAX_SEED)
+
+    # generator = torch.Generator().manual_seed(seed)
+
+    # image = pipe(
+    #     prompt=prompt,
+    #     negative_prompt=negative_prompt,
+    #     guidance_scale=guidance_scale,
+    #     num_inference_steps=num_inference_steps,
+    #     width=width,
+    #     height=height,
+    #     generator=generator,
+    # ).images[0]
+
+    # return image, seed
+
+    width = int(width)
+    height = int(height)
+
+    try:
+        resp = requests.get(image_url)
+        resp.raise_for_status()
+    except Exception as e:
+        raise HTTPException(400, f"Could not download image: {e}")
+
+    # img = Image.open(io.BytesIO(resp.content)).convert("RGB")
+    img = Image.open(requests.get(image_url, stream=True).raw).convert("RGB")
+    # img = img.resize((req.width, req.height))
+    img = img.resize((width, height))
+
+    control_net_image = canny(img).resize((width, height))
+
+    prompt = (
+        "redraw the logo from scratch, clean sharp vector-style, "
+        # + STYLE_PROMPTS[req.style_preset]
+    )
 
-    image = pipe(
+    output = pipe(
         prompt=prompt,
-        negative_prompt=negative_prompt,
+        negative_prompt=NEGATIVE,
+        image=img,
+        control_image=control_net_image,
+        # strength=req.strength,
         guidance_scale=guidance_scale,
         num_inference_steps=num_inference_steps,
-        width=width,
         height=height,
-        generator=generator,
+        width=width,
     ).images[0]
 
-    return image, seed
+    return output
 
 
 examples = [
@@ -64,16 +119,20 @@ css = """
 }
 """
 
+NEGATIVE = "blurry, distorted, messy, gradients, background noise"
+WIDTH = 512
+HEIGHT = 512
+
 with gr.Blocks(css=css) as demo:
     with gr.Column(elem_id="col-container"):
         gr.Markdown(" # Text-to-Image Gradio Template")
 
         with gr.Row():
-            prompt = gr.Text(
-                label="Prompt",
+            image_url = gr.Text(
+                label="Image URL",
                 show_label=False,
-                max_lines=1,
-                placeholder="Enter your prompt",
+                # max_lines=1,
+                placeholder="Provide a image URL",
                 container=False,
             )
 
@@ -82,38 +141,40 @@ with gr.Blocks(css=css) as demo:
         result = gr.Image(label="Result", show_label=False)
 
         with gr.Accordion("Advanced Settings", open=False):
-            negative_prompt = gr.Text(
-                label="Negative prompt",
-                max_lines=1,
-                placeholder="Enter a negative prompt",
-                visible=False,
+            negative_prompt = gr.Label(
+                label="Negative prompts",
+                # max_lines=1,
+                value=NEGATIVE,
+                visible=True,
             )
 
-            seed = gr.Slider(
-                label="Seed",
-                minimum=0,
-                maximum=MAX_SEED,
-                step=1,
-                value=0,
-            )
+            # seed = gr.Slider(
+            #     label="Seed",
+            #     minimum=0,
+            #     maximum=MAX_SEED,
+            #     step=1,
+            #     value=0,
+            # )
 
-            randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
+            # randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
 
             with gr.Row():
-                width = gr.Slider(
+                width = gr.Label(
                     label="Width",
-                    minimum=256,
-                    maximum=MAX_IMAGE_SIZE,
-                    step=32,
-                    value=1024,  # Replace with defaults that work for your model
+                    value=WIDTH,
+                    # minimum=256,
+                    # maximum=MAX_IMAGE_SIZE,
+                    # step=32,
+                    # value=1024,  # Replace with defaults that work for your model
                 )
 
-                height = gr.Slider(
+                height = gr.Label(
                     label="Height",
-                    minimum=256,
-                    maximum=MAX_IMAGE_SIZE,
-                    step=32,
-                    value=1024,  # Replace with defaults that work for your model
+                    value=HEIGHT,
+                    # minimum=256,
+                    # maximum=MAX_IMAGE_SIZE,
+                    # step=32,
+                    # value=1024,  # Replace with defaults that work for your model
                 )
 
             with gr.Row():
@@ -122,7 +183,7 @@ with gr.Blocks(css=css) as demo:
                     minimum=0.0,
                     maximum=10.0,
                     step=0.1,
-                    value=0.0,  # Replace with defaults that work for your model
+                    value=8.5,  # Replace with defaults that work for your model
                 )
 
                 num_inference_steps = gr.Slider(
@@ -130,24 +191,27 @@ with gr.Blocks(css=css) as demo:
                     minimum=1,
                     maximum=50,
                     step=1,
-                    value=2,  # Replace with defaults that work for your model
+                    value=25,  # Replace with defaults that work for your model
                 )
 
-        gr.Examples(examples=examples, inputs=[prompt])
+        # gr.Examples(examples=examples, inputs=[prompt])
     gr.on(
-        triggers=[run_button.click, prompt.submit],
+        triggers=[run_button.click, image_url.submit],
         fn=infer,
         inputs=[
-            prompt,
-            negative_prompt,
-            seed,
-            randomize_seed,
+            image_url,
+            # negative_prompt,
+            # seed,
+            # randomize_seed,
             width,
             height,
             guidance_scale,
             num_inference_steps,
         ],
-        outputs=[result, seed],
+        outputs=[
+            result,
+            # seed,
+        ],
     )
 
 if __name__ == "__main__":

requirements.txt

@@ -3,4 +3,11 @@ diffusers
 invisible_watermark
 torch
 transformers
-xformers
+# xformers
+fastapi
+uvicorn
+pydantic
+requests
+Pillow
+controlnet-aux
+gradio