Update app.py

app.py

@@ -85,23 +85,23 @@ def load_learned_embed_in_clip(learned_embeds_path, text_encoder, tokenizer, tok
 
 # ----- ControlNet Canny Edges Pipe / Setup -----------------------------------------------------------------
 
-import gradio as gr
-from PIL import Image
-import numpy as np
-import cv2
+# import gradio as gr
+# from PIL import Image
+# import numpy as np
+# import cv2
 
-from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
-from diffusers import UniPCMultistepScheduler
-import torch
+# from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
+# from diffusers import UniPCMultistepScheduler
+# import torch
 
-controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
-controlnet_pipe = StableDiffusionControlNetPipeline.from_pretrained(
-    "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
-)
+# controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
+# controlnet_pipe = StableDiffusionControlNetPipeline.from_pretrained(
+#     "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
+# )
 
-controlnet_pipe.scheduler = UniPCMultistepScheduler.from_config(controlnet_pipe.scheduler.config)
-controlnet_pipe.enable_model_cpu_offload()
-controlnet_pipe.enable_xformers_memory_efficient_attention()
+# controlnet_pipe.scheduler = UniPCMultistepScheduler.from_config(controlnet_pipe.scheduler.config)
+# controlnet_pipe.enable_model_cpu_offload()
+# controlnet_pipe.enable_xformers_memory_efficient_attention()
 
 
 
@@ -443,27 +443,27 @@ canny_interface = gr.Interface(fn=canny_process_image, inputs=[canny_input_image
 
 
 
-# ----- ControlNet Canny Gradio Setup -----------------------------------------------------------------
+# ----- New ControlNet Canny Gradio Setup with Block -----------------------------------------------------------------
 
-# import gradio as gr
-# from PIL import Image
-# import numpy as np
-# import cv2
 
-# from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
-# from diffusers import UniPCMultistepScheduler
-# import torch
+from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
+from diffusers import UniPCMultistepScheduler
+from PIL import Image
+import gradio as gr
+import numpy as np
+import torch
+import cv2
 
-# controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
-# controlnet_pipe = StableDiffusionControlNetPipeline.from_pretrained(
-#     "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
-# )
+controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
+controlnet_pipe = StableDiffusionControlNetPipeline.from_pretrained(
+    "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
+)
 
-# controlnet_pipe.scheduler = UniPCMultistepScheduler.from_config(controlnet_pipe.scheduler.config)
-# controlnet_pipe.enable_model_cpu_offload()
-# controlnet_pipe.enable_xformers_memory_efficient_attention()
+controlnet_pipe.scheduler = UniPCMultistepScheduler.from_config(controlnet_pipe.scheduler.config)
+controlnet_pipe.enable_model_cpu_offload()
 
-def controlnet_edges(canny_input_prompt, input_image, input_low_threshold, input_high_threshold, input_invert):
+
+def controlnet_edges(canny_input_prompt, input_image, input_low_threshold, input_high_threshold, input_invert, canny_input_seed):
     np_image = np.array(input_image)
 
     output_image = input_image
@@ -478,25 +478,131 @@ def controlnet_edges(canny_input_prompt, input_image, input_low_threshold, input
       canny_1 = 255 - canny_1
 
     canny_2 = Image.fromarray(canny_1)
+    canny_1 = Image.fromarray(canny_1)
+
+    input_width, input_height = canny_2.size
+
+    limit_size = 768
+    # limit_size = 32
+
+    if input_width > input_height:
+        new_width = min(input_width, limit_size)
+        new_height = int(new_width * input_height / input_width)
+    else:
+        new_height = min(input_height, limit_size)
+        new_width = int(new_height * input_width / input_height)
+
+    canny_2 = canny_2.resize((new_width, new_height))
+    canny_1 = canny_1.resize((new_width, new_height))
 
     prompt = canny_input_prompt
-    generator = torch.Generator(device="cpu").manual_seed(2)
-
-    # output_image = controlnet_pipe(
-    #     prompt,
-    #     canny_2,
-    #     negative_prompt="monochrome, lowres, bad anatomy, worst quality, low quality",
-    #     generator=generator,
-    #     num_inference_steps=20,
-    # )
+    generator = torch.Generator(device="cpu").manual_seed(canny_input_seed)
+
     output_image = controlnet_pipe(
         prompt,
         canny_2,
         negative_prompt="monochrome, lowres, bad anatomy, worst quality, low quality",
+        generator=generator,
         num_inference_steps=20,
     )
 
-    return output_image[0][0]
+    return [canny_1, output_image[0][0]]
+    # return output_image[0][0]
+
+import random 
+def random_seed():
+  return random.randint(0, 99999999999999)
+
+
+with gr.Blocks() as canny_blocks_interface:
+  gr.Markdown('''
+      # <span style="display: inline-block; height: 30px; width: 30px; margin-bottom: -3px; border-radius: 7px; background-size: 50px; background-position: center; background-image: url(http://www.astronaut.horse/thumbnail.jpg)"></span> ControlNet + Canny Edge-Tracing
+      This tool allows you to apply a Stable Diffusion text prompt to an existing image composition using an edge-tracing tool called Canny Edge Detector. Note that you cannot currently apply trained artist concepts from the other tabs in this application to this process currently as they were trained using a more recent version of Stable Diffusion. 
+      <br><br>
+      <a href="https://wikipedia.org/wiki/Canny_edge_detector">https://wikipedia.org/wiki/canny_edge_detector</a>
+      <br>
+      <a href="http://www.astronaut.horse">http://www.astronaut.horse</a>
+      <br>
+      <a href="https://discord.gg/ZctfW4SvGw">https://discord.com</a><br>
+      <br>
+  ''')
+  with gr.Row():
+    with gr.Column():
+      canny_input_prompt = gr.inputs.Textbox(label="Enter a single word or phrase")
+      canny_input_low_threshold = gr.inputs.Slider(minimum=0, maximum=1000, step=1, label="Lower Threshold:", default=100)
+      canny_input_high_threshold = gr.inputs.Slider(minimum=0, maximum=1000, step=1, label="Upper Threshold:", default=120)
+      canny_input_seed = gr.Slider(0, 99999999999999, label="seed", dtype=int, value=random_seed, interactive=True, step=1)
+      canny_input_invert = gr.inputs.Checkbox(label="Invert Image")
+    with gr.Column():
+      canny_input_image = gr.inputs.Image(label="input image")
+      go_button = gr.Button('generate image')
+  # with gr.Row():
+      with gr.Accordion(label='traced edge image', open=False):
+        canny_output_1 = gr.outputs.Image(type="pil", label="traced edges")
+  with gr.Row():
+    canny_output_2 = gr.outputs.Image(type="pil", label="final image")
+  go_button.click(fn=controlnet_edges, inputs=[canny_input_prompt, canny_input_image, canny_input_low_threshold, canny_input_high_threshold, canny_input_invert, canny_input_seed], outputs=[canny_output_1, canny_output_2])
+
+
+# canny_blocks_interface.launch(debug=False)
+
+
+
+
+# ----- Old ControlNet Canny Gradio Setup without Block (working) -----------------------------------------------------------------
+
+# import gradio as gr
+# from PIL import Image
+# import numpy as np
+# import cv2
+
+# from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
+# from diffusers import UniPCMultistepScheduler
+# import torch
+
+# controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
+# controlnet_pipe = StableDiffusionControlNetPipeline.from_pretrained(
+#     "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16
+# )
+
+# controlnet_pipe.scheduler = UniPCMultistepScheduler.from_config(controlnet_pipe.scheduler.config)
+# controlnet_pipe.enable_model_cpu_offload()
+# controlnet_pipe.enable_xformers_memory_efficient_attention()
+
+# def controlnet_edges(canny_input_prompt, input_image, input_low_threshold, input_high_threshold, input_invert):
+#     np_image = np.array(input_image)
+
+#     output_image = input_image
+#     numpy_image = np.array(output_image)
+
+#     low_threshold = 80
+#     high_threshold = 100
+#     canny_1 = cv2.Canny(numpy_image, input_low_threshold, input_high_threshold)
+#     canny_1 = canny_1[:, :, None]
+#     canny_1 = np.concatenate([canny_1, canny_1, canny_1], axis=2)
+#     if input_invert:
+#       canny_1 = 255 - canny_1
+
+#     canny_2 = Image.fromarray(canny_1)
+
+#     prompt = canny_input_prompt
+#     generator = torch.Generator(device="cpu").manual_seed(2)
+
+#     # output_image = controlnet_pipe(
+#     #     prompt,
+#     #     canny_2,
+#     #     negative_prompt="monochrome, lowres, bad anatomy, worst quality, low quality",
+#     #     generator=generator,
+#     #     num_inference_steps=20,
+#     # )
+#     output_image = controlnet_pipe(
+#         prompt,
+#         canny_2,
+#         negative_prompt="monochrome, lowres, bad anatomy, worst quality, low quality",
+#         num_inference_steps=20,
+#     )
+
+#     return output_image[0][0]
 
 
 canny_input_prompt = gr.inputs.Textbox(label="Enter a single word or phrase")
@@ -514,6 +620,6 @@ controlnet_canny_interface = gr.Interface(fn=controlnet_edges, inputs=[canny_inp
 
 # ----- Launch Tabs -----------------------------------------------------------------
 
-tabbed_interface = gr.TabbedInterface([new_welcome, advanced_tab, beta, canny_interface, controlnet_canny_interface], ["Artbots", "Advanced", "Beta", "Edges", "ControlNet"])
+tabbed_interface = gr.TabbedInterface([new_welcome, advanced_tab, beta, canny_blocks_interface], ["Artbots", "Advanced", "Beta", "ControlNet"])
 # tabbed_interface = gr.TabbedInterface([new_welcome, advanced_tab, beta], ["Artbots", "Advanced", "Beta"])
 tabbed_interface.launch()