app.py

import gradio as gr
from huggingface_hub import login
import os
import spaces
from diffusers.schedulers import UniPCMultistepScheduler
from diffusers import StableDiffusionXLAdapterPipeline, T2IAdapter, EulerAncestralDiscreteScheduler, AutoencoderKL
from diffusers.utils import load_image, make_image_grid
import torch

# Log in to Hugging Face with your token
token = os.getenv("HF_TOKEN")
login(token=token)

model_id = 'stabilityai/sdxl-turbo'

adapter = StableDiffusionXLAdapterPipeline.load_ip_adapter("TencentARC/T2I-Adapter",
                     subfolder="models",
                     weight_name="t2iadapter_style_sd14v1.pth")

euler_a = EulerAncestralDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler")
vae=AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix",)
pipe = StableDiffusionXLAdapterPipeline.from_pretrained(
    model_id, vae=vae, adapter=adapter, scheduler=euler_a, variant="fp16",
)
pipe.enable_xformers_memory_efficient_attention()
pipe.to("cuda", torch.float16)

# controlnet = SD3ControlNetModel.from_pretrained("alimama-creative/SD3-Controlnet-Softedge", torch_dtype=torch.float16)
#
# pipe = StableDiffusion3ControlNetPipeline.from_pretrained("stabilityai/stable-diffusion-3-medium-diffusers", controlnet=controlnet)
# pipe.to("cuda", torch.float16)


@spaces.GPU
def generate_image(prompt, reference_image, controlnet_conditioning_scale):

    # Generate the image with ControlNet conditioning
    generated_image = pipe(
        prompt=prompt,
        ip_adapter_image=load_image(reference_image),
        controlnet_conditioning_scale=controlnet_conditioning_scale,
    ).images[0]
    return generated_image

# Set up Gradio interface
interface = gr.Interface(
    fn=generate_image,
    inputs=[
        gr.Textbox(label="Prompt"),
        gr.Image( type= "filepath",label="Reference Image (Style)"),
        gr.Slider(label="Control Net Conditioning Scale", minimum=0, maximum=1.0, step=0.1, value=0.6),
    ],
    outputs="image",
    title="Image Generation with Stable Diffusion 3 medium and ControlNet",
    description="Generates an image based on a text prompt and a reference image using Stable Diffusion 3 medium with ControlNet."
)

interface.launch()