app.py

import torch
from diffusers.models import MotionAdapter
from diffusers import AnimateDiffSDXLPipeline, DDIMScheduler
from diffusers.utils import export_to_gif

import gradio as gr
from huggingface_hub import login
import os
import spaces,tempfile
import torch
from diffusers import StableDiffusionXLPipeline
from PIL import Image
import torch
from diffusers import AutoPipelineForText2Image, DDIMScheduler
from diffusers import AutoPipelineForText2Image
from diffusers.utils import load_image
import torch
from diffusers.models import MotionAdapter
from diffusers import AnimateDiffSDXLPipeline, DDIMScheduler
from diffusers.utils import export_to_gif

token = os.getenv("HF_TOKEN")
login(token=token)


adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-sdxl-beta", torch_dtype=torch.float16)

model_id = "stabilityai/sdxl-turbo"
scheduler = DDIMScheduler.from_pretrained(
    model_id,
    subfolder="scheduler",
    clip_sample=False,
    timestep_spacing="linspace",
    beta_schedule="linear",
    steps_offset=1,
)
pipe = AnimateDiffSDXLPipeline.from_pretrained(
    model_id,
    motion_adapter=adapter,
    scheduler=scheduler,
    torch_dtype=torch.float16,
    variant="fp16",
).to("cuda")
pipe.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin")

# enable memory savings
pipe.enable_vae_slicing()
pipe.enable_vae_tiling()
pipeline = pipe


@spaces.GPU
def generate_image(prompt, reference_image, controlnet_conditioning_scale,num_frames):
    style_images = [load_image(f.name) for f in reference_image]

    pipeline.set_ip_adapter_scale(controlnet_conditioning_scale)

    output = pipeline(
        prompt=prompt,
        ip_adapter_image=[style_images],
        negative_prompt="",
        guidance_scale=5,
        num_inference_steps=30,
        num_frames=num_frames,
    )
    frames = output.frames[0]
    export_to_gif(frames, "animation.gif")

    return "animation.gif"

# Set up Gradio interface
interface = gr.Interface(
    fn=generate_image,
    inputs=[
        gr.Textbox(label="Prompt"),
        # gr.Image( type= "filepath",label="Reference Image (Style)"),
        gr.File(type="file",file_count="multiple",label="Reference Image (Style)"),
        gr.Slider(label="Control Net Conditioning Scale", minimum=0, maximum=1.0, step=0.1, value=1.0),
        gr.Slider(label="Number of frames", minimum=0, maximum=1.0, step=0.1, value=1.0),

    ],
    outputs="image",
    title="Image Generation with Stable Diffusion 3 medium and ControlNet",
    description="Generates an image based on a text prompt and a reference image using Stable Diffusion 3 medium with ControlNet."

)

interface.launch()