test gradio

app.py

@@ -3,8 +3,8 @@ from huggingface_hub import login
 import os
 import spaces,tempfile
 import torch
-from diffusers import AnimateDiffControlNetPipeline
-from diffusers.models import AutoencoderKL, MotionAdapter, ControlNetModel
+from diffusers import AnimateDiffSparseControlNetPipeline
+from diffusers.models import AutoencoderKL, MotionAdapter, SparseControlNetModel
 from diffusers.schedulers import DPMSolverMultistepScheduler
 from diffusers.utils import export_to_gif, load_image
 from diffusers import AutoPipelineForText2Image
@@ -58,10 +58,8 @@ lora_adapter_id = "guoyww/animatediff-motion-lora-v1-5-3"
 vae_id = "stabilityai/sd-vae-ft-mse"
 device = "cuda"
 
-controlnet = ControlNetModel.from_single_file("control_v11f1p_sd15_depth.pth", torch_dtype=torch.float16)
-
 motion_adapter = MotionAdapter.from_pretrained(motion_adapter_id, torch_dtype=torch.float16).to(device)
-controlnet = ControlNetModel.from_pretrained(controlnet_id, torch_dtype=torch.float16).to(device)
+controlnet = SparseControlNetModel.from_pretrained(controlnet_id, torch_dtype=torch.float16).to(device)
 vae = AutoencoderKL.from_pretrained(vae_id, torch_dtype=torch.float16).to(device)
 scheduler = DPMSolverMultistepScheduler.from_pretrained(
     model_id,
@@ -70,7 +68,7 @@ scheduler = DPMSolverMultistepScheduler.from_pretrained(
     algorithm_type="dpmsolver++",
     use_karras_sigmas=True,
 )
-gif_pipe = AnimateDiffControlNetPipeline.from_pretrained(
+gif_pipe = AnimateDiffSparseControlNetPipeline.from_pretrained(
     model_id,
     motion_adapter=motion_adapter,
     controlnet=controlnet,
@@ -79,7 +77,6 @@ gif_pipe = AnimateDiffControlNetPipeline.from_pretrained(
     torch_dtype=torch.float16,
 ).to(device)
 gif_pipe.load_lora_weights(lora_adapter_id, adapter_name="motion_lora")
-gif_pipe.enable_free_noise(context_length=16, context_stride=4)
 
 
 
@@ -152,21 +149,12 @@ This way, each frame represents a distinct scene, and there’s no redundancy be
         controlnet_frame_indices.append(frame['frame_index'])
         long_prompt[frame['frame_index']] = frame['description']
 
-    full_images = []
-    last_image = conditioning_frames[0]
-    sparse_index_pointer = 0
-
-    for frame_index in range(max(controlnet_frame_indices)):
-        if sparse_index_pointer < len(controlnet_frame_indices) and frame_index == controlnet_frame_indices[sparse_index_pointer]:
-            last_image = controlnet_frame_indices[sparse_index_pointer]
-            sparse_index_pointer += 1
-        full_images.append(last_image)
-
     video = gif_pipe(
         prompt=long_prompt,
         negative_prompt="low quality, worst quality",
         num_inference_steps=25,
-        conditioning_frames=full_images,
+        conditioning_frames=conditioning_frames,
+        controlnet_frame_indices=controlnet_frame_indices,
         controlnet_conditioning_scale=float(controlnet_conditioning_scale),
     ).frames[0]
     export_to_gif(video, "output.gif")