app.py

import subprocess
subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)

from huggingface_hub import snapshot_download, hf_hub_download

snapshot_download(
    repo_id="Wan-AI/Wan2.1-T2V-1.3B",
    local_dir="wan_models/Wan2.1-T2V-1.3B",
    local_dir_use_symlinks=False,
    resume_download=True,
    repo_type="model" 
)

hf_hub_download(
    repo_id="gdhe17/Self-Forcing",
    filename="checkpoints/self_forcing_dmd.pt",
    local_dir=".",              
    local_dir_use_symlinks=False 
)

import os
import re
import random
import argparse
import hashlib
import urllib.request
import time
from PIL import Image
import spaces
import numpy as np
import torch
import gradio as gr
from omegaconf import OmegaConf
from tqdm import tqdm
import imageio

# Original project imports
from pipeline import CausalInferencePipeline
from demo_utils.constant import ZERO_VAE_CACHE
from demo_utils.vae_block3 import VAEDecoderWrapper
from utils.wan_wrapper import WanDiffusionWrapper, WanTextEncoder

# --- Argument Parsing ---
parser = argparse.ArgumentParser(description="Gradio Demo for Self-Forcing with Frame Streaming")
parser.add_argument('--port', type=int, default=7860, help="Port to run the Gradio app on.")
parser.add_argument('--host', type=str, default='0.0.0.0', help="Host to bind the Gradio app to.")
parser.add_argument("--checkpoint_path", type=str, default='./checkpoints/self_forcing_dmd.pt', help="Path to the model checkpoint.")
parser.add_argument("--config_path", type=str, default='./configs/self_forcing_dmd.yaml', help="Path to the model config.")
parser.add_argument('--share', action='store_true', help="Create a public Gradio link.")
parser.add_argument('--trt', action='store_true', help="Use TensorRT optimized VAE decoder.")
parser.add_argument('--fps', type=float, default=15.0, help="Playback FPS for frame streaming.")
args = parser.parse_args()

gpu = "cuda"

try:
    config = OmegaConf.load(args.config_path)
    default_config = OmegaConf.load("configs/default_config.yaml")
    config = OmegaConf.merge(default_config, config)
except FileNotFoundError as e:
    print(f"Error loading config file: {e}\n. Please ensure config files are in the correct path.")
    exit(1)

# Initialize Models
print("Initializing models...")
text_encoder = WanTextEncoder()
transformer = WanDiffusionWrapper(is_causal=True)

try:
    state_dict = torch.load(args.checkpoint_path, map_location="cpu")
    transformer.load_state_dict(state_dict.get('generator_ema', state_dict.get('generator')))
except FileNotFoundError as e:
    print(f"Error loading checkpoint: {e}\nPlease ensure the checkpoint '{args.checkpoint_path}' exists.")
    exit(1)

text_encoder.eval().to(dtype=torch.float16).requires_grad_(False)
transformer.eval().to(dtype=torch.float16).requires_grad_(False)

text_encoder.to(gpu)
transformer.to(gpu)

APP_STATE = {
    "torch_compile_applied": False,
    "fp8_applied": False,
    "current_use_taehv": False,
    "current_vae_decoder": None,
}

def initialize_vae_decoder(use_taehv=False, use_trt=False):
    if use_trt:
        from demo_utils.vae import VAETRTWrapper
        print("Initializing TensorRT VAE Decoder...")
        vae_decoder = VAETRTWrapper()
        APP_STATE["current_use_taehv"] = False
    elif use_taehv:
        print("Initializing TAEHV VAE Decoder...")
        from demo_utils.taehv import TAEHV
        taehv_checkpoint_path = "checkpoints/taew2_1.pth"
        if not os.path.exists(taehv_checkpoint_path):
            print(f"Downloading TAEHV checkpoint to {taehv_checkpoint_path}...")
            os.makedirs("checkpoints", exist_ok=True)
            download_url = "https://github.com/madebyollin/taehv/raw/main/taew2_1.pth"
            try:
                urllib.request.urlretrieve(download_url, taehv_checkpoint_path)
            except Exception as e:
                raise RuntimeError(f"Failed to download taew2_1.pth: {e}")
        
        class DotDict(dict): __getattr__ = dict.get
        
        class TAEHVDiffusersWrapper(torch.nn.Module):
            def __init__(self):
                super().__init__()
                self.dtype = torch.float16
                self.taehv = TAEHV(checkpoint_path=taehv_checkpoint_path).to(self.dtype)
                self.config = DotDict(scaling_factor=1.0)
            def decode(self, latents, return_dict=None):
                return self.taehv.decode_video(latents, parallel=not LOW_MEMORY).mul_(2).sub_(1)
        
        vae_decoder = TAEHVDiffusersWrapper()
        APP_STATE["current_use_taehv"] = True
    else:
        print("Initializing Default VAE Decoder...")
        vae_decoder = VAEDecoderWrapper()
        try:
            vae_state_dict = torch.load('wan_models/Wan2.1-T2V-1.3B/Wan2.1_VAE.pth', map_location="cpu")
            decoder_state_dict = {k: v for k, v in vae_state_dict.items() if 'decoder.' in k or 'conv2' in k}
            vae_decoder.load_state_dict(decoder_state_dict)
        except FileNotFoundError:
            print("Warning: Default VAE weights not found.")
        APP_STATE["current_use_taehv"] = False

    vae_decoder.eval().to(dtype=torch.float16).requires_grad_(False).to(gpu)
    APP_STATE["current_vae_decoder"] = vae_decoder
    print(f"✅ VAE decoder initialized: {'TAEHV' if use_taehv else 'Default VAE'}")

# Initialize with default VAE
initialize_vae_decoder(use_taehv=False, use_trt=args.trt)

pipeline = CausalInferencePipeline(
    config, device=gpu, generator=transformer, text_encoder=text_encoder, 
    vae=APP_STATE["current_vae_decoder"]
)

pipeline.to(dtype=torch.float16).to(gpu)

# --- Frame Streaming Video Generation Handler ---
@torch.no_grad()
@spaces.GPU
def video_generation_handler(prompt, seed, fps, progress=gr.Progress()):
    """
    Generator function that yields RGB frames for display in gr.Image.
    Includes timing delays for smooth playback.
    """
    if seed == -1: 
        seed = random.randint(0, 2**32 - 1)
    
    print(f"🎬 Starting video generation with prompt: '{prompt}' and seed: {seed}")
    
    # Calculate frame delay based on FPS
    frame_delay = 1.0 / fps if fps > 0 else 1.0 / 15.0
    
    print("🔤 Encoding text prompt...")
    conditional_dict = text_encoder(text_prompts=[prompt])
    for key, value in conditional_dict.items():
        conditional_dict[key] = value.to(dtype=torch.float16)
    
    # --- Generation Loop ---
    rnd = torch.Generator(gpu).manual_seed(int(seed))
    pipeline._initialize_kv_cache(1, torch.float16, device=gpu)
    pipeline._initialize_crossattn_cache(1, torch.float16, device=gpu)
    noise = torch.randn([1, 21, 16, 60, 104], device=gpu, dtype=torch.float16, generator=rnd)
    
    vae_cache, latents_cache = None, None
    if not APP_STATE["current_use_taehv"] and not args.trt:
        vae_cache = [c.to(device=gpu, dtype=torch.float16) for c in ZERO_VAE_CACHE]

    num_blocks = 7
    current_start_frame = 0
    all_num_frames = [pipeline.num_frame_per_block] * num_blocks
    
    total_frames_yielded = 0
    all_frames_for_video = []
    
    for idx, current_num_frames in enumerate(all_num_frames):
        print(f"📦 Processing block {idx+1}/{num_blocks} with {current_num_frames} frames")
        
        noisy_input = noise[:, current_start_frame : current_start_frame + current_num_frames]

        for step_idx, current_timestep in enumerate(pipeline.denoising_step_list):
            timestep = torch.ones([1, current_num_frames], device=noise.device, dtype=torch.int64) * current_timestep
            _, denoised_pred = pipeline.generator(
                noisy_image_or_video=noisy_input, conditional_dict=conditional_dict,
                timestep=timestep, kv_cache=pipeline.kv_cache1,
                crossattn_cache=pipeline.crossattn_cache,
                current_start=current_start_frame * pipeline.frame_seq_length
            )
            if step_idx < len(pipeline.denoising_step_list) - 1:
                next_timestep = pipeline.denoising_step_list[step_idx + 1]
                noisy_input = pipeline.scheduler.add_noise(
                    denoised_pred.flatten(0, 1), torch.randn_like(denoised_pred.flatten(0, 1)),
                    next_timestep * torch.ones([1 * current_num_frames], device=noise.device, dtype=torch.long)
                ).unflatten(0, denoised_pred.shape[:2])

        if idx < len(all_num_frames) - 1:
            pipeline.generator(
                noisy_image_or_video=denoised_pred, conditional_dict=conditional_dict,
                timestep=torch.zeros_like(timestep), kv_cache=pipeline.kv_cache1,
                crossattn_cache=pipeline.crossattn_cache,
                current_start=current_start_frame * pipeline.frame_seq_length,
            )

        # Decode to pixels
        if args.trt:
            pixels, vae_cache = pipeline.vae.forward(denoised_pred.half(), *vae_cache)
        elif APP_STATE["current_use_taehv"]:
            if latents_cache is None: 
                latents_cache = denoised_pred
            else:
                denoised_pred = torch.cat([latents_cache, denoised_pred], dim=1)
                latents_cache = denoised_pred[:, -3:]
            pixels = pipeline.vae.decode(denoised_pred)
        else:
            pixels, vae_cache = pipeline.vae(denoised_pred.half(), *vae_cache)
            
        # Handle frame skipping for first block
        if idx == 0 and not args.trt: 
            pixels = pixels[:, 3:]
        elif APP_STATE["current_use_taehv"] and idx > 0: 
            pixels = pixels[:, 12:]

        print(f"📹 Decoded pixels shape: {pixels.shape}")
        
        # Yield individual frames with timing delays
        for frame_idx in range(pixels.shape[1]):
            frame_tensor = pixels[0, frame_idx]  # Get single frame [C, H, W]
            
            # Normalize from [-1, 1] to [0, 255]
            frame_np = torch.clamp(frame_tensor.float(), -1., 1.) * 127.5 + 127.5
            frame_np = frame_np.to(torch.uint8).cpu().numpy()
            
            # Convert from CHW to HWC format (RGB)
            frame_np = np.transpose(frame_np, (1, 2, 0))  # CHW -> HWC
            
            all_frames_for_video.append(frame_np)
            total_frames_yielded += 1
            
            # Calculate progress
            total_expected_frames = num_blocks * pipeline.num_frame_per_block
            current_frame_count = (idx * pipeline.num_frame_per_block) + frame_idx + 1
            frame_progress = current_frame_count / total_expected_frames
            
            # Update progress
            progress(frame_progress, desc=f"Frame {total_frames_yielded} | Block {idx+1}/{num_blocks}")
            
            print(f"📺 Yielding frame {total_frames_yielded}: shape {frame_np.shape}")
            
            # Yield frame with timing delay
            yield gr.update(visible=True, frame_np), gr.update(visible=False)
            
            # Sleep between frames for smooth playback (except for the last frame)
            if not (frame_idx == pixels.shape[1] - 1 and idx + 1 == num_blocks):
                time.sleep(frame_delay)
            
        current_start_frame += current_num_frames
    
    print(f"✅ Video generation completed! Total frames yielded: {total_frames_yielded}")
    
    # Save final video
    try:
        video_path = f"gradio_tmp/{seed}_{hashlib.md5(prompt.encode()).hexdigest()}.mp4"
        imageio.mimwrite(video_path, all_frames_for_video, fps=fps, quality=8)
        print(f"✅ Video saved to {video_path}")
        return gr.update(visible=False), gr.update(value=video_path, visible=True)
    except Exception as e:
        print(f"⚠️ Could not save final video: {e}")
        return None, None

# --- Gradio UI Layout ---
with gr.Blocks(theme=gr.themes.Soft(), title="Self-Forcing Frame Streaming Demo") as demo:
    gr.Markdown("# 🚀 Self-Forcing Video Generation with Frame Streaming")
    gr.Markdown("*Real-time video generation with frame-by-frame display*")
    
    with gr.Row():
        with gr.Column(scale=2):
            gr.Markdown("### 📝 Configure Generation")
            with gr.Group():
                prompt = gr.Textbox(
                    label="Prompt", 
                    placeholder="A stylish woman walks down a Tokyo street...", 
                    lines=4,
                    value="A stylish woman walks down a Tokyo street filled with warm glowing neon and animated city signage."
                )
                gr.Examples(
                    examples=[
                        "A stylish woman walks down a Tokyo street filled with warm glowing neon and animated city signage. She wears a black leather jacket, a long red dress, and black boots, and carries a black purse.",
                        "A white and orange tabby cat is seen happily darting through a dense garden, as if chasing something. Its eyes are wide and happy as it jogs forward, scanning the branches, flowers, and leaves.",
                        "A drone shot of a surfer riding a wave on a sunny day. The camera follows the surfer as they carve through the water.",
                    ],
                    inputs=[prompt]
                )
            
            with gr.Row():
                seed = gr.Number(label="Seed", value=-1, info="Use -1 for a random seed.")
                fps = gr.Slider(
                    label="Playback FPS", 
                    minimum=1, 
                    maximum=30, 
                    value=args.fps, 
                    step=1,
                    info="Frames per second for playback"
                )
                
            start_btn = gr.Button("🎬 Start Generation", variant="primary", size="lg")
            
        with gr.Column(scale=3):
            gr.Markdown("### 📺 Live Frame Stream")
            gr.Markdown("*Click 'Start Generation' to begin frame streaming*")
            
            frame_display = gr.Image(
                label="Generated Frames",
                height=480,
                width=832,
                show_label=True,
                container=True
            )
            
            final_video = gr.Video(
                label="Final Rendered Video", 
                visible=True, 
                interactive=False,
                height=400
            )

    # Connect the generator to the image display
    start_btn.click(
        fn=video_generation_handler,
        inputs=[prompt, seed, fps],
        outputs=[frame_display, final_video],
        show_progress="full"
    )

# --- Launch App ---
if __name__ == "__main__":
    if os.path.exists("gradio_tmp"):
        import shutil
        shutil.rmtree("gradio_tmp")
    os.makedirs("gradio_tmp", exist_ok=True)
    
    demo.queue().launch(
        server_name=args.host, 
        server_port=args.port, 
        share=args.share,
        show_error=True
    )