Update app.py

app.py

@@ -1,84 +1,87 @@
 import gradio as gr
-import torch
-from diffusers import StableDiffusionImg2ImgPipeline
-from torchvision import transforms
+import json
+from transformers import pipeline
+from moviepy.editor import ImageClip, concatenate_videoclips
 from PIL import Image
-import io
-import base64
+import numpy as np
+import os
 import spaces
-from functools import lru_cache
 
-# Base64 utilities
-def pil_to_b64(img: Image.Image) -> str:
-    buf = io.BytesIO()
-    img.save(buf, format="PNG")
-    return base64.b64encode(buf.getvalue()).decode()
+# Text Expansion Model (use Mistral or LLaMA on ZeroGPU)
+text_generator = pipeline("text-generation", model="mistralai/Mistral-7B-Instruct")
 
-def b64_to_pil(b64: str) -> Image.Image:
-    return Image.open(io.BytesIO(base64.b64decode(b64))).convert("RGB")
+# Text-to-Speech Model (Bark small)
+tts_pipeline = pipeline("text-to-speech", model="suno/bark-small")
 
-# ✅ Cached Model Loaders (ZeroGPU Safe)
-@lru_cache(maxsize=2)
-def load_ghibli_model():
-    pipe = StableDiffusionImg2ImgPipeline.from_pretrained(
-        "nitrosocke/Ghibli-Diffusion",
-        torch_dtype=torch.float16,
-        use_safetensors=True
-    ).to("cuda")
-    return pipe
+# Image Generation Model (LoRA-based)
+image_generator = pipeline("text-to-image", model="stabilityai/sdxl-turbo")
 
-@lru_cache(maxsize=2)
-def load_animegan_model():
-    model = torch.hub.load(
-        "bryandlee/animegan2-pytorch:main",
-        "generator",
-        pretrained="face_paint_512_v2"
-    ).to("cuda").eval()
-    return model
-
-# ✅ Image Processing (Gradio Image Upload)
+# Main Processing Function
 @spaces.GPU
-def process_image(img: Image.Image, effect: str) -> Image.Image:
-    if effect == "ghibli":
-        pipe = load_ghibli_model()
-        out_img = pipe(prompt="ghibli style", image=img, strength=0.5, guidance_scale=7.5).images[0]
-    else:
-        animegan = load_animegan_model()
-        transform = transforms.Compose([
-            transforms.Resize((512, 512)),
-            transforms.ToTensor()
-        ])
-        img_tensor = transform(img).unsqueeze(0).to("cuda")
-        with torch.no_grad():
-            out = animegan(img_tensor)[0].clamp(0, 1).cpu()
-        out_img = transforms.ToPILImage()(out)
-    return out_img
+def generate_cartoon(script_text):
+    # Step 1: Expand Script
+    prompt = f"""
+    You are a cartoon script writer. Convert the following story into a detailed cartoon scene plan.
+    For each scene, provide:
+    1. Scene description (setting + action)
+    2. Dialogue
+    3. Characters involved
+    4. Background description
 
-# ✅ Base64 API Processing
-@spaces.GPU
-def process_base64(b64: str, effect: str) -> str:
-    img = b64_to_pil(b64)
-    out_img = process_image(img, effect)
-    return pil_to_b64(out_img)
+    Story:
+    """
+    {script_text}
+    """
+    Return result in JSON format.
+    """
+    
+    response = text_generator(prompt, max_new_tokens=1024)[0]['generated_text']
+
+    try:
+        scene_data = json.loads(response)
+    except:
+        return "Script expansion failed. Please refine input."
+
+    characters = set()
+    backgrounds = set()
+    scene_clips = []
+
+    os.makedirs("generated_images", exist_ok=True)
+    os.makedirs("generated_audio", exist_ok=True)
+
+    for idx, scene in enumerate(scene_data.get("scenes", [])):
+        # Generate Background Image
+        background_prompt = scene.get("background_description", "cartoon background")
+        background_image = image_generator(background_prompt).images[0]
+        bg_path = f"generated_images/scene_{idx+1}.png"
+        background_image.save(bg_path)
+
+        # Generate TTS Audio
+        dialogue = scene.get("dialogue", "")
+        audio_output = tts_pipeline(dialogue)
+        audio_path = f"generated_audio/scene_{idx+1}.wav"
+        audio_output['audio'].export(audio_path, format="wav")
+
+        # Create Scene Video Clip
+        image_clip = ImageClip(bg_path).set_duration(5).fadein(1).fadeout(1)
+        image_clip = image_clip.set_audio(audio_path)
+        scene_clips.append(image_clip)
 
-# ✅ Gradio UI
-with gr.Blocks() as demo:
-    gr.Markdown("# 🎨 Ghibli & AnimeGAN Effects (ZeroGPU Compatible)")
+    # Merge Scenes
+    final_video = concatenate_videoclips(scene_clips)
+    final_video_path = "final_cartoon_video.mp4"
+    final_video.write_videofile(final_video_path, fps=24)
 
-    # Image Upload Tab
-    with gr.Tab("Web UI"):
-        img_input = gr.Image(type="pil", label="Upload Image")
-        effect_choice = gr.Radio(["ghibli", "anime"], label="Select Effect")
-        process_btn = gr.Button("Apply Effect")
-        img_output = gr.Image(label="Processed Image")
-        process_btn.click(process_image, [img_input, effect_choice], img_output)
+    return final_video_path
 
-    # Base64 API Tab
-    with gr.Tab("Base64 API"):
-        b64_input = gr.Textbox(label="Input Image (Base64)", lines=5)
-        effect_choice_b64 = gr.Radio(["ghibli", "anime"], label="Select Effect")
-        process_btn_b64 = gr.Button("Run API")
-        b64_output = gr.Textbox(label="Output Image (Base64)", lines=5)
-        process_btn_b64.click(process_base64, [b64_input, effect_choice_b64], b64_output)
+# Gradio UI
+demo = gr.Interface(
+    fn=generate_cartoon,
+    inputs=gr.Textbox(label="Enter Cartoon Script", lines=10),
+    outputs=gr.Video(label="Generated Cartoon Video"),
+    title="Cartoon Video Generator",
+    description="Enter a short cartoon story script and generate a cartoon video (7-10 mins) automatically."
+)
 
-demo.launch()
+if __name__ == "__main__":
+    demo.launch()