Update app.py

app.py

@@ -32,14 +32,21 @@ import asyncio
 ###############################################################################
 
 
+
+
+import uuid
+from diffusers import AnimateDiffPipeline, MotionAdapter, EulerDiscreteScheduler
+from diffusers.utils import export_to_video
+from safetensors.torch import load_file
+
+
 # import spaces  #
 
-import os
+
 import imageio
 import numpy as np
 import torch
 import rembg
-from PIL import Image
 from torchvision.transforms import v2
 from pytorch_lightning import seed_everything
 from omegaconf import OmegaConf
@@ -284,6 +291,86 @@ def make3d(images):
 ############# above part is for 3D generate #############
 ###############################################################################
 
+###############################################################################
+############# this part is for text to video #############
+###############################################################################
+
+
+
+
+MORE = """ ## TRY Other Models
+        ### JARVIS: Your VOICE Assistant -> https://huggingface.co/spaces/KingNish/JARVIS
+        ### Instant Image: 4k images in 5 Second -> https://huggingface.co/spaces/KingNish/Instant-Image
+        """
+
+# Constants
+bases = {
+    "Cartoon": "frankjoshua/toonyou_beta6",
+    "Realistic": "emilianJR/epiCRealism",
+    "3d": "Lykon/DreamShaper",
+    "Anime": "Yntec/mistoonAnime2"
+}
+step_loaded = None
+base_loaded = "Realistic"
+motion_loaded = None
+
+# Ensure model and scheduler are initialized in GPU-enabled function
+if not torch.cuda.is_available():
+    raise NotImplementedError("No GPU detected!")
+
+device = "cuda"
+dtype = torch.float16
+pipe = AnimateDiffPipeline.from_pretrained(bases[base_loaded], torch_dtype=dtype).to(device)
+pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config, timestep_spacing="trailing", beta_schedule="linear")
+
+# Safety checkers
+from transformers import CLIPFeatureExtractor
+
+feature_extractor = CLIPFeatureExtractor.from_pretrained("openai/clip-vit-base-patch32")
+
+# Function 
+#@spaces.GPU(duration=60,queue=False)
+def generate_image(prompt, base="Realistic", motion="", step=8, progress=gr.Progress()):
+    global step_loaded
+    global base_loaded
+    global motion_loaded
+    print(prompt, base, step)
+
+    if step_loaded != step:
+        repo = "ByteDance/AnimateDiff-Lightning"
+        ckpt = f"animatediff_lightning_{step}step_diffusers.safetensors"
+        pipe.unet.load_state_dict(load_file(hf_hub_download(repo, ckpt), device=device), strict=False)
+        step_loaded = step
+
+    if base_loaded != base:
+        pipe.unet.load_state_dict(torch.load(hf_hub_download(bases[base], "unet/diffusion_pytorch_model.bin"), map_location=device), strict=False)
+        base_loaded = base
+
+    if motion_loaded != motion:
+        pipe.unload_lora_weights()
+        if motion != "":
+            pipe.load_lora_weights(motion, adapter_name="motion")
+            pipe.set_adapters(["motion"], [0.7])
+        motion_loaded = motion
+
+    progress((0, step))
+    def progress_callback(i, t, z):
+        progress((i+1, step))
+
+    output = pipe(prompt=prompt, guidance_scale=1.2, num_inference_steps=step, callback=progress_callback, callback_steps=1)
+
+    name = str(uuid.uuid4()).replace("-", "")
+    path = f"/tmp/{name}.mp4"
+    export_to_video(output.frames[0], path, fps=10)
+    return path
+
+
+
+###############################################################################
+############# above part is for text to video #############
+###############################################################################
+
+
 css = """
 #warning {background-color: #FFCCCB}
 .chatbot {
@@ -1265,7 +1352,16 @@ def create_ui():
         # above part is for 3d generate.
         ###############################################################################
        
+        ###############################################################################
+        ############# this part is for text to video #############
+        ###############################################################################
+        
         
+        
+        
+        ###############################################################################
+        ############# above part is for text to video #############
+        ###############################################################################
         def clear_tts_fields():
             return [gr.update(value=""), gr.update(value=""), None, None, gr.update(value=False), gr.update(value=True), None, None]