Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
import torch
|
3 |
+
import spaces
|
4 |
+
from diffusers import LTXConditionPipeline
|
5 |
+
from diffusers.utils import export_to_video
|
6 |
+
from gtts import gTTS
|
7 |
+
from pydub import AudioSegment
|
8 |
+
import whisper
|
9 |
+
import ffmpeg
|
10 |
+
import os
|
11 |
+
|
12 |
+
# Load pipeline
|
13 |
+
pipe = LTXConditionPipeline.from_pretrained(
|
14 |
+
"Lightricks/LTX-Video-0.9.7-distilled", torch_dtype=torch.float16
|
15 |
+
)
|
16 |
+
pipe.to("cuda")
|
17 |
+
|
18 |
+
@spaces.GPU(duration=120)
|
19 |
+
def generate_video(prompt):
|
20 |
+
generator = torch.Generator("cuda").manual_seed(42)
|
21 |
+
|
22 |
+
# Generate latent video
|
23 |
+
latents = pipe(
|
24 |
+
prompt=prompt,
|
25 |
+
width=512,
|
26 |
+
height=512,
|
27 |
+
num_frames=24,
|
28 |
+
output_type="latent",
|
29 |
+
generator=generator,
|
30 |
+
num_inference_steps=7
|
31 |
+
).frames
|
32 |
+
|
33 |
+
# Decode frames
|
34 |
+
frames = pipe(
|
35 |
+
prompt=prompt,
|
36 |
+
latents=latents,
|
37 |
+
num_frames=24,
|
38 |
+
output_type="pil",
|
39 |
+
generator=generator,
|
40 |
+
num_inference_steps=7
|
41 |
+
).frames[0]
|
42 |
+
|
43 |
+
# Save as video
|
44 |
+
video_path = "output.mp4"
|
45 |
+
export_to_video(frames, video_path, fps=12)
|
46 |
+
|
47 |
+
# TTS
|
48 |
+
tts = gTTS(text=prompt, lang='en')
|
49 |
+
tts.save("voice.mp3")
|
50 |
+
AudioSegment.from_mp3("voice.mp3").export("voice.wav", format="wav")
|
51 |
+
|
52 |
+
# Subtitles
|
53 |
+
model = whisper.load_model("base")
|
54 |
+
result = model.transcribe("voice.wav", language="en")
|
55 |
+
with open("subtitles.srt", "w") as f:
|
56 |
+
f.write(result["srt"])
|
57 |
+
|
58 |
+
# Merge audio + subtitles into video
|
59 |
+
ffmpeg.input(video_path).output(
|
60 |
+
"final.mp4",
|
61 |
+
vf="subtitles=subtitles.srt",
|
62 |
+
i="voice.mp3",
|
63 |
+
c="copy",
|
64 |
+
shortest=None,
|
65 |
+
loglevel="error"
|
66 |
+
).run()
|
67 |
+
|
68 |
+
return "final.mp4"
|
69 |
+
|
70 |
+
# Gradio UI
|
71 |
+
demo = gr.Interface(fn=generate_video, inputs="text", outputs=gr.Video())
|
72 |
+
demo.launch()
|