Spaces:
Sleeping
Sleeping
File size: 3,168 Bytes
90dff8a fbf2779 90dff8a 2018ed2 90dff8a 2018ed2 90dff8a 1115063 90dff8a 1115063 90dff8a fbf2779 90dff8a fbf2779 90dff8a fbf2779 90dff8a 2018ed2 90dff8a fbf2779 90dff8a 2018ed2 90dff8a 2018ed2 90dff8a fbf2779 90dff8a fbf2779 90dff8a 2018ed2 90dff8a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 |
import cv2
from scenedetect import open_video, SceneManager
from scenedetect.detectors import ContentDetector
from moviepy.editor import VideoFileClip, concatenate_videoclips
from transformers import CLIPProcessor, CLIPModel
import torch
import yt_dlp
import os
def process_video(video_url, description):
# Download or load the video from the URL
video_path = download_video(video_url)
# Segment video into scenes
scenes = detect_scenes(video_path)
# Extract frames and analyze with CLIP model
best_scene = analyze_scenes(video_path, scenes, description)
# Extract the best scene into a final clip
final_clip = extract_best_scene(video_path, best_scene)
# Ensure the output directory exists
output_dir = "output"
os.makedirs(output_dir, exist_ok=True)
final_clip_path = os.path.join(output_dir, "final_clip.mp4")
# Save and return the final clip
try:
final_clip.write_videofile(final_clip_path)
except Exception as e:
return str(e)
return final_clip_path
def detect_scenes(video_path):
video = open_video(video_path)
scene_manager = SceneManager()
scene_manager.add_detector(ContentDetector())
scene_manager.detect_scenes(video)
scene_list = scene_manager.get_scene_list()
return scene_list
def analyze_scenes(video_path, scenes, description):
# Load CLIP model and processor
model = CLIPModel.from_pretrained("openai/clip-vit-base-patch32")
processor = CLIPProcessor.from_pretrained("openai/clip-vit-base-patch32")
best_scene = None
highest_prob = 0.0
for scene in scenes:
# Extract every 5th frame from the scene
frames = extract_frames(video_path, scene)
# Analyze frames with CLIP
for frame in frames:
inputs = processor(text=description, images=frame, return_tensors="pt", padding=True)
outputs = model(**inputs)
logits_per_image = outputs.logits_per_image
probs = logits_per_image.softmax(dim=1)
max_prob = max(probs[0]).item()
if max_prob > highest_prob:
highest_prob = max_prob
best_scene = scene
return best_scene
def extract_frames(video_path, scene):
frames = []
start_frame, end_frame = scene[0].get_frames(), scene[1].get_frames()
video_clip = VideoFileClip(video_path)
for frame_num in range(start_frame, end_frame, 5):
frame = video_clip.get_frame(frame_num / video_clip.fps)
frames.append(frame)
return frames
def extract_best_scene(video_path, scene):
start_time = scene[0].get_seconds()
end_time = scene[1].get_seconds()
video_clip = VideoFileClip(video_path).subclip(start_time, end_time)
return video_clip
def download_video(video_url):
ydl_opts = {
'format': 'bestvideo[height<=1440]+bestaudio/best[height<=1440]',
'outtmpl': 'downloaded_video.%(ext)s',
'noplaylist': True,
}
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
info_dict = ydl.extract_info(video_url, download=True)
video_file = ydl.prepare_filename(info_dict)
return video_file
|