Spaces:
Build error
Build error
| import gradio as gr | |
| import imageio_ffmpeg | |
| import numpy as np | |
| from PIL import Image, ImageDraw, ImageFont | |
| import math | |
| import dlib | |
| import tempfile | |
| import requests | |
| import os | |
| from transformers import pipeline | |
| import cv2 | |
| import io | |
| detector = dlib.get_frontal_face_detector() | |
| try: | |
| predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat") | |
| except RuntimeError: | |
| print("Downloading shape_predictor_68_face_landmarks.dat...") | |
| landmarks_url = "http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2" | |
| landmarks_compressed = requests.get(landmarks_url).content | |
| import bz2 | |
| landmarks_data = bz2.decompress(landmarks_compressed) | |
| with open("shape_predictor_68_face_landmarks.dat", "wb") as f: | |
| f.write(landmarks_data) | |
| predictor = dlib.shape_predictor("shape_predictor_68_face_landmarks.dat") | |
| API_URL = "https://api-inference.huggingface.co/models/black-forest-labs/flux-1-schnell" | |
| HF_TOKEN = os.getenv("HF_TOKEN") | |
| LLM_API_URL = "https://api-inference.huggingface.co/models/lmsys/fastchat-t5-3b-v1.0" | |
| def query_hf_image_generation(prompt): | |
| headers = {"Authorization": f"Bearer {HF_TOKEN}"} | |
| payload = {"inputs": prompt} | |
| response = requests.post(API_URL, headers=headers, json=payload) | |
| if response.status_code == 200: | |
| image_bytes = response.content | |
| image = Image.open(io.BytesIO(image_bytes)) | |
| return image | |
| else: | |
| raise Exception(f"Image generation failed: {response.content}") | |
| def query_llm(prompt, image_description): | |
| headers = {"Authorization": f"Bearer {HF_TOKEN}"} | |
| system_prompt = "You are an expert in image to video creation, and give only the motion type, intensity, text overlay, text color, text start and end times for the image described below based on user's prompt. Give the response in a JSON format." | |
| prompt_template = f"<|system|>\n{system_prompt}</s>\n<|user|>\nImage Description: {image_description}\nUser Prompt: {prompt}</s>\n<|assistant|>\n" | |
| payload = {"inputs": prompt_template, "max_new_tokens": 200} | |
| response = requests.post(LLM_API_URL, headers=headers, json=payload) | |
| if response.status_code == 200: | |
| return response.json()[0]['generated_text'] | |
| else: | |
| raise Exception(f"LLM query failed: {response.content}") | |
| def extract_motion_params(llm_output): | |
| try: | |
| import json | |
| start_index = llm_output.find('{') | |
| end_index = llm_output.rfind('}') + 1 | |
| json_string = llm_output[start_index:end_index] | |
| params = json.loads(json_string) | |
| return params | |
| except: | |
| return { | |
| "motion_type": "none", | |
| "intensity": 0.25, | |
| "text_overlay": "", | |
| "text_color": "white", | |
| "start_time": 0, | |
| "end_time": 5 | |
| } | |
| def detect_face_landmarks(image): | |
| gray = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2GRAY) | |
| rects = detector(gray, 1) | |
| if len(rects) > 0: | |
| shape = predictor(gray, rects[0]) | |
| shape = np.array([(shape.part(i).x, shape.part(i).y) for i in range(68)]) | |
| return shape | |
| else: | |
| return None | |
| def apply_color_grading(frame, color_preset, intensity): | |
| if color_preset == "sepia": | |
| sepia_matrix = np.array([[0.393, 0.769, 0.189], | |
| [0.349, 0.686, 0.168], | |
| [0.272, 0.534, 0.131]]) | |
| frame_float = frame.astype(np.float32) / 255.0 | |
| sepia_effect = cv2.transform(frame_float, sepia_matrix) | |
| blended_frame = (1 - intensity) * frame_float + intensity * sepia_effect | |
| return (np.clip(blended_frame, 0, 1) * 255).astype(np.uint8) | |
| elif color_preset == "vintage": | |
| frame_float = frame.astype(np.float32) / 255.0 | |
| frame_float[:, :, 0] *= (1 - intensity * 0.6) | |
| frame_float[:, :, 2] *= (1 + intensity * 0.3) | |
| grayscale = cv2.cvtColor(frame_float, cv2.COLOR_RGB2GRAY) | |
| grayscale_rgb = cv2.cvtColor(grayscale, cv2.COLOR_GRAY2RGB) | |
| blended_frame = (1 - intensity * 0.5) * frame_float + intensity * 0.5 * grayscale_rgb | |
| return (np.clip(blended_frame, 0, 1) * 255).astype(np.uint8) | |
| elif color_preset == "black_and_white": | |
| gray_frame = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY) | |
| return cv2.cvtColor(gray_frame, cv2.COLOR_GRAY2RGB) | |
| elif color_preset == "cold": | |
| frame_float = frame.astype(np.float32) / 255.0 | |
| frame_float[:, :, 0] *= (1 + intensity * 0.7) | |
| frame_float[:, :, 2] *= (1 - intensity * 0.2) | |
| return (np.clip(frame_float, 0, 1) * 255).astype(np.uint8) | |
| elif color_preset == "warm": | |
| frame_float = frame.astype(np.float32) / 255.0 | |
| frame_float[:, :, 2] *= (1 + intensity * 0.7) | |
| frame_float[:, :, 0] *= (1 - intensity * 0.2) | |
| return (np.clip(frame_float, 0, 1) * 255).astype(np.uint8) | |
| elif color_preset == "neon": | |
| frame_float = frame.astype(np.float32) / 255.0 | |
| lab = cv2.cvtColor(frame_float, cv2.COLOR_RGB2LAB) | |
| l, a, b = cv2.split(lab) | |
| clahe = cv2.createCLAHE(clipLimit=3.0, tileGridSize=(8,8)) | |
| l = clahe.apply(l) | |
| lab = cv2.merge((l, a, b)) | |
| frame_float = cv2.cvtColor(lab, cv2.COLOR_LAB2RGB) | |
| frame_float[:, :, 0] *= (1 - intensity * 0.4) | |
| frame_float[:, :, 1] *= (1 + intensity * 0.8) | |
| frame_float[:, :, 2] *= (1 - intensity * 0.4) | |
| return (np.clip(frame_float, 0, 1) * 255).astype(np.uint8) | |
| return frame | |
| def apply_vignette(frame, intensity): | |
| width, height = frame.shape[1], frame.shape[0] | |
| x = np.linspace(-1, 1, width) | |
| y = np.linspace(-1, 1, height) | |
| X, Y = np.meshgrid(x, y) | |
| radius = np.sqrt(X**2 + Y**2) | |
| vignette = 1 - intensity * radius**2 | |
| vignette = np.clip(vignette, 0, 1) | |
| vignette = np.stack([vignette] * 3, axis=-1) | |
| frame_float = frame.astype(np.float32) / 255.0 | |
| result = frame_float * vignette | |
| return (np.clip(result, 0, 1) * 255).astype(np.uint8) | |
| def apply_bokeh(frame, intensity, t): | |
| frame_float = frame.astype(np.float32) / 255.0 | |
| gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY) | |
| circles = [] | |
| for _ in range(int(intensity * 30)): | |
| radius = np.random.randint(5, 30) | |
| x = np.random.randint(radius, frame.shape[1] - radius) | |
| y = np.random.randint(radius, frame.shape[0] - radius) | |
| color = frame_float[y, x] | |
| brightness = np.random.uniform(0.5, 1.0) | |
| circles.append((x, y, radius, color, brightness)) | |
| bokeh_effect = np.zeros_like(frame_float) | |
| for x, y, radius, color, brightness in circles: | |
| y_grid, x_grid = np.ogrid[-y:frame.shape[0]-y, -x:frame.shape[1]-x] | |
| mask = x_grid*x_grid + y_grid*y_grid <= radius*radius | |
| bokeh_effect[mask] += np.array(color) * brightness * (0.5 + 0.5 * np.sin(t * 2 * math.pi)) | |
| blended_frame = frame_float + intensity * bokeh_effect | |
| return (np.clip(blended_frame, 0, 1) * 255).astype(np.uint8) | |
| def apply_advanced_motion(image, motion_type, intensity, duration, fps, text_overlay, text_color, font_size, start_time, end_time, color_preset, vignette_intensity): | |
| frames = [] | |
| width, height = image.size | |
| landmarks = detect_face_landmarks(image) | |
| for i in range(int(duration * fps)): | |
| t = i / (duration * fps) | |
| frame = image.copy() | |
| if landmarks is not None: | |
| if motion_type == "head_nod": | |
| top_head = landmarks[27] | |
| bottom_head = landmarks[8] | |
| angle = math.sin(t * 2 * math.pi) * intensity * 8 | |
| center_x = (top_head[0] + bottom_head[0]) // 2 | |
| center_y = (top_head[1] + bottom_head[1]) // 2 | |
| M = cv2.getRotationMatrix2D((center_x, center_y), angle, 1) | |
| rotated_image = cv2.warpAffine(np.array(image), M, (width, height), flags=cv2.INTER_LANCZOS4) | |
| frame = Image.fromarray(rotated_image) | |
| elif motion_type == "head_shake": | |
| top_head = landmarks[27] | |
| left_head = landmarks[0] | |
| right_head = landmarks[16] | |
| angle = math.sin(t * 3 * math.pi) * intensity * 6 | |
| center_x = top_head[0] | |
| center_y = top_head[1] | |
| M = cv2.getRotationMatrix2D((center_x, center_y), angle, 1) | |
| rotated_image = cv2.warpAffine(np.array(image), M, (width, height), flags=cv2.INTER_LANCZOS4) | |
| frame = Image.fromarray(rotated_image) | |
| elif motion_type == "eye_blink": | |
| left_eye_top = landmarks[37] | |
| left_eye_bottom = landmarks[41] | |
| right_eye_top = landmarks[43] | |
| right_eye_bottom = landmarks[47] | |
| blink_progress = abs(math.sin(t * 2 * math.pi)) | |
| if blink_progress > 0.9: | |
| draw = ImageDraw.Draw(frame) | |
| draw.line([tuple(landmarks[36]), tuple(landmarks[39])], fill=text_color, width=2) | |
| draw.line([tuple(landmarks[42]), tuple(landmarks[45])], fill=text_color, width=2) | |
| else: | |
| frame = image.copy() | |
| elif motion_type == "smile": | |
| mouth_left = landmarks[48] | |
| mouth_right = landmarks[54] | |
| mouth_top = landmarks[51] | |
| mouth_bottom = landmarks[57] | |
| smile_progress = intensity * t | |
| draw = ImageDraw.Draw(frame) | |
| curve_points = [ | |
| tuple(mouth_left), | |
| (mouth_left[0] + (mouth_right[0] - mouth_left[0]) // 4, mouth_left[1] + int(20 * smile_progress)), | |
| (mouth_left[0] + 3 * (mouth_right[0] - mouth_left[0]) // 4, mouth_right[1] + int(20 * smile_progress)), | |
| tuple(mouth_right) | |
| ] | |
| draw.line(curve_points, fill=text_color, width=4) | |
| if motion_type == "zoom": | |
| scale = 1 + intensity * t | |
| new_size = (int(width * scale), int(height * scale)) | |
| resized_image = image.resize(new_size, Image.Resampling.LANCZOS) | |
| x_offset = (new_size[0] - width) // 2 | |
| y_offset = (new_size[1] - height) // 2 | |
| frame = resized_image.crop((x_offset, y_offset, x_offset + width, y_offset + height)) | |
| elif motion_type == "pan": | |
| x_offset = int(intensity * t * (width - width)) | |
| y_offset = int(intensity * t * (height - height)) | |
| frame = Image.new("RGB", (width, height)) | |
| frame.paste(image, (-x_offset, -y_offset)) | |
| elif motion_type == "rotate": | |
| angle = intensity * t * 360 | |
| rotated_image = image.rotate(angle, expand=True, resample=Image.Resampling.BICUBIC) | |
| x_offset = (rotated_image.width - width) // 2 | |
| y_offset = (rotated_image.height - height) // 2 | |
| frame = Image.new("RGB", (width, height)) | |
| frame.paste(rotated_image, (-x_offset, -y_offset)) | |
| elif motion_type == "move_right": | |
| x_offset = int(intensity * t * width) | |
| frame = Image.new("RGB", (width, height), "black") | |
| frame.paste(image, (x_offset, 0)) | |
| elif motion_type == "move_left": | |
| x_offset = -int(intensity * t * width) | |
| frame = Image.new("RGB", (width, height), "black") | |
| frame.paste(image, (x_offset, 0)) | |
| elif motion_type == "move_up": | |
| y_offset = -int(intensity * t * height) | |
| frame = Image.new("RGB", (width, height), "black") | |
| frame.paste(image, (0, y_offset)) | |
| elif motion_type == "move_down": | |
| y_offset = int(intensity * t * height) | |
| frame = Image.new("RGB", (width, height), "black") | |
| frame.paste(image, (0, y_offset)) | |
| elif motion_type == "shake": | |
| shake_intensity = intensity * 10 | |
| x_offset = int(shake_intensity * math.sin(t * 2 * math.pi * 5)) | |
| y_offset = int(shake_intensity * math.cos(t * 2 * math.pi * 3)) | |
| frame = Image.new("RGB", (width, height)) | |
| frame.paste(image, (x_offset, y_offset)) | |
| elif motion_type == "fade_in": | |
| alpha = t | |
| frame = Image.blend(Image.new("RGB", (width, height), "black"), image, alpha) | |
| elif motion_type == "fade_out": | |
| alpha = 1 - t | |
| frame = Image.blend(Image.new("RGB", (width, height), "black"), image, alpha) | |
| elif motion_type == "rain": | |
| draw = ImageDraw.Draw(frame) | |
| for _ in range(int(intensity * 5)): | |
| x = np.random.randint(0, width) | |
| y = np.random.randint(0, height) | |
| length = np.random.randint(5, 15) | |
| speed = intensity * 3 | |
| y_end = y + length + i * speed | |
| draw.line([(x, y), (x, y_end)], fill="lightblue", width=1) | |
| elif motion_type == "bokeh": | |
| frame_np = np.array(frame) | |
| frame_np = apply_bokeh(frame_np, intensity, t) | |
| frame = Image.fromarray(frame_np) | |
| frame_np = np.array(frame) | |
| if color_preset: | |
| frame_np = apply_color_grading(frame_np, color_preset, intensity) | |
| if vignette_intensity > 0: | |
| frame_np = apply_vignette(frame_np, vignette_intensity) | |
| frame = Image.fromarray(frame_np) | |
| draw = ImageDraw.Draw(frame) | |
| if text_overlay and start_time <= t <= end_time: | |
| try: | |
| font = ImageFont.truetype("arial.ttf", font_size) | |
| except IOError: | |
| font = ImageFont.load_default() | |
| text_width, text_height = draw.textsize(text_overlay, font=font) | |
| x = (width - text_width) // 2 | |
| y = (height - text_height) // 2 | |
| draw.text((x, y), text_overlay, font=font, fill=text_color) | |
| frames.append(np.array(frame)) | |
| return frames | |
| def create_video_from_frames(frames, duration=5, fps=30): | |
| with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmpfile: | |
| output_filename = tmpfile.name | |
| writer = imageio_ffmpeg.write_frames(output_filename, frames[0].shape[:2], pix_fmt_out='yuv420p', fps=fps, codec='libx264', preset="veryslow") | |
| writer.send(None) | |
| for frame in frames: | |
| writer.send(frame) | |
| writer.close() | |
| return output_filename | |
| def generate_and_animate(prompt): | |
| try: | |
| image = query_hf_image_generation(prompt) | |
| image_description = pipeline("image-to-text", model="Salesforce/blip-image-captioning-base")(image)[0]['generated_text'] | |
| llm_response = query_llm(prompt, image_description) | |
| motion_params = extract_motion_params(llm_response) | |
| frames = apply_advanced_motion( | |
| image, | |
| motion_params["motion_type"], | |
| motion_params["intensity"], | |
| duration=5, | |
| fps=30, | |
| text_overlay=motion_params["text_overlay"], | |
| text_color=motion_params["text_color"], | |
| font_size=50, | |
| start_time=motion_params["start_time"], | |
| end_time=motion_params["end_time"], | |
| color_preset=motion_params.get("color_preset", None), | |
| vignette_intensity=motion_params.get("vignette_intensity", 0) | |
| ) | |
| video_file = create_video_from_frames(frames) | |
| return video_file, gr.Image.update(value=image) | |
| except Exception as e: | |
| return str(e), None | |
| motion_types = [ | |
| "zoom", "pan", "rotate", "move_right", "move_left", "move_up", "move_down", | |
| "shake", "fade_in", "fade_out", "head_nod", "head_shake", "eye_blink", "smile", "rain", "bokeh", "none" | |
| ] | |
| text_colors = ["white", "black", "red", "green", "blue", "yellow"] | |
| color_presets = ["sepia", "vintage", "black_and_white", "cold", "warm", "neon", "none"] | |
| iface = gr.Interface( | |
| fn=generate_and_animate, | |
| inputs=[ | |
| gr.Textbox(label="Prompt"), | |
| ], | |
| outputs=[ | |
| gr.Video(label="Generated Video"), | |
| gr.Image(label="Generated Image") | |
| ], | |
| title="AI Video Generator", | |
| description="Enter a prompt to generate an image and animate it. Uses Flux 1, an LLM, and advanced video processing techniques." | |
| ) | |
| if __name__ == "__main__": | |
| iface.launch(share=True, debug=True) |