🎭 Sonic: Advanced Portrait Animation
Transform still images into dynamic videos synchronized with audio
import spaces import gradio as gr import os import numpy as np from pydub import AudioSegment import hashlib from sonic import Sonic from PIL import Image import torch # 모델 초기화 cmd = ( 'python3 -m pip install "huggingface_hub[cli]"; ' 'huggingface-cli download LeonJoe13/Sonic --local-dir checkpoints; ' 'huggingface-cli download stabilityai/stable-video-diffusion-img2vid-xt --local-dir checkpoints/stable-video-diffusion-img2vid-xt; ' 'huggingface-cli download openai/whisper-tiny --local-dir checkpoints/whisper-tiny;' ) os.system(cmd) pipe = Sonic() def get_md5(content): md5hash = hashlib.md5(content) return md5hash.hexdigest() tmp_path = './tmp_path/' res_path = './res_path/' os.makedirs(tmp_path, exist_ok=True) os.makedirs(res_path, exist_ok=True) @spaces.GPU(duration=300) # 긴 비디오 처리를 위해 duration 300초로 설정 def get_video_res(img_path, audio_path, res_video_path, dynamic_scale=1.0): # ============================ # 1) 4초(프레임 50)로 늘리기 # 2) 원본 비율 유지(크롭 제거) # ============================ ## 수정됨: expand_ratio를 0으로 (기존 0.5) expand_ratio = 0.0 min_resolution = 512 ## 수정됨: 4초 분량 = 50프레임 inference_steps = 100 # 기존 25 -> 50 audio = AudioSegment.from_file(audio_path) duration = len(audio) / 1000.0 # 초 단위 print(f"Audio duration: {duration} seconds, using inference_steps: {inference_steps}") # 얼굴 인식 (face_info는 참고용) face_info = pipe.preprocess(img_path, expand_ratio=expand_ratio) print(f"Face detection info: {face_info}") # 얼굴이 하나라도 검출되면(>0), 기존에는 크롭 과정을 진행했으나, # 원본 이미지 비율 유지를 위해 크롭 부분 제거 if face_info['face_num'] > 0: ## 수정됨: 아래 3줄 크롭 코드 제거 # crop_image_path = img_path + '.crop.png' # pipe.crop_image(img_path, crop_image_path, face_info['crop_bbox']) # img_path = crop_image_path os.makedirs(os.path.dirname(res_video_path), exist_ok=True) # 원본 이미지를 그대로 전달 pipe.process( img_path, audio_path, res_video_path, min_resolution=min_resolution, inference_steps=inference_steps, dynamic_scale=dynamic_scale ) return res_video_path else: return -1 def process_sonic(image, audio, dynamic_scale): # 입력 검증 if image is None: raise gr.Error("Please upload an image") if audio is None: raise gr.Error("Please upload an audio file") img_md5 = get_md5(np.array(image)) audio_md5 = get_md5(audio[1]) print(f"Processing with image hash: {img_md5}, audio hash: {audio_md5}") sampling_rate, arr = audio[:2] if len(arr.shape) == 1: arr = arr[:, None] # numpy array -> AudioSegment 변환 audio_segment = AudioSegment( arr.tobytes(), frame_rate=sampling_rate, sample_width=arr.dtype.itemsize, channels=arr.shape[1] ) audio_segment = audio_segment.set_frame_rate(sampling_rate) # 파일 경로 생성 image_path = os.path.abspath(os.path.join(tmp_path, f'{img_md5}.png')) audio_path = os.path.abspath(os.path.join(tmp_path, f'{audio_md5}.wav')) res_video_path = os.path.abspath(os.path.join(res_path, f'{img_md5}_{audio_md5}_{dynamic_scale}.mp4')) # 이미지/오디오 파일 캐싱 if not os.path.exists(image_path): image.save(image_path) if not os.path.exists(audio_path): audio_segment.export(audio_path, format="wav") # 캐시된 결과가 있으면 바로 사용, 없으면 새로 생성 if os.path.exists(res_video_path): print(f"Using cached result: {res_video_path}") return res_video_path else: print(f"Generating new video with dynamic scale: {dynamic_scale}") return get_video_res(image_path, audio_path, res_video_path, dynamic_scale) # 예시 데이터를 위한 dummy 함수 (필요시 실제 예시 데이터 추가) def get_example(): return [] css = """ .gradio-container { font-family: 'Arial', sans-serif; } .main-header { text-align: center; color: #2a2a2a; margin-bottom: 2em; } .parameter-section { background-color: #f5f5f5; padding: 1em; border-radius: 8px; margin: 1em 0; } .example-section { margin-top: 2em; } """ with gr.Blocks(css=css) as demo: gr.HTML("""
Transform still images into dynamic videos synchronized with audio