import os os.system("git clone https://github.com/google-research/frame-interpolation") import sys sys.path.append("frame-interpolation") import math import cv2 import numpy as np import tensorflow as tf import mediapy from PIL import Image import gradio as gr from huggingface_hub import snapshot_download from image_tools.sizes import resize_and_crop model = snapshot_download(repo_id="akhaliq/frame-interpolation-film-style") from eval import interpolator, util interpolator = interpolator.Interpolator(model, None) ffmpeg_path = util.get_ffmpeg_path() mediapy.set_ffmpeg(ffmpeg_path) fl_ = "" fl_mask = "" def do_interpolation(frame1, frame2, interpolation, n): print("tween frames: " + str(interpolation)) print(frame1, frame2) input_frames = [frame1, frame2] frames = list( util.interpolate_recursively_from_files( input_frames, int(interpolation), interpolator)) #print(frames) mediapy.write_video(f"{n}_to_{n+1}_out.mp4", frames, fps=25) return f"{n}_to_{n+1}_out.mp4" def get_frames(video_in, step, name, n): frames = [] cap = cv2.VideoCapture(video_in) cframes = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) cfps = int(cap.get(cv2.CAP_PROP_FPS)) print(f'frames: {cframes}, fps: {cfps}') #resize the video #clip = VideoFileClip(video_in) #check fps #if cfps > 25: # print("video rate is over 25, resetting to 25") # clip_resized = clip.resize(height=1024) # clip_resized.write_videofile("video_resized.mp4", fps=25) #else: # print("video rate is OK") # clip_resized = clip.resize(height=1024) # clip_resized.write_videofile("video_resized.mp4", fps=cfps) #print("video resized to 1024 height") # Opens the Video file with CV2 #cap = cv2.VideoCapture("video_resized.mp4") fps = cap.get(cv2.CAP_PROP_FPS) print("video fps: " + str(fps)) i=0 while(cap.isOpened()): ret, frame = cap.read() if ret == False: break #if resize_w > 0: #resize_h = resize_w / 2.0 #frame = cv2.resize(frame, (int(resize_w), int(resize_h))) cv2.imwrite(f"{str(n)}_{name}_{step}{str(i)}.png", frame) frames.append(f"{str(n)}_{name}_{step}{str(i)}.png") i+=1 cap.release() cv2.destroyAllWindows() print("broke the video into frames") return frames, fps def create_video(frames, fps, type): print("building video result") imgs = [] for j, img in enumerate(frames): imgs.append(cv2.cvtColor(cv2.imread(img).astype(np.uint8), cv2.COLOR_BGR2RGB)) mediapy.write_video(type + "_result.mp4", imgs, fps=fps) return type + "_result.mp4" def infer(f_in, interpolation, fps_output): fps_output = logscale(fps_output) # 1. break video into frames and get FPS #break_vid = get_frames(url_in, "vid_input_frame", "origin", resize_n) frames_list = f_in #break_vid[0] fps = 1 #break_vid[1] print(f"ORIGIN FPS: {fps}") n_frame = int(300) #limited to 300 frames #n_frame = len(frames_list) if n_frame >= len(frames_list): print("video is shorter than the cut value") n_frame = len(frames_list) # 2. prepare frames result arrays result_frames = [] print("set stop frames to: " + str(n_frame)) for idx, frame in enumerate(frames_list[0:int(n_frame)]): if idx < len(frames_list) - 1: next_frame = frames_list[idx+1] interpolated_frames = do_interpolation(frame, next_frame, interpolation, idx) # should return a list of interpolated frames break_interpolated_video = get_frames(interpolated_frames, "interpol", f"{idx}_", -1) print(break_interpolated_video[0]) for j, img in enumerate(break_interpolated_video[0][0:len(break_interpolated_video[0])-1]): print(f"IMG:{img}") os.rename(img, f"{idx}_to_{idx+1}_{j}.png") result_frames.append(f"{idx}_to_{idx+1}_{j}.png") print("frames " + str(idx) + " & " + str(idx+1) + "/" + str(n_frame) + ": done;") #print(f"CURRENT FRAMES: {result_frames}") result_frames.append(f"{frames_list[n_frame-1]}") final_vid = create_video(result_frames, fps_output, "interpolated") files = final_vid print("interpolated frames: " + str(len(frames_list)) + " -> " + str(len(result_frames))) cv2.destroyAllWindows() return final_vid, files def logscale(linear): return int(math.pow(2, linear)) def linscale(linear): return int(math.log2(linear)) def remove_bg(fl, count): global fl_ fr = cv2.imread(fl) b = 1 element = cv2.getStructuringElement(cv2.MORPH_RECT, (2 * b + 1, 2 * b + 1), (b, b)) n = int((fr.shape[0]*fr.shape[1]) / (256*256)) fr_bg = cv2.medianBlur(fr, 255) for i in range(0, n): fr_bg = cv2.medianBlur(fr_bg, 255) fr_diff = cv2.convertScaleAbs(fr.astype(np.int16)-fr_bg.astype(np.int16)).astype(np.uint8) fr_diff = cv2.cvtColor(fr_diff, cv2.COLOR_BGR2GRAY) md = 12 fr_diff[fr_diff>=md] = 255 fr_diff[fr_diff0] = 0 fr_diff = cv2.dilate(cv2.erode(fr_diff, element), element) if count % 2: # odd: is photo without the flash fr = cv2.imread(fl_).astype(np.uint8) fr = fr_diff - cv2.cvtColor(fr, cv2.COLOR_BGR2GRAY) cv2.imwrite(fl, fr) return fl else: # even: with the flash fl_ = fl.split(".")[0] + "_.png" cv2.imwrite(fl_, fr_diff.astype(np.uint8)) return fl_ def sharpest(fl, i): break_vid = get_frames(fl, "vid_input_frame", "origin", i) frames = [] blur_s = [] for jdx, fr in enumerate(break_vid[0]): frames.append(cv2.imread(fr).astype(np.uint8)) blur_s.append(cv2.Laplacian(cv2.cvtColor(frames[len(frames)-1], cv2.COLOR_BGR2GRAY), cv2.CV_64F).var()) print(str(int(blur_s[jdx]))) indx = np.argmax(blur_s) fl = break_vid[0][indx] n = 25 half = int(n/2) if indx-half < 0: n = indx*2+1 elif indx+half >= len(frames): n = (len(frames)-1-indx)*2+1 #denoise frame = cv2.fastNlMeansDenoisingColoredMulti( srcImgs = frames, imgToDenoiseIndex = indx, temporalWindowSize = n, hColor = 5, templateWindowSize = 21, searchWindowSize = 21) cv2.imwrite(fl, frame) print(str(i) +'th file, sharpest frame: '+str(indx)+', name: '+fl) return fl def sortFiles(e): e = e.split('/') return e[len(e)-1] def loadf(f, r_bg): if f != None and f[0] != None: f.sort(key=sortFiles) fnew = [] for i, fl in enumerate(f): ftype = fl.split('/') if ftype[len(ftype)-1].split('.')[1] == 'mp4': fl = sharpest(fl, i) if r_bg == True: fl = remove_bg(fl, i) fnew.append(fl) return fnew, fnew else: return f, f title="""

Video interpolation from images with FILM

This space uses FILM to generate interpolation frames in a set of image files you need to turn into a video for animation (or 'pixilation'). If .mp4 videos are uploaded instead, selects the sharpest frame of each. Limited to 300 uploaded frames, from the beginning of your input.
Duplicate Space

""" with gr.Blocks() as demo: with gr.Column(): gr.HTML(title) with gr.Row(): with gr.Column(): with gr.Accordion(label="Upload files here", open=True): files_orig = gr.File(file_count="multiple", file_types=['image', '.mp4']) files_input = gr.File(file_count="multiple", visible=False) gallery_input = gr.Gallery(label="Slideshow", preview=True, columns=8192, interactive=False) r_bg = gr.Checkbox(label="Remove background", value=True) files_orig.upload(fn=loadf, inputs=[files_orig, r_bg], outputs=[files_input, gallery_input]) with gr.Row(): interpolation_slider = gr.Slider(minimum=1, maximum=5, step=1, value=1, label="Interpolation Steps: ") interpolation = gr.Number(value=1, show_label=False, interactive=False) interpolation_slider.change(fn=logscale, inputs=[interpolation_slider], outputs=[interpolation]) with gr.Row(): fps_output_slider = gr.Slider(minimum=0, maximum=5, step=1, value=0, label="FPS output: ") fps_output = gr.Number(value=1, show_label=False, interactive=False) fps_output_slider.change(fn=logscale, inputs=[fps_output_slider], outputs=[fps_output]) submit_btn = gr.Button("Submit") with gr.Column(): video_output = gr.Video() file_output = gr.File() gr.Examples( examples=[[ ["./examples/0.png", "./examples/1.png", "./examples/2.png", "./examples/3.png", "./examples/4.png"], True ]], fn=loadf, inputs=[files_orig, r_bg], outputs=[files_input, gallery_input], cache_examples=True ) submit_btn.click(fn=infer, inputs=[files_input, interpolation_slider, fps_output_slider], outputs=[video_output, file_output]) demo.launch()