adding rife and upscale models

controlnet_pipeline.py

@@ -364,7 +364,6 @@ class ControlnetCogVideoXImageToVideoPCDPipeline(DiffusionPipeline, CogVideoXLor
             image_latents = torch.cat([first_frame, image_latents], dim=1)
 
         if latents is None:
-            print("Latent is known")
             latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
         else:
             latents = latents.to(device)

inference/cli_demo_camera_i2v_pcd.py

@@ -5,7 +5,7 @@ sys.path.append('.')
 sys.path.append('..')
 import argparse
 import os
-
+from rife_model import load_rife_model, rife_inference_with_latents
 import torch
 from transformers import T5EncoderModel, T5Tokenizer
 from diffusers import (
@@ -20,7 +20,7 @@ from cogvideo_transformer import CustomCogVideoXTransformer3DModel
 from cogvideo_controlnet_pcd import CogVideoXControlnetPCD
 from training.controlnet_datasets_camera_pcd_mask import RealEstate10KPCDRenderDataset
 from torchvision.transforms.functional import to_pil_image
-
+import utils
 from inference.utils import stack_images_horizontally
 from PIL import Image
 import numpy as np
@@ -36,6 +36,10 @@ import cv2
 import numpy as np
 import torch
 
+device = "cuda" if torch.cuda.is_available() else "cpu"
+upscale_model = utils.load_sd_upscale("model_real_esran/RealESRGAN_x4.pth", device)
+frame_interpolation_model = load_rife_model("model_rife")
+
 def get_black_region_mask_tensor(video_tensor, threshold=2, kernel_size=15):
     """
     Generate cleaned binary masks for black regions in a video tensor.
@@ -82,7 +86,6 @@ def maxpool_mask_tensor(mask_tensor):
     downsampling_factor_h = (H // 8) // 2
     downsampling_factor_w = (W // 8) // 2
 
-
     # Reshape to (B=T, C=1, H, W) for 2D spatial pooling
     x = mask_tensor.unsqueeze(1).float()  # (T, 1, H, W)
     x_pooled = F.max_pool2d(x, kernel_size=(H // downsampling_factor_h, W // downsampling_factor_w))  # → (T, 1, 30, 45)
@@ -222,6 +225,7 @@ def generate_video(
     if controlnet_transformer_out_proj_dim_factor is not None:
         controlnet_kwargs["out_proj_dim"] = num_attention_heads_orig * controlnet_transformer_out_proj_dim_factor
     controlnet_kwargs["out_proj_dim_zero_init"] = controlnet_transformer_out_proj_dim_zero_init
+
     controlnet = CogVideoXControlnetPCD(
         num_layers=controlnet_transformer_num_layers,
         downscale_coef=downscale_coef,
@@ -361,6 +365,20 @@ def generate_video(
             height=height,  # Height of the generated video
             width=width,  # Width of the generated video
         ).frames
+
+        # ++++++++++++++++++++++++++++++++++++++
+        latents = video_generate_all # This is a latent
+
+        scale_status = True
+        rife_status = True
+        if scale_status:
+            latents = utils.upscale_batch_and_concatenate(upscale_model, latents, device)
+        if rife_status:
+            latents = rife_inference_with_latents(frame_interpolation_model, latents)
+
+        video_generate_all = latents
+        # ++++++++++++++++++++++++++++++++++++++
+        
         video_generate = video_generate_all[0]
 
         # 6. Export the generated frames to a video file. fps must be 8 for original video.

inference/rife/IFNet.py

@@ -0,0 +1,123 @@
+from .refine import *
+
+
+def deconv(in_planes, out_planes, kernel_size=4, stride=2, padding=1):
+    return nn.Sequential(
+        torch.nn.ConvTranspose2d(in_channels=in_planes, out_channels=out_planes, kernel_size=4, stride=2, padding=1),
+        nn.PReLU(out_planes),
+    )
+
+
+def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
+    return nn.Sequential(
+        nn.Conv2d(
+            in_planes,
+            out_planes,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=True,
+        ),
+        nn.PReLU(out_planes),
+    )
+
+
+class IFBlock(nn.Module):
+    def __init__(self, in_planes, c=64):
+        super(IFBlock, self).__init__()
+        self.conv0 = nn.Sequential(
+            conv(in_planes, c // 2, 3, 2, 1),
+            conv(c // 2, c, 3, 2, 1),
+        )
+        self.convblock = nn.Sequential(
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+        )
+        self.lastconv = nn.ConvTranspose2d(c, 5, 4, 2, 1)
+
+    def forward(self, x, flow, scale):
+        if scale != 1:
+            x = F.interpolate(x, scale_factor=1.0 / scale, mode="bilinear", align_corners=False)
+        if flow != None:
+            flow = F.interpolate(flow, scale_factor=1.0 / scale, mode="bilinear", align_corners=False) * 1.0 / scale
+            x = torch.cat((x, flow), 1)
+        x = self.conv0(x)
+        x = self.convblock(x) + x
+        tmp = self.lastconv(x)
+        tmp = F.interpolate(tmp, scale_factor=scale * 2, mode="bilinear", align_corners=False)
+        flow = tmp[:, :4] * scale * 2
+        mask = tmp[:, 4:5]
+        return flow, mask
+
+
+class IFNet(nn.Module):
+    def __init__(self):
+        super(IFNet, self).__init__()
+        self.block0 = IFBlock(6, c=240)
+        self.block1 = IFBlock(13 + 4, c=150)
+        self.block2 = IFBlock(13 + 4, c=90)
+        self.block_tea = IFBlock(16 + 4, c=90)
+        self.contextnet = Contextnet()
+        self.unet = Unet()
+
+    def forward(self, x, scale=[4, 2, 1], timestep=0.5):
+        img0 = x[:, :3]
+        img1 = x[:, 3:6]
+        gt = x[:, 6:]  # In inference time, gt is None
+        flow_list = []
+        merged = []
+        mask_list = []
+        warped_img0 = img0
+        warped_img1 = img1
+        flow = None
+        loss_distill = 0
+        stu = [self.block0, self.block1, self.block2]
+        for i in range(3):
+            if flow != None:
+                flow_d, mask_d = stu[i](
+                    torch.cat((img0, img1, warped_img0, warped_img1, mask), 1), flow, scale=scale[i]
+                )
+                flow = flow + flow_d
+                mask = mask + mask_d
+            else:
+                flow, mask = stu[i](torch.cat((img0, img1), 1), None, scale=scale[i])
+            mask_list.append(torch.sigmoid(mask))
+            flow_list.append(flow)
+            warped_img0 = warp(img0, flow[:, :2])
+            warped_img1 = warp(img1, flow[:, 2:4])
+            merged_student = (warped_img0, warped_img1)
+            merged.append(merged_student)
+        if gt.shape[1] == 3:
+            flow_d, mask_d = self.block_tea(
+                torch.cat((img0, img1, warped_img0, warped_img1, mask, gt), 1), flow, scale=1
+            )
+            flow_teacher = flow + flow_d
+            warped_img0_teacher = warp(img0, flow_teacher[:, :2])
+            warped_img1_teacher = warp(img1, flow_teacher[:, 2:4])
+            mask_teacher = torch.sigmoid(mask + mask_d)
+            merged_teacher = warped_img0_teacher * mask_teacher + warped_img1_teacher * (1 - mask_teacher)
+        else:
+            flow_teacher = None
+            merged_teacher = None
+        for i in range(3):
+            merged[i] = merged[i][0] * mask_list[i] + merged[i][1] * (1 - mask_list[i])
+            if gt.shape[1] == 3:
+                loss_mask = (
+                    ((merged[i] - gt).abs().mean(1, True) > (merged_teacher - gt).abs().mean(1, True) + 0.01)
+                    .float()
+                    .detach()
+                )
+                loss_distill += (((flow_teacher.detach() - flow_list[i]) ** 2).mean(1, True) ** 0.5 * loss_mask).mean()
+        c0 = self.contextnet(img0, flow[:, :2])
+        c1 = self.contextnet(img1, flow[:, 2:4])
+        tmp = self.unet(img0, img1, warped_img0, warped_img1, mask, flow, c0, c1)
+        res = tmp[:, :3] * 2 - 1
+        merged[2] = torch.clamp(merged[2] + res, 0, 1)
+        return flow_list, mask_list[2], merged, flow_teacher, merged_teacher, loss_distill

inference/rife/IFNet_2R.py

@@ -0,0 +1,123 @@
+from .refine_2R import *
+
+
+def deconv(in_planes, out_planes, kernel_size=4, stride=2, padding=1):
+    return nn.Sequential(
+        torch.nn.ConvTranspose2d(in_channels=in_planes, out_channels=out_planes, kernel_size=4, stride=2, padding=1),
+        nn.PReLU(out_planes),
+    )
+
+
+def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
+    return nn.Sequential(
+        nn.Conv2d(
+            in_planes,
+            out_planes,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=True,
+        ),
+        nn.PReLU(out_planes),
+    )
+
+
+class IFBlock(nn.Module):
+    def __init__(self, in_planes, c=64):
+        super(IFBlock, self).__init__()
+        self.conv0 = nn.Sequential(
+            conv(in_planes, c // 2, 3, 1, 1),
+            conv(c // 2, c, 3, 2, 1),
+        )
+        self.convblock = nn.Sequential(
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+        )
+        self.lastconv = nn.ConvTranspose2d(c, 5, 4, 2, 1)
+
+    def forward(self, x, flow, scale):
+        if scale != 1:
+            x = F.interpolate(x, scale_factor=1.0 / scale, mode="bilinear", align_corners=False)
+        if flow != None:
+            flow = F.interpolate(flow, scale_factor=1.0 / scale, mode="bilinear", align_corners=False) * 1.0 / scale
+            x = torch.cat((x, flow), 1)
+        x = self.conv0(x)
+        x = self.convblock(x) + x
+        tmp = self.lastconv(x)
+        tmp = F.interpolate(tmp, scale_factor=scale, mode="bilinear", align_corners=False)
+        flow = tmp[:, :4] * scale
+        mask = tmp[:, 4:5]
+        return flow, mask
+
+
+class IFNet(nn.Module):
+    def __init__(self):
+        super(IFNet, self).__init__()
+        self.block0 = IFBlock(6, c=240)
+        self.block1 = IFBlock(13 + 4, c=150)
+        self.block2 = IFBlock(13 + 4, c=90)
+        self.block_tea = IFBlock(16 + 4, c=90)
+        self.contextnet = Contextnet()
+        self.unet = Unet()
+
+    def forward(self, x, scale=[4, 2, 1], timestep=0.5):
+        img0 = x[:, :3]
+        img1 = x[:, 3:6]
+        gt = x[:, 6:]  # In inference time, gt is None
+        flow_list = []
+        merged = []
+        mask_list = []
+        warped_img0 = img0
+        warped_img1 = img1
+        flow = None
+        loss_distill = 0
+        stu = [self.block0, self.block1, self.block2]
+        for i in range(3):
+            if flow != None:
+                flow_d, mask_d = stu[i](
+                    torch.cat((img0, img1, warped_img0, warped_img1, mask), 1), flow, scale=scale[i]
+                )
+                flow = flow + flow_d
+                mask = mask + mask_d
+            else:
+                flow, mask = stu[i](torch.cat((img0, img1), 1), None, scale=scale[i])
+            mask_list.append(torch.sigmoid(mask))
+            flow_list.append(flow)
+            warped_img0 = warp(img0, flow[:, :2])
+            warped_img1 = warp(img1, flow[:, 2:4])
+            merged_student = (warped_img0, warped_img1)
+            merged.append(merged_student)
+        if gt.shape[1] == 3:
+            flow_d, mask_d = self.block_tea(
+                torch.cat((img0, img1, warped_img0, warped_img1, mask, gt), 1), flow, scale=1
+            )
+            flow_teacher = flow + flow_d
+            warped_img0_teacher = warp(img0, flow_teacher[:, :2])
+            warped_img1_teacher = warp(img1, flow_teacher[:, 2:4])
+            mask_teacher = torch.sigmoid(mask + mask_d)
+            merged_teacher = warped_img0_teacher * mask_teacher + warped_img1_teacher * (1 - mask_teacher)
+        else:
+            flow_teacher = None
+            merged_teacher = None
+        for i in range(3):
+            merged[i] = merged[i][0] * mask_list[i] + merged[i][1] * (1 - mask_list[i])
+            if gt.shape[1] == 3:
+                loss_mask = (
+                    ((merged[i] - gt).abs().mean(1, True) > (merged_teacher - gt).abs().mean(1, True) + 0.01)
+                    .float()
+                    .detach()
+                )
+                loss_distill += (((flow_teacher.detach() - flow_list[i]) ** 2).mean(1, True) ** 0.5 * loss_mask).mean()
+        c0 = self.contextnet(img0, flow[:, :2])
+        c1 = self.contextnet(img1, flow[:, 2:4])
+        tmp = self.unet(img0, img1, warped_img0, warped_img1, mask, flow, c0, c1)
+        res = tmp[:, :3] * 2 - 1
+        merged[2] = torch.clamp(merged[2] + res, 0, 1)
+        return flow_list, mask_list[2], merged, flow_teacher, merged_teacher, loss_distill

inference/rife/IFNet_HDv3.py

@@ -0,0 +1,138 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from .warplayer import warp
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
+    return nn.Sequential(
+        nn.Conv2d(
+            in_planes,
+            out_planes,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=True,
+        ),
+        nn.PReLU(out_planes),
+    )
+
+
+def conv_bn(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
+    return nn.Sequential(
+        nn.Conv2d(
+            in_planes,
+            out_planes,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=False,
+        ),
+        nn.BatchNorm2d(out_planes),
+        nn.PReLU(out_planes),
+    )
+
+
+class IFBlock(nn.Module):
+    def __init__(self, in_planes, c=64):
+        super(IFBlock, self).__init__()
+        self.conv0 = nn.Sequential(
+            conv(in_planes, c // 2, 3, 2, 1),
+            conv(c // 2, c, 3, 2, 1),
+        )
+        self.convblock0 = nn.Sequential(conv(c, c), conv(c, c))
+        self.convblock1 = nn.Sequential(conv(c, c), conv(c, c))
+        self.convblock2 = nn.Sequential(conv(c, c), conv(c, c))
+        self.convblock3 = nn.Sequential(conv(c, c), conv(c, c))
+        self.conv1 = nn.Sequential(
+            nn.ConvTranspose2d(c, c // 2, 4, 2, 1),
+            nn.PReLU(c // 2),
+            nn.ConvTranspose2d(c // 2, 4, 4, 2, 1),
+        )
+        self.conv2 = nn.Sequential(
+            nn.ConvTranspose2d(c, c // 2, 4, 2, 1),
+            nn.PReLU(c // 2),
+            nn.ConvTranspose2d(c // 2, 1, 4, 2, 1),
+        )
+
+    def forward(self, x, flow, scale=1):
+        x = F.interpolate(
+            x, scale_factor=1.0 / scale, mode="bilinear", align_corners=False, recompute_scale_factor=False
+        )
+        flow = (
+            F.interpolate(
+                flow, scale_factor=1.0 / scale, mode="bilinear", align_corners=False, recompute_scale_factor=False
+            )
+            * 1.0
+            / scale
+        )
+        feat = self.conv0(torch.cat((x, flow), 1))
+        feat = self.convblock0(feat) + feat
+        feat = self.convblock1(feat) + feat
+        feat = self.convblock2(feat) + feat
+        feat = self.convblock3(feat) + feat
+        flow = self.conv1(feat)
+        mask = self.conv2(feat)
+        flow = (
+            F.interpolate(flow, scale_factor=scale, mode="bilinear", align_corners=False, recompute_scale_factor=False)
+            * scale
+        )
+        mask = F.interpolate(
+            mask, scale_factor=scale, mode="bilinear", align_corners=False, recompute_scale_factor=False
+        )
+        return flow, mask
+
+
+class IFNet(nn.Module):
+    def __init__(self):
+        super(IFNet, self).__init__()
+        self.block0 = IFBlock(7 + 4, c=90)
+        self.block1 = IFBlock(7 + 4, c=90)
+        self.block2 = IFBlock(7 + 4, c=90)
+        self.block_tea = IFBlock(10 + 4, c=90)
+        # self.contextnet = Contextnet()
+        # self.unet = Unet()
+
+    def forward(self, x, scale_list=[4, 2, 1], training=False):
+        if training == False:
+            channel = x.shape[1] // 2
+            img0 = x[:, :channel]
+            img1 = x[:, channel:]
+        flow_list = []
+        merged = []
+        mask_list = []
+        warped_img0 = img0
+        warped_img1 = img1
+        flow = (x[:, :4]).detach() * 0
+        mask = (x[:, :1]).detach() * 0
+        loss_cons = 0
+        block = [self.block0, self.block1, self.block2]
+        for i in range(3):
+            f0, m0 = block[i](torch.cat((warped_img0[:, :3], warped_img1[:, :3], mask), 1), flow, scale=scale_list[i])
+            f1, m1 = block[i](
+                torch.cat((warped_img1[:, :3], warped_img0[:, :3], -mask), 1),
+                torch.cat((flow[:, 2:4], flow[:, :2]), 1),
+                scale=scale_list[i],
+            )
+            flow = flow + (f0 + torch.cat((f1[:, 2:4], f1[:, :2]), 1)) / 2
+            mask = mask + (m0 + (-m1)) / 2
+            mask_list.append(mask)
+            flow_list.append(flow)
+            warped_img0 = warp(img0, flow[:, :2])
+            warped_img1 = warp(img1, flow[:, 2:4])
+            merged.append((warped_img0, warped_img1))
+        """
+        c0 = self.contextnet(img0, flow[:, :2])
+        c1 = self.contextnet(img1, flow[:, 2:4])
+        tmp = self.unet(img0, img1, warped_img0, warped_img1, mask, flow, c0, c1)
+        res = tmp[:, 1:4] * 2 - 1
+        """
+        for i in range(3):
+            mask_list[i] = torch.sigmoid(mask_list[i])
+            merged[i] = merged[i][0] * mask_list[i] + merged[i][1] * (1 - mask_list[i])
+            # merged[i] = torch.clamp(merged[i] + res, 0, 1)
+        return flow_list, mask_list[2], merged

inference/rife/IFNet_m.py

@@ -0,0 +1,127 @@
+from .refine import *
+
+
+def deconv(in_planes, out_planes, kernel_size=4, stride=2, padding=1):
+    return nn.Sequential(
+        torch.nn.ConvTranspose2d(in_channels=in_planes, out_channels=out_planes, kernel_size=4, stride=2, padding=1),
+        nn.PReLU(out_planes),
+    )
+
+
+def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
+    return nn.Sequential(
+        nn.Conv2d(
+            in_planes,
+            out_planes,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=True,
+        ),
+        nn.PReLU(out_planes),
+    )
+
+
+class IFBlock(nn.Module):
+    def __init__(self, in_planes, c=64):
+        super(IFBlock, self).__init__()
+        self.conv0 = nn.Sequential(
+            conv(in_planes, c // 2, 3, 2, 1),
+            conv(c // 2, c, 3, 2, 1),
+        )
+        self.convblock = nn.Sequential(
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+            conv(c, c),
+        )
+        self.lastconv = nn.ConvTranspose2d(c, 5, 4, 2, 1)
+
+    def forward(self, x, flow, scale):
+        if scale != 1:
+            x = F.interpolate(x, scale_factor=1.0 / scale, mode="bilinear", align_corners=False)
+        if flow != None:
+            flow = F.interpolate(flow, scale_factor=1.0 / scale, mode="bilinear", align_corners=False) * 1.0 / scale
+            x = torch.cat((x, flow), 1)
+        x = self.conv0(x)
+        x = self.convblock(x) + x
+        tmp = self.lastconv(x)
+        tmp = F.interpolate(tmp, scale_factor=scale * 2, mode="bilinear", align_corners=False)
+        flow = tmp[:, :4] * scale * 2
+        mask = tmp[:, 4:5]
+        return flow, mask
+
+
+class IFNet_m(nn.Module):
+    def __init__(self):
+        super(IFNet_m, self).__init__()
+        self.block0 = IFBlock(6 + 1, c=240)
+        self.block1 = IFBlock(13 + 4 + 1, c=150)
+        self.block2 = IFBlock(13 + 4 + 1, c=90)
+        self.block_tea = IFBlock(16 + 4 + 1, c=90)
+        self.contextnet = Contextnet()
+        self.unet = Unet()
+
+    def forward(self, x, scale=[4, 2, 1], timestep=0.5, returnflow=False):
+        timestep = (x[:, :1].clone() * 0 + 1) * timestep
+        img0 = x[:, :3]
+        img1 = x[:, 3:6]
+        gt = x[:, 6:]  # In inference time, gt is None
+        flow_list = []
+        merged = []
+        mask_list = []
+        warped_img0 = img0
+        warped_img1 = img1
+        flow = None
+        loss_distill = 0
+        stu = [self.block0, self.block1, self.block2]
+        for i in range(3):
+            if flow != None:
+                flow_d, mask_d = stu[i](
+                    torch.cat((img0, img1, timestep, warped_img0, warped_img1, mask), 1), flow, scale=scale[i]
+                )
+                flow = flow + flow_d
+                mask = mask + mask_d
+            else:
+                flow, mask = stu[i](torch.cat((img0, img1, timestep), 1), None, scale=scale[i])
+            mask_list.append(torch.sigmoid(mask))
+            flow_list.append(flow)
+            warped_img0 = warp(img0, flow[:, :2])
+            warped_img1 = warp(img1, flow[:, 2:4])
+            merged_student = (warped_img0, warped_img1)
+            merged.append(merged_student)
+        if gt.shape[1] == 3:
+            flow_d, mask_d = self.block_tea(
+                torch.cat((img0, img1, timestep, warped_img0, warped_img1, mask, gt), 1), flow, scale=1
+            )
+            flow_teacher = flow + flow_d
+            warped_img0_teacher = warp(img0, flow_teacher[:, :2])
+            warped_img1_teacher = warp(img1, flow_teacher[:, 2:4])
+            mask_teacher = torch.sigmoid(mask + mask_d)
+            merged_teacher = warped_img0_teacher * mask_teacher + warped_img1_teacher * (1 - mask_teacher)
+        else:
+            flow_teacher = None
+            merged_teacher = None
+        for i in range(3):
+            merged[i] = merged[i][0] * mask_list[i] + merged[i][1] * (1 - mask_list[i])
+            if gt.shape[1] == 3:
+                loss_mask = (
+                    ((merged[i] - gt).abs().mean(1, True) > (merged_teacher - gt).abs().mean(1, True) + 0.01)
+                    .float()
+                    .detach()
+                )
+                loss_distill += (((flow_teacher.detach() - flow_list[i]) ** 2).mean(1, True) ** 0.5 * loss_mask).mean()
+        if returnflow:
+            return flow
+        else:
+            c0 = self.contextnet(img0, flow[:, :2])
+            c1 = self.contextnet(img1, flow[:, 2:4])
+            tmp = self.unet(img0, img1, warped_img0, warped_img1, mask, flow, c0, c1)
+            res = tmp[:, :3] * 2 - 1
+            merged[2] = torch.clamp(merged[2] + res, 0, 1)
+        return flow_list, mask_list[2], merged, flow_teacher, merged_teacher, loss_distill

inference/rife/RIFE.py

@@ -0,0 +1,95 @@
+from torch.optim import AdamW
+from torch.nn.parallel import DistributedDataParallel as DDP
+from .IFNet import *
+from .IFNet_m import *
+from .loss import *
+from .laplacian import *
+from .refine import *
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class Model:
+    def __init__(self, local_rank=-1, arbitrary=False):
+        if arbitrary == True:
+            self.flownet = IFNet_m()
+        else:
+            self.flownet = IFNet()
+        self.device()
+        self.optimG = AdamW(
+            self.flownet.parameters(), lr=1e-6, weight_decay=1e-3
+        )  # use large weight decay may avoid NaN loss
+        self.epe = EPE()
+        self.lap = LapLoss()
+        self.sobel = SOBEL()
+        if local_rank != -1:
+            self.flownet = DDP(self.flownet, device_ids=[local_rank], output_device=local_rank)
+
+    def train(self):
+        self.flownet.train()
+
+    def eval(self):
+        self.flownet.eval()
+
+    def device(self):
+        self.flownet.to(device)
+
+    def load_model(self, path, rank=0):
+        def convert(param):
+            return {k.replace("module.", ""): v for k, v in param.items() if "module." in k}
+
+        if rank <= 0:
+            self.flownet.load_state_dict(convert(torch.load("{}/flownet.pkl".format(path))))
+
+    def save_model(self, path, rank=0):
+        if rank == 0:
+            torch.save(self.flownet.state_dict(), "{}/flownet.pkl".format(path))
+
+    def inference(self, img0, img1, scale=1, scale_list=[4, 2, 1], TTA=False, timestep=0.5):
+        for i in range(3):
+            scale_list[i] = scale_list[i] * 1.0 / scale
+        imgs = torch.cat((img0, img1), 1)
+        flow, mask, merged, flow_teacher, merged_teacher, loss_distill = self.flownet(
+            imgs, scale_list, timestep=timestep
+        )
+        if TTA == False:
+            return merged[2]
+        else:
+            flow2, mask2, merged2, flow_teacher2, merged_teacher2, loss_distill2 = self.flownet(
+                imgs.flip(2).flip(3), scale_list, timestep=timestep
+            )
+            return (merged[2] + merged2[2].flip(2).flip(3)) / 2
+
+    def update(self, imgs, gt, learning_rate=0, mul=1, training=True, flow_gt=None):
+        for param_group in self.optimG.param_groups:
+            param_group["lr"] = learning_rate
+        img0 = imgs[:, :3]
+        img1 = imgs[:, 3:]
+        if training:
+            self.train()
+        else:
+            self.eval()
+        flow, mask, merged, flow_teacher, merged_teacher, loss_distill = self.flownet(
+            torch.cat((imgs, gt), 1), scale=[4, 2, 1]
+        )
+        loss_l1 = (self.lap(merged[2], gt)).mean()
+        loss_tea = (self.lap(merged_teacher, gt)).mean()
+        if training:
+            self.optimG.zero_grad()
+            loss_G = (
+                loss_l1 + loss_tea + loss_distill * 0.01
+            )  # when training RIFEm, the weight of loss_distill should be 0.005 or 0.002
+            loss_G.backward()
+            self.optimG.step()
+        else:
+            flow_teacher = flow[2]
+        return merged[2], {
+            "merged_tea": merged_teacher,
+            "mask": mask,
+            "mask_tea": mask,
+            "flow": flow[2][:, :2],
+            "flow_tea": flow_teacher,
+            "loss_l1": loss_l1,
+            "loss_tea": loss_tea,
+            "loss_distill": loss_distill,
+        }

inference/rife/RIFE_HDv3.py

@@ -0,0 +1,86 @@
+import torch
+import torch.nn as nn
+import numpy as np
+from torch.optim import AdamW
+import torch.optim as optim
+import itertools
+from .warplayer import warp
+from torch.nn.parallel import DistributedDataParallel as DDP
+from .IFNet_HDv3 import *
+import torch.nn.functional as F
+from .loss import *
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class Model:
+    def __init__(self, local_rank=-1):
+        self.flownet = IFNet()
+        self.device()
+        self.optimG = AdamW(self.flownet.parameters(), lr=1e-6, weight_decay=1e-4)
+        self.epe = EPE()
+        # self.vgg = VGGPerceptualLoss().to(device)
+        self.sobel = SOBEL()
+        if local_rank != -1:
+            self.flownet = DDP(self.flownet, device_ids=[local_rank], output_device=local_rank)
+
+    def train(self):
+        self.flownet.train()
+
+    def eval(self):
+        self.flownet.eval()
+
+    def device(self):
+        self.flownet.to(device)
+
+    def load_model(self, path, rank=0):
+        def convert(param):
+            if rank == -1:
+                return {k.replace("module.", ""): v for k, v in param.items() if "module." in k}
+            else:
+                return param
+
+        if rank <= 0:
+            if torch.cuda.is_available():
+                self.flownet.load_state_dict(convert(torch.load("{}/flownet.pkl".format(path))))
+            else:
+                self.flownet.load_state_dict(convert(torch.load("{}/flownet.pkl".format(path), map_location="cpu")))
+
+    def save_model(self, path, rank=0):
+        if rank == 0:
+            torch.save(self.flownet.state_dict(), "{}/flownet.pkl".format(path))
+
+    def inference(self, img0, img1, scale=1.0):
+        imgs = torch.cat((img0, img1), 1)
+        scale_list = [4 / scale, 2 / scale, 1 / scale]
+        flow, mask, merged = self.flownet(imgs, scale_list)
+        return merged[2]
+
+    def update(self, imgs, gt, learning_rate=0, mul=1, training=True, flow_gt=None):
+        for param_group in self.optimG.param_groups:
+            param_group["lr"] = learning_rate
+        img0 = imgs[:, :3]
+        img1 = imgs[:, 3:]
+        if training:
+            self.train()
+        else:
+            self.eval()
+        scale = [4, 2, 1]
+        flow, mask, merged = self.flownet(torch.cat((imgs, gt), 1), scale=scale, training=training)
+        loss_l1 = (merged[2] - gt).abs().mean()
+        loss_smooth = self.sobel(flow[2], flow[2] * 0).mean()
+        # loss_vgg = self.vgg(merged[2], gt)
+        if training:
+            self.optimG.zero_grad()
+            loss_G = loss_cons + loss_smooth * 0.1
+            loss_G.backward()
+            self.optimG.step()
+        else:
+            flow_teacher = flow[2]
+        return merged[2], {
+            "mask": mask,
+            "flow": flow[2][:, :2],
+            "loss_l1": loss_l1,
+            "loss_cons": loss_cons,
+            "loss_smooth": loss_smooth,
+        }

inference/rife/laplacian.py

@@ -0,0 +1,69 @@
+import torch
+import numpy as np
+import torch.nn as nn
+import torch.nn.functional as F
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+import torch
+
+
+def gauss_kernel(size=5, channels=3):
+    kernel = torch.tensor(
+        [
+            [1.0, 4.0, 6.0, 4.0, 1],
+            [4.0, 16.0, 24.0, 16.0, 4.0],
+            [6.0, 24.0, 36.0, 24.0, 6.0],
+            [4.0, 16.0, 24.0, 16.0, 4.0],
+            [1.0, 4.0, 6.0, 4.0, 1.0],
+        ]
+    )
+    kernel /= 256.0
+    kernel = kernel.repeat(channels, 1, 1, 1)
+    kernel = kernel.to(device)
+    return kernel
+
+
+def downsample(x):
+    return x[:, :, ::2, ::2]
+
+
+def upsample(x):
+    cc = torch.cat([x, torch.zeros(x.shape[0], x.shape[1], x.shape[2], x.shape[3]).to(device)], dim=3)
+    cc = cc.view(x.shape[0], x.shape[1], x.shape[2] * 2, x.shape[3])
+    cc = cc.permute(0, 1, 3, 2)
+    cc = torch.cat([cc, torch.zeros(x.shape[0], x.shape[1], x.shape[3], x.shape[2] * 2).to(device)], dim=3)
+    cc = cc.view(x.shape[0], x.shape[1], x.shape[3] * 2, x.shape[2] * 2)
+    x_up = cc.permute(0, 1, 3, 2)
+    return conv_gauss(x_up, 4 * gauss_kernel(channels=x.shape[1]))
+
+
+def conv_gauss(img, kernel):
+    img = torch.nn.functional.pad(img, (2, 2, 2, 2), mode="reflect")
+    out = torch.nn.functional.conv2d(img, kernel, groups=img.shape[1])
+    return out
+
+
+def laplacian_pyramid(img, kernel, max_levels=3):
+    current = img
+    pyr = []
+    for level in range(max_levels):
+        filtered = conv_gauss(current, kernel)
+        down = downsample(filtered)
+        up = upsample(down)
+        diff = current - up
+        pyr.append(diff)
+        current = down
+    return pyr
+
+
+class LapLoss(torch.nn.Module):
+    def __init__(self, max_levels=5, channels=3):
+        super(LapLoss, self).__init__()
+        self.max_levels = max_levels
+        self.gauss_kernel = gauss_kernel(channels=channels)
+
+    def forward(self, input, target):
+        pyr_input = laplacian_pyramid(img=input, kernel=self.gauss_kernel, max_levels=self.max_levels)
+        pyr_target = laplacian_pyramid(img=target, kernel=self.gauss_kernel, max_levels=self.max_levels)
+        return sum(torch.nn.functional.l1_loss(a, b) for a, b in zip(pyr_input, pyr_target))

inference/rife/loss.py

@@ -0,0 +1,130 @@
+import torch
+import numpy as np
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision.models as models
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+class EPE(nn.Module):
+    def __init__(self):
+        super(EPE, self).__init__()
+
+    def forward(self, flow, gt, loss_mask):
+        loss_map = (flow - gt.detach()) ** 2
+        loss_map = (loss_map.sum(1, True) + 1e-6) ** 0.5
+        return loss_map * loss_mask
+
+
+class Ternary(nn.Module):
+    def __init__(self):
+        super(Ternary, self).__init__()
+        patch_size = 7
+        out_channels = patch_size * patch_size
+        self.w = np.eye(out_channels).reshape((patch_size, patch_size, 1, out_channels))
+        self.w = np.transpose(self.w, (3, 2, 0, 1))
+        self.w = torch.tensor(self.w).float().to(device)
+
+    def transform(self, img):
+        patches = F.conv2d(img, self.w, padding=3, bias=None)
+        transf = patches - img
+        transf_norm = transf / torch.sqrt(0.81 + transf**2)
+        return transf_norm
+
+    def rgb2gray(self, rgb):
+        r, g, b = rgb[:, 0:1, :, :], rgb[:, 1:2, :, :], rgb[:, 2:3, :, :]
+        gray = 0.2989 * r + 0.5870 * g + 0.1140 * b
+        return gray
+
+    def hamming(self, t1, t2):
+        dist = (t1 - t2) ** 2
+        dist_norm = torch.mean(dist / (0.1 + dist), 1, True)
+        return dist_norm
+
+    def valid_mask(self, t, padding):
+        n, _, h, w = t.size()
+        inner = torch.ones(n, 1, h - 2 * padding, w - 2 * padding).type_as(t)
+        mask = F.pad(inner, [padding] * 4)
+        return mask
+
+    def forward(self, img0, img1):
+        img0 = self.transform(self.rgb2gray(img0))
+        img1 = self.transform(self.rgb2gray(img1))
+        return self.hamming(img0, img1) * self.valid_mask(img0, 1)
+
+
+class SOBEL(nn.Module):
+    def __init__(self):
+        super(SOBEL, self).__init__()
+        self.kernelX = torch.tensor(
+            [
+                [1, 0, -1],
+                [2, 0, -2],
+                [1, 0, -1],
+            ]
+        ).float()
+        self.kernelY = self.kernelX.clone().T
+        self.kernelX = self.kernelX.unsqueeze(0).unsqueeze(0).to(device)
+        self.kernelY = self.kernelY.unsqueeze(0).unsqueeze(0).to(device)
+
+    def forward(self, pred, gt):
+        N, C, H, W = pred.shape[0], pred.shape[1], pred.shape[2], pred.shape[3]
+        img_stack = torch.cat([pred.reshape(N * C, 1, H, W), gt.reshape(N * C, 1, H, W)], 0)
+        sobel_stack_x = F.conv2d(img_stack, self.kernelX, padding=1)
+        sobel_stack_y = F.conv2d(img_stack, self.kernelY, padding=1)
+        pred_X, gt_X = sobel_stack_x[: N * C], sobel_stack_x[N * C :]
+        pred_Y, gt_Y = sobel_stack_y[: N * C], sobel_stack_y[N * C :]
+
+        L1X, L1Y = torch.abs(pred_X - gt_X), torch.abs(pred_Y - gt_Y)
+        loss = L1X + L1Y
+        return loss
+
+
+class MeanShift(nn.Conv2d):
+    def __init__(self, data_mean, data_std, data_range=1, norm=True):
+        c = len(data_mean)
+        super(MeanShift, self).__init__(c, c, kernel_size=1)
+        std = torch.Tensor(data_std)
+        self.weight.data = torch.eye(c).view(c, c, 1, 1)
+        if norm:
+            self.weight.data.div_(std.view(c, 1, 1, 1))
+            self.bias.data = -1 * data_range * torch.Tensor(data_mean)
+            self.bias.data.div_(std)
+        else:
+            self.weight.data.mul_(std.view(c, 1, 1, 1))
+            self.bias.data = data_range * torch.Tensor(data_mean)
+        self.requires_grad = False
+
+
+class VGGPerceptualLoss(torch.nn.Module):
+    def __init__(self, rank=0):
+        super(VGGPerceptualLoss, self).__init__()
+        blocks = []
+        pretrained = True
+        self.vgg_pretrained_features = models.vgg19(pretrained=pretrained).features
+        self.normalize = MeanShift([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], norm=True).cuda()
+        for param in self.parameters():
+            param.requires_grad = False
+
+    def forward(self, X, Y, indices=None):
+        X = self.normalize(X)
+        Y = self.normalize(Y)
+        indices = [2, 7, 12, 21, 30]
+        weights = [1.0 / 2.6, 1.0 / 4.8, 1.0 / 3.7, 1.0 / 5.6, 10 / 1.5]
+        k = 0
+        loss = 0
+        for i in range(indices[-1]):
+            X = self.vgg_pretrained_features[i](X)
+            Y = self.vgg_pretrained_features[i](Y)
+            if (i + 1) in indices:
+                loss += weights[k] * (X - Y.detach()).abs().mean() * 0.1
+                k += 1
+        return loss
+
+
+if __name__ == "__main__":
+    img0 = torch.zeros(3, 3, 256, 256).float().to(device)
+    img1 = torch.tensor(np.random.normal(0, 1, (3, 3, 256, 256))).float().to(device)
+    ternary_loss = Ternary()
+    print(ternary_loss(img0, img1).shape)

inference/rife/pytorch_msssim/__init__.py

@@ -0,0 +1,203 @@
+import torch
+import torch.nn.functional as F
+from math import exp
+import numpy as np
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def gaussian(window_size, sigma):
+    gauss = torch.Tensor([exp(-((x - window_size // 2) ** 2) / float(2 * sigma**2)) for x in range(window_size)])
+    return gauss / gauss.sum()
+
+
+def create_window(window_size, channel=1):
+    _1D_window = gaussian(window_size, 1.5).unsqueeze(1)
+    _2D_window = _1D_window.mm(_1D_window.t()).float().unsqueeze(0).unsqueeze(0).to(device)
+    window = _2D_window.expand(channel, 1, window_size, window_size).contiguous()
+    return window
+
+
+def create_window_3d(window_size, channel=1):
+    _1D_window = gaussian(window_size, 1.5).unsqueeze(1)
+    _2D_window = _1D_window.mm(_1D_window.t())
+    _3D_window = _2D_window.unsqueeze(2) @ (_1D_window.t())
+    window = _3D_window.expand(1, channel, window_size, window_size, window_size).contiguous().to(device)
+    return window
+
+
+def ssim(img1, img2, window_size=11, window=None, size_average=True, full=False, val_range=None):
+    # Value range can be different from 255. Other common ranges are 1 (sigmoid) and 2 (tanh).
+    if val_range is None:
+        if torch.max(img1) > 128:
+            max_val = 255
+        else:
+            max_val = 1
+
+        if torch.min(img1) < -0.5:
+            min_val = -1
+        else:
+            min_val = 0
+        L = max_val - min_val
+    else:
+        L = val_range
+
+    padd = 0
+    (_, channel, height, width) = img1.size()
+    if window is None:
+        real_size = min(window_size, height, width)
+        window = create_window(real_size, channel=channel).to(img1.device)
+
+    # mu1 = F.conv2d(img1, window, padding=padd, groups=channel)
+    # mu2 = F.conv2d(img2, window, padding=padd, groups=channel)
+    mu1 = F.conv2d(F.pad(img1, (5, 5, 5, 5), mode="replicate"), window, padding=padd, groups=channel)
+    mu2 = F.conv2d(F.pad(img2, (5, 5, 5, 5), mode="replicate"), window, padding=padd, groups=channel)
+
+    mu1_sq = mu1.pow(2)
+    mu2_sq = mu2.pow(2)
+    mu1_mu2 = mu1 * mu2
+
+    sigma1_sq = F.conv2d(F.pad(img1 * img1, (5, 5, 5, 5), "replicate"), window, padding=padd, groups=channel) - mu1_sq
+    sigma2_sq = F.conv2d(F.pad(img2 * img2, (5, 5, 5, 5), "replicate"), window, padding=padd, groups=channel) - mu2_sq
+    sigma12 = F.conv2d(F.pad(img1 * img2, (5, 5, 5, 5), "replicate"), window, padding=padd, groups=channel) - mu1_mu2
+
+    C1 = (0.01 * L) ** 2
+    C2 = (0.03 * L) ** 2
+
+    v1 = 2.0 * sigma12 + C2
+    v2 = sigma1_sq + sigma2_sq + C2
+    cs = torch.mean(v1 / v2)  # contrast sensitivity
+
+    ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2)
+
+    if size_average:
+        ret = ssim_map.mean()
+    else:
+        ret = ssim_map.mean(1).mean(1).mean(1)
+
+    if full:
+        return ret, cs
+    return ret
+
+
+def ssim_matlab(img1, img2, window_size=11, window=None, size_average=True, full=False, val_range=None):
+    # Value range can be different from 255. Other common ranges are 1 (sigmoid) and 2 (tanh).
+    if val_range is None:
+        if torch.max(img1) > 128:
+            max_val = 255
+        else:
+            max_val = 1
+
+        if torch.min(img1) < -0.5:
+            min_val = -1
+        else:
+            min_val = 0
+        L = max_val - min_val
+    else:
+        L = val_range
+
+    padd = 0
+    (_, _, height, width) = img1.size()
+    if window is None:
+        real_size = min(window_size, height, width)
+        window = create_window_3d(real_size, channel=1).to(img1.device, dtype=img1.dtype)
+        # Channel is set to 1 since we consider color images as volumetric images
+
+    img1 = img1.unsqueeze(1)
+    img2 = img2.unsqueeze(1)
+
+    mu1 = F.conv3d(F.pad(img1, (5, 5, 5, 5, 5, 5), mode="replicate"), window, padding=padd, groups=1)
+    mu2 = F.conv3d(F.pad(img2, (5, 5, 5, 5, 5, 5), mode="replicate"), window, padding=padd, groups=1)
+
+    mu1_sq = mu1.pow(2)
+    mu2_sq = mu2.pow(2)
+    mu1_mu2 = mu1 * mu2
+
+    sigma1_sq = F.conv3d(F.pad(img1 * img1, (5, 5, 5, 5, 5, 5), "replicate"), window, padding=padd, groups=1) - mu1_sq
+    sigma2_sq = F.conv3d(F.pad(img2 * img2, (5, 5, 5, 5, 5, 5), "replicate"), window, padding=padd, groups=1) - mu2_sq
+    sigma12 = F.conv3d(F.pad(img1 * img2, (5, 5, 5, 5, 5, 5), "replicate"), window, padding=padd, groups=1) - mu1_mu2
+
+    C1 = (0.01 * L) ** 2
+    C2 = (0.03 * L) ** 2
+
+    v1 = 2.0 * sigma12 + C2
+    v2 = sigma1_sq + sigma2_sq + C2
+    cs = torch.mean(v1 / v2)  # contrast sensitivity
+
+    ssim_map = ((2 * mu1_mu2 + C1) * v1) / ((mu1_sq + mu2_sq + C1) * v2)
+
+    if size_average:
+        ret = ssim_map.mean()
+    else:
+        ret = ssim_map.mean(1).mean(1).mean(1)
+
+    if full:
+        return ret, cs
+    return ret
+
+
+def msssim(img1, img2, window_size=11, size_average=True, val_range=None, normalize=False):
+    device = img1.device
+    weights = torch.FloatTensor([0.0448, 0.2856, 0.3001, 0.2363, 0.1333]).to(device)
+    levels = weights.size()[0]
+    mssim = []
+    mcs = []
+    for _ in range(levels):
+        sim, cs = ssim(img1, img2, window_size=window_size, size_average=size_average, full=True, val_range=val_range)
+        mssim.append(sim)
+        mcs.append(cs)
+
+        img1 = F.avg_pool2d(img1, (2, 2))
+        img2 = F.avg_pool2d(img2, (2, 2))
+
+    mssim = torch.stack(mssim)
+    mcs = torch.stack(mcs)
+
+    # Normalize (to avoid NaNs during training unstable models, not compliant with original definition)
+    if normalize:
+        mssim = (mssim + 1) / 2
+        mcs = (mcs + 1) / 2
+
+    pow1 = mcs**weights
+    pow2 = mssim**weights
+    # From Matlab implementation https://ece.uwaterloo.ca/~z70wang/research/iwssim/
+    output = torch.prod(pow1[:-1] * pow2[-1])
+    return output
+
+
+# Classes to re-use window
+class SSIM(torch.nn.Module):
+    def __init__(self, window_size=11, size_average=True, val_range=None):
+        super(SSIM, self).__init__()
+        self.window_size = window_size
+        self.size_average = size_average
+        self.val_range = val_range
+
+        # Assume 3 channel for SSIM
+        self.channel = 3
+        self.window = create_window(window_size, channel=self.channel)
+
+    def forward(self, img1, img2):
+        (_, channel, _, _) = img1.size()
+
+        if channel == self.channel and self.window.dtype == img1.dtype:
+            window = self.window
+        else:
+            window = create_window(self.window_size, channel).to(img1.device).type(img1.dtype)
+            self.window = window
+            self.channel = channel
+
+        _ssim = ssim(img1, img2, window=window, window_size=self.window_size, size_average=self.size_average)
+        dssim = (1 - _ssim) / 2
+        return dssim
+
+
+class MSSSIM(torch.nn.Module):
+    def __init__(self, window_size=11, size_average=True, channel=3):
+        super(MSSSIM, self).__init__()
+        self.window_size = window_size
+        self.size_average = size_average
+        self.channel = channel
+
+    def forward(self, img1, img2):
+        return msssim(img1, img2, window_size=self.window_size, size_average=self.size_average)

inference/rife/refine.py

@@ -0,0 +1,107 @@
+import torch
+import torch.nn as nn
+from .warplayer import warp
+import torch.nn.functional as F
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
+    return nn.Sequential(
+        nn.Conv2d(
+            in_planes,
+            out_planes,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=True,
+        ),
+        nn.PReLU(out_planes),
+    )
+
+
+def deconv(in_planes, out_planes, kernel_size=4, stride=2, padding=1):
+    return nn.Sequential(
+        torch.nn.ConvTranspose2d(
+            in_channels=in_planes, out_channels=out_planes, kernel_size=4, stride=2, padding=1, bias=True
+        ),
+        nn.PReLU(out_planes),
+    )
+
+
+class Conv2(nn.Module):
+    def __init__(self, in_planes, out_planes, stride=2):
+        super(Conv2, self).__init__()
+        self.conv1 = conv(in_planes, out_planes, 3, stride, 1)
+        self.conv2 = conv(out_planes, out_planes, 3, 1, 1)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.conv2(x)
+        return x
+
+
+c = 16
+
+
+class Contextnet(nn.Module):
+    def __init__(self):
+        super(Contextnet, self).__init__()
+        self.conv1 = Conv2(3, c)
+        self.conv2 = Conv2(c, 2 * c)
+        self.conv3 = Conv2(2 * c, 4 * c)
+        self.conv4 = Conv2(4 * c, 8 * c)
+
+    def forward(self, x, flow):
+        x = self.conv1(x)
+        flow = (
+            F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False, recompute_scale_factor=False)
+            * 0.5
+        )
+        f1 = warp(x, flow)
+        x = self.conv2(x)
+        flow = (
+            F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False, recompute_scale_factor=False)
+            * 0.5
+        )
+        f2 = warp(x, flow)
+        x = self.conv3(x)
+        flow = (
+            F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False, recompute_scale_factor=False)
+            * 0.5
+        )
+        f3 = warp(x, flow)
+        x = self.conv4(x)
+        flow = (
+            F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False, recompute_scale_factor=False)
+            * 0.5
+        )
+        f4 = warp(x, flow)
+        return [f1, f2, f3, f4]
+
+
+class Unet(nn.Module):
+    def __init__(self):
+        super(Unet, self).__init__()
+        self.down0 = Conv2(17, 2 * c)
+        self.down1 = Conv2(4 * c, 4 * c)
+        self.down2 = Conv2(8 * c, 8 * c)
+        self.down3 = Conv2(16 * c, 16 * c)
+        self.up0 = deconv(32 * c, 8 * c)
+        self.up1 = deconv(16 * c, 4 * c)
+        self.up2 = deconv(8 * c, 2 * c)
+        self.up3 = deconv(4 * c, c)
+        self.conv = nn.Conv2d(c, 3, 3, 1, 1)
+
+    def forward(self, img0, img1, warped_img0, warped_img1, mask, flow, c0, c1):
+        s0 = self.down0(torch.cat((img0, img1, warped_img0, warped_img1, mask, flow), 1))
+        s1 = self.down1(torch.cat((s0, c0[0], c1[0]), 1))
+        s2 = self.down2(torch.cat((s1, c0[1], c1[1]), 1))
+        s3 = self.down3(torch.cat((s2, c0[2], c1[2]), 1))
+        x = self.up0(torch.cat((s3, c0[3], c1[3]), 1))
+        x = self.up1(torch.cat((x, s2), 1))
+        x = self.up2(torch.cat((x, s1), 1))
+        x = self.up3(torch.cat((x, s0), 1))
+        x = self.conv(x)
+        return torch.sigmoid(x)

inference/rife/refine_2R.py

@@ -0,0 +1,104 @@
+import torch
+import torch.nn as nn
+from .warplayer import warp
+import torch.nn.functional as F
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+
+def conv(in_planes, out_planes, kernel_size=3, stride=1, padding=1, dilation=1):
+    return nn.Sequential(
+        nn.Conv2d(
+            in_planes,
+            out_planes,
+            kernel_size=kernel_size,
+            stride=stride,
+            padding=padding,
+            dilation=dilation,
+            bias=True,
+        ),
+        nn.PReLU(out_planes),
+    )
+
+
+def deconv(in_planes, out_planes, kernel_size=4, stride=2, padding=1):
+    return nn.Sequential(
+        torch.nn.ConvTranspose2d(
+            in_channels=in_planes, out_channels=out_planes, kernel_size=4, stride=2, padding=1, bias=True
+        ),
+        nn.PReLU(out_planes),
+    )
+
+
+class Conv2(nn.Module):
+    def __init__(self, in_planes, out_planes, stride=2):
+        super(Conv2, self).__init__()
+        self.conv1 = conv(in_planes, out_planes, 3, stride, 1)
+        self.conv2 = conv(out_planes, out_planes, 3, 1, 1)
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.conv2(x)
+        return x
+
+
+c = 16
+
+
+class Contextnet(nn.Module):
+    def __init__(self):
+        super(Contextnet, self).__init__()
+        self.conv1 = Conv2(3, c, 1)
+        self.conv2 = Conv2(c, 2 * c)
+        self.conv3 = Conv2(2 * c, 4 * c)
+        self.conv4 = Conv2(4 * c, 8 * c)
+
+    def forward(self, x, flow):
+        x = self.conv1(x)
+        # flow = F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False, recompute_scale_factor=False) * 0.5
+        f1 = warp(x, flow)
+        x = self.conv2(x)
+        flow = (
+            F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False, recompute_scale_factor=False)
+            * 0.5
+        )
+        f2 = warp(x, flow)
+        x = self.conv3(x)
+        flow = (
+            F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False, recompute_scale_factor=False)
+            * 0.5
+        )
+        f3 = warp(x, flow)
+        x = self.conv4(x)
+        flow = (
+            F.interpolate(flow, scale_factor=0.5, mode="bilinear", align_corners=False, recompute_scale_factor=False)
+            * 0.5
+        )
+        f4 = warp(x, flow)
+        return [f1, f2, f3, f4]
+
+
+class Unet(nn.Module):
+    def __init__(self):
+        super(Unet, self).__init__()
+        self.down0 = Conv2(17, 2 * c, 1)
+        self.down1 = Conv2(4 * c, 4 * c)
+        self.down2 = Conv2(8 * c, 8 * c)
+        self.down3 = Conv2(16 * c, 16 * c)
+        self.up0 = deconv(32 * c, 8 * c)
+        self.up1 = deconv(16 * c, 4 * c)
+        self.up2 = deconv(8 * c, 2 * c)
+        self.up3 = deconv(4 * c, c)
+        self.conv = nn.Conv2d(c, 3, 3, 2, 1)
+
+    def forward(self, img0, img1, warped_img0, warped_img1, mask, flow, c0, c1):
+        s0 = self.down0(torch.cat((img0, img1, warped_img0, warped_img1, mask, flow), 1))
+        s1 = self.down1(torch.cat((s0, c0[0], c1[0]), 1))
+        s2 = self.down2(torch.cat((s1, c0[1], c1[1]), 1))
+        s3 = self.down3(torch.cat((s2, c0[2], c1[2]), 1))
+        x = self.up0(torch.cat((s3, c0[3], c1[3]), 1))
+        x = self.up1(torch.cat((x, s2), 1))
+        x = self.up2(torch.cat((x, s1), 1))
+        x = self.up3(torch.cat((x, s0), 1))
+        x = self.conv(x)
+        return torch.sigmoid(x)

inference/rife/warplayer.py

@@ -0,0 +1,34 @@
+import torch
+import torch.nn as nn
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+backwarp_tenGrid = {}
+
+
+def warp(tenInput, tenFlow):
+    k = (str(tenFlow.device), str(tenFlow.size()))
+    if k not in backwarp_tenGrid:
+        tenHorizontal = (
+            torch.linspace(-1.0, 1.0, tenFlow.shape[3], device=device)
+            .view(1, 1, 1, tenFlow.shape[3])
+            .expand(tenFlow.shape[0], -1, tenFlow.shape[2], -1)
+        )
+        tenVertical = (
+            torch.linspace(-1.0, 1.0, tenFlow.shape[2], device=device)
+            .view(1, 1, tenFlow.shape[2], 1)
+            .expand(tenFlow.shape[0], -1, -1, tenFlow.shape[3])
+        )
+        backwarp_tenGrid[k] = torch.cat([tenHorizontal, tenVertical], 1).to(device)
+
+    tenFlow = torch.cat(
+        [
+            tenFlow[:, 0:1, :, :] / ((tenInput.shape[3] - 1.0) / 2.0),
+            tenFlow[:, 1:2, :, :] / ((tenInput.shape[2] - 1.0) / 2.0),
+        ],
+        1,
+    )
+
+    g = (backwarp_tenGrid[k] + tenFlow).permute(0, 2, 3, 1)
+    return torch.nn.functional.grid_sample(
+        input=tenInput, grid=g, mode="bilinear", padding_mode="border", align_corners=True
+    )

inference/rife_model.py

@@ -0,0 +1,129 @@
+import torch
+from diffusers.image_processor import VaeImageProcessor
+from torch.nn import functional as F
+import cv2
+import utils
+from rife.pytorch_msssim import ssim_matlab
+import numpy as np
+import logging
+import skvideo.io
+from rife.RIFE_HDv3 import Model
+
+logger = logging.getLogger(__name__)
+device = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+def pad_image(img, scale):
+    _, _, h, w = img.shape
+    tmp = max(32, int(32 / scale))
+    ph = ((h - 1) // tmp + 1) * tmp
+    pw = ((w - 1) // tmp + 1) * tmp
+    padding = (0, 0, pw - w, ph - h)
+    return F.pad(img, padding)
+
+
+def make_inference(model, I0, I1, upscale_amount, n):
+    middle = model.inference(I0, I1, upscale_amount)
+    if n == 1:
+        return [middle]
+    first_half = make_inference(model, I0, middle, upscale_amount, n=n // 2)
+    second_half = make_inference(model, middle, I1, upscale_amount, n=n // 2)
+    if n % 2:
+        return [*first_half, middle, *second_half]
+    else:
+        return [*first_half, *second_half]
+
+
+@torch.inference_mode()
+def ssim_interpolation_rife(model, samples, exp=1, upscale_amount=1, output_device="cpu"):
+
+    output = []
+    # [f, c, h, w]
+    for b in range(samples.shape[0]):
+        frame = samples[b : b + 1]
+        _, _, h, w = frame.shape
+        I0 = samples[b : b + 1]
+        I1 = samples[b + 1 : b + 2] if b + 2 < samples.shape[0] else samples[-1:]
+        I1 = pad_image(I1, upscale_amount)
+        # [c, h, w]
+        I0_small = F.interpolate(I0, (32, 32), mode="bilinear", align_corners=False)
+        I1_small = F.interpolate(I1, (32, 32), mode="bilinear", align_corners=False)
+
+        ssim = ssim_matlab(I0_small[:, :3], I1_small[:, :3])
+
+        if ssim > 0.996:
+            I1 = I0
+            I1 = pad_image(I1, upscale_amount)
+            I1 = make_inference(model, I0, I1, upscale_amount, 1)
+
+            I1_small = F.interpolate(I1[0], (32, 32), mode="bilinear", align_corners=False)
+            ssim = ssim_matlab(I0_small[:, :3], I1_small[:, :3])
+            frame = I1[0]
+            I1 = I1[0]
+
+        tmp_output = []
+        if ssim < 0.2:
+            for i in range((2**exp) - 1):
+                tmp_output.append(I0)
+
+        else:
+            tmp_output = make_inference(model, I0, I1, upscale_amount, 2**exp - 1) if exp else []
+
+        frame = pad_image(frame, upscale_amount)
+        tmp_output = [frame] + tmp_output
+        for i, frame in enumerate(tmp_output):
+            output.append(frame.to(output_device))
+    return output
+
+
+def load_rife_model(model_path):
+    model = Model()
+    model.load_model(model_path, -1)
+    model.eval()
+    return model
+
+
+# Create a generator that yields each frame, similar to cv2.VideoCapture
+def frame_generator(video_capture):
+    while True:
+        ret, frame = video_capture.read()
+        if not ret:
+            break
+        yield frame
+    video_capture.release()
+
+
+def rife_inference_with_path(model, video_path):
+    video_capture = cv2.VideoCapture(video_path)
+    tot_frame = video_capture.get(cv2.CAP_PROP_FRAME_COUNT)
+    pt_frame_data = []
+    pt_frame = skvideo.io.vreader(video_path)
+    for frame in pt_frame:
+        pt_frame_data.append(
+            torch.from_numpy(np.transpose(frame, (2, 0, 1))).to("cpu", non_blocking=True).float() / 255.0
+        )
+
+    pt_frame = torch.from_numpy(np.stack(pt_frame_data))
+    pt_frame = pt_frame.to(device)
+    pbar = utils.ProgressBar(tot_frame, desc="RIFE inference")
+    frames = ssim_interpolation_rife(model, pt_frame)
+    pt_image = torch.stack([frames[i].squeeze(0) for i in range(len(frames))])
+    image_np = VaeImageProcessor.pt_to_numpy(pt_image)  # (to [49, 512, 480, 3])
+    image_pil = VaeImageProcessor.numpy_to_pil(image_np)
+    video_path = utils.save_video(image_pil, fps=16)
+    if pbar:
+        pbar.update(1)
+    return video_path
+
+
+def rife_inference_with_latents(model, latents):
+    rife_results = []
+    latents = latents.to(device)
+    for i in range(latents.size(0)):
+        #  [f, c, w, h]
+        latent = latents[i]
+        frames = ssim_interpolation_rife(model, latent)
+        pt_image = torch.stack([frames[i].squeeze(0) for i in range(len(frames))])  # (to [f, c, w, h])
+        rife_results.append(pt_image)
+
+    return torch.stack(rife_results)

inference/utils.py

@@ -1,5 +1,21 @@
+import math
+from typing import Union, List
+
+import torch
+import os
+from datetime import datetime
+import numpy as np
+import itertools
+import PIL.Image
+import safetensors.torch
+import tqdm
+import logging
+from diffusers.utils import export_to_video
+from spandrel import ModelLoader
 from PIL import Image
 
+logger = logging.getLogger(__file__)
+
 def stack_images_horizontally(image1: Image.Image, image2: Image.Image) -> Image.Image:
     # Ensure both images have the same height
     height = max(image1.height, image2.height)
@@ -12,4 +28,208 @@ def stack_images_horizontally(image1: Image.Image, image2: Image.Image) -> Image
     new_image.paste(image1, (0, 0))
     new_image.paste(image2, (image1.width, 0))
     
-    return new_image
+    return new_image
+
+def load_torch_file(ckpt, device=None, dtype=torch.float16):
+    if device is None:
+        device = torch.device("cpu")
+    if ckpt.lower().endswith(".safetensors") or ckpt.lower().endswith(".sft"):
+        sd = safetensors.torch.load_file(ckpt, device=device.type)
+    else:
+        if not "weights_only" in torch.load.__code__.co_varnames:
+            logger.warning(
+                "Warning torch.load doesn't support weights_only on this pytorch version, loading unsafely."
+            )
+
+        pl_sd = torch.load(ckpt, map_location=device, weights_only=True)
+        if "global_step" in pl_sd:
+            logger.debug(f"Global Step: {pl_sd['global_step']}")
+        if "state_dict" in pl_sd:
+            sd = pl_sd["state_dict"]
+        elif "params_ema" in pl_sd:
+            sd = pl_sd["params_ema"]
+        else:
+            sd = pl_sd
+
+    sd = {k: v.to(dtype) for k, v in sd.items()}
+    return sd
+
+
+def state_dict_prefix_replace(state_dict, replace_prefix, filter_keys=False):
+    if filter_keys:
+        out = {}
+    else:
+        out = state_dict
+    for rp in replace_prefix:
+        replace = list(
+            map(
+                lambda a: (a, "{}{}".format(replace_prefix[rp], a[len(rp) :])),
+                filter(lambda a: a.startswith(rp), state_dict.keys()),
+            )
+        )
+        for x in replace:
+            w = state_dict.pop(x[0])
+            out[x[1]] = w
+    return out
+
+
+def module_size(module):
+    module_mem = 0
+    sd = module.state_dict()
+    for k in sd:
+        t = sd[k]
+        module_mem += t.nelement() * t.element_size()
+    return module_mem
+
+
+def get_tiled_scale_steps(width, height, tile_x, tile_y, overlap):
+    return math.ceil((height / (tile_y - overlap))) * math.ceil((width / (tile_x - overlap)))
+
+
+@torch.inference_mode()
+def tiled_scale_multidim(
+    samples, function, tile=(64, 64), overlap=8, upscale_amount=4, out_channels=3, output_device="cpu", pbar=None
+):
+    dims = len(tile)
+    print(f"samples dtype:{samples.dtype}")
+    output = torch.empty(
+        [samples.shape[0], out_channels] + list(map(lambda a: round(a * upscale_amount), samples.shape[2:])),
+        device=output_device,
+    )
+
+    for b in range(samples.shape[0]):
+        s = samples[b : b + 1]
+        out = torch.zeros(
+            [s.shape[0], out_channels] + list(map(lambda a: round(a * upscale_amount), s.shape[2:])),
+            device=output_device,
+        )
+        out_div = torch.zeros(
+            [s.shape[0], out_channels] + list(map(lambda a: round(a * upscale_amount), s.shape[2:])),
+            device=output_device,
+        )
+
+        for it in itertools.product(*map(lambda a: range(0, a[0], a[1] - overlap), zip(s.shape[2:], tile))):
+            s_in = s
+            upscaled = []
+
+            for d in range(dims):
+                pos = max(0, min(s.shape[d + 2] - overlap, it[d]))
+                l = min(tile[d], s.shape[d + 2] - pos)
+                s_in = s_in.narrow(d + 2, pos, l)
+                upscaled.append(round(pos * upscale_amount))
+
+            ps = function(s_in).to(output_device)
+            mask = torch.ones_like(ps)
+            feather = round(overlap * upscale_amount)
+            for t in range(feather):
+                for d in range(2, dims + 2):
+                    m = mask.narrow(d, t, 1)
+                    m *= (1.0 / feather) * (t + 1)
+                    m = mask.narrow(d, mask.shape[d] - 1 - t, 1)
+                    m *= (1.0 / feather) * (t + 1)
+
+            o = out
+            o_d = out_div
+            for d in range(dims):
+                o = o.narrow(d + 2, upscaled[d], mask.shape[d + 2])
+                o_d = o_d.narrow(d + 2, upscaled[d], mask.shape[d + 2])
+
+            o += ps * mask
+            o_d += mask
+
+            if pbar is not None:
+                pbar.update(1)
+
+        output[b : b + 1] = out / out_div
+    return output
+
+
+def tiled_scale(
+    samples,
+    function,
+    tile_x=64,
+    tile_y=64,
+    overlap=8,
+    upscale_amount=4,
+    out_channels=3,
+    output_device="cpu",
+    pbar=None,
+):
+    return tiled_scale_multidim(
+        samples, function, (tile_y, tile_x), overlap, upscale_amount, out_channels, output_device, pbar
+    )
+
+
+def load_sd_upscale(ckpt, inf_device):
+    sd = load_torch_file(ckpt, device=inf_device)
+    if "module.layers.0.residual_group.blocks.0.norm1.weight" in sd:
+        sd = state_dict_prefix_replace(sd, {"module.": ""})
+    out = ModelLoader().load_from_state_dict(sd).half()
+    return out
+
+
+def upscale(upscale_model, tensor: torch.Tensor, inf_device, output_device="cpu") -> torch.Tensor:
+    memory_required = module_size(upscale_model.model)
+    memory_required += (
+        (512 * 512 * 3) * tensor.element_size() * max(upscale_model.scale, 1.0) * 384.0
+    )  # The 384.0 is an estimate of how much some of these models take, TODO: make it more accurate
+    memory_required += tensor.nelement() * tensor.element_size()
+    print(f"UPScaleMemory required: {memory_required / 1024 / 1024 / 1024} GB")
+
+    upscale_model.to(inf_device)
+    tile = 512
+    overlap = 32
+
+    steps = tensor.shape[0] * get_tiled_scale_steps(
+        tensor.shape[3], tensor.shape[2], tile_x=tile, tile_y=tile, overlap=overlap
+    )
+
+    pbar = ProgressBar(steps, desc="Tiling and Upscaling")
+
+    s = tiled_scale(
+        samples=tensor.to(torch.float16),
+        function=lambda a: upscale_model(a),
+        tile_x=tile,
+        tile_y=tile,
+        overlap=overlap,
+        upscale_amount=upscale_model.scale,
+        pbar=pbar,
+    )
+
+    upscale_model.to(output_device)
+    return s
+
+
+def upscale_batch_and_concatenate(upscale_model, latents, inf_device, output_device="cpu") -> torch.Tensor:
+    upscaled_latents = []
+    for i in range(latents.size(0)):
+        latent = latents[i]
+        upscaled_latent = upscale(upscale_model, latent, inf_device, output_device)
+        upscaled_latents.append(upscaled_latent)
+    return torch.stack(upscaled_latents)
+
+
+def save_video(tensor: Union[List[np.ndarray], List[PIL.Image.Image]], fps: int = 8):
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    video_path = f"./output/{timestamp}.mp4"
+    os.makedirs(os.path.dirname(video_path), exist_ok=True)
+    export_to_video(tensor, video_path, fps=fps)
+    return video_path
+
+
+class ProgressBar:
+    def __init__(self, total, desc=None):
+        self.total = total
+        self.current = 0
+        self.b_unit = tqdm.tqdm(total=total, desc="ProgressBar context index: 0" if desc is None else desc)
+
+    def update(self, value):
+        if value > self.total:
+            value = self.total
+        self.current = value
+        if self.b_unit is not None:
+            self.b_unit.set_description("ProgressBar context index: {}".format(self.current))
+            self.b_unit.refresh()
+
+            # 更新进度
+            self.b_unit.update(self.current)