passing correct arguments

cogvideo_controlnet_pcd.py

@@ -36,7 +36,7 @@ class CogVideoXControlnetPCD(ModelMixin, ConfigMixin, PeftAdapterMixin):
         attention_bias: bool = True,
         sample_width: int = 90,
         sample_height: int = 60,
-        sample_frames: int = 97,
+        sample_frames: int = 49,
         patch_size: int = 2,
         temporal_compression_ratio: int = 4,
         max_text_seq_length: int = 226,

controlnet_pipeline.py

@@ -562,7 +562,7 @@ class ControlnetCogVideoXImageToVideoPCDPipeline(DiffusionPipeline, CogVideoXLor
         negative_prompt: Optional[Union[str, List[str]]] = None,
         height: Optional[int] = None,
         width: Optional[int] = None,
-        num_frames: int = 97,
+        num_frames: int = 49,
         num_inference_steps: int = 50,
         timesteps: Optional[List[int]] = None,
         guidance_scale: float = 6,

gradio_app.py

@@ -193,7 +193,7 @@ with demo:
                         seed_input2 = gr.Number(value=42, label="Seed")
                         height_input = gr.Number(value=480, label="Height")
                         width_input = gr.Number(value=720, label="Width")
-                        num_frames_input2 = gr.Number(value=97, label="Num Frames")
+                        num_frames_input2 = gr.Number(value=49, label="Num Frames")
                         fps_input2 = gr.Number(value=8, label="FPS")
                         downscale_coef_input = gr.Number(value=8, label="Downscale Coef")
                         vae_channels_input = gr.Number(value=16, label="VAE Channels")

inference/cli_demo_camera_i2v_pcd.py

@@ -145,7 +145,7 @@ def generate_video(
     num_videos_per_prompt: int = 1,
     dtype: torch.dtype = torch.bfloat16,
     seed: int = 42,
-    num_frames: int = 97,
+    num_frames: int = 49,
     height: int = 480,
     width: int = 720,
     start_camera_idx: int = 0,
@@ -382,7 +382,7 @@ if __name__ == "__main__":
     )
     parser.add_argument("--controlnet_weights", type=float, default=0.5, help="Strenght of controlnet")
     parser.add_argument("--use_zero_conv", action="store_true", default=False, help="Use zero conv")
-    parser.add_argument("--infer_with_mask", action="store_true", default=False, help="add mask to controlnet")
+    parser.add_argument("--infer_with_mask", action="store_true", default=True, help="add mask to controlnet")
     parser.add_argument("--pool_style", default='max', help="max pool or avg pool")
     parser.add_argument("--controlnet_guidance_start", type=float, default=0.0, help="The stage when the controlnet starts to be applied")
     parser.add_argument("--controlnet_guidance_end", type=float, default=0.5, help="The stage when the controlnet end to be applied")
@@ -403,17 +403,17 @@ if __name__ == "__main__":
     parser.add_argument("--seed", type=int, default=42, help="The seed for reproducibility")
     parser.add_argument("--height", type=int, default=480)
     parser.add_argument("--width", type=int, default=720)
-    parser.add_argument("--num_frames", type=int, default=97)
+    parser.add_argument("--num_frames", type=int, default=49)
     parser.add_argument("--start_camera_idx", type=int, default=0)
     parser.add_argument("--end_camera_idx", type=int, default=1)
-    parser.add_argument("--controlnet_transformer_num_attn_heads", type=int, default=None)
-    parser.add_argument("--controlnet_transformer_attention_head_dim", type=int, default=None)
-    parser.add_argument("--controlnet_transformer_out_proj_dim_factor", type=int, default=None)
-    parser.add_argument("--controlnet_transformer_out_proj_dim_zero_init", action="store_true", default=False, help=("Init project zero."),
+    parser.add_argument("--controlnet_transformer_num_attn_heads", type=int, default=4)
+    parser.add_argument("--controlnet_transformer_attention_head_dim", type=int, default=64)
+    parser.add_argument("--controlnet_transformer_out_proj_dim_factor", type=int, default=64)
+    parser.add_argument("--controlnet_transformer_out_proj_dim_zero_init", action="store_true", default=True, help=("Init project zero."),
     )
     parser.add_argument("--downscale_coef", type=int, default=8)
     parser.add_argument("--vae_channels", type=int, default=16)
-    parser.add_argument("--controlnet_input_channels", type=int, default=6)
+    parser.add_argument("--controlnet_input_channels", type=int, default=3)
     parser.add_argument("--controlnet_transformer_num_layers", type=int, default=8)
     parser.add_argument("--enable_model_cpu_offload", action="store_true", default=False, help="Enable model CPU offload")
     parser.add_argument("--fps", type=int, default=8, help="Frames per second for the output video")

preprocess/get_vae_latent.py

@@ -88,12 +88,12 @@ def main(args):
 
         try:
             vr = VideoReader(source_video_file)
-            video = torch.from_numpy(vr.get_batch(np.arange(97)).asnumpy()).permute(0, 3, 1, 2).contiguous()
+            video = torch.from_numpy(vr.get_batch(np.arange(49)).asnumpy()).permute(0, 3, 1, 2).contiguous()
             video = (video / 255.0) * 2 - 1
             source_latent = encode_video(video, vae)
 
             vr = VideoReader(masked_video_file)
-            video = torch.from_numpy(vr.get_batch(np.arange(97)).asnumpy()).permute(0, 3, 1, 2).contiguous()
+            video = torch.from_numpy(vr.get_batch(np.arange(49)).asnumpy()).permute(0, 3, 1, 2).contiguous()
             video = (video / 255.0) * 2 - 1
             video = add_dashed_rays_to_video(video)
             masked_latent = encode_video(video, vae)

scripts/inference.sh

@@ -27,7 +27,7 @@ CUDA_VISIBLE_DEVICES=0 python inference/cli_demo_camera_i2v_pcd.py \
     --controlnet_transformer_out_proj_dim_factor 64 \
     --controlnet_transformer_out_proj_dim_zero_init \
     --vae_channels 16 \
-    --num_frames 97 \
+    --num_frames 49 \
     --controlnet_transformer_num_layers 8 \
     --infer_with_mask \
     --pool_style 'max' \

scripts/train.sh

@@ -23,7 +23,7 @@ CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" accelerate launch --config_file accelerat
   --height 480 \
   --width 720 \
   --fps 8 \
-  --max_num_frames 97 \
+  --max_num_frames 49 \
   --video_root_dir $video_root_dir \
   --hflip_p 0.0 \
   --controlnet_transformer_num_layers 8 \

scripts/train_with_latent.sh

@@ -23,7 +23,7 @@ CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7" accelerate launch --config_file accelerat
   --height 480 \
   --width 720 \
   --fps 8 \
-  --max_num_frames 97 \
+  --max_num_frames 49 \
   --video_root_dir $video_root_dir \
   --hflip_p 0.0 \
   --controlnet_transformer_num_layers 8 \

training/controlnet_datasets_camera_pcd_mask.py

@@ -39,7 +39,7 @@ class RealEstate10KPCDRenderDataset(Dataset):
     def __init__(
             self,
             video_root_dir,
-            sample_n_frames=97,
+            sample_n_frames=49,
             image_size=[480, 720],
             shuffle_frames=False,
             hflip_p=0.0,
@@ -135,7 +135,7 @@ class RealEstate10KPCDRenderCapEmbDataset(RealEstate10KPCDRenderDataset):
             self,
             video_root_dir,
             text_embedding_path,
-            sample_n_frames=97,
+            sample_n_frames=49,
             image_size=[480, 720],
             shuffle_frames=False,
             hflip_p=0.0,

training/train_controlnet_i2v_pcd_render_mask_aware_add_dash.py

@@ -255,7 +255,7 @@ def get_args():
     )
     parser.add_argument("--fps", type=int, default=8, help="All input videos will be used at this FPS.")
     parser.add_argument(
-        "--max_num_frames", type=int, default=97, help="All input videos will be truncated to these many frames."
+        "--max_num_frames", type=int, default=49, help="All input videos will be truncated to these many frames."
     )
     parser.add_argument(
         "--train_batch_size", type=int, default=4, help="Batch size (per device) for the training dataloader."
@@ -479,7 +479,7 @@ def get_args():
     return parser.parse_args()
 
 
-def read_video(video_path, start_index=0, frames_count=97, stride=1):
+def read_video(video_path, start_index=0, frames_count=49, stride=1):
     video_reader = VideoReader(video_path)
     end_index = min(start_index + frames_count * stride, len(video_reader)) - 1
     batch_index = np.linspace(start_index, end_index, frames_count, dtype=int)

training/train_controlnet_i2v_pcd_render_mask_aware_add_dash_use_latent.py

@@ -255,7 +255,7 @@ def get_args():
     )
     parser.add_argument("--fps", type=int, default=8, help="All input videos will be used at this FPS.")
     parser.add_argument(
-        "--max_num_frames", type=int, default=97, help="All input videos will be truncated to these many frames."
+        "--max_num_frames", type=int, default=49, help="All input videos will be truncated to these many frames."
     )
     parser.add_argument(
         "--train_batch_size", type=int, default=4, help="Batch size (per device) for the training dataloader."
@@ -479,7 +479,7 @@ def get_args():
     return parser.parse_args()
 
 
-def read_video(video_path, start_index=0, frames_count=97, stride=1):
+def read_video(video_path, start_index=0, frames_count=49, stride=1):
     video_reader = VideoReader(video_path)
     end_index = min(start_index + frames_count * stride, len(video_reader)) - 1
     batch_index = np.linspace(start_index, end_index, frames_count, dtype=int)