adding CogVideox1.5-5B-I2V

cogvideo_controlnet_pcd.py

@@ -46,7 +46,7 @@ class CogVideoXControlnetPCD(ModelMixin, ConfigMixin, PeftAdapterMixin):
         norm_eps: float = 1e-5,
         spatial_interpolation_scale: float = 1.875,
         temporal_interpolation_scale: float = 1.0,
-        use_rotary_positional_embeddings: bool = False,
+        use_rotary_positional_embeddings: bool = True,
         use_learned_positional_embeddings: bool = False,
         out_proj_dim: int = None,
         out_proj_dim_zero_init: bool = False,
@@ -95,8 +95,8 @@ class CogVideoXControlnetPCD(ModelMixin, ConfigMixin, PeftAdapterMixin):
             temporal_compression_ratio=temporal_compression_ratio,
             spatial_interpolation_scale=spatial_interpolation_scale,
             temporal_interpolation_scale=temporal_interpolation_scale,
-            use_positional_embeddings=False,
-            use_learned_positional_embeddings=False,
+            use_positional_embeddings=not use_rotary_positional_embeddings,
+            use_learned_positional_embeddings=use_learned_positional_embeddings,
         )
         self.embedding_dropout = nn.Dropout(dropout)
 

inference/cli_demo_camera_i2v_pcd.py

@@ -227,6 +227,7 @@ def generate_video(
         controlnet_kwargs["out_proj_dim"] = num_attention_heads_orig * controlnet_transformer_out_proj_dim_factor
     controlnet_kwargs["out_proj_dim_zero_init"] = controlnet_transformer_out_proj_dim_zero_init
 
+
     controlnet = CogVideoXControlnetPCD(
         num_layers=controlnet_transformer_num_layers,
         downscale_coef=downscale_coef,
@@ -367,7 +368,6 @@ def generate_video(
             height=height,  # Height of the generated video
             width=width,  # Width of the generated video
         ).frames
-
         video_generate = video_generate_all[0]
 
         # 6. Export the generated frames to a video file. fps must be 8 for original video.

training/controlnet_datasets_camera_pcd_mask.py

@@ -164,6 +164,7 @@ class RealEstate10KPCDRenderCapEmbDataset(RealEstate10KPCDRenderDataset):
         
         anchor_pixels = torch.from_numpy(mask_video_reader.get_batch(indices).asnumpy()).permute(0, 3, 1, 2).contiguous()
         anchor_pixels = anchor_pixels / 255.
+        
         try:
             masks = np.load(os.path.join(self.mask_root, clip_name + '.npz'))['mask']*1.0
             masks = torch.from_numpy(masks).unsqueeze(1)