Create app2.py

app2.py

@@ -0,0 +1,119 @@
+import subprocess
+import os
+import gradio as gr
+
+
+subprocess.run(["git", "clone", "https://github.com/fat-ai/MuseV.git"])
+
+os.chdir("./MuseV")
+subprocess.run(["pip", "install", "-r", "requirements.txt"])
+subprocess.run(["pip", "install", "--no-cache-dir", "-U", "openmim"])
+subprocess.run(["mim", "install", "mmengine"])
+subprocess.run(["mim", "install", "mmcv>=2.0.1"])
+subprocess.run(["mim", "install", "mmdet>=3.1.0"])
+subprocess.run(["mim", "install", "mmpose>=1.1.0"])
+subprocess.run(["git", "clone", "--recursive", "https://github.com/fat-ai/MuseV.git"])
+subprocess.run(["git", "clone", "https://huggingface.co/TMElyralab/MuseV", "./checkpoints"])
+
+os.chdir("..")
+command = "\"import sys; sys.path.append('./MuseV/MuseV'); sys.path.append('./MuseV/MuseV/MMCM'); sys.path.append('./MuseV/MuseV/diffusers/src'); sys.path.append('./MuseV/MuseV/controlnet_aux/src')\""
+subprocess.run(["python","-c",command])
+
+subprocess.run(["mv", "./MuseV/scripts/inference/text2video.py", "./MuseV/text2video.py"])
+subprocess.run(["mv", "./MuseV/scripts/inference/video2video.py", "./MuseV/video2video.py"])
+
+with open ("./MuseV/configs/model/motion_model.py","r+") as scrip:
+    s = scrip.read()
+    s = s.replace('/content/MuseV/checkpoints', "/home/user/app/MuseV/checkpoints")
+    scrip.write(s)
+    scrip.truncate()
+    scrip.seek(0)
+    
+with open ("./MuseV/configs/model/ip_adapter.py","r+") as scrip:
+    s = scrip.read()
+    s = s.replace('/content/MuseV/checkpoints', "/home/user/app/MuseV/checkpoints")
+    scrip.write(s)
+    scrip.truncate()
+    scrip.seek(0)
+
+with open ("./MuseV/configs/model/T2I_all_model.py","r+") as scrip:
+    s = scrip.read()
+    s = s.replace('/content/MuseV/checkpoints', "/home/user/app/MuseV/checkpoints")
+    scrip.write(s)
+    scrip.truncate()
+    scrip.seek(0)
+
+from PIL import Image
+
+def add_new_image(image):
+    image = Image.fromarray(image)
+    height = image.height
+    width = image.width
+    lr = 0.5
+    ip_img = "${.condition_images}"
+    image.save("/content/img.png")
+    img_settings = f"""- condition_images: /content/img.png
+  eye_blinks_factor: 1.8
+  height: {height}
+  img_length_ratio: {lr}
+  ipadapter_image: {ip_img}
+  name: image
+  prompt: (masterpiece, best quality, highres:1),(1person, solo:1),(eye blinks:1.8),(head wave:1.3)
+  refer_image: {ip_img}
+  video_path: null
+  width: {width}"""
+    print(img_settings)
+    with open ("/home/user/app/MuseV/configs/tasks/example.yaml","r+") as configs:
+        configs.write(img_settings)
+        configs.truncate()
+        configs.seek(0)
+
+def add_new_video(video):
+    print(video)
+    lr = 1.0
+    ip_img = "${.condition_images}"
+    img_settings = f"""- name: "dance2"
+  prompt: "(best quality), ((masterpiece)), (highres), illustration, original, extremely detailed wallpaper"
+  video_path:  ./MuseV/data/source_video/video1_girl_poseseq.mp4 
+  condition_images: ./MuseV/data/images/cyber_girl.png
+  refer_image: {ip_img}
+  ipadapter_image: {ip_img}
+  height: 960
+  width: 512
+  img_length_ratio: 1.0
+  video_is_middle: True """
+    print(img_settings)
+    with open ("/home/user/app/MuseV/configs/tasks/example.yaml","r+") as configs:
+        configs.write(img_settings)
+        configs.truncate()
+        configs.seek(0)
+        
+
+def run():
+    #subprocess.run(["python", "./MuseV/text2video.py", "--sd_model_name", "majicmixRealv6Fp16", "--unet_model_name", "musev", "-test_data_path", "./MuseV/configs/tasks/example.yaml", "--n_batch", "1", "--target_datas", "image", "--vae_model_path", "./MuseV/checkpoints/vae/sd-vae-ft-mse", "--motion_speed", "5.0", "--time_size", "12", "--fps", "12"])
+    subprocess.run(["python", "./MuseV/text2video.py", "--sd_model_name", "majicmixRealv6Fp16", "--unet_model_name", "musev_referencenet", "--referencenet_model_name", "musev_referencenet", "--ip_adapter_model_name", "musev_referencenet", "-test_data_path", "./MuseV/configs/tasks/example.yaml", "--output_dir", "./MuseV",  "--n_batch", "1",  "--target_datas", "image",  "--vision_clip_extractor_class_name", "ImageClipVisionFeatureExtractor", "--vision_clip_model_path", "./MuseV/checkpoints/IP-Adapter/models/image_encoder", "--motion_speed", "5.0", "--vae_model_path", "./MuseV/checkpoints/vae/sd-vae-ft-mse", "--time_size", "12", "--fps", "12"])
+    return "./output.mp4"
+
+
+def run_video():
+    subprocess.run(["python", "./MuseV/video2video.py", "--sd_model_name", "fantasticmix_v10",  "--unet_model_name", "musev", "-test_data_path", "./MuseV/configs/tasks/example.yaml", "--output_dir", "./output",  "--n_batch", "1", "--controlnet_name", "dwpose_body_hand",  "--which2video", "video_middle",  "--target_datas",  "dance1", "--fps", "12", "--time_size", "12"])
+    return "./output.mp4"
+    
+with gr.Blocks() as demo:
+  title = gr.Markdown("""# MuseV Image2Vid & Vid2Vid """)
+  subtitle1 = gr.Markdown("""Image2Vid""")
+  image = gr.Image()
+  button1 = gr.Button(value="Save Image")
+  button1.click(fn=add_new_image,inputs=[image])
+  button2 = gr.Button(value="Generate Img2Vid")
+  video = gr.Video()
+  button2.click(fn=run,outputs=video)
+  subtitle2 = gr.Markdown("""Vid2Vid""")
+  video_in = gr.Video()
+  button3 = gr.Button(value="Save Video")
+  button3.click(fn=add_new_video,inputs=[video_in])
+  button4 = gr.Button(value="Generate Vid2Vid")
+  video_out = gr.Video()
+  button4.click(fn=run_video,outputs=video_out)  
+
+demo.launch(share=True)