update

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.mp4 filter=lfs diff=lfs merge=lfs -text

README.md

@@ -5,7 +5,7 @@ colorFrom: green
 colorTo: yellow
 sdk: gradio
 sdk_version: 4.24.0
-app_file: app.py
+app_file: app_high_res.py
 pinned: false
 license: apache-2.0
 short_description: Multimodal Language Model

app.py

@@ -10,8 +10,8 @@ import functools
 from transformers import AutoProcessor, Idefics2ForConditionalGeneration
 from models.conversation import conv_templates
 from typing import List
-processor = AutoProcessor.from_pretrained("Mantis-VL/mantis-8b-idefics2-video-eval-high-res-35k-mantis-2epoch_4096")
-model = Idefics2ForConditionalGeneration.from_pretrained("Mantis-VL/mantis-8b-idefics2-video-eval-high-res-35k-mantis-2epoch_4096", device_map="auto")
+processor = AutoProcessor.from_pretrained("Mantis-VL/mantis-8b-idefics2-video-eval-95k-mantis-2epoch_4096")
+model = Idefics2ForConditionalGeneration.from_pretrained("Mantis-VL/mantis-8b-idefics2-video-eval-95k-mantis-2epoch_4096", device_map="auto")
 MAX_NUM_FRAMES = 24
 conv_template = conv_templates["idefics_2"]
 
@@ -23,6 +23,14 @@ for item in examples:
     item['images'] = [os.path.join("./examples", video_id, x) for x in item['images']]
     item['video'] = os.path.join("./examples", item['video'])
 
+with open("./examples/hd.json", 'r') as f:
+    hd_examples = json.load(f)
+
+for item in hd_examples:
+    item['video'] = os.path.join("./examples", item['video'])
+
+examples = hd_examples + examples
+
 VIDEO_EVAL_PROMPT = """
 Suppose you are an expert in judging and evaluating the quality of AI-generated videos, 
 please watch the following frames of a given video and see the text prompt for generating the video, 
@@ -155,6 +163,8 @@ def eval_video(prompt, video:str):
     video_frames = read_video_pyav(container, indices)
 
     frames = [Image.fromarray(x) for x in video_frames]
+    # resize to 256 x 256
+    frames = [x.resize((256, 256)) for x in frames]
     
     eval_prompt = VIDEO_EVAL_PROMPT.format(text_prompt=prompt)
     eval_prompt += "<video>"

app_high_res.py

@@ -0,0 +1,237 @@
+import gradio as gr
+import spaces
+import os
+import time
+import json
+import numpy as np
+import av
+import torch
+from PIL import Image
+import functools
+from transformers import AutoProcessor, Idefics2ForConditionalGeneration
+from models.conversation import conv_templates
+from typing import List
+processor = AutoProcessor.from_pretrained("Mantis-VL/mantis-8b-idefics2-video-eval-high-res-35k-mantis-2epoch_4096")
+model = Idefics2ForConditionalGeneration.from_pretrained("Mantis-VL/mantis-8b-idefics2-video-eval-high-res-35k-mantis-2epoch_4096", device_map="auto", torch_dtype=torch.float16)
+MAX_NUM_FRAMES = 24
+conv_template = conv_templates["idefics_2"]
+
+with open("./examples/all_subsets.json", 'r') as f:
+    examples = json.load(f)
+
+for item in examples:
+    video_id = item['images'][0].split("_")[0]
+    item['images'] = [os.path.join("./examples", video_id, x) for x in item['images']]
+    item['video'] = os.path.join("./examples", item['video'])
+    
+with open("./examples/hd.json", 'r') as f:
+    hd_examples = json.load(f)
+
+for item in hd_examples:
+    item['video'] = os.path.join("./examples", item['video'])
+
+examples = hd_examples + examples
+
+VIDEO_EVAL_PROMPT = """
+Suppose you are an expert in judging and evaluating the quality of AI-generated videos,
+please watch the following frames of a given video and see the text prompt for generating the video,
+then give scores from 7 different dimensions:
+(1) visual quality: the quality of the video in terms of clearness, resolution, brightness, and color
+(2) object consistency, the consistency of objects or humans in video
+(3) dynamic degree, the degree of dynamic changes
+(4) motion smoothness, the smoothness of motion or movements
+(5) text-to-video alignment, the alignment between the text prompt and the video content
+(6) factual consistency, the consistency of the video content with the common-sense and factual knowledge
+(7) overall score, the overall quality of the video
+for each dimension, output a number from [1,2,3,4],
+in which '1' is 'Bad', '2' is 'Average', '3' is 'Good', '4' is 'Perfect'
+Here is an output example:
+visual quality: 3
+object consistency: 4
+dynamic degree: 4
+motion smoothness: 1
+text-to-video alignment: 1
+factual consistency: 2
+overall score: 1
+
+For this video, the text prompt is "{text_prompt}",
+all the frames of video are as follows:
+
+"""
+@spaces.GPU
+def generate(text:str, images:List[Image.Image], history: List[dict], **kwargs):
+    global processor, model
+    model = model.to("cuda") if model.device.type != "cuda" else model
+    if not images:
+        images = None
+    
+    user_role = conv_template.roles[0]
+    assistant_role = conv_template.roles[1]
+    
+    idefics_2_message = []
+    cur_img_idx = 0
+    cur_vid_idx = 0
+    all_videos = [x for x in images if isinstance(x, list)]
+    flatten_images = []
+    for x in images:
+        if isinstance(x, list):
+            flatten_images.extend(x)
+        else:
+            flatten_images.append(x)
+    
+    print(history)
+    for i, message in enumerate(history):
+        if message["role"] == user_role:
+            idefics_2_message.append({
+                "role": user_role,
+                "content": []
+            })
+            message_text = message["text"]
+            num_video_tokens_in_text = message_text.count("<video>")
+            if num_video_tokens_in_text > 0:
+                for _ in range(num_video_tokens_in_text):
+                    message_text = message_text.replace("<video>", "<image> " * len(all_videos[cur_vid_idx]), 1)
+                    cur_vid_idx += 1
+            num_image_tokens_in_text = message_text.count("<image>")
+            if num_image_tokens_in_text > 0:
+                sub_texts = [x.strip() for x in message_text.split("<image>")]
+                if sub_texts[0]:
+                    idefics_2_message[-1]["content"].append({"type": "text", "text": sub_texts[0]})
+                for sub_text in sub_texts[1:]:
+                    idefics_2_message[-1]["content"].append({"type": "image"})
+                    if sub_text:
+                        idefics_2_message.append({
+                            "role": user_role,
+                            "content": [{"type": "text", "text": sub_text}]
+                        })
+            else:
+                idefics_2_message[-1]["content"].append({"type": "text", "text": message_text})
+        elif message["role"] == assistant_role:
+            if i == len(history) - 1 and not message["text"]:
+                break
+            idefics_2_message.append({
+                "role": assistant_role,
+                "content": [{"type": "text", "text": message["text"]}]
+            })
+    if text:
+        assert idefics_2_message[-1]["role"] == assistant_role and not idefics_2_message[-1]["content"], "Internal error"
+        idefics_2_message.append({
+            "role": user_role,
+            "content": [{"type": "text", "text": text}]
+        })
+    
+    print(idefics_2_message)
+    prompt = processor.apply_chat_template(idefics_2_message, add_generation_prompt=True) 
+    
+    images = [Image.open(x) if isinstance(x, str) else x for x in flatten_images]
+    inputs = processor(text=prompt, images=images, return_tensors="pt")
+    inputs = {k: v.to(model.device) for k, v in inputs.items()}
+    outputs = model.generate(**inputs, max_new_tokens=1024)
+    generated_text = processor.decode(outputs[0, inputs["input_ids"].shape[-1]:], skip_special_tokens=True)
+    return generated_text
+
+    
+def read_video_pyav(container, indices):
+    '''
+    Decode the video with PyAV decoder.
+
+    Args:
+        container (av.container.input.InputContainer): PyAV container.
+        indices (List[int]): List of frame indices to decode.
+
+    Returns:
+        np.ndarray: np array of decoded frames of shape (num_frames, height, width, 3).
+    '''
+    frames = []
+    container.seek(0)
+    start_index = indices[0]
+    end_index = indices[-1]
+    for i, frame in enumerate(container.decode(video=0)):
+        if i > end_index:
+            break
+        if i >= start_index and i in indices:
+            frames.append(frame)
+    return np.stack([x.to_ndarray(format="rgb24") for x in frames])
+
+def eval_video(prompt, video:str):
+    container = av.open(video)
+
+    # sample uniformly 8 frames from the video
+    total_frames = container.streams.video[0].frames
+    if total_frames > MAX_NUM_FRAMES:
+        indices = np.arange(0, total_frames, total_frames / MAX_NUM_FRAMES).astype(int)
+    else:
+        indices = np.arange(total_frames)
+    video_frames = read_video_pyav(container, indices)
+
+    frames = [Image.fromarray(x) for x in video_frames]
+    
+    eval_prompt = VIDEO_EVAL_PROMPT.format(text_prompt=prompt)
+    eval_prompt += "<video>"
+    user_role = conv_template.roles[0]
+    assistant_role = conv_template.roles[1]
+    chat_messages = [
+        {
+            "role": user_role,
+            "text": eval_prompt
+        },
+        {
+            "role": assistant_role,
+            "text": ""
+        }
+    ]
+    response = generate(None, [frames], chat_messages)
+    return response
+
+def build_demo():
+    with gr.Blocks() as demo:
+        gr.Markdown("""
+            ## Video Evaluation
+            upload a video along with a text prompt when generating the video, this model will evaluate the video's quality from 7 different dimensions.
+        """)
+        with gr.Row():
+            video = gr.Video(width=500, label="Video")
+            with gr.Column():
+                eval_prompt_template = gr.Textbox(VIDEO_EVAL_PROMPT.strip(' \n'), label="Evaluation Prompt Template", interactive=False, max_lines=26)
+                video_prompt = gr.Textbox(label="Text Prompt", lines=1)
+                with gr.Row():
+                    eval_button = gr.Button("Evaluate Video")
+                    clear_button = gr.ClearButton([video, video_prompt])
+                eval_result = gr.Textbox(label="Evaluation result", interactive=False, lines=7)
+        
+        eval_button.click(
+            eval_video, [video_prompt, video], [eval_result]
+        )
+        
+        dummy_id = gr.Textbox("id", label="id", visible=False, min_width=50)
+        dummy_output = gr.Textbox("reference score", label="reference scores", visible=False, lines=7)
+        
+        gr.Examples(
+            examples=
+            [
+                [
+                    item['id'], 
+                    item['prompt'],
+                    item['video'],
+                    item['conversations'][1]['value']
+                ] for item in examples
+            ],
+            inputs=[dummy_id, video_prompt, video, dummy_output],
+        )        
+        
+#         gr.Markdown("""
+# ## Citation
+# ```
+# @article{jiang2024mantis,
+#   title={MANTIS: Interleaved Multi-Image Instruction Tuning},
+#   author={Jiang, Dongfu and He, Xuan and Zeng, Huaye and Wei, Con and Ku, Max and Liu, Qian and Chen, Wenhu},
+#   journal={arXiv preprint arXiv:2405.01483},
+#   year={2024}
+# }
+# ```""")
+    return demo    
+    
+
+if __name__ == "__main__":
+    demo = build_demo()
+    demo.launch(share=True)

examples/hd.json

@@ -0,0 +1,62 @@
+[
+    {
+        "id": "hd1",
+        "conversations": [
+            {
+                "from": "human",
+                "value": ""
+            },
+            {
+                "from": "gpt",
+                "value": ""
+            }
+        ],
+        "video": "hd1.mp4",
+        "prompt": "An indoor gym"
+    },
+    {
+        "id": "hd2",
+        "conversations": [
+            {
+                "from": "human",
+                "value": ""
+            },
+            {
+                "from": "gpt",
+                "value": ""
+            }
+        ],
+        "video": "hd2.mp4",
+        "prompt": "None"
+    },
+    {
+        "id": "hd3",
+        "conversations": [
+            {
+                "from": "human",
+                "value": ""
+            },
+            {
+                "from": "gpt",
+                "value": ""
+            }
+        ],
+        "video": "hd3.mp4",
+        "prompt": "A person barbecuing"
+    },
+    {
+        "id": "hd4",
+        "conversations": [
+            {
+                "from": "human",
+                "value": ""
+            },
+            {
+                "from": "gpt",
+                "value": ""
+            }
+        ],
+        "video": "hd4.mp4",
+        "prompt": "A child eating"
+    }
+]

examples/hd1.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:057409b842da7f266df0f46639a729a69eff2819e1cd0b567815bdfd93b59343
+size 97124

examples/hd2.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:970d1797c5e357672a3f2cfcfabcacb6eeced4032811551166cfcb7eb63a4814
+size 311262

examples/hd3.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91ebff6f8e885d97bfddac3cd39c56957de269c552dc012fa711624860a7f1d6
+size 223946

examples/hd4.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbd672b2647c473997ef2edca63607367e2768d8de8b0683d1b504977c305c49
+size 3860683