Upload 27 files

libra/__init__.py

@@ -0,0 +1 @@
+from .model import LibraLlamaForCausalLM

libra/constants.py

@@ -0,0 +1,12 @@
+CONTROLLER_HEART_BEAT_EXPIRATION = 30
+WORKER_HEART_BEAT_INTERVAL = 15
+
+LOGDIR = "."
+
+# Model Constants
+IGNORE_INDEX = -100
+IMAGE_TOKEN_INDEX = -200
+DEFAULT_IMAGE_TOKEN = "<image>"
+DEFAULT_IMAGE_PATCH_TOKEN = "<im_patch>"
+DEFAULT_IM_START_TOKEN = "<im_start>"
+DEFAULT_IM_END_TOKEN = "<im_end>"

libra/conversation.py

@@ -0,0 +1,416 @@
+import dataclasses
+from enum import auto, Enum
+from typing import List, Tuple
+
+
+class SeparatorStyle(Enum):
+    """Different separator style."""
+    SINGLE = auto()
+    TWO = auto()
+    MPT = auto()
+    PLAIN = auto()
+    LLAMA_2 = auto()
+    LLAMA_3 = auto()
+
+
+@dataclasses.dataclass
+class Conversation:
+    """A class that keeps all conversation history."""
+    system: str
+    roles: List[str]
+    messages: List[List[str]]
+    offset: int
+    sep_style: SeparatorStyle = SeparatorStyle.SINGLE
+    sep: str = "###"
+    sep2: str = None
+    version: str = "Unknown"
+
+    skip_next: bool = False
+
+    def get_prompt(self):
+        messages = self.messages
+        if len(messages) > 0 and type(messages[0][1]) is tuple:
+            messages = self.messages.copy()
+            init_role, init_msg = messages[0].copy()
+            init_msg = init_msg[0].replace("<image>", "").strip()
+            if 'mmtag' in self.version:
+                messages[0] = (init_role, init_msg)
+                messages.insert(0, (self.roles[0], "<Image><image></Image>"))
+                messages.insert(1, (self.roles[1], "Received."))
+            else:
+                messages[0] = (init_role, "<image>\n" + init_msg)
+
+        if self.sep_style == SeparatorStyle.SINGLE:
+            ret = self.system + self.sep
+            for role, message in messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + ": " + message + self.sep
+                else:
+                    ret += role + ":"
+                    
+        elif self.sep_style == SeparatorStyle.TWO:
+            seps = [self.sep, self.sep2]
+            ret = self.system + seps[0]
+            for i, (role, message) in enumerate(messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + ": " + message + seps[i % 2]
+                else:
+                    ret += role + ":"
+                    
+        elif self.sep_style == SeparatorStyle.MPT:
+            ret = self.system + self.sep
+            for role, message in messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + message + self.sep
+                else:
+                    ret += role
+                    
+        elif self.sep_style == SeparatorStyle.LLAMA_2:
+            wrap_sys = lambda msg: f"<<SYS>>\n{msg}\n<</SYS>>\n\n" if len(msg) > 0 else msg
+            wrap_inst = lambda msg: f"[INST] {msg} [/INST]"
+            ret = ""
+
+            for i, (role, message) in enumerate(messages):
+                if i == 0:
+                    assert message, "first message should not be none"
+                    assert role == self.roles[0], "first message should come from user"
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    if i == 0: 
+                        message = wrap_sys(self.system) + message
+                    if i % 2 == 0:
+                        message = wrap_inst(message)
+                        ret += self.sep + message
+                    else:
+                        ret += " " + message + " " + self.sep2
+                else:
+                    ret += ""
+                    
+            ret = ret.lstrip(self.sep)
+            
+        elif self.sep_style == SeparatorStyle.LLAMA_3:
+            wrap_sys = lambda msg: f"<|start_header_id|>system<|end_header_id|>\n\n{msg}{self.sep2}" if len(msg) > 0 else ""
+            wrap_role = lambda role, msg: f"<|start_header_id|>{role}<|end_header_id|>\n\n{msg}{self.sep2}\n"
+
+            ret = "" # "<|begin_of_text|>" 
+
+            for i, (role, message) in enumerate(messages):
+                if i == 0:
+                    assert message, "first message should not be none"
+                    assert role == self.roles[0], "first message should come from user"
+                    
+                if message:  
+                    if isinstance(message, tuple):
+                        message, _, _ = message  
+
+                    
+                    if i == 0:
+                        ret += wrap_sys(self.system)  
+                        ret += wrap_role("user", message)  
+                    else:
+                        role_name = "user" if role == self.roles[0] else "assistant"
+                        ret += wrap_role(role_name, message)  
+                else:
+                    ret += f"<|start_header_id|>{role}<|end_header_id|>\n\n"
+                    
+            ret = ret.strip()
+            
+        elif self.sep_style == SeparatorStyle.PLAIN:
+            seps = [self.sep, self.sep2]
+            ret = self.system
+            for i, (role, message) in enumerate(messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += message + seps[i % 2]
+                else:
+                    ret += ""
+        else:
+            raise ValueError(f"Invalid style: {self.sep_style}")
+
+        return ret
+
+    def append_message(self, role, message):
+        self.messages.append([role, message])
+        
+    def process_image(self, image, image_process_mode, return_pil=False, image_format='PNG', max_len=1344, min_len=672):
+        if image_process_mode == "Pad":
+            def expand2square(pil_img, background_color=(0, 0, 0)):
+                width, height = pil_img.size
+                if width == height:
+                    return pil_img
+                elif width > height:
+                    result = Image.new(pil_img.mode, (width, width), background_color)
+                    result.paste(pil_img, (0, (width - height) // 2))
+                    return result
+                else:
+                    result = Image.new(pil_img.mode, (height, height), background_color)
+                    result.paste(pil_img, ((height - width) // 2, 0))
+                    return result
+            image = expand2square(image)
+        elif image_process_mode in ["Default", "Crop"]:
+            pass
+        elif image_process_mode == "Resize":
+            image = image.resize((518, 518))
+        else:
+            raise ValueError(f"Invalid image_process_mode: {image_process_mode}")
+        if max(image.size) > max_len:
+            max_hw, min_hw = max(image.size), min(image.size)
+            aspect_ratio = max_hw / min_hw
+            shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+            longest_edge = int(shortest_edge * aspect_ratio)
+            W, H = image.size
+            if H > W:
+                H, W = longest_edge, shortest_edge
+            else:
+                H, W = shortest_edge, longest_edge
+            image = image.resize((W, H))
+        if return_pil:
+            return image
+        else:
+            buffered = BytesIO()
+            image.save(buffered, format=image_format)
+            img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+            return img_b64_str
+        
+    def get_images(self, return_pil=False):
+        images = []
+        for i, (role, msg) in enumerate(self.messages[self.offset:]):
+            if i % 2 == 0:
+                if type(msg) is tuple:
+                    msg, image, image_process_mode = msg
+                    image = self.process_image(image, image_process_mode, return_pil=return_pil)
+                    images.append(image)
+        return images
+    
+
+    def copy(self):
+        return Conversation(
+            system=self.system,
+            roles=self.roles,
+            messages=[[x, y] for x, y in self.messages],
+            offset=self.offset,
+            sep_style=self.sep_style,
+            sep=self.sep,
+            sep2=self.sep2,
+            version=self.version)
+
+    def dict(self):
+        if len(self.get_images()) > 0:
+            return {
+                "system": self.system,
+                "roles": self.roles,
+                "messages": [[x, y[0] if type(y) is tuple else y] for x, y in self.messages],
+                "offset": self.offset,
+                "sep": self.sep,
+                "sep2": self.sep2,
+            }
+        return {
+            "system": self.system,
+            "roles": self.roles,
+            "messages": self.messages,
+            "offset": self.offset,
+            "sep": self.sep,
+            "sep2": self.sep2,
+        }
+
+    
+conv_mpt = Conversation(
+    system="""<|im_start|>system
+A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.""",
+    roles=("<|im_start|>user\n", "<|im_start|>assistant\n"),
+    version="mpt",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.MPT,
+    sep="<|im_end|>",
+)
+
+conv_vicuna_v0 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("Human", "Assistant"),
+    messages=(
+        ("Human", "What are the key differences between renewable and non-renewable energy sources?"),
+        ("Assistant",
+            "Renewable energy sources are those that can be replenished naturally in a relatively "
+            "short amount of time, such as solar, wind, hydro, geothermal, and biomass. "
+            "Non-renewable energy sources, on the other hand, are finite and will eventually be "
+            "depleted, such as coal, oil, and natural gas. Here are some key differences between "
+            "renewable and non-renewable energy sources:\n"
+            "1. Availability: Renewable energy sources are virtually inexhaustible, while non-renewable "
+            "energy sources are finite and will eventually run out.\n"
+            "2. Environmental impact: Renewable energy sources have a much lower environmental impact "
+            "than non-renewable sources, which can lead to air and water pollution, greenhouse gas emissions, "
+            "and other negative effects.\n"
+            "3. Cost: Renewable energy sources can be more expensive to initially set up, but they typically "
+            "have lower operational costs than non-renewable sources.\n"
+            "4. Reliability: Renewable energy sources are often more reliable and can be used in more remote "
+            "locations than non-renewable sources.\n"
+            "5. Flexibility: Renewable energy sources are often more flexible and can be adapted to different "
+            "situations and needs, while non-renewable sources are more rigid and inflexible.\n"
+            "6. Sustainability: Renewable energy sources are more sustainable over the long term, while "
+            "non-renewable sources are not, and their depletion can lead to economic and social instability.\n")
+    ),
+    offset=2,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="###",
+)
+
+conv_vicuna_v1 = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+    "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+    roles=("USER", "ASSISTANT"),
+    version="v1",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+)
+
+conv_llama_2 = Conversation(
+    system="""You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
+
+If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""",
+    roles=("USER", "ASSISTANT"),
+    version="llama_v2",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA_2,
+    sep="<s>",
+    sep2="</s>",
+)
+
+conv_llama_3 = Conversation(
+    system="""You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe.  Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure that your responses are socially unbiased and positive in nature.
+
+If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information.""",
+    roles=("USER", "ASSISTANT"),
+    version="llama_v3",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA_3,
+    sep=" ",
+    sep2="<|eot_id|>",
+)
+
+conv_libra_llama_2 = Conversation(
+    system="You are a helpful language and vision assistant. "
+           "You are able to understand the visual content that the user provides, "
+           "and assist the user with a variety of tasks using natural language.",
+    roles=("USER", "ASSISTANT"),
+    version="llama_v2",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA_2,
+    sep="<s>",
+    sep2="</s>",
+)
+
+conv_libra_llama_3 = Conversation(
+    system="You are a helpful language and vision assistant. "
+           "You are able to understand the visual content that the user provides, "
+           "and assist the user with a variety of tasks using natural language.",
+    roles=("USER", "ASSISTANT"),
+    version="llama_v3",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.LLAMA_3,
+    sep=" ",
+    sep2="<|eot_id|>",
+)
+
+conv_libra_plain = Conversation(
+    system="",
+    roles=("", ""),
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.PLAIN,
+    sep="\n",
+)
+
+conv_libra_v0 = Conversation(
+    system="A chat between a curious human and an artificial intelligence assistant. "
+           "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+    roles=("Human", "Assistant"),
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="###",
+)
+
+conv_libra_v0_mmtag = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+           "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
+           "The visual content will be provided with the following format: <Image>visual content</Image>.",
+    roles=("Human", "Assistant"),
+    messages=(
+    ),
+    offset=0,
+    sep_style=SeparatorStyle.SINGLE,
+    sep="###",
+    version="v0_mmtag",
+)
+
+conv_libra_v1 = Conversation(
+    system="The assistant specialized in comparing Chest X-ray images, identifying differences, and noting temporal changes.",
+    roles=("USER", "ASSISTANT"),
+    version="v1",
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+)
+
+
+conv_libra_v1_mmtag = Conversation(
+    system="A chat between a curious user and an artificial intelligence assistant. "
+           "The assistant is able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language."
+           "The visual content will be provided with the following format: <Image>visual content</Image>.",
+    roles=("USER", "ASSISTANT"),
+    messages=(),
+    offset=0,
+    sep_style=SeparatorStyle.TWO,
+    sep=" ",
+    sep2="</s>",
+    version="v1_mmtag",
+)
+
+default_conversation = conv_vicuna_v1
+conv_templates = {
+    "default": conv_libra_v1,
+    
+    "v0": conv_vicuna_v0,
+    "v1": conv_vicuna_v1,
+    "vicuna_v1": conv_vicuna_v1,
+    
+    "plain": conv_libra_plain,
+    "libra_v0": conv_libra_v0,
+    "libra_v1": conv_libra_v1,
+    
+    "libra_v0_mmtag": conv_libra_v0_mmtag,
+    "libra_v1_mmtag": conv_libra_v1_mmtag,
+    
+    "llama_2": conv_llama_2,
+    "libra_llama_2": conv_libra_llama_2,
+    
+    "llama_3": conv_llama_3,
+    "libra_llama_3": conv_libra_llama_3,
+    
+    "mpt": conv_mpt,
+    
+}
+
+if __name__ == "__main__":
+    print(default_conversation.get_prompt())

libra/eval/__init__.py

@@ -0,0 +1,6 @@
+try:
+    from .run_libra import libra_eval
+    from .temporal_f1 import temporal_f1_score
+    from .radiology_report import evaluate_report
+except:
+    pass

libra/eval/eval_vqa_libra.py

@@ -0,0 +1,264 @@
+import argparse
+import torch
+import os
+import json
+from tqdm import tqdm
+import shortuuid
+import numpy as np
+import re
+
+from libra.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from libra.conversation import conv_templates, SeparatorStyle
+from libra.model.builder import load_pretrained_model
+from libra.utils import disable_torch_init
+from libra.mm_utils import tokenizer_image_token, process_images, get_model_name_from_path, KeywordsStoppingCriteria
+
+import math
+import pydicom
+from PIL import Image
+from io import BytesIO
+from pydicom.pixel_data_handlers.util import apply_voi_lut
+
+def split_list(lst, n):
+    """Split a list into n (roughly) equal-sized chunks"""
+    chunk_size = math.ceil(len(lst) / n)  # integer division
+    return [lst[i:i+chunk_size] for i in range(0, len(lst), chunk_size)]
+
+
+def get_chunk(lst, n, k):
+    chunks = split_list(lst, n)
+    return chunks[k]
+
+def load_images(image_file):
+    """
+    Load an image from a local file, a URL, or a DICOM file.
+
+    Args:
+        image_file (str): The path or URL of the image file to load.
+
+    Returns:
+        PIL.Image.Image: The loaded image in RGB format.
+
+    Raises:
+        ValueError: If the DICOM file does not contain image data.
+        TypeError: If the input is neither a valid file path nor a URL.
+    """
+    if isinstance(image_file, str):
+        # Case 1: Load from URL
+        if image_file.startswith(('http://', 'https://')):
+            try:
+                response = requests.get(image_file)
+                response.raise_for_status()
+                image = Image.open(BytesIO(response.content)).convert('RGB')
+            except Exception as e:
+                raise ValueError(f"Error loading image from URL: {image_file}\n{e}")
+
+        # Case 2: Load from DICOM file
+        elif image_file.lower().endswith('.dcm'):
+            try:
+                dicom = pydicom.dcmread(image_file)
+                if 'PixelData' in dicom:
+                    data = apply_voi_lut(dicom.pixel_array, dicom)
+
+                    # Handle MONOCHROME1 images
+                    if dicom.PhotometricInterpretation == "MONOCHROME1":
+                        data = np.max(data) - data
+
+                    # Normalize the image data
+                    data = data - np.min(data)
+                    data = data / np.max(data)
+                    data = (data * 255).astype(np.uint8)
+
+                    # Convert to 3-channel RGB if necessary
+                    if data.ndim == 2:
+                        data = np.stack([data] * 3, axis=-1)
+
+                    image = Image.fromarray(data).convert('RGB')
+                else:
+                    raise ValueError("DICOM file does not contain image data")
+            except Exception as e:
+                raise ValueError(f"Error loading DICOM file: {image_file}\n{e}")
+
+        # Case 3: Load standard image files (e.g., PNG, JPG)
+        else:
+            try:
+                image = Image.open(image_file).convert('RGB')
+            except Exception as e:
+                raise ValueError(f"Error loading standard image file: {image_file}\n{e}")
+
+    else:
+        raise TypeError("image_file must be a string representing a file path or URL")
+
+    return image
+
+def get_image_tensors(image_file, image_folder, image_processor, model, device='cuda'):
+    # Load and preprocess the images
+    if isinstance(image_file, str):
+        image = []
+        image_path = os.path.join(image_folder, image_file)
+        img = load_images(image_path)
+        image.append(img)
+    elif isinstance(image_file, (list, tuple)):
+        image = []
+        image_paths = [os.path.join(image_folder, file_name) for file_name in image_file]
+        for path in image_paths:
+            img = load_images(path)
+            image.append(img)
+    else:
+        raise TypeError("image_file must be a string or a str/list/tuple of strings")
+
+    # Ensure two images are present
+    if len(image) != 2:
+        image.append(image[0])  
+        if model.config.mm_projector_type == "TAC":
+            print("Contains only current image. Adding a dummy prior image for TAC.")
+
+    # Process each image
+    processed_images = []
+    for img_data in image:
+        image_temp = process_images([img_data], image_processor, model.config)[0]
+        image_temp = image_temp.to(device=device, non_blocking=True)
+        processed_images.append(image_temp)
+
+    # Separate current and prior images
+    cur_images = [processed_images[0]]
+    prior_images = [processed_images[1]]
+
+    # Stack and return as batched tensor
+    batch_images = torch.stack([torch.stack(cur_images), torch.stack(prior_images)])
+
+    return batch_images
+
+def eval_model(args):
+    """
+    Evaluate a pre-trained model on a set of questions and images.
+    Args:
+        args (Namespace): A namespace object containing the following attributes:
+            - model_path (str): Path to the pre-trained model.
+            - model_base (str): Base model name.
+            - question_file (str): Path to the JSON file containing questions.
+            - num_chunks (int): Number of chunks to split the questions into.
+            - chunk_idx (int): Index of the chunk to process.
+            - answers_file (str): Path to the file where answers will be saved.
+            - image_folder (str): Folder containing the images.
+            - conv_mode (str): Conversation mode to use.
+            - temperature (float): Sampling temperature for generation.
+            - top_p (float): Top-p sampling parameter.
+            - num_beams (int): Number of beams for beam search.
+            - max_new_tokens (int): Maximum number of new tokens to generate.
+            - length_penalty (float): Length penalty for beam search.
+            - num_return_sequences (int): Number of sequences to return.
+    Raises:
+        TypeError: If `image_file` is neither a string nor a list/tuple of strings.
+    Returns:
+        None
+    """
+    # Model
+    disable_torch_init()
+    model_path = os.path.expanduser(args.model_path)
+    model_name = get_model_name_from_path(model_path)
+    tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, args.model_base, model_name)
+    
+    questions = [json.loads(q) for q in open(os.path.expanduser(args.question_file), "r")]
+    questions = get_chunk(questions, args.num_chunks, args.chunk_idx)
+    answers_file = os.path.expanduser(args.answers_file)
+    os.makedirs(os.path.dirname(answers_file), exist_ok=True)
+    ans_file = open(answers_file, "w")
+
+    for line in tqdm(questions):
+        idx = line["question_id"]
+        image_file = line["image"]
+        qs = line["text"]
+        cur_prompt = qs
+        if model.config.mm_use_im_start_end:
+            qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
+        else:
+            qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
+
+        conv = conv_templates[args.conv_mode].copy()
+        conv.append_message(conv.roles[0], qs)
+        conv.append_message(conv.roles[1], None)
+        prompt = conv.get_prompt()
+
+        input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
+        
+        attention_mask = torch.ones(input_ids.shape, dtype=torch.long)
+        pad_token_id = tokenizer.pad_token_id
+        
+        image_tensors = get_image_tensors(image_file, args.image_folder, image_processor, model)
+
+        stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+        keywords = [stop_str]
+        stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+
+        with torch.inference_mode():
+            torch.cuda.empty_cache() 
+            if args.num_beams > 1:
+                output_ids = model.generate(
+                    input_ids=input_ids,
+                    images=image_tensors,
+                    do_sample=False,
+                    num_beams=args.num_beams,
+                    no_repeat_ngram_size=3,
+                    max_new_tokens=args.max_new_tokens,
+                    stopping_criteria=[stopping_criteria],
+                    use_cache=True,
+                    length_penalty=args.length_penalty, 
+                    output_scores=True,  
+                    num_return_sequences = args.num_return_sequences,
+                    attention_mask=attention_mask, 
+                    pad_token_id=pad_token_id)
+            else:
+                output_ids = model.generate(
+                    input_ids,
+                    images=image_tensors,
+                    do_sample= True,
+                    temperature=args.temperature,
+                    top_p=args.top_p,
+                    num_beams=args.num_beams,
+                    no_repeat_ngram_size=3,
+                    max_new_tokens=args.max_new_tokens,
+                    stopping_criteria=[stopping_criteria],
+                    use_cache=True,
+                    attention_mask=attention_mask, 
+                    pad_token_id=pad_token_id)
+            
+        torch.cuda.empty_cache() 
+        input_token_len = input_ids.shape[1]
+        n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
+        
+        if n_diff_input_output > 0:
+            print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
+            
+        outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
+        outputs = outputs.strip()
+        
+        ans_id = shortuuid.uuid()
+        ans_file.write(json.dumps({"question_id": idx,
+                                   "prompt": cur_prompt,
+                                   "text": outputs,
+                                   "answer_id": ans_id,
+                                   "model_id": model_name,
+                                   "metadata": {}}) + "\n")
+        ans_file.flush()
+    ans_file.close()
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-path", type=str, default="libra")
+    parser.add_argument("--model-base", type=str, default=None)
+    parser.add_argument("--image-folder", type=str, default="")
+    parser.add_argument("--question-file", type=str, default="question.jsonl")
+    parser.add_argument("--answers-file", type=str, default="answer.jsonl")
+    parser.add_argument("--conv-mode", type=str, default="libra_v1")
+    parser.add_argument("--num-chunks", type=int, default=1)
+    parser.add_argument("--chunk-idx", type=int, default=0)
+    parser.add_argument("--temperature", type=float, default=0.2)
+    parser.add_argument("--top_p", type=float, default=None)
+    parser.add_argument("--num_beams", type=int, default=1)
+    parser.add_argument("--num_return_sequences", type=int, default=None)
+    parser.add_argument("--length_penalty", type=float, default=1.0)
+    parser.add_argument("--max_new_tokens", type=int, default=128)
+    args = parser.parse_args()
+
+    eval_model(args)

libra/eval/radiology_report.py

@@ -0,0 +1,222 @@
+import argparse
+import json
+import os
+import re
+import sys
+
+import evaluate
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from libra.eval import temporal_f1_score
+
+# Pre-load metrics
+bertscore_metric = evaluate.load("bertscore")
+rouge_metric = evaluate.load('rouge')
+bleu_metric = evaluate.load("bleu")
+meteor_metric = evaluate.load('meteor')
+
+
+def clean_text(text: str) -> str:
+    """
+    Perform basic cleanup of text by removing newlines, dashes, and some special patterns.
+    """
+    text = re.sub(r'\n+', ' ', text)
+    text = re.sub(r'[_-]+', ' ', text)
+    text = re.sub(r'\(___, __, __\)', '', text)
+    text = re.sub(r'---, ---, ---', '', text)
+    text = re.sub(r'\(__, __, ___\)', '', text)
+    text = re.sub(r'[_-]+', ' ', text)
+    text = re.sub(r'[^\w\s.,:;()\-]', '', text)
+    text = re.sub(r'\s{2,}', ' ', text).strip()
+    return text
+
+
+def load_json(path: str) -> list:
+    """
+    Load a JSONL file and return a list of parsed objects.
+    Each line should be a valid JSON object.
+    """
+    content = []
+    with open(path, 'r', encoding='utf-8') as file:
+        for line in file:
+            content.append(json.loads(line))
+    return content
+
+
+def extract_sections(data: list) -> list:
+    """
+    Extract relevant text sections (e.g., findings, impression, text)
+    from a list of JSON objects and clean each item.
+    """
+    sections_list = []
+    for item in data:
+        if 'reference' in item:
+            cleaned_text = clean_text(item['reference'].lower())
+            sections_list.append(cleaned_text)
+        elif 'findings' in item:
+            cleaned_text = clean_text(item['findings'].lower())
+            sections_list.append(cleaned_text)
+        elif 'impression' in item:
+            cleaned_text = clean_text(item['impression'].lower())
+            sections_list.append(cleaned_text)
+        elif 'text' in item:
+            cleaned_text = clean_text(item['text'].lower())
+            sections_list.append(cleaned_text)
+    return sections_list
+
+
+def append_results_to_csv(results: dict, model_name: str, csv_path: str) -> None:
+    """
+    Convert the results dictionary into a DataFrame and append it to a CSV file.
+    Inserts 'Model Name' at the first column if it doesn't exist.
+    Creates a new CSV if it doesn't exist, otherwise appends.
+    """
+    df = pd.DataFrame([results])
+    df.insert(0, "Model Name", model_name)
+
+    header = not os.path.isfile(csv_path)  # If file doesn't exist, write the header
+    df.to_csv(csv_path, mode='a', header=header, index=False)
+
+
+def evaluate_report(
+    references: str,
+    predictions: str,
+) -> dict:
+    """
+    Evaluate the model outputs against reference texts using multiple metrics:
+    - BLEU (1–4)
+    - METEOR
+    - ROUGE-L
+    - BERTScore (F1)
+    - Temporal F1
+
+    Returns a dictionary of computed metrics.
+    """
+    # Load data
+    references_data = load_json(references)
+    predictions_data = load_json(predictions)
+
+    # Basic validation: question_id alignment
+    gt_ids = [item['question_id'] for item in references_data]
+    pred_ids = [item['question_id'] for item in predictions_data]
+    assert gt_ids == pred_ids, "Please make sure predictions and references are perfectly matched by question_id."
+
+    # Extract text sections
+    references_list = extract_sections(references_data)
+    predictions_list = extract_sections(predictions_data)
+
+    # Calculate metrics
+    with tqdm(total=8, desc="Calculating metrics") as pbar:
+        # BLEU-1
+        bleu1 = bleu_metric.compute(
+            predictions=predictions_list, 
+            references=references_list, 
+            max_order=1
+        )['bleu']
+        print(f"BLEU-1 Score: {round(bleu1 * 100, 2)}")
+        pbar.update(1)
+
+        # BLEU-2
+        bleu2 = bleu_metric.compute(
+            predictions=predictions_list, 
+            references=references_list, 
+            max_order=2
+        )['bleu']
+        print(f"BLEU-2 Score: {round(bleu2 * 100, 2)}")
+        pbar.update(1)
+
+        # BLEU-3
+        bleu3 = bleu_metric.compute(
+            predictions=predictions_list, 
+            references=references_list, 
+            max_order=3
+        )['bleu']
+        print(f"BLEU-3 Score: {round(bleu3 * 100, 2)}")
+        pbar.update(1)
+
+        # BLEU-4
+        bleu4 = bleu_metric.compute(
+            predictions=predictions_list, 
+            references=references_list, 
+            max_order=4
+        )['bleu']
+        print(f"BLEU-4 Score: {round(bleu4 * 100, 2)}")
+        pbar.update(1)
+
+        # ROUGE-L
+        rougel = rouge_metric.compute(
+            predictions=predictions_list, 
+            references=references_list
+        )['rougeL']
+        print(f"ROUGE-L Score: {round(rougel * 100, 2)}")
+        pbar.update(1)
+
+        # METEOR
+        meteor = meteor_metric.compute(
+            predictions=predictions_list, 
+            references=references_list
+        )['meteor']
+        print(f"METEOR Score: {round(meteor * 100, 2)}")
+        pbar.update(1)
+
+        # BERTScore (mean F1)
+        bert_f1 = bertscore_metric.compute(
+            predictions=predictions_list, 
+            references=references_list, 
+            model_type='distilbert-base-uncased'
+        )['f1']
+        bert_score = float(np.mean(bert_f1))
+        print(f"Bert Score: {round(bert_score * 100, 2)}")
+        pbar.update(1)
+
+        # Temporal F1
+        tem_f1 = temporal_f1_score(
+            predictions=predictions_list, 
+            references=references_list
+        )["f1"]
+        print(f"Temporal F1 Score: {round(tem_f1 * 100, 2)}")
+        pbar.update(1)
+
+    return {
+        'BLEU1': round(bleu1 * 100, 2),
+        'BLEU2': round(bleu2 * 100, 2),
+        'BLEU3': round(bleu3 * 100, 2),
+        'BLEU4': round(bleu4 * 100, 2),
+        'METEOR': round(meteor * 100, 2),
+        'ROUGE-L': round(rougel * 100, 2),
+        'Bert_score': round(bert_score * 100, 2),
+        'Temporal_entity_score': round(tem_f1 * 100, 2)
+    }
+
+
+def main():
+    """
+    Parse arguments, compute evaluation metrics, and append the results to a CSV file.
+    """
+    parser = argparse.ArgumentParser(
+        description='Evaluation for Libra Generated Outputs'
+    )
+    parser.add_argument('--references', type=str, required=True,
+                        help='Path to the ground truth file (JSONL).')
+    parser.add_argument('--predictions', type=str, required=True,
+                        help='Path to the prediction file (JSONL).')
+    parser.add_argument('--model-name', type=str, required=True,
+                        help='Unique model identifier for tracking in the results CSV.')
+    parser.add_argument('--save-to-csv', type=str, required=True,
+                        help='Path of the CSV file where results will be saved/appended.')
+    args = parser.parse_args()
+
+    # Calculate metrics
+    scores_results = evaluate_report(
+        references=args.references, 
+        predictions=args.predictions
+    )
+
+    # Append results to CSV
+    append_results_to_csv(scores_results, args.model_name, args.save_to_csv)
+
+
+if __name__ == "__main__":
+    main()

libra/eval/run_libra.py

@@ -0,0 +1,226 @@
+import argparse
+import torch
+
+from libra.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from libra.conversation import conv_templates, SeparatorStyle
+from libra.model.builder import load_pretrained_model
+from libra.utils import disable_torch_init
+from libra.mm_utils import tokenizer_image_token, process_images, get_model_name_from_path, KeywordsStoppingCriteria
+
+import requests
+import pydicom
+from PIL import Image
+from io import BytesIO
+from pydicom.pixel_data_handlers.util import apply_voi_lut
+import datetime
+
+
+def load_images(image_file):
+    """
+    Load an image from a local file, a URL, or a DICOM file.
+
+    Args:
+        image_file (str): The path or URL of the image file to load.
+
+    Returns:
+        PIL.Image.Image: The loaded image in RGB format.
+
+    Raises:
+        ValueError: If the DICOM file does not contain image data.
+        TypeError: If the input is neither a valid file path nor a URL.
+    """
+    if isinstance(image_file, str):
+        # Case 1: Load from URL
+        if image_file.startswith(('http://', 'https://')):
+            try:
+                response = requests.get(image_file)
+                response.raise_for_status()
+                image = Image.open(BytesIO(response.content)).convert('RGB')
+            except Exception as e:
+                raise ValueError(f"Error loading image from URL: {image_file}\n{e}")
+
+        # Case 2: Load from DICOM file
+        elif image_file.lower().endswith('.dcm'):
+            try:
+                dicom = pydicom.dcmread(image_file)
+                if 'PixelData' in dicom:
+                    data = apply_voi_lut(dicom.pixel_array, dicom)
+
+                    # Handle MONOCHROME1 images
+                    if dicom.PhotometricInterpretation == "MONOCHROME1":
+                        data = np.max(data) - data
+
+                    # Normalize the image data
+                    data = data - np.min(data)
+                    data = data / np.max(data)
+                    data = (data * 255).astype(np.uint8)
+
+                    # Convert to 3-channel RGB if necessary
+                    if data.ndim == 2:
+                        data = np.stack([data] * 3, axis=-1)
+
+                    image = Image.fromarray(data).convert('RGB')
+                else:
+                    raise ValueError("DICOM file does not contain image data")
+            except Exception as e:
+                raise ValueError(f"Error loading DICOM file: {image_file}\n{e}")
+
+        # Case 3: Load standard image files (e.g., PNG, JPG)
+        else:
+            try:
+                image = Image.open(image_file).convert('RGB')
+            except Exception as e:
+                raise ValueError(f"Error loading standard image file: {image_file}\n{e}")
+
+    else:
+        raise TypeError("image_file must be a string representing a file path or URL")
+
+    return image
+
+def get_image_tensors(image_path, image_processor, model, device='cuda'):
+    # Load and preprocess the images
+    if isinstance(image_path, str):
+        image = []
+        img = load_images(image_path)
+        image.append(img)
+    elif isinstance(image_path, (list, tuple)):
+        image = []
+        for path in image_path:
+            img = load_images(path)
+            image.append(img)
+    else:
+        raise TypeError("image_file must be a string or a str/list/tuple of strings")
+
+    # Ensure two images are present
+    if len(image) != 2:
+        image.append(image[0])  
+        if model.config.mm_projector_type == "TAC":
+            print("Contains only current image. Adding a dummy prior image for TAC.")
+
+    # Process each image
+    processed_images = []
+    for img_data in image:
+        image_temp = process_images([img_data], image_processor, model.config)[0]
+        image_temp = image_temp.to(device=device, non_blocking=True)
+        processed_images.append(image_temp)
+
+    # Separate current and prior images
+    cur_images = [processed_images[0]]
+    prior_images = [processed_images[1]]
+
+    # Stack and return as batched tensor
+    batch_images = torch.stack([torch.stack(cur_images), torch.stack(prior_images)])
+
+    return batch_images
+
+def libra_eval(
+    model_path=None,
+    model_base=None,
+    image_file=None,
+    query=None,
+    conv_mode="libra_v1",
+    temperature=0.2,
+    top_p=None,
+    num_beams=1,
+    num_return_sequences=None,
+    length_penalty=1.0,
+    max_new_tokens=128
+):
+    # Model
+    disable_torch_init()
+
+    model_name = get_model_name_from_path(model_path)
+    tokenizer, model, image_processor, context_len = load_pretrained_model(model_path, model_base, model_name)
+
+    qs = query
+    if model.config.mm_use_im_start_end:
+        qs = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + qs
+    else:
+        qs = DEFAULT_IMAGE_TOKEN + '\n' + qs
+
+    if 'libra' in model_name.lower():
+        mode_conv = "libra_v1"
+
+    if conv_mode is not None and mode_conv != conv_mode:
+        print('[WARNING] the auto inferred conversation mode is {}, while `--conv-mode` is {}, using {}'.format(mode_conv, conv_mode, conv_mode))
+    else:
+        conv_mode = mode_conv
+
+    conv = conv_templates[conv_mode].copy()
+    conv.append_message(conv.roles[0], qs)
+    conv.append_message(conv.roles[1], None)
+    prompt = conv.get_prompt()
+
+    input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).cuda()
+    attention_mask = torch.ones(input_ids.shape, dtype=torch.long)
+    pad_token_id = tokenizer.pad_token_id
+
+    image_tensor = get_image_tensors(image_file, image_processor, model)
+    
+    stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+    keywords = [stop_str]
+    stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+
+    with torch.inference_mode():
+        torch.cuda.empty_cache()
+        if num_beams > 1:
+            output_ids = model.generate(
+                input_ids=input_ids,
+                images=image_tensor,
+                do_sample=False,
+                num_beams=num_beams,
+                no_repeat_ngram_size=3,
+                max_new_tokens=max_new_tokens,
+                stopping_criteria=[stopping_criteria],
+                use_cache=True,
+                length_penalty=length_penalty, 
+                output_scores=True,  
+                attention_mask=attention_mask, 
+                pad_token_id=pad_token_id,
+                num_return_sequences = num_return_sequences)
+        else:
+            output_ids = model.generate(
+                input_ids,
+                images=image_tensor,
+                do_sample= True,
+                temperature=temperature,
+                top_p=top_p,
+                num_beams=num_beams,
+                no_repeat_ngram_size=3,
+                max_new_tokens=max_new_tokens,
+                attention_mask=attention_mask, 
+                pad_token_id=pad_token_id,
+                stopping_criteria=[stopping_criteria],
+                use_cache=True)
+
+    input_token_len = input_ids.shape[1]
+    n_diff_input_output = (input_ids != output_ids[:, :input_token_len]).sum().item()
+
+    if n_diff_input_output > 0:
+        print(f'[Warning] {n_diff_input_output} output_ids are not the same as the input_ids')
+
+    outputs = tokenizer.batch_decode(output_ids[:, input_token_len:], skip_special_tokens=True)[0]
+    outputs = outputs.strip()
+    
+    if outputs.endswith(stop_str):
+        outputs = outputs[:-len(stop_str)]
+    outputs = outputs.strip()
+    
+    return outputs
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-path", type=str, default="X-iZhang/libra-v1.0-7b")
+    parser.add_argument("--model-base", type=str, default=None)
+    parser.add_argument("--image-file", type=str, required=True)
+    parser.add_argument("--query", type=str, required=True)
+    parser.add_argument("--conv-mode", type=str, default="libra_v1")
+    parser.add_argument("--temperature", type=float, default=0.2)
+    parser.add_argument("--top_p", type=float, default=None)
+    parser.add_argument("--num_beams", type=int, default=1)
+    parser.add_argument("--num_return_sequences", type=int, default=None)
+    parser.add_argument("--length_penalty", type=float, default=1.0)
+    parser.add_argument("--max_new_tokens", type=int, default=128)
+    args = parser.parse_args()
+
+    libra_eval(**vars(args))

libra/eval/temporal_f1.py

@@ -0,0 +1,153 @@
+import re
+import argparse
+from typing import List, Union
+
+# Keywords used for entity extraction
+KEYWORDS = {
+    "bigger", "change", "cleared", "constant", "decrease", "decreased", "decreasing", "elevated", "elevation", 
+    "enlarged", "enlargement", "enlarging", "expanded", "greater", "growing", "improved", "improvement", 
+    "improving", "increase", "increased", "increasing", "larger", "new", "persistence", "persistent", 
+    "persisting", "progression", "progressive", "reduced", "removal", "resolution", "resolved", "resolving", 
+    "smaller", "stability", "stable", "stably", "unchanged", "unfolded", "worse", "worsen", "worsened", 
+    "worsening", "unaltered"
+}
+
+def clean_text(text: str) -> str:
+    """
+    Clean the input text by removing special characters and redundant spaces or newlines.
+
+    Args:
+        text (str): Input text.
+
+    Returns:
+        str: Cleaned text.
+    """
+    # Remove special characters and redundant newlines
+    text = re.sub(r'\n+', ' ', text)  # Replace multiple newlines with a single space
+    text = re.sub(r'[_-]+', ' ', text)  # Replace underscores and dashes with spaces
+    text = re.sub(r'\(___, __, __\)', '', text)  # Remove irrelevant underscore patterns
+    text = re.sub(r'---, ---, ---', '', text)  # Remove dashed patterns
+    text = re.sub(r'\(__, __, ___\)', '', text)  # Remove similar underscore patterns
+    text = re.sub(r'[_-]+', ' ', text)  # Replace underscores and dashes again (if any remain)
+    text = re.sub(r'[^\w\s.,:;()-]', '', text)  # Remove non-alphanumeric characters except common punctuation
+    
+    # Remove extra spaces
+    text = re.sub(r'\s{2,}', ' ', text).strip()
+    return text
+
+def extract_entities(text: str, keywords: set) -> set:
+    """
+    Extract entities from the given text based on the provided keywords.
+
+    Args:
+        text (str): Input text.
+        keywords (set): Set of keywords to extract entities.
+
+    Returns:
+        set: Set of matched keywords found in the text.
+    """
+    # Clean the text before extracting entities
+    text = clean_text(text)
+    
+    # Create a regex pattern that matches any of the keywords as whole words
+    pattern = r'\b(' + '|'.join(re.escape(word) for word in keywords) + r')\b'
+    
+    # Find all matches and return them as a set
+    return {match.group().lower() for match in re.finditer(pattern, text.lower())}
+
+def calculate_tem_score(prediction_text: str, reference_text: Union[str, List[str]], epsilon: float = 1e-10) -> float:
+    """
+    Calculate the Temporal Entity Matching (TEM) score (similar to F1-score).
+
+    Args:
+        reference_text (Union[str, List[str]]): Reference text or a list of reference texts.
+        prediction_text (str): Prediction text.
+        epsilon (float): Small value to avoid division by zero.
+
+    Returns:
+        float: TEM score.
+    """
+    if isinstance(reference_text, list):
+        reference_entities = set()
+        for ref in reference_text:
+            reference_entities.update(extract_entities(ref, KEYWORDS))
+    else:
+        reference_entities = extract_entities(reference_text, KEYWORDS)
+
+    prediction_entities = extract_entities(prediction_text, KEYWORDS)
+
+    if len(reference_entities) == 0:
+        if len(prediction_entities) == 0:
+            return {
+                "f1": 1.0,
+                "prediction_entities": prediction_entities,
+                "reference_entities": reference_entities
+            }  # Perfect match when both are empty
+        else:
+            return {
+                "f1": epsilon,
+                "prediction_entities": prediction_entities,
+                "reference_entities": reference_entities
+            }  # Minimal score when reference is empty but prediction is not
+
+    # Calculate intersection of entities
+    true_positives = len(prediction_entities & reference_entities) 
+
+    # Calculate precision and recall with epsilon to avoid division by zero
+    precision = (true_positives + epsilon) / (len(prediction_entities) + epsilon)
+    recall = (true_positives + epsilon) / (len(reference_entities) + epsilon)
+
+    # Calculate TEM score (F1 score)
+    tem_score = (2 * precision * recall) / (precision + recall + epsilon)
+    
+    return {
+        "f1": tem_score,
+        "prediction_entities": prediction_entities,
+        "reference_entities": reference_entities
+    }
+
+def temporal_f1_score(predictions: List[str], references: List[Union[str, List[str]]], epsilon: float = 1e-10) -> float:
+    """
+    Calculate the average TEM score over a list of reference and prediction texts.
+
+    Args:
+        references (List[Union[str, List[str]]]): List of reference texts or lists of reference texts.
+        predictions (List[str]): List of prediction texts.
+        epsilon (float): Small value to avoid division by zero.
+
+    Returns:
+        float: Average TEM score.
+    """
+    assert len(references) == len(predictions), "Reference and prediction lists must have the same length."
+    
+    tem_scores = []
+    prediction_entities = []
+    reference_entities = []
+
+    for pred, ref in zip(predictions, references):
+        result = calculate_tem_score(pred, ref, epsilon)
+        tem_scores.append(result["f1"])
+        prediction_entities.append(result["prediction_entities"])
+        reference_entities.append(result["reference_entities"])
+
+    average_f1 = sum(tem_scores) / len(tem_scores)
+
+    return {
+        "f1": average_f1,
+        "prediction_entities": prediction_entities,
+        "reference_entities": reference_entities
+    }
+
+# Command-line interface
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Calculate the average TEM score for reference and prediction texts.")
+    parser.add_argument("--predictions", nargs='+', required=True, help="List of prediction texts.")
+    parser.add_argument("--reference", nargs='+', required=True, help="List of reference texts or lists of reference texts.")
+
+    args = parser.parse_args()
+
+    # Convert references into a nested list if necessary
+    reference_list = [eval(ref) if ref.startswith('[') else ref for ref in args.reference]
+
+    # Calculate the average TEM score
+    temporal_f1_score(predictions=args.predictions, references=reference_list)

libra/mm_utils.py

@@ -0,0 +1,116 @@
+from PIL import Image
+from io import BytesIO
+import base64
+
+import torch
+from transformers import StoppingCriteria
+from libra.constants import IMAGE_TOKEN_INDEX
+
+
+def load_image_from_base64(image):
+    return Image.open(BytesIO(base64.b64decode(image)))
+
+def expand2square(pil_img, background_color=(0, 0, 0)):
+    width, height = pil_img.size
+    if width == height:
+        return pil_img
+    elif width > height:
+        result = Image.new(pil_img.mode, (width, width), background_color)
+        result.paste(pil_img, (0, (width - height) // 2))
+        return result
+    else:
+        result = Image.new(pil_img.mode, (height, height), background_color)
+        result.paste(pil_img, ((height - width) // 2, 0))
+        return result
+    
+def process_images(images, image_processor, model_cfg):
+    image_aspect_ratio = getattr(model_cfg, "image_aspect_ratio", None)
+    new_images = []
+
+    try:
+        if not images:
+            raise ValueError("Input images list is empty.")
+
+        if image_aspect_ratio == 'pad':
+            for image in images:
+                if not isinstance(image, Image.Image):
+                    raise TypeError("All input images must be of type PIL.Image.")
+                image = expand2square(image, (0, 0, 0))
+                image = image_processor.preprocess(image, return_tensors='pt')['pixel_values'][0]
+                new_images.append(image)
+        else:
+            return image_processor(images, return_tensors='pt')['pixel_values']
+
+        if new_images and all(x is not None and x.shape == new_images[0].shape for x in new_images):
+            new_images = torch.stack(new_images, dim=0)
+        
+        return new_images
+    except Exception as e:
+        print(f"Error processing images: {e}")
+        return None
+
+
+def tokenizer_image_token(prompt, tokenizer, image_token_index=IMAGE_TOKEN_INDEX, return_tensors=None):
+    prompt_chunks = [tokenizer(chunk).input_ids for chunk in prompt.split('<image>')]
+    #  使用分词器将输入文本prompt按<image>标记分割，然后对每个分割后的文本块进行分词处理，获取对应的输入ID列表。
+    def insert_separator(X, sep):
+        return [ele for sublist in zip(X, [sep]*len(X)) for ele in sublist][:-1]
+
+    input_ids = []
+    offset = 0
+    if len(prompt_chunks) > 0 and len(prompt_chunks[0]) > 0 and prompt_chunks[0][0] == tokenizer.bos_token_id:
+        offset = 1
+        input_ids.append(prompt_chunks[0][0])
+
+    for x in insert_separator(prompt_chunks, [image_token_index] * (offset + 1)):
+        input_ids.extend(x[offset:])
+
+    if return_tensors is not None:
+        if return_tensors == 'pt':
+            return torch.tensor(input_ids, dtype=torch.long)
+        raise ValueError(f'Unsupported tensor type: {return_tensors}')
+    return input_ids
+
+
+def get_model_name_from_path(model_path):
+    model_path = model_path.strip("/")
+    model_paths = model_path.split("/")
+    if model_paths[-1].startswith('checkpoint-'):
+        return model_paths[-2] + "_" + model_paths[-1]
+    else:
+        return model_paths[-1]
+
+    
+class KeywordsStoppingCriteria(StoppingCriteria):
+    def __init__(self, keywords, tokenizer, input_ids):
+        self.keywords = keywords
+        self.keyword_ids = []
+        self.max_keyword_len = 0
+        for keyword in keywords:
+            cur_keyword_ids = tokenizer(keyword).input_ids
+            if len(cur_keyword_ids) > 1 and cur_keyword_ids[0] == tokenizer.bos_token_id:
+                cur_keyword_ids = cur_keyword_ids[1:]
+            if len(cur_keyword_ids) > self.max_keyword_len:
+                self.max_keyword_len = len(cur_keyword_ids)
+            self.keyword_ids.append(torch.tensor(cur_keyword_ids))
+        self.tokenizer = tokenizer
+        self.start_len = input_ids.shape[1]
+    
+    def call_for_batch(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        offset = min(output_ids.shape[1] - self.start_len, self.max_keyword_len)
+        self.keyword_ids = [keyword_id.to(output_ids.device) for keyword_id in self.keyword_ids]
+        for keyword_id in self.keyword_ids:
+            truncated_output_ids = output_ids[0, -keyword_id.shape[0]:]
+            if torch.equal(truncated_output_ids, keyword_id):
+                return True
+        outputs = self.tokenizer.batch_decode(output_ids[:, -offset:], skip_special_tokens=True)[0]
+        for keyword in self.keywords:
+            if keyword in outputs:
+                return True
+        return False
+    
+    def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool:
+        outputs = []
+        for i in range(output_ids.shape[0]):
+            outputs.append(self.call_for_batch(output_ids[i].unsqueeze(0), scores))
+        return all(outputs)

libra/model/__init__.py

@@ -0,0 +1,4 @@
+try:
+    from .language_model.libra_llama import LibraLlamaForCausalLM, LibraConfig
+except:
+    pass

libra/model/builder.py

@@ -0,0 +1,135 @@
+
+#    Copyright 2024 Xi Zhang
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import os
+import warnings
+import shutil
+
+from transformers import AutoTokenizer, AutoModelForCausalLM, AutoConfig, BitsAndBytesConfig
+import torch
+from libra.model import *
+from libra.constants import DEFAULT_IMAGE_PATCH_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+
+
+def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, load_4bit=False, device_map="auto", device="cuda"):
+    kwargs = {"device_map": device_map}
+
+    if device != "cuda":
+        kwargs['device_map'] = {"": device}
+        
+    if load_8bit:
+        kwargs['load_in_8bit'] = True
+    elif load_4bit:
+        kwargs['load_in_4bit'] = True
+        kwargs['quantization_config'] = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_compute_dtype=torch.float16,
+            bnb_4bit_use_double_quant=True,
+            bnb_4bit_quant_type='nf4'
+        )
+    else:
+        kwargs['torch_dtype'] = torch.float16
+
+    if 'libra' in model_name.lower():
+        # Load Libra model
+        if 'lora' in model_name.lower() and model_base is None:
+            warnings.warn('There is `lora` in model name but no `model_base` is provided. If you are loading a LoRA model, please provide the `model_base` argument.')
+        if 'lora' in model_name.lower() and model_base is not None:
+            from libra.model.language_model.libra_llama import LibraConfig
+            lora_cfg_pretrained = LibraConfig.from_pretrained(model_path)
+            tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
+            print('Loading libra from base model...')
+            model = LibraLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=lora_cfg_pretrained, **kwargs)
+            token_num, tokem_dim = model.lm_head.out_features, model.lm_head.in_features
+            if model.lm_head.weight.shape[0] != token_num:
+                model.lm_head.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))
+                model.model.embed_tokens.weight = torch.nn.Parameter(torch.empty(token_num, tokem_dim, device=model.device, dtype=model.dtype))
+
+            print('Loading additional Libra weights...')
+            if os.path.exists(os.path.join(model_path, 'non_lora_trainables.bin')):
+                non_lora_trainables = torch.load(os.path.join(model_path, 'non_lora_trainables.bin'), map_location='cpu')
+            else:
+                from huggingface_hub import hf_hub_download
+                def load_from_hf(repo_id, filename, subfolder=None):
+                    cache_file = hf_hub_download(
+                        repo_id=repo_id,
+                        filename=filename,
+                        subfolder=subfolder)
+                    return torch.load(cache_file, map_location='cpu')
+                non_lora_trainables = load_from_hf(model_path, 'non_lora_trainables.bin')
+            non_lora_trainables = {(k[11:] if k.startswith('base_model.') else k): v for k, v in non_lora_trainables.items()}
+            if any(k.startswith('model.model.') for k in non_lora_trainables):
+                non_lora_trainables = {(k[6:] if k.startswith('model.') else k): v for k, v in non_lora_trainables.items()}
+            model.load_state_dict(non_lora_trainables, strict=False)
+
+            from peft import PeftModel
+            print('Loading LoRA weights...')
+            model = PeftModel.from_pretrained(model, model_path)
+            print('Merging LoRA weights...')
+            model = model.merge_and_unload()
+            print('Model is loaded...')
+        elif model_base is not None:
+            print('Loading Libra from base model...')
+            tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
+            cfg_pretrained = AutoConfig.from_pretrained(model_path)
+            model = LibraLlamaForCausalLM.from_pretrained(model_base, low_cpu_mem_usage=True, config=cfg_pretrained, **kwargs)
+
+            mm_projector_weights = torch.load(os.path.join(model_path, 'mm_projector.bin'), map_location='cpu')
+            mm_projector_weights = {k: v.to(torch.float16) for k, v in mm_projector_weights.items()}
+            model.load_state_dict(mm_projector_weights, strict=False)
+        else:
+            tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
+            model = LibraLlamaForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
+    else:
+        # Load language model
+        if model_base is not None:
+            # PEFT model
+            from peft import PeftModel
+            tokenizer = AutoTokenizer.from_pretrained(model_base, use_fast=False)
+            model = AutoModelForCausalLM.from_pretrained(model_base, torch_dtype=torch.float16, low_cpu_mem_usage=True, device_map="auto")
+            print(f"Loading LoRA weights from {model_path}")
+            model = PeftModel.from_pretrained(model, model_path)
+            print(f"Merging weights")
+            model = model.merge_and_unload()
+            print('Convert to FP16...')
+            model.to(torch.float16)
+        else:
+            use_fast = False
+            tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False)
+            model = AutoModelForCausalLM.from_pretrained(model_path, low_cpu_mem_usage=True, **kwargs)
+
+    image_processor = None
+
+    if 'libra' in model_name.lower():
+        mm_use_im_start_end = getattr(model.config, "mm_use_im_start_end", False)
+        mm_use_im_patch_token = getattr(model.config, "mm_use_im_patch_token", True)
+        if mm_use_im_patch_token:
+            tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
+        if mm_use_im_start_end:
+            tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
+        model.resize_token_embeddings(len(tokenizer))
+
+        vision_tower = model.get_vision_tower()
+        if not vision_tower.is_loaded:
+            vision_tower.load_model()
+        vision_tower.to(device=device, dtype=torch.float16)
+        image_processor = vision_tower.image_processor
+
+    if hasattr(model.config, "max_sequence_length"):
+        context_len = model.config.max_sequence_length
+    else:
+        context_len = 2048
+
+    return tokenizer, model, image_processor, context_len

libra/model/language_model/libra_llama.py

@@ -0,0 +1,141 @@
+#    Copyright 2024 Xi Zhang
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+from torch.nn import CrossEntropyLoss
+
+from transformers import AutoConfig, AutoModelForCausalLM, LlamaConfig, LlamaModel, LlamaForCausalLM
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from ..libra_arch import LibraMetaModel, LibraMetaForCausalLM
+
+
+class LibraConfig(LlamaConfig):
+    model_type = "libra"
+
+class LibraLlamaModel(LibraMetaModel, LlamaModel):
+    config_class = LibraConfig
+
+    def __init__(self, config: LlamaConfig):
+        super(LibraLlamaModel, self).__init__(config)
+
+
+class LibraLlamaForCausalLM(LlamaForCausalLM, LibraMetaForCausalLM):
+    config_class = LibraConfig
+
+    def __init__(self, config):
+        super(LlamaForCausalLM, self).__init__(config)
+        self.model = LibraLlamaModel(config)
+        self.vocab_size = config.vocab_size
+
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_model(self):
+        return self.model
+   
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        images: Optional[torch.FloatTensor] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        input_ids, attention_mask, past_key_values, inputs_embeds, labels = self.prepare_inputs_labels_for_multimodal(input_ids, attention_mask, past_key_values, labels, images)
+
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict
+        )
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+
+        loss = None
+        #Adopted from https://github.com/huggingface/transformers/blob/v4.21.0/src/transformers/models/gptj/modeling_gptj.py#L847
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits=shift_logits.view(-1, shift_logits.size(-1))
+            shift_labels = shift_labels.view(-1)
+            # Enable model/pipeline parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            
+            loss = loss_fct(shift_logits, shift_labels)
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
+    ):
+        if past_key_values:
+            input_ids = input_ids[:, -1:]
+
+
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+        
+        model_inputs.update(
+            {
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+                "images": kwargs.get("images", None),
+            }
+        )
+        
+        return model_inputs
+
+AutoConfig.register("libra", LibraConfig)  # Register the LibraConfig to the AutoConfig registry
+AutoModelForCausalLM.register(LibraConfig, LibraLlamaForCausalLM)  # Register the LibraLlamaForCausalLM to the AutoModel registry

libra/model/libra_arch.py

@@ -0,0 +1,333 @@
+#    Copyright 2024 Xi Zhang
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+from abc import ABC, abstractmethod
+
+import torch
+import torch.nn as nn
+
+from .multimodal_encoder.builder import build_vision_tower
+from .multimodal_projector.builder import build_vision_projector
+
+from libra.constants import (
+    IGNORE_INDEX,  
+    IMAGE_TOKEN_INDEX,  
+    DEFAULT_IMAGE_PATCH_TOKEN,  
+    DEFAULT_IM_START_TOKEN, 
+    DEFAULT_IM_END_TOKEN, 
+)
+
+
+class LibraMetaModel:
+    """
+    LibraMetaModel is a class that initializes and manages a multi-modal model with vision and projection modules.
+
+    Attributes:
+        config (object): Configuration object containing model parameters.
+        vision_tower (object): Vision model component.
+        mm_projector (object): Multi-modal projection module.
+
+    Methods:
+        __init__(config):
+            Initializes the LibraMetaModel with the given configuration.
+        
+        get_vision_tower():
+            Retrieves the vision model component. If the vision model is a list, returns the first element.
+        
+        initialize_vision_modules(model_args, fsdp=None):
+            Initializes the vision and projection modules based on the provided model arguments.
+            Loads pre-trained weights for the multi-modal MLP adapter if available.
+    """
+    def __init__(self, config):
+ 
+        super(LibraMetaModel, self).__init__(config)
+
+        if hasattr(config, "mm_vision_tower"):
+
+            self.vision_tower = build_vision_tower(config, delay_load=True)
+            self.mm_projector = build_vision_projector(config)
+
+    def get_vision_tower(self):
+        vision_tower = getattr(self, 'vision_tower', None)
+        if type(vision_tower) is list:
+            vision_tower = vision_tower[0]
+        return vision_tower
+
+    def initialize_vision_modules(self, model_args, fsdp=None):
+        vision_tower = model_args.vision_tower
+        mm_vision_select_layer = model_args.mm_vision_select_layer
+        mm_vision_select_feature = model_args.mm_vision_select_feature
+        pretrain_mm_mlp_adapter = model_args.pretrain_mm_mlp_adapter 
+
+        self.config.mm_vision_tower = vision_tower
+
+        if self.get_vision_tower() is None:
+            vision_tower = build_vision_tower(model_args)
+
+            if fsdp is not None and len(fsdp) > 0:
+                self.vision_tower = [vision_tower]
+            else:
+                self.vision_tower = vision_tower
+        else:
+            if fsdp is not None and len(fsdp) > 0:
+                vision_tower = self.vision_tower[0]
+            else:
+                vision_tower = self.vision_tower
+            vision_tower.load_model()
+
+        self.config.use_mm_proj = True
+        self.config.mm_projector_type = getattr(model_args, 'mm_projector_type', 'linear') 
+        self.config.mm_hidden_size = vision_tower.hidden_size 
+        self.config.mm_vision_select_layer = mm_vision_select_layer 
+        self.config.mm_vision_select_feature = mm_vision_select_feature 
+
+        if getattr(self, 'mm_projector', None) is None:   
+            self.mm_projector = build_vision_projector(self.config)
+        else:
+            for p in self.mm_projector.parameters():
+                p.requires_grad = True
+
+        if pretrain_mm_mlp_adapter is not None:
+            mm_projector_weights = torch.load(pretrain_mm_mlp_adapter, map_location='cpu') 
+
+            def get_w(weights, keyword):
+                return {k.split(keyword + '.')[1]: v for k, v in weights.items() if keyword in k}
+
+            self.mm_projector.load_state_dict(get_w(mm_projector_weights, 'mm_projector'))
+
+
+class LibraMetaForCausalLM(ABC):
+
+    @abstractmethod
+    def get_model(self):
+        pass
+
+    def get_vision_tower(self):
+        return self.get_model().get_vision_tower()
+
+    def encode_images(self, images):
+        image_features_temp = self.get_model().get_vision_tower()(images)
+        image_features = self.get_model().mm_projector(image_features_temp)
+        
+        return image_features
+
+    def prepare_inputs_labels_for_multimodal(
+        self, input_ids, attention_mask, past_key_values, labels, images
+    ):
+        """
+        Prepare inputs and labels for multimodal tasks, applying different logic based on training or inference phase.
+        
+        Args:
+            input_ids (Tensor): IDs of the input token sequence.
+            attention_mask (Tensor): Attention mask.
+            past_key_values (Tensor): Cached key and value for attention mechanism.
+            labels (Tensor): Target labels.
+            images (Tensor): Image inputs.
+
+        Returns:
+            Tuple: Processed input_ids, attention_mask, past_key_values, multimodal_features, labels
+        """
+
+        vision_tower = self.get_vision_tower()
+
+        if vision_tower is None or images is None or input_ids.shape[1] == 1:
+            if past_key_values is not None and vision_tower is not None and images is not None and input_ids.shape[1] == 1:
+                attention_mask = torch.ones(
+                    (attention_mask.shape[0], past_key_values[-1][-1].shape[-2] + 1),
+                    dtype=attention_mask.dtype,
+                    device=attention_mask.device
+                )
+            return input_ids, attention_mask, past_key_values, None, labels
+        
+        if input_ids.size(0) != images.size(0) and input_ids.size(0) != images.size(1):
+            # print(
+            #     "Warning: Dimension mismatch detected. Adjust dimensions for beam-search.\n"
+            #     "Program continues..."
+            # )
+            num_groups = input_ids.size(0)
+            images_1 = images[:num_groups]
+            images_2 = images[num_groups:]
+            images = torch.cat((images_1, images_2), dim=1)
+            images = images.permute(1, 0, 2, 3, 4).contiguous()
+               
+        image_features = self.encode_images(images)  
+        
+        new_input_embeds = []
+        new_labels = [] if labels is not None else None
+        cur_image_idx = 0
+        for batch_idx, cur_input_ids in enumerate(input_ids):
+            if (cur_input_ids == IMAGE_TOKEN_INDEX).sum() == 0:
+
+                cur_image_features = image_features[cur_image_idx]
+                cur_input_embeds_temp = self.get_model().embed_tokens(cur_input_ids)
+                cur_input_embeds = torch.cat([cur_input_embeds_temp, cur_image_features[0:0]], dim=0)
+            
+                new_input_embeds.append(cur_input_embeds)
+                if labels is not None:
+                    new_labels.append(labels[batch_idx])
+                cur_image_idx += 1
+
+                continue
+                
+            image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
+            cur_new_input_embeds = []
+            
+            if labels is not None:
+                cur_labels = labels[batch_idx]
+                cur_new_labels = [] 
+
+                assert cur_labels.shape == cur_input_ids.shape
+            
+            while image_token_indices.numel() > 0:
+
+                cur_image_features = image_features[cur_image_idx]
+                image_token_start = image_token_indices[0]
+
+                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
+                    cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids[:image_token_start-1]).detach())
+                    cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids[image_token_start-1:image_token_start]))
+                    cur_new_input_embeds.append(cur_image_features)
+                    cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids[image_token_start+1:image_token_start+2]))
+                    if labels is not None:
+                        cur_new_labels.append(cur_labels[:image_token_start])
+                        cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=labels.device, dtype=labels.dtype))
+                        cur_new_labels.append(cur_labels[image_token_start:image_token_start+1])
+                        cur_labels_temp = cur_labels[image_token_start+2:]
+                else:
+                    cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids[:image_token_start])) 
+                    cur_new_input_embeds.append(cur_image_features) 
+
+                    if labels is not None:
+                        cur_new_labels.append(cur_labels[:image_token_start]) 
+                        cur_new_labels.append(torch.full((cur_image_features.shape[0],), IGNORE_INDEX, device=labels.device, dtype=labels.dtype)) 
+                        cur_labels_temp = cur_labels[image_token_start+1:] 
+                
+                cur_image_idx += 1
+                
+                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
+                    cur_input_ids = cur_input_ids[image_token_start+2:]
+                else:
+                    cur_input_ids = cur_input_ids[image_token_start+1:]
+                image_token_indices = torch.where(cur_input_ids == IMAGE_TOKEN_INDEX)[0]
+
+            if cur_input_ids.numel() > 0:
+                if getattr(self.config, 'tune_mm_mlp_adapter', False) and getattr(self.config, 'mm_use_im_start_end', False):
+                    cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids).detach())
+                else:
+                    cur_new_input_embeds.append(self.get_model().embed_tokens(cur_input_ids))
+                if labels is not None:
+                    cur_new_labels.append(cur_labels_temp)
+            
+            cur_new_input_embeds = [x.to(device=self.device) for x in cur_new_input_embeds]
+
+            cur_new_input_embeds = torch.cat(cur_new_input_embeds, dim=0) 
+
+            new_input_embeds.append(cur_new_input_embeds) 
+
+            if labels is not None:
+                cur_new_labels = torch.cat(cur_new_labels, dim=0) 
+                new_labels.append(cur_new_labels) 
+       
+                
+        if any(x.shape != new_input_embeds[0].shape for x in new_input_embeds):
+            max_len = max(x.shape[0] for x in new_input_embeds)
+
+            new_input_embeds_align = []
+            for cur_new_embed in new_input_embeds:
+                cur_new_embed = torch.cat((cur_new_embed, torch.zeros((max_len - cur_new_embed.shape[0], cur_new_embed.shape[1]), dtype=cur_new_embed.dtype, device=cur_new_embed.device)), dim=0)
+                new_input_embeds_align.append(cur_new_embed)
+            new_input_embeds = torch.stack(new_input_embeds_align, dim=0)
+
+            if labels is not None:
+                new_labels_align = []
+                _new_labels = new_labels
+                for cur_new_label in new_labels:
+                    cur_new_label = torch.cat((cur_new_label, torch.full((max_len - cur_new_label.shape[0],), IGNORE_INDEX, dtype=cur_new_label.dtype, device=cur_new_label.device)), dim=0)
+                    new_labels_align.append(cur_new_label)
+                new_labels = torch.stack(new_labels_align, dim=0)
+
+            if attention_mask is not None:
+                new_attention_mask = []
+                for cur_attention_mask, cur_new_labels, cur_new_labels_align in zip(attention_mask, _new_labels, new_labels):
+                    new_attn_mask_pad_left = torch.full((cur_new_labels.shape[0] - labels.shape[1],), True, dtype=attention_mask.dtype, device=attention_mask.device)
+                    new_attn_mask_pad_right = torch.full((cur_new_labels_align.shape[0] - cur_new_labels.shape[0],), False, dtype=attention_mask.dtype, device=attention_mask.device)
+                    cur_new_attention_mask = torch.cat((new_attn_mask_pad_left, cur_attention_mask, new_attn_mask_pad_right), dim=0)
+                    new_attention_mask.append(cur_new_attention_mask)
+                attention_mask = torch.stack(new_attention_mask, dim=0)
+                assert attention_mask.shape == new_labels.shape
+        else:
+            
+            new_input_embeds = torch.stack(new_input_embeds, dim=0) 
+            
+            if labels is not None:
+                new_labels  = torch.stack(new_labels, dim=0)
+
+            if attention_mask is not None:
+                new_attn_mask_pad_left = torch.full((attention_mask.shape[0], new_input_embeds.shape[1] - input_ids.shape[1]), True, dtype=attention_mask.dtype, device=attention_mask.device)
+                attention_mask = torch.cat((new_attn_mask_pad_left, attention_mask), dim=1)
+
+                assert attention_mask.shape == new_input_embeds.shape[:2]
+        
+        return None, attention_mask, past_key_values, new_input_embeds, new_labels 
+
+    def initialize_vision_tokenizer(self, model_args, tokenizer):
+
+        if model_args.mm_use_im_patch_token:
+            
+            tokenizer.add_tokens([DEFAULT_IMAGE_PATCH_TOKEN], special_tokens=True)
+            self.resize_token_embeddings(len(tokenizer))
+            
+        if model_args.mm_use_im_start_end:
+            num_new_tokens = tokenizer.add_tokens([DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN], special_tokens=True)
+            self.resize_token_embeddings(len(tokenizer))
+
+            if num_new_tokens > 0:
+                input_embeddings = self.get_input_embeddings().weight.data
+                output_embeddings = self.get_output_embeddings().weight.data
+
+                input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(
+                    dim=0, keepdim=True)
+                output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(
+                    dim=0, keepdim=True)
+
+                input_embeddings[-num_new_tokens:] = input_embeddings_avg
+                output_embeddings[-num_new_tokens:] = output_embeddings_avg
+
+            if model_args.tune_mm_mlp_adapter:
+                for p in self.get_input_embeddings().parameters():
+                    p.requires_grad = True
+                for p in self.get_output_embeddings().parameters():
+                    p.requires_grad = False
+
+            if model_args.pretrain_mm_mlp_adapter:
+                mm_projector_weights = torch.load(model_args.pretrain_mm_mlp_adapter, map_location='cpu')
+                embed_tokens_weight = mm_projector_weights['model.embed_tokens.weight']
+                assert num_new_tokens == 2
+                if input_embeddings.shape == embed_tokens_weight.shape:
+                    input_embeddings[-num_new_tokens:] = embed_tokens_weight[-num_new_tokens:]
+                elif embed_tokens_weight.shape[0] == num_new_tokens:
+                    input_embeddings[-num_new_tokens:] = embed_tokens_weight
+                else:
+                    raise ValueError(f"Unexpected embed_tokens_weight shape. Pretrained: {embed_tokens_weight.shape}. Current: {input_embeddings.shape}. Numer of new tokens: {num_new_tokens}.")
+        
+
+        elif model_args.mm_use_im_patch_token:
+            
+            if model_args.tune_mm_mlp_adapter:
+                
+                for p in self.get_input_embeddings().parameters():
+                    p.requires_grad = False
+                
+                for p in self.get_output_embeddings().parameters():
+                    p.requires_grad = False

libra/model/multimodal_encoder/builder.py

@@ -0,0 +1,38 @@
+#    Copyright 2024 Xi Zhang
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import os
+from .clip_encoder import CLIPVisionTower
+from .dino_encoder import DINOVisionTower
+
+def build_vision_tower(vision_tower_cfg, **kwargs):
+
+    vision_tower = getattr(vision_tower_cfg, 'mm_vision_tower', getattr(vision_tower_cfg, 'vision_tower', None))
+    
+    if vision_tower is None:
+        raise ValueError("No vision tower specified in configuration.")
+
+    is_absolute_path_exists = os.path.exists(vision_tower)
+
+    if is_absolute_path_exists or vision_tower.startswith("openai") or \
+       vision_tower.startswith("facebook") or vision_tower.startswith("microsoft"):
+        
+        if "clip" in vision_tower.lower():
+            return CLIPVisionTower(vision_tower, args=vision_tower_cfg, **kwargs)
+        elif "dino" in vision_tower.lower():
+            return DINOVisionTower(vision_tower, args=vision_tower_cfg, **kwargs)
+        else:
+            raise ValueError(f'Unknown vision model type in vision_tower: {vision_tower}')
+    
+    raise ValueError(f'Unknown vision tower: {vision_tower}')

libra/model/multimodal_encoder/clip_encoder.py

@@ -0,0 +1,126 @@
+#    Copyright 2024 Xi Zhang
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import torch
+import torch.nn as nn
+
+from transformers import AutoImageProcessor, AutoModel, AutoConfig
+        
+class CLIPVisionTower(nn.Module):
+    def __init__(self, vision_tower, args, delay_load=False):
+        super().__init__()
+
+        self.is_loaded = False
+
+        self.vision_tower_name = vision_tower
+        self.select_layer = args.mm_vision_select_layer 
+        self.select_feature = getattr(args, 'mm_vision_select_feature', 'patch') 
+
+        if not delay_load:
+            self.load_model()
+        elif getattr(args, 'unfreeze_mm_vision_tower', False):
+            self.load_model()
+        else:
+            self.cfg_only = AutoConfig.from_pretrained(self.vision_tower_name)
+
+    def load_model(self):
+        if self.is_loaded:
+            print('{} is already loaded, `load_model` called again, skipping.'.format(self.vision_tower_name))
+            return
+        
+        self.image_processor = AutoImageProcessor.from_pretrained(self.vision_tower_name)
+        self.vision_tower = AutoModel.from_pretrained(self.vision_tower_name)
+        self.vision_tower.requires_grad_(False)
+
+        self.is_loaded = True
+    
+    def get_features(self, images):
+        outputs = self.vision_tower(images, output_hidden_states=True)
+        hidden_states = outputs.hidden_states
+        
+        if self.select_layer == "all":
+            if self.select_feature == "patch":
+                all_layers_features = [hidden_state[:, 1:, :].contiguous() for hidden_state in hidden_states[1:]]
+            elif self.select_feature == "cls_patch":
+                all_layers_features = [hidden_state.contiguous() for hidden_state in hidden_states[1:]]
+            else:
+                raise ValueError(f"Unexpected select feature: {self.select_feature}")
+
+            return torch.stack(all_layers_features) 
+        else:
+            selected_layer_features = hidden_states[int(self.select_layer)]
+
+            if self.select_feature == "patch":
+                selected_layer_features = selected_layer_features[:, 1:]
+            elif self.select_feature == "cls_patch":
+                selected_layer_features = selected_layer_features
+            else:
+                raise ValueError(f"Unexpected select feature: {self.select_feature}")
+
+            return selected_layer_features
+    
+    @torch.no_grad()
+    def forward(self, images):
+        
+        if images.shape[0] != 2:
+            raise ValueError(
+                f"Expected images.shape[0] == 2, but got {images.shape[0]}. "
+                "Ensure the input includes both current and previous images."
+            )
+
+        cur_images = images[0]  
+        prev_images = images[1]  
+
+        cur_features = self.get_features(cur_images)  
+        prev_features = self.get_features(prev_images)  
+
+        cur_features = cur_features.permute(1, 0, 2, 3) 
+        prev_features = prev_features.permute(1, 0, 2, 3) 
+
+        # Stack current and previous images along a new dimension
+        images_features = torch.stack([cur_features, prev_features])  
+
+        return images_features
+
+    @property
+    def dummy_feature(self):
+        return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)
+
+    @property
+    def dtype(self):
+        
+        return self.vision_tower.dtype 
+
+    @property
+    def device(self):
+        return self.vision_tower.device 
+
+    @property
+    def config(self):
+        if self.is_loaded:
+            return self.vision_tower.config
+        else:
+            return self.cfg_only
+
+    @property
+    def hidden_size(self):
+        return self.config.hidden_size 
+
+    @property
+    def num_patches(self):
+        return (self.config.image_size // self.config.patch_size) ** 2
+
+    @property
+    def num_layers(self):
+        return self.config.num_hidden_layers 

libra/model/multimodal_encoder/dino_encoder.py

@@ -0,0 +1,126 @@
+#    Copyright 2024 Xi Zhang
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import torch
+import torch.nn as nn
+
+from transformers import AutoImageProcessor, AutoModel, AutoConfig
+        
+class DINOVisionTower(nn.Module):
+    def __init__(self, vision_tower, args, delay_load=False):
+        super().__init__()
+
+        self.is_loaded = False
+
+        self.vision_tower_name = vision_tower
+        self.select_layer = args.mm_vision_select_layer 
+        self.select_feature = getattr(args, 'mm_vision_select_feature', 'patch') 
+
+        if not delay_load:
+            self.load_model()
+        elif getattr(args, 'unfreeze_mm_vision_tower', False):
+            self.load_model()
+        else:
+            self.cfg_only = AutoConfig.from_pretrained(self.vision_tower_name)
+
+    def load_model(self):
+        if self.is_loaded:
+            print('{} is already loaded, `load_model` called again, skipping.'.format(self.vision_tower_name))
+            return
+        
+        self.image_processor = AutoImageProcessor.from_pretrained(self.vision_tower_name)
+        self.vision_tower = AutoModel.from_pretrained(self.vision_tower_name)
+        self.vision_tower.requires_grad_(False)
+
+        self.is_loaded = True
+    
+    def get_features(self, images):
+        outputs = self.vision_tower(images, output_hidden_states=True)
+        hidden_states = outputs.hidden_states
+        
+        if self.select_layer == "all":
+            if self.select_feature == "patch":
+                all_layers_features = [hidden_state[:, 1:, :].contiguous() for hidden_state in hidden_states[1:]]
+            elif self.select_feature == "cls_patch":
+                all_layers_features = [hidden_state.contiguous() for hidden_state in hidden_states[1:]]
+            else:
+                raise ValueError(f"Unexpected select feature: {self.select_feature}")
+
+            return torch.stack(all_layers_features)  
+        else:
+            selected_layer_features = hidden_states[int(self.select_layer)]
+
+            if self.select_feature == "patch":
+                selected_layer_features = selected_layer_features[:, 1:]
+            elif self.select_feature == "cls_patch":
+                selected_layer_features = selected_layer_features
+            else:
+                raise ValueError(f"Unexpected select feature: {self.select_feature}")
+
+            return torch.stack([selected_layer_features])
+    
+    @torch.no_grad()
+    def forward(self, images):
+        
+        if images.shape[0] != 2:
+            raise ValueError(
+                f"Expected images.shape[0] == 2, but got {images.shape}. "
+                "Ensure the input includes both current and previous images."
+            )
+
+        cur_images = images[0]  
+        prev_images = images[1]  
+
+        cur_features = self.get_features(cur_images) 
+        prev_features = self.get_features(prev_images) 
+        
+        cur_features = cur_features.permute(1, 0, 2, 3) 
+        prev_features = prev_features.permute(1, 0, 2, 3) 
+
+        # Stack current and previous images along a new dimension
+        images_features = torch.stack([cur_features, prev_features])  
+        
+        return images_features
+
+    @property
+    def dummy_feature(self):
+        return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)
+
+    @property
+    def dtype(self):
+        
+        return self.vision_tower.dtype 
+
+    @property
+    def device(self):
+        return self.vision_tower.device 
+
+    @property
+    def config(self):
+        if self.is_loaded:
+            return self.vision_tower.config
+        else:
+            return self.cfg_only
+
+    @property
+    def hidden_size(self):
+        return self.config.hidden_size 
+    
+    @property
+    def num_patches(self):
+        return (self.config.image_size // self.config.patch_size) ** 2 
+    
+    @property
+    def num_layers(self):
+        return self.config.num_hidden_layers 

libra/model/multimodal_projector/builder.py

@@ -0,0 +1,167 @@
+#    Copyright 2024 Xi Zhang
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import torch
+import torch.nn as nn
+import torchvision.ops as ops
+import re
+
+
+class TAC(nn.Module):
+    def __init__(self, config):
+        super(TAC,self).__init__()
+
+        self.mm_hidden_size = config.mm_hidden_size
+        self.hidden_size = config.hidden_size
+        self.num_attention_heads = config.num_attention_heads
+        self.dropout = 0.1
+        self.layers_number = 12 # RAD-DINO hidden layers
+
+        # LFE
+        self.LFE = nn.Sequential(
+            ops.SqueezeExcitation(self.layers_number,self.layers_number // 2,activation=nn.GELU),
+            nn.Conv2d(self.layers_number,self.layers_number // 2,kernel_size=1,bias=False),
+            ops.SqueezeExcitation(self.layers_number // 2,self.layers_number // 4,activation=nn.GELU),
+            nn.Conv2d(self.layers_number // 2,self.layers_number // 4,kernel_size=1,bias=False),
+            ops.SqueezeExcitation(self.layers_number // 4,1,activation=nn.GELU),
+            nn.Conv2d(self.layers_number // 4,1,kernel_size=1,bias=False)
+        )
+  
+        self.LFE_prior_bias = nn.Parameter(torch.tensor(0.0, device=torch.device("cuda" if torch.cuda.is_available() else "cpu")))
+        self.LFE_cos = nn.CosineSimilarity(dim=-1, eps=1e-6)
+        
+        # self-attention
+        self.cur_self_attention = nn.MultiheadAttention(embed_dim=(self.mm_hidden_size), num_heads=self.num_attention_heads,batch_first=True,add_bias_kv=True)
+        self.prior_self_attention = nn.MultiheadAttention(embed_dim=(self.mm_hidden_size), num_heads=self.num_attention_heads,batch_first=True,add_bias_kv=True)
+        self.cros_attention = nn.MultiheadAttention(embed_dim=(self.mm_hidden_size), num_heads=self.num_attention_heads,batch_first=True,add_bias_kv=True)
+        
+        self.norm1 = nn.LayerNorm(self.mm_hidden_size)
+        self.norm2 = nn.LayerNorm(self.mm_hidden_size)
+        self.norm3 = nn.LayerNorm(self.mm_hidden_size)
+        self.norm4 = nn.LayerNorm(self.mm_hidden_size)
+        
+        self.mlp_attn = nn.Sequential(
+            nn.Linear(self.mm_hidden_size, self.mm_hidden_size),
+            nn.GELU(),
+            nn.Dropout(self.dropout),
+            nn.Linear(self.mm_hidden_size, self.mm_hidden_size),
+            nn.Dropout(self.dropout)
+        )
+
+        self.mlp_final = nn.Sequential(
+            nn.Linear(self.mm_hidden_size, self.hidden_size),
+            nn.GELU(),
+            nn.Linear(self.hidden_size, self.hidden_size),
+            nn.GELU(),
+            nn.Linear(self.hidden_size, self.hidden_size),
+            nn.GELU(),
+            nn.Linear(self.hidden_size, self.hidden_size)
+        )
+
+        self.dropout1 = nn.Dropout(self.dropout)
+        self.dropout2 = nn.Dropout(self.dropout)
+        self.dropout3 = nn.Dropout(self.dropout)
+    
+    def calculate_cosine_similarity(self, tensor1, tensor2):
+
+        assert tensor1.shape == tensor2.shape, "The shapes of the two tensors must be the same"
+
+        tensor1_flat = tensor1.view(tensor1.size(0), -1)
+        tensor2_flat = tensor2.view(tensor2.size(0), -1)
+
+        tensor1_flat_normalized = tensor1_flat / tensor1_flat.norm(dim=-1, keepdim=True)
+        tensor2_flat_normalized = tensor2_flat / tensor2_flat.norm(dim=-1, keepdim=True)
+
+        cosine_similarities = self.LFE_cos(tensor1_flat_normalized, tensor2_flat_normalized)
+        cosine_similarities_normalized = ((cosine_similarities + 1) / 2).pow(8)
+        cosine_similarities_normalized = cosine_similarities_normalized.view(-1, 1, 1)
+
+        return cosine_similarities_normalized 
+    
+    # self-attention block   
+    def cur_self_att_block(self,x):
+        x = self.cur_self_attention(x,x,x)[0]
+        return self.dropout1(x)
+    # self-attention block   
+    def prior_self_att_block(self,x):
+        x = self.prior_self_attention(x,x,x)[0]
+        return self.dropout2(x)
+    # cross attention block 
+    def cros_att_block(self,x,y):
+        x = self.cros_attention(x,y,y)[0]
+        return self.dropout3(x)
+    
+    #TFM
+    def TFM(self,cur_features,prev_features):
+        
+        cur_features_temp = cur_features
+        prev_features_temp = prev_features
+        
+        cos= self.calculate_cosine_similarity(cur_features_temp,prev_features_temp)
+        prev_weight = cos * self.LFE_prior_bias
+        prev_features_temp = prev_features_temp + prev_weight
+        
+        cur_features = self.norm1(cur_features_temp + self.cur_self_att_block(cur_features_temp))
+        prev_features = self.norm2(prev_features_temp + self.prior_self_att_block(prev_features_temp))
+        combined_features = self.norm3(cur_features + self.cros_att_block(cur_features,prev_features))
+
+        output = self.norm4(cur_features_temp + self.mlp_attn(combined_features))
+        output = self.mlp_final(output)
+
+        return output   
+    
+    def forward(self, image_features, *args, **kwargs): 
+        cur_features, prev_features = image_features
+        
+        cur_features = self.LFE(cur_features).squeeze(1)
+        prev_features= self.LFE(prev_features).squeeze(1)
+        
+        output = self.TFM(cur_features,prev_features)
+
+        return output    
+    
+    @property
+    def config(self):
+        return {"mm_projector_type": 'TAC'}
+
+class Projector(nn.Module):
+    def __init__(self, base_projector):
+        super().__init__()
+        self.projector = base_projector
+
+    def forward(self, image_features, *args, **kwargs):
+        temp_features = image_features[0].squeeze(1)
+        return self.projector(temp_features)
+
+
+def build_vision_projector(config, delay_load=False, *args,**kwargs):
+    projector_type = getattr(config, 'mm_projector_type', 'linear')
+    
+    if projector_type == 'linear':
+        linear_layer = nn.Linear(config.mm_hidden_size, config.hidden_size)
+        return Projector(linear_layer)
+
+    mlp_gelu_match = re.match(r'^mlp(\d+)x_gelu$', projector_type)
+    if mlp_gelu_match:
+        mlp_depth = int(mlp_gelu_match.group(1))
+        modules = [nn.Linear(config.mm_hidden_size, config.hidden_size)] 
+        for _ in range(1, mlp_depth):
+            modules.append(nn.GELU())
+            modules.append(nn.Linear(config.hidden_size, config.hidden_size))
+        return Projector(nn.Sequential(*modules))
+    
+    if projector_type == 'TAC':
+        return TAC(config)
+
+    raise ValueError(f'Unknown projector type: {projector_type}')

libra/serve/cli.py

@@ -0,0 +1,201 @@
+import argparse
+import torch
+
+from libra.constants import IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from libra.conversation import conv_templates, SeparatorStyle
+from libra.model.builder import load_pretrained_model
+from libra.utils import disable_torch_init
+from libra.mm_utils import process_images, tokenizer_image_token, get_model_name_from_path, KeywordsStoppingCriteria
+
+import requests
+import pydicom
+from PIL import Image
+from io import BytesIO
+from pydicom.pixel_data_handlers.util import apply_voi_lut
+from transformers import TextStreamer
+
+
+def load_images(image_file):
+    """
+    Load an image from a local file, a URL, or a DICOM file.
+
+    Args:
+        image_file (str): The path or URL of the image file to load.
+
+    Returns:
+        PIL.Image.Image: The loaded image in RGB format.
+
+    Raises:
+        ValueError: If the DICOM file does not contain image data.
+        TypeError: If the input is neither a valid file path nor a URL.
+    """
+    if isinstance(image_file, str):
+        # Case 1: Load from URL
+        if image_file.startswith(('http://', 'https://')):
+            try:
+                response = requests.get(image_file)
+                response.raise_for_status()
+                image = Image.open(BytesIO(response.content)).convert('RGB')
+            except Exception as e:
+                raise ValueError(f"Error loading image from URL: {image_file}\n{e}")
+
+        # Case 2: Load from DICOM file
+        elif image_file.lower().endswith('.dcm'):
+            try:
+                dicom = pydicom.dcmread(image_file)
+                if 'PixelData' in dicom:
+                    data = apply_voi_lut(dicom.pixel_array, dicom)
+
+                    # Handle MONOCHROME1 images
+                    if dicom.PhotometricInterpretation == "MONOCHROME1":
+                        data = np.max(data) - data
+
+                    # Normalize the image data
+                    data = data - np.min(data)
+                    data = data / np.max(data)
+                    data = (data * 255).astype(np.uint8)
+
+                    # Convert to 3-channel RGB if necessary
+                    if data.ndim == 2:
+                        data = np.stack([data] * 3, axis=-1)
+
+                    image = Image.fromarray(data).convert('RGB')
+                else:
+                    raise ValueError("DICOM file does not contain image data")
+            except Exception as e:
+                raise ValueError(f"Error loading DICOM file: {image_file}\n{e}")
+
+        # Case 3: Load standard image files (e.g., PNG, JPG)
+        else:
+            try:
+                image = Image.open(image_file).convert('RGB')
+            except Exception as e:
+                raise ValueError(f"Error loading standard image file: {image_file}\n{e}")
+
+    else:
+        raise TypeError("image_file must be a string representing a file path or URL")
+
+    return image
+
+def main(args):
+    """
+    Main function to load a pretrained model, process images, and interact with the user through a conversation loop.
+    Args:
+        args (Namespace): A namespace object containing the following attributes:
+            model_path (str): Path to the pretrained model.
+            model_base (str): Base model name.
+            load_8bit (bool): Flag to load the model in 8-bit precision.
+            load_4bit (bool): Flag to load the model in 4-bit precision.
+            device (str): Device to load the model on (e.g., 'cuda', 'cpu').
+            conv_mode (str, optional): Conversation mode to use. If None, it will be inferred from the model name.
+            image_file (list): List of paths to image files to be processed.
+            temperature (float): Sampling temperature for text generation.
+            max_new_tokens (int): Maximum number of new tokens to generate.
+            debug (bool): Flag to enable debug mode for additional output.
+    Raises:
+        EOFError: If an EOFError is encountered during user input, the loop will exit.
+    """
+    # Model
+    disable_torch_init()
+
+    model_name = get_model_name_from_path(args.model_path)
+    tokenizer, model, image_processor, context_len = load_pretrained_model(args.model_path, args.model_base, model_name, args.load_8bit, args.load_4bit, device=args.device)
+
+    if 'libra' in model_name.lower():
+        conv_mode = "libra_v1"
+
+    if args.conv_mode is not None and conv_mode != args.conv_mode:
+        print('[WARNING] the auto inferred conversation mode is {}, while `--conv-mode` is {}, using {}'.format(conv_mode, args.conv_mode, args.conv_mode))
+    else:
+        args.conv_mode = conv_mode
+
+    conv = conv_templates[args.conv_mode].copy()
+    roles = conv.roles
+    
+    image=[]
+    for path in args.image_file:
+        img = load_images(path)
+        image.append(img)
+
+    # set dummy prior image
+    if len(image) == 1:
+        print("Contains only current image. Adding a dummy prior image.")
+        image.append(image[0])
+
+    processed_images = []
+    for img_data in image:
+        image_temp = process_images([img_data], image_processor, model.config)[0]
+        image_temp = image_temp.to(device='cuda',non_blocking=True) 
+        processed_images.append(image_temp)
+        
+    cur_images = [processed_images[0]]
+    prior_images = [processed_images[1]]
+    image_tensor = torch.stack([torch.stack(cur_images), torch.stack(prior_images)])
+    image_tensor = image_tensor.to(model.device, dtype=torch.float16)
+
+    while True:
+        try:
+            inp = input(f"{roles[0]}: ")
+        except EOFError:
+            inp = ""
+        if not inp:
+            print("exit...")
+            break
+
+        print(f"{roles[1]}: ", end="")
+
+        if image is not None:
+            # first message
+            if model.config.mm_use_im_start_end:
+                inp = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN + '\n' + inp
+            else:
+                inp = DEFAULT_IMAGE_TOKEN + '\n' + inp
+            image = None
+
+        conv.append_message(conv.roles[0], inp)
+        conv.append_message(conv.roles[1], None)
+        prompt = conv.get_prompt()
+
+        input_ids = tokenizer_image_token(prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors='pt').unsqueeze(0).to(model.device)
+        stop_str = conv.sep if conv.sep_style != SeparatorStyle.TWO else conv.sep2
+        keywords = [stop_str]
+        stopping_criteria = KeywordsStoppingCriteria(keywords, tokenizer, input_ids)
+        streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+        
+        attention_mask = torch.ones(input_ids.shape, dtype=torch.long)
+        pad_token_id = tokenizer.pad_token_id
+        
+        with torch.inference_mode():
+            output_ids = model.generate(
+                input_ids,
+                images=image_tensor,
+                do_sample=True if args.temperature > 0 else False,
+                temperature=args.temperature,
+                max_new_tokens=args.max_new_tokens,
+                streamer=streamer,
+                use_cache=True,
+                attention_mask=attention_mask, 
+                pad_token_id=pad_token_id,
+                stopping_criteria=[stopping_criteria])
+        
+        outputs = tokenizer.decode(output_ids[0, input_ids.shape[1]:],skip_special_tokens=True).strip()
+        conv.messages[-1][-1] = outputs
+
+        if args.debug:
+            print("\n", {"prompt": prompt, "outputs": outputs}, "\n")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model-path", type=str, default="X-iZhang/libra-v1.0-7b")
+    parser.add_argument("--model-base", type=str, default=None)
+    parser.add_argument("--image-file", type=str, nargs="+", required=True, help="List of image files to process.")
+    parser.add_argument("--device", type=str, default="cuda")
+    parser.add_argument("--conv-mode", type=str, default="libra_v1")
+    parser.add_argument("--temperature", type=float, default=0.5)
+    parser.add_argument("--max-new-tokens", type=int, default=512)
+    parser.add_argument("--load-8bit", action="store_true")
+    parser.add_argument("--load-4bit", action="store_true")
+    parser.add_argument("--debug", action="store_true")
+    args = parser.parse_args()
+    
+    main(args)

libra/train/libra_trainer.py

@@ -0,0 +1,258 @@
+import os
+import torch
+
+from torch.utils.data import Sampler
+
+from transformers import Trainer
+
+from transformers.trainer import (
+    is_sagemaker_mp_enabled,
+    get_parameter_names,
+    has_length,
+    ALL_LAYERNORM_LAYERS,
+    logger,
+)
+from typing import List, Optional
+
+
+def maybe_zero_3(param, ignore_status=False, name=None):
+    from deepspeed import zero
+    from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus
+    if hasattr(param, "ds_id"):
+        if param.ds_status == ZeroParamStatus.NOT_AVAILABLE:
+            if not ignore_status:
+                print(name, 'no ignore status')
+        with zero.GatheredParameters([param]):
+            param = param.data.detach().cpu().clone()
+    else:
+        param = param.detach().cpu().clone()
+    return param
+
+
+def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match):
+    to_return = {k: t for k, t in named_params if any(key_match in k for key_match in keys_to_match)}
+    to_return = {k: maybe_zero_3(v, ignore_status=True, name=k).cpu() for k, v in to_return.items()}
+    return to_return
+
+
+def split_to_even_chunks(indices, lengths, num_chunks):
+    """
+    Split a list of indices into `chunks` chunks of roughly equal lengths.
+    """
+
+    if len(indices) % num_chunks != 0:
+        return [indices[i::num_chunks] for i in range(num_chunks)]
+
+    num_indices_per_chunk = len(indices) // num_chunks
+
+    chunks = [[] for _ in range(num_chunks)]
+    chunks_lengths = [0 for _ in range(num_chunks)]
+    for index in indices:
+        shortest_chunk = chunks_lengths.index(min(chunks_lengths))
+        chunks[shortest_chunk].append(index)
+        chunks_lengths[shortest_chunk] += lengths[index]
+        if len(chunks[shortest_chunk]) == num_indices_per_chunk:
+            chunks_lengths[shortest_chunk] = float("inf")
+
+    return chunks
+
+
+def get_modality_length_grouped_indices(lengths, batch_size, world_size, generator=None):
+    # We need to use torch for the random part as a distributed sampler will set the random seed for torch.
+    assert all(l != 0 for l in lengths), "Should not have zero length."
+    if all(l > 0 for l in lengths) or all(l < 0 for l in lengths):
+        # all samples are in the same modality
+        return get_length_grouped_indices(lengths, batch_size, world_size, generator=generator)
+    mm_indices, mm_lengths = zip(*[(i, l) for i, l in enumerate(lengths) if l > 0])
+    lang_indices, lang_lengths = zip(*[(i, -l) for i, l in enumerate(lengths) if l < 0])
+
+    mm_shuffle = [mm_indices[i] for i in get_length_grouped_indices(mm_lengths, batch_size, world_size, generator=None)]
+    lang_shuffle = [lang_indices[i] for i in get_length_grouped_indices(lang_lengths, batch_size, world_size, generator=None)]
+    megabatch_size = world_size * batch_size
+    mm_megabatches = [mm_shuffle[i : i + megabatch_size] for i in range(0, len(mm_shuffle), megabatch_size)]
+    lang_megabatches = [lang_shuffle[i : i + megabatch_size] for i in range(0, len(lang_shuffle), megabatch_size)]
+
+    last_mm = mm_megabatches[-1]
+    last_lang = lang_megabatches[-1]
+    additional_batch = last_mm + last_lang
+    megabatches = mm_megabatches[:-1] + lang_megabatches[:-1]
+    megabatch_indices = torch.randperm(len(megabatches), generator=generator)
+    megabatches = [megabatches[i] for i in megabatch_indices]
+
+    if len(additional_batch) > 0:
+        megabatches.append(sorted(additional_batch))
+
+    return [i for megabatch in megabatches for i in megabatch]
+
+def get_length_grouped_indices(lengths, batch_size, world_size, generator=None, merge=True):
+    # We need to use torch for the random part as a distributed sampler will set the random seed for torch.
+    indices = torch.randperm(len(lengths), generator=generator)
+    megabatch_size = world_size * batch_size
+    megabatches = [indices[i : i + megabatch_size].tolist() for i in range(0, len(lengths), megabatch_size)]
+    megabatches = [sorted(megabatch, key=lambda i: lengths[i], reverse=True) for megabatch in megabatches]
+    megabatches = [split_to_even_chunks(megabatch, lengths, world_size) for megabatch in megabatches]
+
+    return [i for megabatch in megabatches for batch in megabatch for i in batch]
+
+
+class LengthGroupedSampler(Sampler):
+    r"""
+    Sampler that samples indices in a way that groups together features of the dataset of roughly the same length while
+    keeping a bit of randomness.
+    """
+
+    def __init__(
+        self,
+        batch_size: int,
+        world_size: int,
+        lengths: Optional[List[int]] = None,
+        generator=None,
+        group_by_modality: bool = False,
+    ):
+        if lengths is None:
+            raise ValueError("Lengths must be provided.")
+
+        self.batch_size = batch_size
+        self.world_size = world_size
+        self.lengths = lengths
+        self.generator = generator
+        self.group_by_modality = group_by_modality
+
+    def __len__(self):
+        return len(self.lengths)
+
+    def __iter__(self):
+        if self.group_by_modality:
+            indices = get_modality_length_grouped_indices(self.lengths, self.batch_size, self.world_size, generator=self.generator)
+        else:
+            indices = get_length_grouped_indices(self.lengths, self.batch_size, self.world_size, generator=self.generator)
+        return iter(indices)
+
+
+class LibraTrainer(Trainer):
+    
+    def _get_train_sampler(self) -> Optional[torch.utils.data.Sampler]:
+        if self.train_dataset is None or not has_length(self.train_dataset):
+            return None
+
+        if self.args.group_by_modality_length:
+            lengths = self.train_dataset.modality_lengths
+            return LengthGroupedSampler(
+                self.args.train_batch_size,
+                world_size=self.args.world_size * self.args.gradient_accumulation_steps,
+                lengths=lengths,
+                group_by_modality=True,
+            )
+        else:
+            return super()._get_train_sampler()
+
+    
+    def create_optimizer(self):
+        """
+        Setup the optimizer.
+
+        We provide a reasonable default that works well. If you want to use something else, you can pass a tuple in the
+        Trainer's init through `optimizers`, or subclass and override this method in a subclass.
+        """
+        if is_sagemaker_mp_enabled():
+            return super().create_optimizer()
+
+        opt_model = self.model
+
+        if self.optimizer is None:
+            decay_parameters = get_parameter_names(opt_model, ALL_LAYERNORM_LAYERS)
+            decay_parameters = [name for name in decay_parameters if "bias" not in name]
+            if self.args.mm_projector_lr is not None:
+                projector_parameters = [name for name, _ in opt_model.named_parameters() if "mm_projector" in name]
+                optimizer_grouped_parameters = [
+                    {
+                        "params": [
+                            p for n, p in opt_model.named_parameters() if (n in decay_parameters and n not in projector_parameters and p.requires_grad)
+                        ],
+                        "weight_decay": self.args.weight_decay,
+                    },
+                    {
+                        "params": [
+                            p for n, p in opt_model.named_parameters() if (n not in decay_parameters and n not in projector_parameters and p.requires_grad)
+                        ],
+                        "weight_decay": 0.0,
+                    },
+                    {
+                        "params": [
+                            p for n, p in opt_model.named_parameters() if (n in decay_parameters and n in projector_parameters and p.requires_grad)
+                        ],
+                        "weight_decay": self.args.weight_decay,
+                        "lr": self.args.mm_projector_lr,
+                    },
+                    {
+                        "params": [
+                            p for n, p in opt_model.named_parameters() if (n not in decay_parameters and n in projector_parameters and p.requires_grad)
+                        ],
+                        "weight_decay": 0.0,
+                        "lr": self.args.mm_projector_lr,
+                    },
+                ]
+            else:
+                optimizer_grouped_parameters = [
+                    {
+                        "params": [
+                            p for n, p in opt_model.named_parameters() if (n in decay_parameters and p.requires_grad)
+                        ],
+                        "weight_decay": self.args.weight_decay,
+                    },
+                    {
+                        "params": [
+                            p for n, p in opt_model.named_parameters() if (n not in decay_parameters and p.requires_grad)
+                        ],
+                        "weight_decay": 0.0,
+                    },
+                ]
+
+            optimizer_cls, optimizer_kwargs = Trainer.get_optimizer_cls_and_kwargs(self.args)
+
+            self.optimizer = optimizer_cls(optimizer_grouped_parameters, **optimizer_kwargs)
+            if optimizer_cls.__name__ == "Adam8bit":
+                import bitsandbytes
+
+                manager = bitsandbytes.optim.GlobalOptimManager.get_instance()
+
+                skipped = 0
+                for module in opt_model.modules():
+                    if isinstance(module, nn.Embedding):
+                        skipped += sum({p.data_ptr(): p.numel() for p in module.parameters()}.values())
+                        logger.info(f"skipped {module}: {skipped/2**20}M params")
+                        manager.register_module_override(module, "weight", {"optim_bits": 32})
+                        logger.debug(f"bitsandbytes: will optimize {module} in fp32")
+                logger.info(f"skipped: {skipped/2**20}M params")
+
+
+        return self.optimizer
+
+    def _save_checkpoint(self, model, trial, metrics=None):
+        if getattr(self.args, 'tune_mm_mlp_adapter', False):
+            from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
+            checkpoint_folder = f"{PREFIX_CHECKPOINT_DIR}-{self.state.global_step}"
+
+            run_dir = self._get_output_dir(trial=trial)
+            output_dir = os.path.join(run_dir, checkpoint_folder)
+
+            # Only save Adapter
+            keys_to_match = ['mm_projector', 'vision_resampler']
+            if getattr(self.args, "use_im_start_end", False):
+                keys_to_match.extend(['embed_tokens', 'embed_in'])
+
+            weight_to_save = get_mm_adapter_state_maybe_zero_3(self.model.named_parameters(), keys_to_match)
+
+            if self.args.local_rank == 0 or self.args.local_rank == -1:
+                self.model.config.save_pretrained(output_dir)
+                torch.save(weight_to_save, os.path.join(output_dir, f'mm_projector.bin'))
+                
+            super(LibraTrainer, self)._save_checkpoint(model, trial, metrics)
+        else:
+            super(LibraTrainer, self)._save_checkpoint(model, trial, metrics)
+
+    def _save(self, output_dir: Optional[str] = None, state_dict=None):
+        if getattr(self.args, 'tune_mm_mlp_adapter', False):
+            pass
+        else:
+            super(LibraTrainer, self)._save(output_dir, state_dict)

libra/train/llama2_flash_attn_monkey_patch.py

@@ -0,0 +1,241 @@
+"""
+Directly copied the code from https://github.com/lm-sys/FastChat/blob/main/fastchat/train/llama2_flash_attn_monkey_patch.py and made some adjustments
+"""
+import warnings
+from typing import Optional, Tuple
+
+import torch
+from flash_attn import __version__ as flash_attn_version
+from flash_attn.bert_padding import pad_input, unpad_input
+from flash_attn.flash_attn_interface import (
+    flash_attn_func,
+    flash_attn_varlen_kvpacked_func,
+)
+from transformers.models.llama.modeling_llama import (
+    LlamaAttention,
+    LlamaModel,
+    rotate_half,
+)
+
+
+def apply_rotary_pos_emb(q, k, cos_sin, position_ids):
+    gather_indices = position_ids[:, :, None, None]  # [bsz, seq_len, 1, 1]
+    gather_indices = gather_indices.repeat(
+        1, 1, cos_sin[0].shape[1], cos_sin[0].shape[3]
+    )
+    bsz = gather_indices.shape[0]
+    cos, sin = (
+        torch.gather(x.transpose(1, 2).repeat(bsz, 1, 1, 1), 1, gather_indices)
+        for x in cos_sin
+    )
+    q, k = ((x * cos) + (rotate_half(x) * sin) for x in (q, k))
+    return q, k
+
+
+def forward(
+    self,
+    hidden_states: torch.Tensor,
+    attention_mask: Optional[torch.Tensor] = None,
+    position_ids: Optional[torch.Tensor] = None,
+    past_key_value: Optional[Tuple[torch.Tensor]] = None,
+    output_attentions: bool = False,
+    use_cache: bool = False,
+    padding_mask: Optional[torch.Tensor] = None,
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+    if output_attentions:
+        warnings.warn(
+            "Output attentions is not supported for patched `LlamaAttention`, returning `None` instead."
+        )
+
+    bsz, q_len, _ = hidden_states.size()
+    kv_heads = getattr(self, "num_key_value_heads", self.num_heads)
+
+    q, k, v = (
+        op(hidden_states).view(bsz, q_len, nh, self.head_dim)
+        for op, nh in (
+            (self.q_proj, self.num_heads),
+            (self.k_proj, kv_heads),
+            (self.v_proj, kv_heads),
+        )
+    )
+    # shape: (b, s, num_heads, head_dim)
+
+    kv_seq_len = k.shape[1]
+    past_kv_len = 0
+    if past_key_value is not None:
+        past_kv_len = past_key_value[0].shape[2]
+        kv_seq_len += past_kv_len
+
+    cos_sin = self.rotary_emb(v, seq_len=kv_seq_len)
+    q, k = apply_rotary_pos_emb(q, k, cos_sin, position_ids)
+
+    if past_key_value is not None:
+        assert (
+            flash_attn_version >= "2.1.0"
+        ), "past_key_value support requires flash-attn >= 2.1.0"
+        # reuse k, v
+        k = torch.cat([past_key_value[0].transpose(1, 2), k], dim=1)
+        v = torch.cat([past_key_value[1].transpose(1, 2), v], dim=1)
+
+    past_key_value = (k.transpose(1, 2), v.transpose(1, 2)) if use_cache else None
+
+    if attention_mask is None:
+        output = flash_attn_func(q, k, v, 0.0, softmax_scale=None, causal=True).view(
+            bsz, q_len, -1
+        )
+    else:
+        q, indices, cu_q_lens, max_s = unpad_input(q, attention_mask[:, -q_len:])
+        # We can skip concat and call unpad twice but seems better to call unpad only once.
+        kv, _, cu_k_lens, max_k = unpad_input(
+            torch.stack((k, v), dim=2), attention_mask
+        )
+        output_unpad = flash_attn_varlen_kvpacked_func(
+            q,
+            kv,
+            cu_q_lens,
+            cu_k_lens,
+            max_s,
+            max_k,
+            0.0,
+            softmax_scale=None,
+            causal=True,
+        )
+        output_unpad = output_unpad.reshape(-1, self.num_heads * self.head_dim)
+        output = pad_input(output_unpad, indices, bsz, q_len)
+
+    return self.o_proj(output), None, past_key_value
+
+
+# Disable the transformation of the attention mask in LlamaModel as flash attention
+# takes a boolean key_padding_mask. Fills in the past kv length for use in forward.
+def _prepare_decoder_attention_mask(
+    self, attention_mask, input_shape, inputs_embeds, past_key_values_length
+):
+    # [bsz, seq_len]
+    if past_key_values_length > 0 and attention_mask is not None:
+        attention_mask = torch.cat(
+            (
+                torch.full(
+                    (input_shape[0], past_key_values_length),
+                    True,
+                    dtype=attention_mask.dtype,
+                    device=attention_mask.device,
+                ),
+                attention_mask,
+            ),
+            dim=-1,
+        )
+
+    if attention_mask is not None and torch.all(attention_mask):
+        return None  # This uses the faster call when training with full samples
+
+    return attention_mask
+
+
+def replace_llama_attn_with_flash_attn():
+    cuda_major, cuda_minor = torch.cuda.get_device_capability()
+    if cuda_major < 8:
+        warnings.warn(
+            "Flash attention is only supported on A100 or H100 GPU during training due to head dim > 64 backward."
+            "ref: https://github.com/HazyResearch/flash-attention/issues/190#issuecomment-1523359593"
+        )
+
+    LlamaModel._prepare_decoder_attention_mask = _prepare_decoder_attention_mask
+    LlamaAttention.forward = forward
+
+
+def test():
+    from fastchat.train.llama_flash_attn_monkey_patch import forward as fastchat_forward
+    from transformers.models.llama.configuration_llama import LlamaConfig
+
+    config = LlamaConfig(
+        hidden_size=1024,
+        intermediate_size=128,
+        num_hidden_layers=1,
+        num_attention_heads=8,
+        max_position_embeddings=16,
+    )
+    device = torch.device("cuda")
+    model = LlamaModel(config)
+    attn = LlamaAttention(config).to(device).half()
+    bsz, hs, seqlen = 2, config.hidden_size, config.max_position_embeddings
+    position_ids = torch.arange(seqlen, dtype=torch.long, device=device).view(
+        -1, seqlen
+    )
+
+    mask = torch.full((bsz, seqlen), True, dtype=torch.bool, device=device)
+    for i in range(4):
+        hidden = torch.rand((bsz, seqlen, hs), dtype=torch.float16, device=device)
+        if i:
+            mask[0, -i:] = False
+            mask[1, :i] = False
+
+        lmask = model._prepare_decoder_attention_mask(mask, hidden.shape[:2], hidden, 0)
+        ref, _, _ = attn.forward(
+            hidden, attention_mask=lmask, position_ids=position_ids
+        )
+
+        fast, _, _ = fastchat_forward(
+            attn, hidden, attention_mask=mask, position_ids=position_ids
+        )
+
+        lmask = _prepare_decoder_attention_mask(
+            model, mask, hidden.shape[:2], hidden, 0
+        )
+        test, _, _ = forward(
+            attn, hidden, attention_mask=lmask, position_ids=position_ids
+        )
+
+        print(f"Mean(abs(ref)) = {torch.mean(torch.abs(ref))}")
+        print(f"Mean(abs(ref - fast)) = {torch.mean(torch.abs(ref - fast))}")
+        print(f"Mean(abs(ref - test)) = {torch.mean(torch.abs(ref - test))}")
+        print(f"Mean(abs(fast - test)) = {torch.mean(torch.abs(fast - test))}")
+        print(f"allclose(fast, test) = {torch.allclose(fast, test)}")
+
+    with torch.no_grad():
+        # Also check that past_kv is handled properly
+        hidden = torch.rand((bsz, seqlen, hs), dtype=torch.float16, device=device)
+        part_len = seqlen // 4
+        assert part_len * 4 == seqlen
+        mask = torch.full((bsz, seqlen), True, dtype=torch.bool, device=device)
+        mask[0, -2:] = False
+        lmask = _prepare_decoder_attention_mask(
+            model, mask, hidden.shape[:2], hidden, 0
+        )
+        oneshot, _, _ = forward(
+            attn, hidden, attention_mask=lmask, position_ids=position_ids
+        )
+        parts = []
+        past_kv, past_kv_len = None, 0
+        for i in range(4):
+            start = part_len * i
+            end = start + part_len
+            hidden_part = hidden[:, start:end, ...]
+            lmask = _prepare_decoder_attention_mask(
+                model,
+                mask[:, start:end],
+                hidden_part.shape[:2],
+                hidden_part,
+                past_kv_len,
+            )
+            part, _, past_kv = forward(
+                attn,
+                hidden_part.clone(),
+                attention_mask=lmask,
+                position_ids=position_ids[:, start:end],
+                past_key_value=past_kv,
+                use_cache=True,
+            )
+            parts.append(part)
+            past_kv_len = past_kv[0].shape[2]
+
+        print(
+            f"allclose(oneshot[:, 0], parts[0]) = {torch.allclose(oneshot[:, :part_len], parts[0])}"
+        )
+        print(
+            f"allclose(oneshot, parts) = {torch.allclose(oneshot, torch.cat(parts, dim=1))}"
+        )
+
+
+if __name__ == "__main__":
+    test()

libra/train/llama_flash_attn_monkey_patch.py

@@ -0,0 +1,119 @@
+"""
+Directly copied the code from https://github.com/lm-sys/FastChat/blob/main/fastchat/train/llama_flash_attn_monkey_patch.py and made some adjustments
+"""
+
+from typing import Optional, Tuple
+import warnings
+
+import torch
+
+import transformers
+from transformers.models.llama.modeling_llama import apply_rotary_pos_emb, repeat_kv
+
+try:
+    from flash_attn.flash_attn_interface import flash_attn_unpadded_qkvpacked_func
+except ImportError:
+    from flash_attn.flash_attn_interface import flash_attn_varlen_qkvpacked_func as flash_attn_unpadded_qkvpacked_func
+from flash_attn.bert_padding import unpad_input, pad_input
+
+
+def forward(
+    self,
+    hidden_states: torch.Tensor,
+    attention_mask: Optional[torch.Tensor] = None,
+    position_ids: Optional[torch.Tensor] = None,
+    past_key_value: Optional[Tuple[torch.Tensor]] = None,
+    output_attentions: bool = False,
+    use_cache: bool = False,
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+    if output_attentions:
+        warnings.warn(
+            "Output attentions is not supported for patched `LlamaAttention`, returning `None` instead."
+        )
+
+    bsz, q_len, _ = hidden_states.size()
+
+    query_states = (
+        self.q_proj(hidden_states)
+        .view(bsz, q_len, self.num_heads, self.head_dim)
+        .transpose(1, 2)
+    )
+    key_states = (
+        self.k_proj(hidden_states)
+        .view(bsz, q_len, self.num_key_value_heads, self.head_dim)
+        .transpose(1, 2)
+    )
+    value_states = (
+        self.v_proj(hidden_states)
+        .view(bsz, q_len, self.num_key_value_heads, self.head_dim)
+        .transpose(1, 2)
+    )  # shape: (b, num_heads, s, head_dim)
+
+    kv_seq_len = key_states.shape[-2]
+    if past_key_value is not None:
+        kv_seq_len += past_key_value[0].shape[-2]
+
+    cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+    query_states, key_states = apply_rotary_pos_emb(
+        query_states, key_states, cos, sin, position_ids
+    )
+
+    if past_key_value is not None:
+        # reuse k, v
+        key_states = torch.cat([past_key_value[0], key_states], dim=2)
+        value_states = torch.cat([past_key_value[1], value_states], dim=2)
+
+    past_key_value = (key_states, value_states) if use_cache else None
+
+    # repeat k/v heads if n_kv_heads < n_heads
+    key_states = repeat_kv(key_states, self.num_key_value_groups)
+    value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+    # Transform the data into the format required by flash attention
+    qkv = torch.stack([query_states, key_states, value_states], dim=2)
+    qkv = qkv.transpose(1, 3)  # shape: [b, s, 3, num_heads, head_dim]
+    key_padding_mask = attention_mask
+
+    if key_padding_mask is None:
+        qkv = qkv.reshape(-1, 3, self.num_heads, self.head_dim)
+        cu_q_lens = torch.arange(
+            0, (bsz + 1) * q_len, step=q_len, dtype=torch.int32, device=qkv.device
+        )
+        max_s = q_len
+        output = flash_attn_unpadded_qkvpacked_func(
+            qkv, cu_q_lens, max_s, 0.0, softmax_scale=None, causal=True
+        )
+        output = output.view(bsz, q_len, -1)
+    else:
+        qkv = qkv.reshape(bsz, q_len, -1)
+        qkv, indices, cu_q_lens, max_s = unpad_input(qkv, key_padding_mask)
+        qkv = qkv.view(-1, 3, self.num_heads, self.head_dim)
+        output_unpad = flash_attn_unpadded_qkvpacked_func(
+            qkv, cu_q_lens, max_s, 0.0, softmax_scale=None, causal=True
+        )
+        output_unpad = output_unpad.reshape(-1, self.num_heads * self.head_dim)
+        output = pad_input(output_unpad, indices, bsz, q_len)
+
+    return self.o_proj(output), None, past_key_value
+
+
+# Disable the transformation of the attention mask in LlamaModel as the flash attention
+# requires the attention mask to be the same as the key_padding_mask
+def _prepare_decoder_attention_mask(
+    self, attention_mask, input_shape, inputs_embeds, past_key_values_length
+):
+    # [bsz, seq_len]
+    return attention_mask
+
+
+def replace_llama_attn_with_flash_attn():
+    cuda_major, cuda_minor = torch.cuda.get_device_capability()
+    if cuda_major < 8:
+        warnings.warn(
+            "Flash attention is only supported on A100 or H100 GPU during training due to head dim > 64 backward."
+            "ref: https://github.com/HazyResearch/flash-attention/issues/190#issuecomment-1523359593"
+        )
+    transformers.models.llama.modeling_llama.LlamaModel._prepare_decoder_attention_mask = (
+        _prepare_decoder_attention_mask
+    )
+    transformers.models.llama.modeling_llama.LlamaAttention.forward = forward

libra/train/llama_xformers_attn_monkey_patch.py

@@ -0,0 +1,129 @@
+"""
+Directly copied the code from https://github.com/lm-sys/FastChat/blob/main/fastchat/train/llama_xformers_attn_monkey_patch.py and made some adjustments
+"""
+
+import logging
+import math
+from typing import Optional, Tuple
+
+import torch
+import transformers.models.llama.modeling_llama
+from torch import nn
+
+try:
+    import xformers.ops
+except ImportError:
+    logging.error("xformers not found! Please install it before trying to use it.")
+
+
+def replace_llama_attn_with_xformers_attn():
+    transformers.models.llama.modeling_llama.LlamaAttention.forward = xformers_forward
+
+
+def xformers_forward(
+    self,
+    hidden_states: torch.Tensor,
+    attention_mask: Optional[torch.Tensor] = None,
+    position_ids: Optional[torch.LongTensor] = None,
+    past_key_value: Optional[Tuple[torch.Tensor]] = None,
+    output_attentions: bool = False,
+    use_cache: bool = False,
+) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+    # pylint: disable=duplicate-code
+    bsz, q_len, _ = hidden_states.size()
+
+    query_states = (
+        self.q_proj(hidden_states)
+        .view(bsz, q_len, self.num_heads, self.head_dim)
+        .transpose(1, 2)
+    )
+    key_states = (
+        self.k_proj(hidden_states)
+        .view(bsz, q_len, self.num_heads, self.head_dim)
+        .transpose(1, 2)
+    )
+    value_states = (
+        self.v_proj(hidden_states)
+        .view(bsz, q_len, self.num_heads, self.head_dim)
+        .transpose(1, 2)
+    )
+
+    kv_seq_len = key_states.shape[-2]
+    if past_key_value is not None:
+        kv_seq_len += past_key_value[0].shape[-2]
+    cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+    (
+        query_states,
+        key_states,
+    ) = transformers.models.llama.modeling_llama.apply_rotary_pos_emb(
+        query_states, key_states, cos, sin, position_ids
+    )
+    # [bsz, nh, t, hd]
+
+    if past_key_value is not None:
+        # reuse k, v, self_attention
+        key_states = torch.cat([past_key_value[0], key_states], dim=2)
+        value_states = torch.cat([past_key_value[1], value_states], dim=2)
+
+    past_key_value = (key_states, value_states) if use_cache else None
+
+    # We only apply xformers optimizations if we don't need to output the whole attention matrix
+    if not output_attentions:
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        # This is a nasty hack. We know attention_mask in transformers is either LowerTriangular or all Zeros.
+        # We therefore check if one element in the upper triangular portion is zero. If it is, then the mask is all zeros.
+        if attention_mask is None or attention_mask[0, 0, 0, 1] == 0:
+            # input and output should be of form (bsz, q_len, num_heads, head_dim)
+            attn_output = xformers.ops.memory_efficient_attention(
+                query_states, key_states, value_states, attn_bias=None
+            )
+        else:
+            # input and output should be of form (bsz, q_len, num_heads, head_dim)
+            attn_output = xformers.ops.memory_efficient_attention(
+                query_states,
+                key_states,
+                value_states,
+                attn_bias=xformers.ops.LowerTriangularMask(),
+            )
+        attn_weights = None
+    else:
+        attn_weights = torch.matmul(
+            query_states, key_states.transpose(2, 3)
+        ) / math.sqrt(self.head_dim)
+
+        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(bsz * self.num_heads, q_len, kv_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                )
+            attn_weights = attn_weights + attention_mask
+            attn_weights = torch.max(
+                attn_weights, torch.tensor(torch.finfo(attn_weights.dtype).min)
+            )
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(
+            attn_weights, dim=-1, dtype=torch.float32
+        ).to(query_states.dtype)
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2)
+
+    attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+    attn_output = self.o_proj(attn_output)
+    return attn_output, attn_weights, past_key_value

libra/train/train.py

@@ -0,0 +1,1434 @@
+#    Copyright 2024 Xi Zhang
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import os
+import copy
+import numpy as np
+from dataclasses import dataclass, field
+import json
+import logging
+import pathlib
+from typing import Dict, Optional, Sequence, List, Union
+
+import random
+import torch
+import shutil
+import evaluate
+
+import transformers
+import tokenizers
+from transformers import EvalPrediction, TrainerCallback
+
+from libra.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
+from torch.utils.data import Dataset
+from libra.train.libra_trainer import LibraTrainer
+
+from libra import conversation as conversation_lib
+from libra.model import *
+from libra.mm_utils import tokenizer_image_token
+from libra.eval import temporal_f1_score
+
+from PIL import Image
+import pydicom
+from pydicom.pixel_data_handlers.util import apply_voi_lut
+
+local_rank = None
+
+
+def rank0_print(*args):
+    if local_rank == 0:
+        print(*args)
+
+from packaging import version
+IS_TOKENIZER_GREATER_THAN_0_14 = version.parse(tokenizers.__version__) >= version.parse('0.14')
+
+@dataclass
+class ModelArguments:
+    model_name_or_path: Optional[str] = field(default="libra")  
+    version: Optional[str] = field(default="libra_v1")  
+    freeze_backbone: bool = field(default=False)  
+    tune_mm_mlp_adapter: bool = field(default=False)  
+    vision_tower: Optional[str] = field(default=None)  
+    mm_vision_select_layer: Optional[Union[int, str]] = field(
+        default=-1, 
+        metadata={"help": "Select specific vision layer (e.g., -1, -2) or 'all' for all layers."}
+    )  
+    pretrain_mm_mlp_adapter: Optional[str] = field(default=None)  
+    mm_projector_type: Optional[str] = field(default='linear')  
+    mm_use_im_start_end: bool = field(default=False) 
+    mm_use_im_patch_token: bool = field(default=True)  
+    mm_vision_select_feature: Optional[str] = field(
+        default="patch",
+        metadata={"help": "Select feature type: 'patch' or 'cls_patch'."}
+    )  
+    compute_metrics: bool = field(
+        default=False,
+        metadata={"help": "Optional callable for computing metrics during evaluation during training."}
+    )  
+
+@dataclass
+class DataArguments:
+    data_path: str = field(default=None,
+                           metadata={"help": "Path to the training data."})
+    lazy_preprocess: bool = False
+    is_multimodal: bool = False
+    image_folder: Optional[str] = field(default=None)
+    image_aspect_ratio: str = 'square'
+    validation_data_path: Optional[str] = field(
+            default=None,
+            metadata={"help": "Path to the validation data."}
+        )
+
+
+@dataclass
+class TrainingArguments(transformers.TrainingArguments):
+    cache_dir: Optional[str] = field(default=None)
+    optim: str = field(default="adamw_torch")
+    remove_unused_columns: bool = field(default=False)
+    freeze_mm_mlp_adapter: bool = field(default=False)
+    mpt_attn_impl: Optional[str] = field(default="triton")
+    model_max_length: int = field(
+        default=512,
+        metadata={
+            "help":
+            "Maximum sequence length. Sequences will be right padded (and possibly truncated)."
+        },
+    )
+    double_quant: bool = field(
+        default=True,
+        metadata={"help": "Compress the quantization statistics through double quantization."}
+    )
+    quant_type: str = field(
+        default="nf4",
+        metadata={"help": "Quantization data type to use. Should be one of `fp4` or `nf4`."}
+    )
+    bits: int = field(
+        default=16,
+        metadata={"help": "How many bits to use."}
+    )
+    lora_enable: bool = False
+    lora_r: int = 64
+    lora_alpha: int = 16
+    lora_dropout: float = 0.05
+    lora_weight_path: str = ""
+    lora_bias: str = "none"
+    mm_projector_lr: Optional[float] = None
+    group_by_modality_length: bool = field(default=False)
+
+
+def maybe_zero_3(param, ignore_status=False, name=None):
+    from deepspeed import zero
+    from deepspeed.runtime.zero.partition_parameters import ZeroParamStatus
+    if hasattr(param, "ds_id"):
+        if param.ds_status == ZeroParamStatus.NOT_AVAILABLE:
+            if not ignore_status:
+                logging.warning(f"{name}: param.ds_status != ZeroParamStatus.NOT_AVAILABLE: {param.ds_status}")
+        with zero.GatheredParameters([param]):
+            param = param.data.detach().cpu().clone()
+    else:
+        param = param.detach().cpu().clone()
+    return param
+
+
+# Borrowed from peft.utils.get_peft_model_state_dict
+def get_peft_state_maybe_zero_3(named_params, bias):
+    if bias == "none":
+        to_return = {k: t for k, t in named_params if "lora_" in k}
+    elif bias == "all":
+        to_return = {k: t for k, t in named_params if "lora_" in k or "bias" in k}
+    elif bias == "lora_only":
+        to_return = {}
+        maybe_lora_bias = {}
+        lora_bias_names = set()
+        for k, t in named_params:
+            if "lora_" in k:
+                to_return[k] = t
+                bias_name = k.split("lora_")[0] + "bias"
+                lora_bias_names.add(bias_name)
+            elif "bias" in k:
+                maybe_lora_bias[k] = t
+        for k, t in maybe_lora_bias:
+            if bias_name in lora_bias_names:
+                to_return[bias_name] = t
+    else:
+        raise NotImplementedError
+    to_return = {k: maybe_zero_3(v, ignore_status=True) for k, v in to_return.items()}
+    return to_return
+
+
+def get_peft_state_non_lora_maybe_zero_3(named_params, require_grad_only=True):
+    to_return = {k: t for k, t in named_params if "lora_" not in k}
+    if require_grad_only:
+        to_return = {k: t for k, t in to_return.items() if t.requires_grad}
+    to_return = {k: maybe_zero_3(v, ignore_status=True).cpu() for k, v in to_return.items()}
+    return to_return
+
+
+def get_non_vision_tower_state_maybe_zero_3(named_params):
+
+    to_return = {k: t for k, t in named_params if "vision_tower" not in k}
+    to_return = {k: maybe_zero_3(v, ignore_status=True).cpu() for k, v in to_return.items()}
+
+    return to_return
+
+
+def get_mm_adapter_state_maybe_zero_3(named_params, keys_to_match):
+    to_return = {k: t for k, t in named_params if any(key_match in k for key_match in keys_to_match)}
+    to_return = {k: maybe_zero_3(v, ignore_status=True).cpu() for k, v in to_return.items()}
+    return to_return
+
+
+def find_all_linear_names(model):
+    cls = torch.nn.Linear
+    lora_module_names = set()
+    multimodal_keywords = ['mm_projector', 'vision_tower', 'vision_resampler']
+    for name, module in model.named_modules():
+        if any(mm_keyword in name for mm_keyword in multimodal_keywords):
+            continue
+        if isinstance(module, cls):
+            names = name.split('.')
+            lora_module_names.add(names[0] if len(names) == 1 else names[-1])
+
+    if 'lm_head' in lora_module_names: # needed for 16-bit
+        lora_module_names.remove('lm_head')
+    return list(lora_module_names)
+
+
+def safe_save_model_for_hf_trainer(trainer: transformers.Trainer,
+                                   output_dir: str):
+    """Collects the state dict and dump to disk."""
+    
+    if getattr(trainer.args, "tune_mm_mlp_adapter", False):
+        # Only save Adapter
+        keys_to_match = ['mm_projector']
+        if getattr(trainer.args, "use_im_start_end", False):
+            keys_to_match.extend(['embed_tokens', 'embed_in'])
+
+        weight_to_save = get_mm_adapter_state_maybe_zero_3(trainer.model.named_parameters(), keys_to_match)
+        trainer.model.config.save_pretrained(output_dir)
+
+        current_folder = output_dir.split('/')[-1]
+        parent_folder = os.path.dirname(output_dir)
+        if trainer.args.local_rank == 0 or trainer.args.local_rank == -1:
+            if current_folder.startswith('checkpoint-'):
+                mm_projector_folder = os.path.join(parent_folder, "mm_projector")
+                os.makedirs(mm_projector_folder, exist_ok=True)
+                torch.save(weight_to_save, os.path.join(mm_projector_folder, f'{current_folder}.bin'))
+            else:
+                torch.save(weight_to_save, os.path.join(output_dir, f'mm_projector.bin'))
+        return
+
+    if trainer.deepspeed:
+        torch.cuda.synchronize()
+        trainer.save_model(output_dir)
+        return
+
+    state_dict = trainer.model.state_dict()
+    if trainer.args.should_save:
+        cpu_state_dict = {
+            key: value.cpu()
+            for key, value in state_dict.items()
+        }
+        del state_dict
+        trainer._save(output_dir, state_dict=cpu_state_dict) 
+
+
+def smart_tokenizer_and_embedding_resize(
+    special_tokens_dict: Dict,
+    tokenizer: transformers.PreTrainedTokenizer,
+    model: transformers.PreTrainedModel,
+):
+    """Resize tokenizer and embedding. You can add some new tokens <video> etc
+    
+    Note: This is the unoptimized version that may make your embedding size not be divisible by 64.
+    """
+    num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict)
+    model.resize_token_embeddings(len(tokenizer))
+
+    if num_new_tokens > 0:
+        input_embeddings = model.get_input_embeddings().weight.data
+        output_embeddings = model.get_output_embeddings().weight.data
+
+        input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(
+            dim=0, keepdim=True)
+        output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(
+            dim=0, keepdim=True)
+
+        input_embeddings[-num_new_tokens:] = input_embeddings_avg
+        output_embeddings[-num_new_tokens:] = output_embeddings_avg
+
+
+def _tokenize_fn(strings: Sequence[str],
+                 tokenizer: transformers.PreTrainedTokenizer) -> Dict:
+    """
+    Tokenizes a list of input strings and returns tokenized results along with sequence lengths.
+    """
+    tokenized_list = [
+        tokenizer(
+            text,
+            return_tensors="pt",
+            padding="longest",
+            max_length=tokenizer.model_max_length,
+            truncation=True,
+        ) for text in strings
+    ]
+    input_ids = labels = [
+        tokenized.input_ids[0] for tokenized in tokenized_list
+    ]
+    input_ids_lens = labels_lens = [
+        tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item()
+        for tokenized in tokenized_list
+    ]
+    return dict(
+        input_ids=input_ids,
+        labels=labels,
+        input_ids_lens=input_ids_lens,
+        labels_lens=labels_lens,
+    )
+
+
+def _mask_targets(target, tokenized_lens, speakers):
+    # cur_idx = 0
+    cur_idx = tokenized_lens[0]
+    tokenized_lens = tokenized_lens[1:]
+    target[:cur_idx] = IGNORE_INDEX
+    for tokenized_len, speaker in zip(tokenized_lens, speakers):
+        if speaker == "human":
+            target[cur_idx+2:cur_idx + tokenized_len] = IGNORE_INDEX
+        cur_idx += tokenized_len
+
+
+def _add_speaker_and_signal(header, source, get_conversation=True):
+    """Add speaker and start/end signal on each round."""
+    BEGIN_SIGNAL = "### "
+    END_SIGNAL = "\n"
+    conversation = header
+    for sentence in source:
+        from_str = sentence["from"]
+        if from_str.lower() == "human":
+            from_str = conversation_lib.default_conversation.roles[0]
+        elif from_str.lower() == "gpt":
+            from_str = conversation_lib.default_conversation.roles[1]
+        else:
+            from_str = 'unknown'
+        sentence["value"] = (BEGIN_SIGNAL + from_str + ": " +
+                             sentence["value"] + END_SIGNAL)
+        if get_conversation:
+            conversation += sentence["value"]
+    conversation += BEGIN_SIGNAL
+    return conversation
+
+
+def preprocess_multimodal(
+    sources: Sequence[str],
+    data_args: DataArguments
+) -> Dict:
+    is_multimodal = data_args.is_multimodal
+    if not is_multimodal:
+        return sources
+
+    for source in sources:
+        for sentence in source:
+            if DEFAULT_IMAGE_TOKEN in sentence['value']:
+                sentence['value'] = sentence['value'].replace(DEFAULT_IMAGE_TOKEN, '').strip()
+                sentence['value'] = DEFAULT_IMAGE_TOKEN + '\n' + sentence['value']
+                sentence['value'] = sentence['value'].strip()
+                if "mmtag" in conversation_lib.default_conversation.version:
+                    sentence['value'] = sentence['value'].replace(DEFAULT_IMAGE_TOKEN, '<Image>' + DEFAULT_IMAGE_TOKEN + '</Image>')
+            replace_token = DEFAULT_IMAGE_TOKEN
+            if data_args.mm_use_im_start_end:
+                replace_token = DEFAULT_IM_START_TOKEN + replace_token + DEFAULT_IM_END_TOKEN
+            sentence["value"] = sentence["value"].replace(DEFAULT_IMAGE_TOKEN, replace_token)
+    return sources
+
+
+def preprocess_llama_2(
+    sources,
+    tokenizer: transformers.PreTrainedTokenizer,
+    has_image: bool = False
+) -> Dict:
+    conv = conversation_lib.default_conversation.copy()
+    roles = {"human": conv.roles[0], "gpt": conv.roles[1]}
+
+    # Apply prompt templates
+    conversations = []
+    for i, source in enumerate(sources):
+        if roles[source[0]["from"]] != conv.roles[0]:
+            # Skip the first one if it is not from human
+            source = source[1:]
+
+        conv.messages = []
+        for j, sentence in enumerate(source):
+            role = roles[sentence["from"]]
+            assert role == conv.roles[j % 2], f"{i}"
+            conv.append_message(role, sentence["value"])
+        conversations.append(conv.get_prompt())
+
+    # Tokenize conversations
+
+    if has_image:
+        input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)
+    else:
+        input_ids = tokenizer(
+            conversations,
+            return_tensors="pt",
+            padding="longest",
+            max_length=tokenizer.model_max_length,
+            truncation=True,
+        ).input_ids
+
+    targets = input_ids.clone()
+
+    assert conv.sep_style == conversation_lib.SeparatorStyle.LLAMA_2
+
+    # Mask targets
+    sep = "[/INST] "
+    for conversation, target in zip(conversations, targets):
+        total_len = int(target.ne(tokenizer.pad_token_id).sum())
+
+        rounds = conversation.split(conv.sep2)
+        cur_len = 1
+        target[:cur_len] = IGNORE_INDEX
+        for i, rou in enumerate(rounds):
+            if rou == "":
+                break
+
+            parts = rou.split(sep)
+            if len(parts) != 2:
+                break
+            parts[0] += sep
+
+            if has_image:
+                round_len = len(tokenizer_image_token(rou, tokenizer))
+                instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 2
+            else:
+                round_len = len(tokenizer(rou).input_ids)
+                instruction_len = len(tokenizer(parts[0]).input_ids) - 2
+
+            target[cur_len : cur_len + instruction_len] = IGNORE_INDEX
+
+            cur_len += round_len
+        target[cur_len:] = IGNORE_INDEX
+
+        if cur_len < tokenizer.model_max_length:
+            if cur_len != total_len:
+                target[:] = IGNORE_INDEX
+                print(
+                    f"WARNING: tokenization mismatch: {cur_len} vs. {total_len}."
+                    f" (ignored)"
+                )
+
+    return dict(
+        input_ids=input_ids,
+        labels=targets,
+    )
+
+# llama_3
+def preprocess_llama_3(
+    sources,
+    tokenizer: transformers.PreTrainedTokenizer,
+    has_image: bool = False
+) -> Dict:
+    
+    special_token = "<|finetune_right_pad_id|>"
+
+    if tokenizer.pad_token_id is None:
+        
+        pad_token_id = tokenizer.convert_tokens_to_ids(special_token)
+        if pad_token_id is None:
+            raise ValueError(f"Cannot find ID for {special_token}. Please check the tokenizer.")
+        
+        tokenizer.pad_token_id = pad_token_id
+        
+    conv = conversation_lib.default_conversation.copy()
+    roles = {"human": conv.roles[0], "gpt": conv.roles[1]}
+
+    # Apply prompt templates
+    conversations = []
+    for i, source in enumerate(sources):
+        if roles[source[0]["from"]] != conv.roles[0]:
+            # Skip the first one if it is not from human
+            source = source[1:]
+
+        conv.messages = []
+        for j, sentence in enumerate(source):
+            role = roles[sentence["from"]]
+            assert role == conv.roles[j % 2], f"{i}"
+            conv.append_message(role, sentence["value"])
+        conversations.append(conv.get_prompt())
+
+    if has_image:
+        input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)
+    else:
+        input_ids = tokenizer(
+            conversations,
+            return_tensors="pt",
+            padding="longest",
+            max_length=tokenizer.model_max_length,
+            truncation=True,
+        ).input_ids
+
+    targets = input_ids.clone()
+
+    assert conv.sep_style == conversation_lib.SeparatorStyle.LLAMA_3
+
+    sep_round = "<|eot_id|>\n<|start_header_id|>user<|end_header_id|>"
+    sep_user = "<|eot_id|>\n<|start_header_id|>assistant<|end_header_id|>\n\n"
+    for conversation, target in zip(conversations, targets):
+        total_len = int(target.ne(tokenizer.pad_token_id).sum())
+        rounds = conversation.split(sep_round)
+        cur_len = 1
+        target[:cur_len] = IGNORE_INDEX
+        for i, rou in enumerate(rounds):
+            if rou == "":
+                break
+
+            parts = rou.split(sep_user)
+            if len(parts) != 2:
+                break
+            parts[0] += sep_user
+
+            if has_image:
+                round_len = len(tokenizer_image_token(rou, tokenizer)) - 1 
+                instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 1
+            else:
+                round_len = len(tokenizer(rou).input_ids) - 1
+                instruction_len = len(tokenizer(parts[0]).input_ids) - 1
+
+            target[cur_len : cur_len + instruction_len] = IGNORE_INDEX
+
+            cur_len += round_len
+        target[cur_len:] = IGNORE_INDEX
+
+        if cur_len < tokenizer.model_max_length:
+            if cur_len != total_len:
+                target[:] = IGNORE_INDEX
+                print(
+                    f"WARNING: tokenization mismatch: {cur_len} vs. {total_len}."
+                    f" (ignored)"
+                )
+
+    return dict(
+        input_ids=input_ids,
+        labels=targets,
+    )
+
+def preprocess_libra(
+    sources,
+    tokenizer: transformers.PreTrainedTokenizer,
+    has_image: bool = False
+) -> Dict:
+    conv = conversation_lib.default_conversation.copy()
+    roles = {"human": conv.roles[0], "gpt": conv.roles[1]}
+
+    # Apply prompt templates
+    conversations = []
+    for i, source in enumerate(sources):
+        if roles[source[0]["from"]] != conv.roles[0]:
+            # Skip the first one if it is not from human
+            source = source[1:]
+
+        conv.messages = []
+        for j, sentence in enumerate(source):
+            role = roles[sentence["from"]]
+            assert role == conv.roles[j % 2], f"{i}"
+            conv.append_message(role, sentence["value"])
+        conversations.append(conv.get_prompt())
+
+    # Tokenize conversations
+    if has_image:
+        input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)
+    else:
+        input_ids = tokenizer(
+            conversations,
+            return_tensors="pt",
+            padding="longest",
+            max_length=tokenizer.model_max_length,
+            truncation=True,
+        ).input_ids
+
+    targets = input_ids.clone()
+    
+    assert conv.sep_style == conversation_lib.SeparatorStyle.TWO
+
+    # Mask targets
+    sep = conv.sep + conv.roles[1] + ": "
+    for conversation, target in zip(conversations, targets):
+        total_len = int(target.ne(tokenizer.pad_token_id).sum())
+        
+
+        rounds = conversation.split(conv.sep2)
+        cur_len = 1
+        target[:cur_len] = IGNORE_INDEX
+        for i, rou in enumerate(rounds):
+            if rou == "":
+                break
+
+            parts = rou.split(sep)
+            if len(parts) != 2:
+                break
+            parts[0] += sep
+
+            if has_image:
+                round_len = len(tokenizer_image_token(rou, tokenizer))
+                instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 2
+            else:
+                round_len = len(tokenizer(rou).input_ids)
+                instruction_len = len(tokenizer(parts[0]).input_ids) - 2
+                
+            if i != 0 and not tokenizer.legacy and IS_TOKENIZER_GREATER_THAN_0_14:
+                round_len -= 1
+                instruction_len -= 1
+                
+            target[cur_len : cur_len + instruction_len] = IGNORE_INDEX
+            
+            cur_len += round_len
+        target[cur_len:] = IGNORE_INDEX
+
+        if cur_len < tokenizer.model_max_length:
+            if cur_len != total_len:
+                target[:] = IGNORE_INDEX
+                print(
+                    f"WARNING: tokenization mismatch: {cur_len} vs. {total_len}."
+                    f" (ignored)"
+                )
+
+    return dict(
+        input_ids=input_ids,
+        labels=targets,
+    )
+
+
+def preprocess_mpt(
+    sources,
+    tokenizer: transformers.PreTrainedTokenizer,
+    has_image: bool = False
+) -> Dict:
+    conv = conversation_lib.default_conversation.copy()
+    roles = {"human": conv.roles[0], "gpt": conv.roles[1]}
+
+    # Apply prompt templates
+    conversations = []
+    for i, source in enumerate(sources):
+        if roles[source[0]["from"]] != conv.roles[0]:
+            # Skip the first one if it is not from human
+            source = source[1:]
+
+        conv.messages = []
+        for j, sentence in enumerate(source):
+            role = roles[sentence["from"]]
+            assert role == conv.roles[j % 2], f"{i}"
+            conv.append_message(role, sentence["value"])
+        conversations.append(conv.get_prompt())
+
+    # Tokenize conversations
+
+    if has_image:
+        input_ids = torch.stack([tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations], dim=0)
+    else:
+        input_ids = tokenizer(
+            conversations,
+            return_tensors="pt",
+            padding="longest",
+            max_length=tokenizer.model_max_length,
+            truncation=True,
+        ).input_ids
+
+    targets = input_ids.clone()
+    assert conv.sep_style == conversation_lib.SeparatorStyle.MPT
+
+    # Mask targets
+    sep = conv.sep + conv.roles[1]
+    for conversation, target in zip(conversations, targets):
+        total_len = int(target.ne(tokenizer.pad_token_id).sum())
+
+        rounds = conversation.split(conv.sep)
+        re_rounds = [conv.sep.join(rounds[:3])] # system + user + gpt
+        for conv_idx in range(3, len(rounds), 2):
+            re_rounds.append(conv.sep.join(rounds[conv_idx:conv_idx+2]))    # user + gpt
+        cur_len = 0
+        target[:cur_len] = IGNORE_INDEX
+        for i, rou in enumerate(re_rounds):
+            if rou == "":
+                break
+
+            parts = rou.split(sep)
+            if len(parts) != 2:
+                break
+            parts[0] += sep
+
+            if has_image:
+                round_len = len(tokenizer_image_token(rou, tokenizer))
+                instruction_len = len(tokenizer_image_token(parts[0], tokenizer)) - 1
+            else:
+                round_len = len(tokenizer(rou).input_ids)
+                instruction_len = len(tokenizer(parts[0]).input_ids) - 1
+
+            if i != 0 and getattr(tokenizer, 'legacy', False) and IS_TOKENIZER_GREATER_THAN_0_14:
+                round_len += 1
+                instruction_len += 1
+
+            target[cur_len : cur_len + instruction_len] = IGNORE_INDEX
+
+            cur_len += round_len
+        target[cur_len:] = IGNORE_INDEX
+
+        if cur_len < tokenizer.model_max_length:
+            if cur_len != total_len:
+                target[:] = IGNORE_INDEX
+                print(
+                    f"WARNING: tokenization mismatch: {cur_len} vs. {total_len}."
+                    f" (ignored)"
+                )
+
+    return dict(
+        input_ids=input_ids,
+        labels=targets,
+    )
+
+
+def preprocess_plain(
+    sources: Sequence[str],
+    tokenizer: transformers.PreTrainedTokenizer,
+) -> Dict:
+    # add end signal and concatenate together
+    conversations = []
+    for source in sources:
+        assert len(source) == 2
+        assert DEFAULT_IMAGE_TOKEN in source[0]['value']
+        source[0]['value'] = DEFAULT_IMAGE_TOKEN
+        conversation = source[0]['value'] + source[1]['value'] + conversation_lib.default_conversation.sep
+        conversations.append(conversation)
+    # tokenize conversations
+    input_ids = [tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations]
+    targets = copy.deepcopy(input_ids)
+    for target, source in zip(targets, sources):
+        tokenized_len = len(tokenizer_image_token(source[0]['value'], tokenizer))
+        target[:tokenized_len] = IGNORE_INDEX
+
+    return dict(input_ids=input_ids, labels=targets)
+
+
+def load_images(image_file):
+    """
+    Load an image from a local file, a URL, or a DICOM file.
+
+    Args:
+        image_file (str): The path or URL of the image file to load.
+
+    Returns:
+        PIL.Image.Image: The loaded image in RGB format.
+
+    Raises:
+        ValueError: If the DICOM file does not contain image data.
+        TypeError: If the input is neither a valid file path nor a URL.
+    """
+    if isinstance(image_file, str):
+        # Case 1: Load from URL
+        if image_file.startswith(('http://', 'https://')):
+            try:
+                response = requests.get(image_file)
+                response.raise_for_status()
+                image = Image.open(BytesIO(response.content)).convert('RGB')
+            except Exception as e:
+                raise ValueError(f"Error loading image from URL: {image_file}\n{e}")
+
+        # Case 2: Load from DICOM file
+        elif image_file.lower().endswith('.dcm'):
+            try:
+                dicom = pydicom.dcmread(image_file)
+                if 'PixelData' in dicom:
+                    data = apply_voi_lut(dicom.pixel_array, dicom)
+
+                    # Handle MONOCHROME1 images
+                    if dicom.PhotometricInterpretation == "MONOCHROME1":
+                        data = np.max(data) - data
+
+                    # Normalize the image data
+                    data = data - np.min(data)
+                    data = data / np.max(data)
+                    data = (data * 255).astype(np.uint8)
+
+                    # Convert to 3-channel RGB if necessary
+                    if data.ndim == 2:
+                        data = np.stack([data] * 3, axis=-1)
+
+                    image = Image.fromarray(data).convert('RGB')
+                else:
+                    raise ValueError("DICOM file does not contain image data")
+            except Exception as e:
+                raise ValueError(f"Error loading DICOM file: {image_file}\n{e}")
+
+        # Case 3: Load standard image files (e.g., PNG, JPG)
+        else:
+            try:
+                image = Image.open(image_file).convert('RGB')
+            except Exception as e:
+                raise ValueError(f"Error loading standard image file: {image_file}\n{e}")
+
+    else:
+        raise TypeError("image_file must be a string representing a file path or URL")
+
+    return image
+
+
+def preprocess(
+    sources: Sequence[str],
+    tokenizer: transformers.PreTrainedTokenizer,
+    has_image: bool = False
+) -> Dict:
+    """
+    Given a list of sources, each is a conversation list. This transform:
+    1. Add signal '### ' at the beginning each sentence, with end signal '\n';
+    2. Concatenate conversations together;
+    3. Tokenize the concatenated conversation;
+    4. Make a deepcopy as the target. Mask human words with IGNORE_INDEX.
+    """
+    if conversation_lib.default_conversation.sep_style == conversation_lib.SeparatorStyle.PLAIN:
+        return preprocess_plain(sources, tokenizer)
+    if conversation_lib.default_conversation.sep_style == conversation_lib.SeparatorStyle.LLAMA_2:
+        return preprocess_llama_2(sources, tokenizer, has_image=has_image)
+    if conversation_lib.default_conversation.sep_style == conversation_lib.SeparatorStyle.LLAMA_3:
+        return preprocess_llama_3(sources, tokenizer, has_image=has_image)
+    if conversation_lib.default_conversation.version.startswith("v1"):
+        return preprocess_libra(sources, tokenizer, has_image=has_image)
+    if conversation_lib.default_conversation.version == "mpt":
+        return preprocess_mpt(sources, tokenizer)
+
+    conversations = []
+    for source in sources:
+        header = f"{conversation_lib.default_conversation.system}\n\n"
+        conversation = _add_speaker_and_signal(header, source)
+        conversations.append(conversation)
+    # tokenize conversations
+    def get_tokenize_len(prompts):
+        return [len(tokenizer_image_token(prompt, tokenizer)) for prompt in prompts]
+
+    if has_image:
+        input_ids = [tokenizer_image_token(prompt, tokenizer, return_tensors='pt') for prompt in conversations]
+    else:
+        conversations_tokenized = _tokenize_fn(conversations, tokenizer)
+        input_ids = conversations_tokenized["input_ids"]
+
+    targets = copy.deepcopy(input_ids)
+    for target, source in zip(targets, sources):
+        if has_image:
+            tokenized_lens = get_tokenize_len([header] + [s["value"] for s in source])
+        else:
+            tokenized_lens = _tokenize_fn([header] + [s["value"] for s in source], tokenizer)["input_ids_lens"]
+        speakers = [sentence["from"] for sentence in source]
+        _mask_targets(target, tokenized_lens, speakers)
+        
+    return dict(input_ids=input_ids, labels=targets)
+
+
+def create_compute_metrics(tokenizer, num_patches: int, sep2: str):
+    """
+    Creates a function to compute evaluation metrics (e.g., BLEU, ROUGE-L, Temple-F1) for the model.
+    based on the given tokenizer and 'num_patches' parameter.
+
+    Args:
+        tokenizer: The tokenizer used for encoding/decoding text.
+        num_patches (int): The number of patches to be adjusted in the labels.
+        sep2 (str): A separator token used to identify a special token ID.
+
+    Returns:
+        A callable function 'compute_metrics(eval_pred)' that computes evaluation metrics.
+    """
+    # Pre-fetch special token IDs to avoid repeated calls
+    bos_token_id = tokenizer.convert_tokens_to_ids(sep2)
+    newline_token_id = tokenizer.convert_tokens_to_ids('<0x0A>')
+    # 0 is commonly used as the <pad> token ID
+    special_token_ids = [bos_token_id, newline_token_id, 0]
+
+    # Pre-load evaluation metrics (adjust if needed for your scenario)
+    bleu_metric = evaluate.load("bleu")
+    rouge_metric = evaluate.load("rouge")
+    
+    def compute_metrics(eval_pred: EvalPrediction) -> dict:
+        """
+        Compute various evaluation metrics including BLEU, ROUGE, F1 for RadGraph and CheXbert, and BERTScore.
+
+        Args:
+            eval_pred (EvalPrediction): Contains model predictions and true labels.
+
+        Returns:
+            dict: Dictionary containing evaluation metric scores.
+        """
+        logits, labels = eval_pred.predictions, eval_pred.label_ids
+        predicted_ids = np.argmax(logits, axis=-1)
+        
+        # Store processed predicted token IDs
+        processed_predicted_ids = []
+
+        for label, predicted in zip(labels, predicted_ids):
+            # (1) Find ignore_count: the position of the first non-IGNORE_INDEX token in the label
+            ignore_count = next(
+                (i for i, token in enumerate(label) if token != IGNORE_INDEX),
+                len(label)  # If all are -100, use the length of the label
+            )
+            
+            # (2) Calculate the truncation start index
+            #     This depends on 'num_patches' and the ignored tokens.
+            start_index = ignore_count + num_patches - 2
+
+            # If start_index exceeds the predicted sequence length, append an empty list
+            if start_index >= len(predicted):
+                processed_predicted_ids.append([])
+                continue
+            
+            # (3) Slice the prediction from 'start_index' onwards
+            temp_predicted = predicted[start_index:]
+            
+            # (4) Find the earliest occurrence of any special token to truncate
+            matching_indices = []
+            for token_id in special_token_ids:
+                idx = np.where(temp_predicted == token_id)[0]
+                if idx.size > 0:
+                    matching_indices.append(idx)
+            
+            if matching_indices:
+                # Merge all matching indices and take the smallest
+                all_indices = np.concatenate(matching_indices)
+                first_match_index = np.min(all_indices)
+                # Truncate up to the first special token
+                temp_predicted = temp_predicted[:first_match_index]
+
+            # Append the processed sequence to the results
+            processed_predicted_ids.append(temp_predicted)
+            
+        # Decode the processed prediction IDs
+        decoded_preds = tokenizer.batch_decode(
+            processed_predicted_ids, 
+            skip_special_tokens=True
+        )
+        
+        # Filter labels by removing IGNORE_INDEX tokens
+        filtered_labels = [
+            [token for token in label_group if token != IGNORE_INDEX]
+            for label_group in labels
+        ]
+        
+        decoded_labels = tokenizer.batch_decode(
+            filtered_labels, 
+            skip_special_tokens=True
+        )
+        
+        references = [[lbl] for lbl in decoded_labels]
+
+        # Calculate BLEU score
+        bleu_score = bleu_metric.compute(
+            predictions=decoded_preds,
+            references=references,
+            max_order=4
+        )["bleu"]
+
+        # Calculate ROUGE-L score
+        rouge_score = rouge_metric.compute(
+            predictions=decoded_preds,
+            references=references
+        )["rougeL"]
+        
+        # Calculate Temporal-F1 score
+        tem_f1_score = temporal_f1_score(
+            predictions=decoded_preds,
+            references=references
+        )["f1"]
+        
+        # Clean up memory
+        del logits, labels, decoded_preds, decoded_labels, references
+        torch.cuda.empty_cache()
+
+        # Return metrics
+        return {
+            "BLEU4": bleu_score,
+            "ROUGE-L": rouge_score,
+            "TEM-F1": tem_f1_score
+        }
+    
+    return compute_metrics
+
+def check_trainable_parameters(model: torch.nn.Module) -> None:
+    """
+    Print the names, shapes, and data types of all trainable parameters in the model.
+
+    Args:
+        model (torch.nn.Module): The model to inspect.
+    """
+    total_params = sum(
+        p.numel() for p in model.parameters() if p.requires_grad
+    )
+
+    print(f"Total number of trainable parameters: {total_params:,d}\n")
+    
+    # (Optional) Print the model structure for reference
+    print("Overall model structure:")
+    print(model)
+    print("\nTrainable parameters:")
+
+    # Print details of each trainable parameter
+    for name, param in model.named_parameters():
+        if param.requires_grad:
+            param_info = (
+                f"Shape: {list(param.shape)}, "
+                f"Dtype: {param.dtype}"
+            )
+            print(f" - {name} -> {param_info}")           
+                
+class LazySupervisedDataset(Dataset):
+    """Dataset for supervised fine-tuning."""
+
+    def __init__(self, data_path: str,
+                 tokenizer: transformers.PreTrainedTokenizer,
+                 data_args: DataArguments,
+                 sample_rate=1.0):
+        super(LazySupervisedDataset, self).__init__()
+        list_data_dict = json.load(open(data_path, "r"))
+        
+        # Apply sampling if sample_rate < 1.0
+        if 0 < sample_rate < 1.0:
+            random.seed(27) # Fixed seed for consistent behavior across different runs
+            sampled_size = int(len(list_data_dict) * sample_rate)
+            list_data_dict = random.sample(list_data_dict, sampled_size)
+
+        rank0_print("Formatting inputs...Skip in lazy mode")
+        self.tokenizer = tokenizer
+        self.list_data_dict = list_data_dict
+        self.data_args = data_args
+
+    def __len__(self):
+        return len(self.list_data_dict)
+
+    @property
+    def lengths(self):
+        length_list = []
+        for sample in self.list_data_dict:
+            img_tokens = 128 if 'image' in sample else 0
+            length_list.append(sum(len(conv['value'].split()) for conv in sample['conversations']) + img_tokens)
+        return length_list
+
+    @property
+    def modality_lengths(self):
+        length_list = []
+        for sample in self.list_data_dict:
+            cur_len = sum(len(conv['value'].split()) for conv in sample['conversations'])
+            cur_len = cur_len if 'image' in sample else -cur_len
+            length_list.append(cur_len)
+        return length_list
+    
+    def __getitem__(self, i) -> Dict[str, torch.Tensor]:
+        sources = self.list_data_dict[i]
+        if isinstance(i, int):
+            sources = [sources]
+        assert len(sources) == 1, "Don't know why it is wrapped to a list"  # FIXME
+        if 'image' in sources[0]:
+            image_file = self.list_data_dict[i]['image']
+            image_folder = self.data_args.image_folder
+            processor = self.data_args.image_processor
+            
+            if isinstance(image_file, str):
+                image=[]
+                image_path = os.path.join(image_folder, image_file)
+                img = load_images(image_path)
+                image.append(img)
+                # set dummy prior image
+                image.append(img)
+            
+            elif isinstance(image_file, (list, tuple)):
+                image=[]
+                image_paths = [os.path.join(image_folder, file_name) for file_name in image_file]
+                for path in image_paths:
+                    img = load_images(path)
+                    image.append(img)
+                # set dummy prior image
+                if len(image) == 1:
+                    print("Contains only current image. Adding a dummy prior image.")
+                    image.append(image[0])
+                    
+            else:
+                raise TypeError("image_file must be a string or a list/tuple of strings")
+        
+            if self.data_args.image_aspect_ratio == 'pad':
+                def expand2square(pil_img, background_color=(0, 0, 0)):
+                    width, height = pil_img.size
+                    if width == height:
+                        return pil_img
+                    elif width > height:
+                        result = Image.new(pil_img.mode, (width, width), background_color)
+                        result.paste(pil_img, (0, (width - height) // 2))
+                        return result
+                    else:
+                        result = Image.new(pil_img.mode, (height, height), background_color)
+                        result.paste(pil_img, ((height - width) // 2, 0))
+                        return result
+                    
+                processed_images = []
+                for img_data in image:
+                    pad_image = expand2square(img_data, (0, 0, 0))  
+                    image_temp = processor.preprocess(pad_image, return_tensors='pt')['pixel_values'][0]
+                    processed_images.append(image_temp)      
+                image = processed_images
+                
+            else:
+                processed_images = []
+                for img_data in image:
+                    image_temp = processor.preprocess(img_data, return_tensors='pt')['pixel_values'][0]
+                    processed_images.append(image_temp)    
+                image = processed_images
+                
+            sources = preprocess_multimodal(
+                copy.deepcopy([e["conversations"] for e in sources]),
+                self.data_args)
+        else:
+            sources = copy.deepcopy([e["conversations"] for e in sources])
+        
+        data_dict = preprocess(
+            sources,
+            self.tokenizer,
+            has_image=('image' in self.list_data_dict[i]))
+        if isinstance(i, int):
+            data_dict = dict(input_ids=data_dict["input_ids"][0],
+                             labels=data_dict["labels"][0])
+
+        # image exist in the data
+        if 'image' in self.list_data_dict[i]:
+            data_dict['image'] = image
+        elif self.data_args.is_multimodal:
+            # image does not exist in the data, but the model is multimodal
+            crop_size = self.data_args.image_processor.crop_size
+            data_dict['image'] = torch.zeros(3, crop_size['height'], crop_size['width'])
+        return data_dict
+
+
+@dataclass
+class DataCollatorForSupervisedDataset(object):
+    """Collate examples for supervised fine-tuning."""
+
+    tokenizer: transformers.PreTrainedTokenizer
+
+    def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]:
+        input_ids, labels = tuple([instance[key] for instance in instances]
+                                  for key in ("input_ids", "labels"))
+        input_ids = torch.nn.utils.rnn.pad_sequence(
+            input_ids,
+            batch_first=True,
+            padding_value=self.tokenizer.pad_token_id)
+        labels = torch.nn.utils.rnn.pad_sequence(labels,
+                                                 batch_first=True,
+                                                 padding_value=IGNORE_INDEX)
+        input_ids = input_ids[:, :self.tokenizer.model_max_length]
+        labels = labels[:, :self.tokenizer.model_max_length]
+        batch = dict(
+            input_ids=input_ids,
+            labels=labels,
+            attention_mask=input_ids.ne(self.tokenizer.pad_token_id),
+        )
+
+        if 'image' in instances[0]:
+            
+            if not all(len(instance['image']) == 2 for instance in instances):
+                raise ValueError("Each instance['image'] must contain exactly two type images.")
+                
+            cur_images = [instance['image'][0] for instance in instances]
+            prior_images = [instance['image'][1] for instance in instances]
+            
+            
+            if all(x is not None and x.shape == cur_images[0].shape for x in cur_images) and \
+               all(x is not None and x.shape == prior_images[0].shape for x in prior_images):
+                
+                batch['images'] = torch.stack([torch.stack(cur_images), torch.stack(prior_images)])
+            else:
+                print("Warning: Image shapes are inconsistent. Using lists for images.")
+                batch['images'] = [cur_images, prior_images]
+   
+        return batch
+
+
+
+def make_supervised_data_module(tokenizer: transformers.PreTrainedTokenizer,
+                                data_args) -> Dict:
+    """Make dataset and collator for supervised fine-tuning."""
+    train_dataset = LazySupervisedDataset(tokenizer=tokenizer,
+                                data_path=data_args.data_path,
+                                data_args=data_args)
+    
+    eval_dataset = LazySupervisedDataset(tokenizer=tokenizer,
+                                data_path=data_args.validation_data_path,
+                                data_args=data_args,
+                                sample_rate=1.0)
+    
+    data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer)
+    return dict(train_dataset=train_dataset,
+                eval_dataset=eval_dataset,
+                data_collator=data_collator)
+
+def train():
+    global local_rank
+
+    parser = transformers.HfArgumentParser(
+        (ModelArguments, DataArguments, TrainingArguments))
+    model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+    local_rank = training_args.local_rank
+    compute_dtype = (torch.float16 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32))
+
+    bnb_model_from_pretrained_args = {}
+    if training_args.bits in [4, 8]:
+        from transformers import BitsAndBytesConfig
+        bnb_model_from_pretrained_args.update(dict(
+            device_map={"": training_args.device},
+            load_in_4bit=training_args.bits == 4,
+            load_in_8bit=training_args.bits == 8,
+            quantization_config=BitsAndBytesConfig(
+                load_in_4bit=training_args.bits == 4,
+                load_in_8bit=training_args.bits == 8,
+                llm_int8_skip_modules=["mm_projector"],
+                llm_int8_threshold=6.0,
+                llm_int8_has_fp16_weight=False,
+                bnb_4bit_compute_dtype=compute_dtype,
+                bnb_4bit_use_double_quant=training_args.double_quant,
+                bnb_4bit_quant_type=training_args.quant_type # {'fp4', 'nf4'}
+            )
+        ))
+
+    if model_args.vision_tower is not None:
+        model = LibraLlamaForCausalLM.from_pretrained(
+            model_args.model_name_or_path,
+            cache_dir=training_args.cache_dir,
+            torch_dtype=(torch.bfloat16 if training_args.bf16 else None),
+            **bnb_model_from_pretrained_args
+        )
+    else:
+        model = transformers.LlamaForCausalLM.from_pretrained(
+            model_args.model_name_or_path,
+            cache_dir=training_args.cache_dir,
+            torch_dtype=(torch.bfloat16 if training_args.bf16 else None),
+            **bnb_model_from_pretrained_args
+        )
+    model.config.use_cache = False
+
+    if model_args.freeze_backbone:
+        model.model.requires_grad_(False)
+    
+    if training_args.bits in [4, 8]:
+        from peft import prepare_model_for_kbit_training
+        model.config.torch_dtype=(torch.float32 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32))
+        model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=training_args.gradient_checkpointing)
+
+    if training_args.gradient_checkpointing:
+        if hasattr(model, "enable_input_require_grads"):
+            model.enable_input_require_grads()
+        else:
+            def make_inputs_require_grad(module, input, output):
+                output.requires_grad_(True)
+            model.get_input_embeddings().register_forward_hook(make_inputs_require_grad)
+
+    if training_args.lora_enable:
+        from peft import LoraConfig, get_peft_model
+        lora_config = LoraConfig(
+            r=training_args.lora_r,
+            lora_alpha=training_args.lora_alpha,
+            target_modules=find_all_linear_names(model),
+            lora_dropout=training_args.lora_dropout,
+            bias=training_args.lora_bias,
+            task_type="CAUSAL_LM",
+        )
+        if training_args.bits == 16:
+            if training_args.bf16:
+                model.to(torch.bfloat16)
+            if training_args.fp16:
+                model.to(torch.float16)
+        rank0_print("Adding LoRA adapters...")
+        model = get_peft_model(model, lora_config)
+
+    if 'mpt' in model_args.model_name_or_path:
+        tokenizer = transformers.AutoTokenizer.from_pretrained(
+            model_args.model_name_or_path,
+            cache_dir=training_args.cache_dir,
+            model_max_length=training_args.model_max_length,
+            padding_side="right"
+        )
+    else:
+        tokenizer = transformers.AutoTokenizer.from_pretrained(
+            model_args.model_name_or_path,
+            cache_dir=training_args.cache_dir,
+            model_max_length=training_args.model_max_length,
+            padding_side="right",
+            use_fast=False,
+        )
+    
+    if model_args.version == "v0":
+        if tokenizer.pad_token is None:
+            smart_tokenizer_and_embedding_resize(
+                special_tokens_dict=dict(pad_token="[PAD]"),
+                tokenizer=tokenizer,
+                model=model,
+            )
+            
+    elif model_args.version == "v0.5":
+        tokenizer.pad_token = tokenizer.unk_token
+    else:
+        tokenizer.pad_token = tokenizer.unk_token
+        if model_args.version in conversation_lib.conv_templates:
+            conversation_lib.default_conversation = conversation_lib.conv_templates[model_args.version]
+        else:
+            conversation_lib.default_conversation = conversation_lib.conv_templates["vicuna_v1"]
+
+    if model_args.vision_tower is not None:
+        model.get_model().initialize_vision_modules(
+            model_args=model_args,
+            fsdp=training_args.fsdp
+        )
+        
+        vision_tower = model.get_vision_tower()
+        vision_tower.to(dtype=torch.bfloat16 if training_args.bf16 else torch.float16, device=training_args.device)
+
+        data_args.image_processor = vision_tower.image_processor
+        data_args.is_multimodal = True
+
+        model.config.image_aspect_ratio = data_args.image_aspect_ratio
+        model.config.tokenizer_padding_side = tokenizer.padding_side
+        model.config.tokenizer_model_max_length = tokenizer.model_max_length
+
+        model.config.tune_mm_mlp_adapter = training_args.tune_mm_mlp_adapter = model_args.tune_mm_mlp_adapter
+        if model_args.tune_mm_mlp_adapter:
+            model.requires_grad_(False)
+            for p in model.get_model().mm_projector.parameters():
+                p.requires_grad = True
+        
+        model.config.freeze_mm_mlp_adapter = training_args.freeze_mm_mlp_adapter
+        if training_args.freeze_mm_mlp_adapter:
+            for p in model.get_model().mm_projector.parameters():
+                p.requires_grad = False
+
+        if training_args.bits in [4, 8]:
+            model.get_model().mm_projector.to(dtype=compute_dtype, device=training_args.device)
+
+        model.config.mm_use_im_start_end = data_args.mm_use_im_start_end = model_args.mm_use_im_start_end
+        model.config.mm_projector_lr = training_args.mm_projector_lr
+        training_args.use_im_start_end = model_args.mm_use_im_start_end
+        model.config.mm_use_im_patch_token = model_args.mm_use_im_patch_token
+        model.initialize_vision_tokenizer(model_args, tokenizer=tokenizer)
+
+    if training_args.bits in [4, 8]:
+        from peft.tuners.lora import LoraLayer
+        for name, module in model.named_modules():
+            if isinstance(module, LoraLayer):
+                if training_args.bf16:
+                    module = module.to(torch.bfloat16)
+            if 'norm' in name:
+                module = module.to(torch.float32)
+            if 'lm_head' in name or 'embed_tokens' in name:
+                if hasattr(module, 'weight'):
+                    if training_args.bf16 and module.weight.dtype == torch.float32:
+                        module = module.to(torch.bfloat16)
+
+    data_module = make_supervised_data_module(tokenizer=tokenizer,
+                                              data_args=data_args)
+    
+    
+    # SaveCallback
+    class SaveCallback(TrainerCallback):
+
+        def __init__(self):
+            super().__init__()
+            self.best_metric = None
+
+        def on_evaluate(self, args, state, control, metrics, **kwargs):
+            """
+            Custom logic for evaluating and saving the best model based on a chosen metric.
+
+            Saves the model and configuration if a better metric is achieved during evaluation.
+            """
+            metric_for_best_model = 'eval_loss'  # Metric used to determine the best model (e.g., eval_loss, eval_bleu, eval_rouge)
+            metric_value = metrics.get(metric_for_best_model)
+                
+            if self.best_metric is None or metric_value < self.best_metric:
+                self.best_metric = metric_value
+                best_model_dir = os.path.join(args.output_dir, 'best_eval_model')
+                
+                # Save generation configuration if present
+                if hasattr(model, 'generation_config'):
+                    model.generation_config.save_pretrained(best_model_dir)
+
+                # Save model configuration
+                model.config.save_pretrained(best_model_dir) 
+              
+                if tokenizer is not None:
+                    tokenizer.save_pretrained(best_model_dir)
+
+                # Save the best model
+                if args.lora_enable:
+                    # Save LoRA-specific parameters
+                    state_dict = get_peft_state_maybe_zero_3(
+                        model.named_parameters(), args.lora_bias
+                    )
+                    non_lora_state_dict = get_peft_state_non_lora_maybe_zero_3(
+                        model.named_parameters()
+                    )
+                    if args.local_rank in [-1, 0]:
+                        model.save_pretrained(best_model_dir, state_dict=state_dict)
+                        torch.save(non_lora_state_dict, os.path.join(best_model_dir, 'non_lora_trainables.bin'))
+                else:
+                    # Save full model state when not using LoRA
+                    state_dict = get_non_vision_tower_state_maybe_zero_3(
+                        model.named_parameters()
+                    )
+                    if args.local_rank in [-1, 0]:
+                        model.save_pretrained(best_model_dir, state_dict=state_dict)
+                    # Save mm_projector state when tuning mm_mlp_adapter
+                    safe_save_model_for_hf_trainer(trainer=trainer, output_dir=best_model_dir)
+              
+    check_trainable_parameters(model)
+
+    compute_metrics_func = None 
+    
+    if model_args.compute_metrics:
+        compute_metrics_func = create_compute_metrics(tokenizer,vision_tower.num_patches,conversation_lib.default_conversation.sep2)
+
+    model.to(training_args.device)
+
+    trainer = LibraTrainer(model=model,
+                    tokenizer=tokenizer,
+                    args=training_args,
+                    callbacks=[SaveCallback()],
+                    compute_metrics=compute_metrics_func,       
+                    **data_module)
+    
+    if list(pathlib.Path(training_args.output_dir).glob("checkpoint-*")):
+        trainer.train(resume_from_checkpoint=True)
+    else:
+        trainer.train()
+        
+    trainer.save_state()
+
+    model.config.use_cache = True
+
+    if training_args.lora_enable:
+        state_dict = get_peft_state_maybe_zero_3(
+            model.named_parameters(), training_args.lora_bias
+        )
+        non_lora_state_dict = get_peft_state_non_lora_maybe_zero_3(
+            model.named_parameters()
+        )
+        if training_args.local_rank == 0 or training_args.local_rank == -1:
+            model.config.save_pretrained(training_args.output_dir)
+            model.save_pretrained(training_args.output_dir, state_dict=state_dict)
+            torch.save(non_lora_state_dict, os.path.join(training_args.output_dir, 'non_lora_trainables.bin'))
+    else:
+        safe_save_model_for_hf_trainer(trainer=trainer,
+                                       output_dir=training_args.output_dir)
+
+
+if __name__ == "__main__":
+    train()

libra/train/train_mem.py

@@ -0,0 +1,19 @@
+# Adopted from https://github.com/lm-sys/FastChat. Below is the original copyright:
+# Adopted from tatsu-lab@stanford_alpaca. Below is the original copyright:
+# Make it more memory efficient by monkey patching the LLaMA model with FlashAttn.
+
+# # Need to call this before importing transformers.
+
+# from libra.train.llama_flash_attn_monkey_patch import replace_llama_attn_with_flash_attn
+# replace_llama_attn_with_flash_attn()
+
+from libra.train.llama2_flash_attn_monkey_patch import (
+    replace_llama_attn_with_flash_attn,
+)
+
+replace_llama_attn_with_flash_attn()
+
+from libra.train.train import train
+
+if __name__ == "__main__":
+    train()

libra/train/train_xformers.py

@@ -0,0 +1,15 @@
+# Adopted from https://github.com/lm-sys/FastChat. Below is the original copyright:
+# Adopted from tatsu-lab@stanford_alpaca. Below is the original copyright:
+# Make it more memory efficient by monkey patching the LLaMA model with xformers attention.
+
+# Need to call this before importing transformers.
+from libra.train.llama_xformers_attn_monkey_patch import (
+    replace_llama_attn_with_xformers_attn,
+)
+
+replace_llama_attn_with_xformers_attn()
+
+from libra.train.train import train
+
+if __name__ == "__main__":
+    train()

libra/utils.py

@@ -0,0 +1,24 @@
+import datetime
+import logging
+import logging.handlers
+import os
+import sys
+
+import requests
+
+from libra.constants import LOGDIR
+
+server_error_msg = "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"
+moderation_msg = "YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES. PLEASE TRY AGAIN."
+
+handler = None
+
+
+def disable_torch_init():
+    """
+    Disable the redundant torch default initialization to accelerate model creation.
+    """
+    import torch
+    setattr(torch.nn.Linear, "reset_parameters", lambda self: None)
+    setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None)
+