code: def preprocess(
      sources: Sequence[str],
      tokenizer: transformers.PreTrainedTokenizer,
      has_image: bool = False,
      prompt: str = None,
      refine_prompt: bool = False,
  ) -> Dict:
      """
      Given a list of sources, each is a conversation list. This transform:
      1. Add signal '### ' at the beginning each sentence, with end signal '\n';
      2. Concatenate conversations together;
      3. Tokenize the concatenated conversation;
      4. Make a deepcopy as the target. Mask human words with IGNORE_INDEX.
      """
      if conversation_lib.default_conversation.version.startswith(
          "plain_guided"
      ):
          return preprocess_plain_guided(
              sources, tokenizer, prompt=prompt
          )
      elif (
          conversation_lib.default_conversation.sep_style
          == conversation_lib.SeparatorStyle.PLAIN
      ):
          return preprocess_plain(sources, tokenizer)
      if (
          conversation_lib.default_conversation.sep_style
          == conversation_lib.SeparatorStyle.LLAMA_2
      ):
          return preprocess_llama_2(
              sources, tokenizer, has_image=has_image
          )
      if conversation_lib.default_conversation.version.startswith("v1"):
          return preprocess_v1(sources, tokenizer, has_image=has_image)
      elif conversation_lib.default_conversation.version.startswith(
          "imgsp"
      ):
          return preprocess_imgsp_v1(
              sources,
              tokenizer,
              has_image=has_image,
              refine_prompt=refine_prompt,
          )
      if conversation_lib.default_conversation.version == "mpt":
          return preprocess_mpt(sources, tokenizer)
      # add end signal and concatenate together
      conversations = []
      for source in sources:
          header = f"{conversation_lib.default_conversation.system}\n\n"
          conversation = _add_speaker_and_signal(header, source)
          conversations.append(conversation)
  
      # tokenize conversations
      def get_tokenize_len(prompts):
          return [
              len(tokenizer_image_token(prompt, tokenizer))
              for prompt in prompts
          ]
  
      if has_image:
          input_ids = [
              tokenizer_image_token(
                  prompt, tokenizer, return_tensors="pt"
              )
              for prompt in conversations
          ]
      else:
          conversations_tokenized = _tokenize_fn(
              conversations, tokenizer
          )
          input_ids = conversations_tokenized["input_ids"]
  
      targets = copy.deepcopy(input_ids)
      for target, source in zip(targets, sources):
          if has_image:
              tokenized_lens = get_tokenize_len(
                  [header] + [s["value"] for s in source]
              )
          else:
              tokenized_lens = _tokenize_fn(
                  [header] + [s["value"] for s in source], tokenizer
              )["input_ids_lens"]
          speakers = [sentence["from"] for sentence in source]
          _mask_targets(target, tokenized_lens, speakers)
  
      return dict(input_ids=input_ids, labels=targets)
  
imports: import os
  import copy
  import random
  from dataclasses import dataclass, field
  import json
  import logging
  import pathlib
  from typing import Dict, Optional, Sequence, List
  import pickle
  import math
  import torch
  import transformers
  from llamavid.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN
  from torch.utils.data import Dataset
  from llamavid.train.llava_trainer import LLaVATrainer
  from llamavid import conversation
  from llamavid.model import *
  from llava.mm_utils import tokenizer_image_token
  from PIL import Image
  from decord import VideoReader, cpu
label_desc: Use image node_id 20 for calling a global function with example usage: preprocess(sources, tokenizer, has_image, prompt, refine_prompt) and returns: dict, preprocess_plain_guided, preprocess_llama_2, preprocess_v1, preprocess_mpt, unknown, preprocess_plain, preprocess_imgsp_v1