code: def preprocess(
      sources: Sequence[str],
      tokenizer: transformers.PreTrainedTokenizer,
      has_image: bool = False,
  ) -> Dict:
      """
      Given a list of sources, each is a conversation list. This transform:
      1. Add signal '### ' at the beginning each sentence, with end signal '\n';
      2. Concatenate conversations together;
      3. Tokenize the concatenated conversation;
      4. Make a deepcopy as the target. Mask human words with IGNORE_INDEX.
      """
      if (
          conversation_lib.default_conversation.sep_style
          == conversation_lib.SeparatorStyle.PLAIN
      ):
          return preprocess_plain(sources, tokenizer)
      if (
          conversation_lib.default_conversation.sep_style
          == conversation_lib.SeparatorStyle.LLAMA_2
      ):
          return preprocess_llama_2(
              sources, tokenizer, has_image=has_image
          )
      if conversation_lib.default_conversation.version.startswith("v1"):
          return preprocess_v1(sources, tokenizer, has_image=has_image)
      if conversation_lib.default_conversation.version == "mpt":
          return preprocess_mpt(sources, tokenizer)
      # add end signal and concatenate together
      conversations = []
      for source in sources:
          header = f"{conversation_lib.default_conversation.system}\n\n"
          conversation = _add_speaker_and_signal(header, source)
          conversations.append(conversation)
  
      # tokenize conversations
      def get_tokenize_len(prompts):
          return [
              len(tokenizer_image_token(prompt, tokenizer))
              for prompt in prompts
          ]
  
      if has_image:
          input_ids = [
              tokenizer_image_token(
                  prompt, tokenizer, return_tensors="pt"
              )
              for prompt in conversations
          ]
      else:
          conversations_tokenized = _tokenize_fn(
              conversations, tokenizer
          )
          input_ids = conversations_tokenized["input_ids"]
  
      targets = copy.deepcopy(input_ids)
      for target, source in zip(targets, sources):
          if has_image:
              tokenized_lens = get_tokenize_len(
                  [header] + [s["value"] for s in source]
              )
          else:
              tokenized_lens = _tokenize_fn(
                  [header] + [s["value"] for s in source], tokenizer
              )["input_ids_lens"]
          speakers = [sentence["from"] for sentence in source]
          _mask_targets(target, tokenized_lens, speakers)
  
      return dict(input_ids=input_ids, labels=targets)
  
imports: import os
  import copy
  from dataclasses import dataclass, field
  import json
  import logging
  import pathlib
  from typing import Dict, Optional, Sequence, List
  import torch
  import transformers
  from llmga.llava.constants import IGNORE_INDEX, IMAGE_TOKEN_INDEX, DEFAULT_IMAGE_TOKEN, DEFAULT_IM_START_TOKEN, DEFAULT_IM_END_TOKEN, DEFAULT_OUTPUT_START_TOKEN, DEFAULT_OUTPUT_END_TOKEN
  from torch.utils.data import Dataset
  from llmga.llava.train.llava_trainer import LLaVATrainer
  from llmga.llava import conversation
  from llmga.llava.model import *
  from llmga.llava.mm_utils import tokenizer_image_token
  from PIL import Image
  from llmga.llava.masks.make_mask import get_mask_generator
  from llmga.llava.prompt_temp import outpaint_prompt, inpaint_prompt, textextend_prompt, regen_prompt, ques_prompt, textextend_prompt2
  import random
  import numpy
  import lmdb
  import io
  import pytextrank
  from torchvision import transforms
label_desc: Use image node_id 17 for calling a global function with example usage: preprocess(sources, tokenizer, has_image) and returns: dict, preprocess_plain, preprocess_llama_2, preprocess_v1, preprocess_mpt, unknown