code: def preprocess(
      sources: Sequence[str],
      targets: Sequence[str],
      tokenizer: AutoTokenizer,
      max_length: int = 512,
  ) -> Dict:
      """Preprocess the data by tokenizing."""
      examples = [s + t for s, t in zip(sources, targets)]
      examples_tokenized, sources_tokenized = [
          _tokenize_fn(strings, tokenizer, max_length)
          for strings in (examples, sources)
      ]
      input_ids = examples_tokenized["input_ids"]
      labels = copy.deepcopy(input_ids)
      for label, source_len in zip(
          labels, sources_tokenized["input_ids_lens"]
      ):
          label[:source_len] = IGNORE_INDEX
      return dict(input_ids=input_ids, labels=labels)
  
imports: import copy
  import json
  from typing import Dict, Sequence
  import torch
  from torch.utils.data import Dataset
  from tqdm import tqdm
  from transformers import AutoTokenizer
label_desc: Use image node_id 2 for calling a global function with example usage: preprocess(sources, targets, tokenizer, max_length) and returns: dict