code: def preprocess(
      source: dict,
      tokenizer: TokenizerSpec,
      name_end_token_ids: int,
      label_start_ids: list,
      special_tokens: dict,
      num_turn_start_tokens: int,
  ):
      """
      Given a conversation list. This transform:
      1. Add signal '### ' at the beginning each sentence, with end signal '\n';
      2. Concatenate conversations together;
      3. Tokenize the concatenated conversation;
      4. Make a deepcopy as the target. Mask human words with IGNORE_INDEX.
      """
      (
          header,
          conversation,
          data_type,
          mask_role,
      ) = _get_header_conversation_type_mask_role(
          source, special_tokens
      )
      # tokenize conversations
      input_ids = tokenizer.text_to_ids(conversation)
      target = copy.deepcopy(input_ids)
      header_tokens = tokenizer.text_to_ids(header)
      header_len = len(header_tokens)
  
      ids = []
      tokenized_lens = []
      assert torch.equal(
          torch.tensor(target[:header_len]), torch.tensor(header_tokens)
      )
      for s in source["conversations"]:
          # hack to remove the extra empty token in front
          id1 = tokenizer.text_to_ids(PREFIX_STR + s["value"])
          id2 = tokenizer.text_to_ids(PREFIX_STR)
          tokenized_sentence = id1[len(id2) :]
          ids.append(torch.tensor(tokenized_sentence))
          tokenized_lens.append(len(tokenized_sentence))
      speakers = [
          sentence["from"] for sentence in source["conversations"]
      ]
      assert mask_role in speakers, "mask role not in the conversation"
      target = torch.LongTensor(target)
      # not going to train on the header
      target[:header_len] = IGNORE_INDEX
      input_ids = torch.LongTensor(input_ids)
      _mask_targets(
          target,
          tokenized_lens,
          speakers,
          header_len,
          ids,
          tokenizer,
          mask_role,
          data_type,
          name_end_token_ids,
          special_tokens,
          label_start_ids,
          num_turn_start_tokens,
      )
      mask = (target != IGNORE_INDEX).bool()
      assert mask.sum().item() != 0, "mask is empty"
      # Choose the last conversation as answer other history are context
      last_ignore_index_pos = (
          torch.nonzero(target == IGNORE_INDEX)[-1].item() + 1
      )
      context_ids = input_ids[:last_ignore_index_pos]
      answer_ids = input_ids[last_ignore_index_pos:]
      return dict(
          input_ids=input_ids,
          mask=mask,
          context_ids=context_ids,
          answer_ids=answer_ids,
      )
  
file_path: NeMo/nemo/collections/nlp/data/language_modeling/megatron/gpt_sft_chat_dataset.py
imports: import copy
  import torch
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.language_modeling.megatron.gpt_sft_dataset import GPTSFTDataset
  from nemo.utils import logging
label_desc: Use image node_id 7 for calling a global function with example usage: preprocess(source, tokenizer, name_end_token_ids, label_start_ids, special_tokens, num_turn_start_tokens) and returns: dict