code: def generate_and_tokenize_prompt(
      data_point,
      prompt_type=None,
      train_on_inputs=False,
      add_eos_token=False,
      cutoff_len=None,
      tokenizer=None,
  ):
      assert prompt_type is not None
      assert cutoff_len is not None
      assert tokenizer is not None
      prompt_dict = ""  # only for custom prompt_type
      assert (
          prompt_type != PromptType.custom.name
      ), "custom not setup for finetune"
      full_prompt, _, _, _, _ = generate_prompt(
          data_point, prompt_type, prompt_dict, False, False
      )
      tokenized_full_prompt = tokenize(
          full_prompt,
          tokenizer,
          cutoff_len,
          add_eos_token=add_eos_token,
      )
      if not train_on_inputs:
          user_prompt, _, _, _, _ = generate_prompt(
              {**data_point, "output": ""},
              prompt_type,
              prompt_dict,
              False,
              False,
          )
          tokenized_user_prompt = tokenize(
              user_prompt,
              tokenizer,
              cutoff_len,
              add_eos_token=add_eos_token,
          )
          user_prompt_len = len(tokenized_user_prompt["input_ids"])
          if add_eos_token:
              user_prompt_len -= 1
  
          # ignore_index=-100 ensures torch/tf don't include padding token id in CrossEntropyLoss
          tokenized_full_prompt["labels"] = [
              -100
          ] * user_prompt_len + tokenized_full_prompt["labels"][
              user_prompt_len:
          ]  # could be sped up, probably
      return tokenized_full_prompt
  
file_path: h2ogpt/finetune.py
imports: import os
  import sys
  from functools import partial
  from typing import List, Union
  import numpy
  from src.loaders import get_loaders, get_tokenizer
  from src.prompter import generate_prompt, prompt_types, PromptType
  from src.utils import get_githash, copy_code, H2O_Fire
  import torch
label_desc: Use image node_id 5 for calling a global function with example usage: generate_and_tokenize_prompt(data_point, prompt_type, train_on_inputs, add_eos_token, cutoff_len, tokenizer) and returns: tokenized_full_prompt