code: def tokenize(prompt, tokenizer, cutoff_len, add_eos_token=False):
      # there's probably a way to do this with the tokenizer settings
      # but again, gotta move fast
      result = tokenizer(
          prompt,
          truncation=True,
          max_length=cutoff_len,
          padding=False,
          return_tensors=None,
      )
      if (
          result["input_ids"][-1] != tokenizer.eos_token_id
          and len(result["input_ids"]) < cutoff_len
          and add_eos_token
      ):
          result["input_ids"].append(tokenizer.eos_token_id)
          result["attention_mask"].append(1)
  
      result["labels"] = result["input_ids"].copy()
  
      return result
  
file_path: h2ogpt/finetune.py
imports: import os
  import sys
  from functools import partial
  from typing import List, Union
  import numpy
  from src.loaders import get_loaders, get_tokenizer
  from src.prompter import generate_prompt, prompt_types, PromptType
  from src.utils import get_githash, copy_code, H2O_Fire
  import torch
label_desc: Use image node_id 3 for calling a global function with example usage: tokenize(prompt, tokenizer, cutoff_len, add_eos_token) and returns: result