code: def pre_process_datasets(
      encoded_datasets,
      input_len,
      cap_length,
      start_token,
      delimiter_token,
      clf_token,
  ):
      """Pre-process datasets containing lists of tuples(story, 1st continuation, 2nd continuation, label)
  
      To Transformer inputs of shape (n_batch, n_alternative, length) comprising for each batch, continuation:
      input_ids[batch, alternative, :] = [start_token] + story[:cap_length] + [delimiter_token] + cont1[:cap_length] + [clf_token]
      """
      tensor_datasets = []
      for dataset in encoded_datasets:
          n_batch = len(dataset)
          input_ids = np.zeros((n_batch, 2, input_len), dtype=np.int64)
          mc_token_ids = np.zeros((n_batch, 2), dtype=np.int64)
          lm_labels = np.full(
              (n_batch, 2, input_len), fill_value=-100, dtype=np.int64
          )
          mc_labels = np.zeros((n_batch,), dtype=np.int64)
          for (
              i,
              (story, cont1, cont2, mc_label),
          ) in enumerate(dataset):
              with_cont1 = (
                  [start_token]
                  + story[:cap_length]
                  + [delimiter_token]
                  + cont1[:cap_length]
                  + [clf_token]
              )
              with_cont2 = (
                  [start_token]
                  + story[:cap_length]
                  + [delimiter_token]
                  + cont2[:cap_length]
                  + [clf_token]
              )
              input_ids[i, 0, : len(with_cont1)] = with_cont1
              input_ids[i, 1, : len(with_cont2)] = with_cont2
              mc_token_ids[i, 0] = len(with_cont1) - 1
              mc_token_ids[i, 1] = len(with_cont2) - 1
              lm_labels[i, 0, : len(with_cont1)] = with_cont1
              lm_labels[i, 1, : len(with_cont2)] = with_cont2
              mc_labels[i] = mc_label
          all_inputs = (input_ids, mc_token_ids, lm_labels, mc_labels)
          tensor_datasets.append(
              tuple(torch.tensor(t) for t in all_inputs)
          )
      return tensor_datasets
  
file_path: H2O/h2o_flexgen/benchmark/third_party/transformers/examples/legacy/run_openai_gpt.py
imports: import argparse
  import csv
  import logging
  import os
  import random
  import numpy
  import torch
  from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
  from tqdm import tqdm, trange
  from transformers import CONFIG_NAME, WEIGHTS_NAME, AdamW, OpenAIGPTDoubleHeadsModel, OpenAIGPTTokenizer, get_linear_schedule_with_warmup
label_desc: Use image node_id 3 for calling a global function with example usage: pre_process_datasets(encoded_datasets, input_len, cap_length, start_token, delimiter_token, clf_token) and returns: tensor_datasets