code: def dataio_prep(hparams, tokenizer):
      """This function prepares the datasets to be used in the brain class.
      It also defines the data processing pipeline through user-defined
      functions. We expect `prepare_multiwoz` to have been called before
      this, so that the `train.json`, `dev.json`,  and `test.json` manifest
      files are available.
      Arguments
      ---------
      hparams : dict
          This dictionary is loaded from the `train.yaml` file, and it includes
          all the hyperparameters needed for dataset construction and loading.
      Returns
      -------
      datasets : dict
          Contains two keys, "train" and "valid" that correspond
          to the appropriate DynamicItemDataset object.
      """
  
      # convert special tokens to their ids
      bos, eos, system, user = tokenizer.convert_tokens_to_ids(
          hparams["special_tokens"]
      )
      # history_window, i.e. how many user-system exchanges consider as context (+1 to consider at least the last user turn)
      history_window = 2 * hparams["max_history"] + 1
  
      #  Define histoy pipeline:
      @sb.utils.data_pipeline.takes("history")
      @sb.utils.data_pipeline.provides(
          "history",
          "history_tokens_lists",
          "history_ids",
          "history_bos",
          "history_token_type",
      )
      def history_pipeline(history):
          yield history
  
          # encode each turn of the history
          history_tokens_lists = [
              tokenizer.encode(turn) for turn in history
          ]
          yield history_tokens_lists
  
          # add speaker tokens to the history turns (user is even, system is odd)
          # BEFORE:  [Hi how are you?], [I'm fine, thanks]
          # AFTER:   [SPK_1 Hi how are you?], [SPK_2 I'm fine, thanks]
          history_input_lists = [
              [user if i % 2 == 0 else system] + encoded_turn
              for i, encoded_turn in enumerate(history_tokens_lists)
          ]
  
          history_ids = history_input_lists[-history_window:]
          # concatenate every token into a single list
          # list(chain(*[[1, 2], [3, 4], [5]]))
          # >>> [1, 2, 3, 4, 5]
          history_ids = torch.LongTensor(list(chain(*history_ids)))
          # without bos for lm_labels
          yield history_ids
  
          # create bos version for the input
          history_bos = torch.cat((torch.tensor([bos]), history_ids))
          yield history_bos
  
          # create a mapping that associates each token in the input to a speaker
          # INPUT: [SPK_1 Hi    how   are   you? ], [SPK_2 I'm   fine, thanks]
          # TYPE:  [SPK_1 SPK_1 SPK_1 SPK_1 SPK_1], [SPK_2 SPK_2 SPK_2 SPK_2 ]
          history_token_type_lists = [
              [user if i % 2 == 0 else system] * len(encoded_turn)
              for i, encoded_turn in enumerate(history_input_lists)
          ]
          history_token_type = torch.LongTensor(
              list(
                  chain(
                      *(
                          [[system]]
                          + history_token_type_lists[-history_window:]
                      )
                  )
              )
          )
  
          yield history_token_type
  
      #  Define reply pipeline:
      @sb.utils.data_pipeline.takes("reply")
      @sb.utils.data_pipeline.provides(
          "reply",
          "reply_tokens_list",
          "reply_ids",
          "reply_eos",
          "reply_token_type",
      )
      def reply_pipeline(reply):
          yield reply
  
          reply_tokens_list = tokenizer.encode(reply)
          yield reply_tokens_list
  
          # specify that the system will say the reply
          reply_input_list = [system] + reply_tokens_list
          reply_ids = torch.LongTensor(reply_input_list)
          yield reply_ids
  
          # create eos version of the reply for lm_labels
          reply_eos = torch.cat((reply_ids, torch.tensor([eos])))
          yield reply_eos
  
          # specify the speaker for each token in the reply
          reply_token_type = torch.LongTensor(
              [system] * len(reply_input_list)
          )
          yield reply_token_type
  
      # Define input_and_token_type_pipeline
      @sb.utils.data_pipeline.takes(
          "history_ids",
          "history_bos",
          "history_token_type",
          "reply_ids",
          "reply_eos",
          "reply_token_type",
      )
      @sb.utils.data_pipeline.provides(
          "input_ids", "token_type_ids", "lm_labels"
      )
      def input_and_token_type_pipeline(
          history_ids,
          history_bos,
          history_token_type,
          reply_ids,
          reply_eos,
          reply_token_type,
      ):
          # put history and reply together
          # N.B. input_sequence = history_bos + reply_ids, we don't have eos in the input
          input_ids = torch.cat((history_bos, reply_ids), -1)
          yield input_ids
  
          token_type_ids = torch.cat(
              (history_token_type, reply_token_type), -1
          )
          yield token_type_ids
  
          # create the language model label (ground truth) for the current input
          # -100 is a special tokens that is ignored during the loss computation
          # the idea is to mask everything except the reply (withouth the speaker token)
          # N.B. we don't have bos in the input
          lm_labels = (
              [hparams["ignore_index"]] * history_ids.shape[0]
              + [hparams["ignore_index"]]
              + reply_eos[1:].tolist()
          )
          lm_labels = torch.LongTensor(lm_labels)
  
          yield lm_labels
  
      # Define datasets. We also connect the dataset with the data processing
      # functions defined above.
      datasets = {}
      data_info = {
          "train": hparams["train_annotation"],
          "valid": hparams["valid_annotation"],
          "test": hparams["test_annotation"],
      }
      for dataset in data_info:
          datasets[
              dataset
          ] = sb.dataio.dataset.DynamicItemDataset.from_json(
              json_path=data_info[dataset],
              replacements={"data_root": hparams["data_folder"]},
              dynamic_items=[
                  reply_pipeline,
                  history_pipeline,
                  input_and_token_type_pipeline,
              ],
              output_keys=[
                  "id",
                  "input_ids",
                  "token_type_ids",
                  "history_bos",
                  "reply_eos",
                  "history_token_type",
                  "reply_token_type",
                  "lm_labels",
              ],
          )
  
      return datasets
  
file_path: speechbrain/recipes/MultiWOZ/response_generation/gpt/train_with_gpt.py
imports: import sys
  import speechbrain
  import torch
  from itertools import chain
  from hyperpyyaml import load_hyperpyyaml
  from speechbrain.utils.distributed import run_on_main
  import math
  from speechbrain.dataio.batch import PaddedBatch
label_desc: Use image node_id 2 for calling a global function with example usage: dataio_prep(hparams, tokenizer) and returns: datasets