code: def dataio_prep(hparams, tokenizer):
      """This function prepares the datasets to be used in the brain class.
      It also defines the data processing pipeline through user-defined
      functions. We expect `prepare_multiwoz` to have been called before
      this, so that the `train.json`, `dev.json`,  and `test.json` manifest
      files are available.
      Arguments
      ---------
      hparams : dict
          This dictionary is loaded from the `train.yaml` file, and it includes
          all the hyperparameters needed for dataset construction and loading.
      Returns
      -------
      datasets : dict
          Contains two keys, "train" and "valid" that correspond
          to the appropriate DynamicItemDataset object.
      """
  
      # history_window, i.e. how many user-system exchanges consider as context (+1 to consider at least the last user turn)
      history_window = 2 * hparams["max_history"] + 1
  
      #  Define histoy pipeline:
      @sb.utils.data_pipeline.takes("history")
      @sb.utils.data_pipeline.provides(
          "prompts",
          "propmt_tokens_lists",
          "prompt_ids",
          "prompt_bos",
      )
      def history_pipeline(history):
          # add INST tokens to the history turns for turns associated with user.
          # BEFORE:  [ Hi how are you? ], [I'm fine, thanks]
          # AFTER:   [[INST] Hi how are you? [/INST]], [I'm fine, thanks]
  
          def generate_prompt(idx_and_item):
              index, item = idx_and_item
              if index % 2 == 0:
                  return "[INST] " + item + " [/INST]"
              else:
                  return item
  
          prompts = list(map(generate_prompt, enumerate(history)))
          yield prompts
  
          # encode each turn of the history
          propmt_tokens_lists = [
              tokenizer.encode(turn) for turn in prompts
          ]
          yield propmt_tokens_lists
  
          prompt_ids = propmt_tokens_lists[-history_window:]
          # concatenate every token into a single list
          # list(chain(*[[1, 2], [3, 4], [5]]))
          # >>> [1, 2, 3, 4, 5]
          prompt_ids = torch.LongTensor(list(chain(*prompt_ids)))
          # without bos for lm_labels
          yield prompt_ids
  
          # # create bos version for the input
          prompt_bos = torch.cat(
              (torch.tensor([tokenizer.bos_token_id]), prompt_ids)
          )
          yield prompt_bos
  
      #  Define reply pipeline:
      @sb.utils.data_pipeline.takes("reply")
      @sb.utils.data_pipeline.provides(
          "reply",
          "reply_tokens_list",
          "reply_ids",
          "reply_eos",
      )
      def reply_pipeline(reply):
          yield reply
  
          reply_tokens_list = tokenizer.encode(reply)
          yield reply_tokens_list
  
          reply_ids = torch.LongTensor(reply_tokens_list)
          yield reply_ids
  
          # create eos version of the reply for lm_labels
          reply_eos = torch.cat(
              (reply_ids, torch.tensor([tokenizer.eos_token_id]))
          )
          yield reply_eos
  
      # Define input_and_token_type_pipeline
      @sb.utils.data_pipeline.takes(
          "prompt_ids",
          "prompt_bos",
          "reply_ids",
          "reply_eos",
      )
      @sb.utils.data_pipeline.provides("input_ids", "lm_labels")
      def input_and_token_type_pipeline(
          prompt_ids,
          prompt_bos,
          reply_ids,
          reply_eos,
      ):
          # put history and reply together
          # N.B. input_sequence = history_ids + reply_ids, we don't have eos in the input
          input_ids = torch.cat((prompt_bos, reply_ids), -1)
          yield input_ids
  
          # create the language model label (ground truth) for the current input
          # -100 is a special tokens that is ignored during the loss computation
          # the idea is to mask everything except the reply (withouth the speaker token)
          # N.B. we don't have bos in the input
          lm_labels = [hparams["ignore_index"]] * prompt_ids.shape[
              0
          ] + reply_eos.tolist()
          lm_labels = torch.LongTensor(lm_labels)
  
          yield lm_labels
  
      # Define datasets. We also connect the dataset with the data processing
      # functions defined above.
      datasets = {}
      data_info = {
          "train": hparams["train_annotation"],
          "valid": hparams["valid_annotation"],
          "test": hparams["test_annotation"],
      }
      for dataset in data_info:
          datasets[
              dataset
          ] = sb.dataio.dataset.DynamicItemDataset.from_json(
              json_path=data_info[dataset],
              replacements={"data_root": hparams["data_folder"]},
              dynamic_items=[
                  reply_pipeline,
                  history_pipeline,
                  input_and_token_type_pipeline,
              ],
              output_keys=[
                  "id",
                  "input_ids",
                  "prompt_bos",
                  "reply_eos",
                  "lm_labels",
              ],
          )
  
      return datasets
  
file_path: speechbrain/recipes/MultiWOZ/response_generation/llama2/train_with_llama2.py
imports: import sys
  import speechbrain
  import torch
  from itertools import chain
  from hyperpyyaml import load_hyperpyyaml
  from speechbrain.utils.distributed import run_on_main
  import math
  from speechbrain.dataio.batch import PaddedBatch
label_desc: Use image node_id 2 for calling a global function with example usage: dataio_prep(hparams, tokenizer) and returns: datasets, unknown, item