code: def dataio_prepare(hparams):
      """This function prepares the datasets to be used in the brain class.
      It also defines the data processing pipeline through user-defined functions.
      """
  
      data_folder = hparams["data_folder"]
  
      train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
          csv_path=hparams["csv_train"],
          replacements={"data_root": data_folder},
      )
  
      if hparams["sorting"] == "ascending":
          # we sort training data to speed up training and get better results.
          train_data = train_data.filtered_sorted(sort_key="duration")
          # when sorting do not shuffle in dataloader ! otherwise is pointless
          hparams["dataloader_opts"]["shuffle"] = False
  
      elif hparams["sorting"] == "descending":
          train_data = train_data.filtered_sorted(
              sort_key="duration", reverse=True
          )
          # when sorting do not shuffle in dataloader ! otherwise is pointless
          hparams["dataloader_opts"]["shuffle"] = False
  
      elif hparams["sorting"] == "random":
          pass
  
      else:
          raise NotImplementedError(
              "sorting must be random, ascending or descending"
          )
  
      valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
          csv_path=hparams["csv_valid"],
          replacements={"data_root": data_folder},
      )
      valid_data = valid_data.filtered_sorted(sort_key="duration")
  
      test_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
          csv_path=hparams["csv_test"],
          replacements={"data_root": data_folder},
      )
      test_data = test_data.filtered_sorted(sort_key="duration")
  
      datasets = [train_data, valid_data, test_data]
  
      asr_tokenizer = hparams["asr_tokenizer"]
      slu_tokenizer = hparams["slu_tokenizer"]
  
      # 2. Define input pipeline:
      @sb.utils.data_pipeline.takes("transcript")
      @sb.utils.data_pipeline.provides(
          "transcript", "transcript_tokens"
      )
      def transcript_pipeline(transcript):
          yield transcript
          transcript_tokens_list = asr_tokenizer.encode_as_ids(
              transcript
          )
          transcript_tokens = torch.LongTensor(transcript_tokens_list)
          yield transcript_tokens
  
      sb.dataio.dataset.add_dynamic_item(datasets, transcript_pipeline)
  
      # 3. Define output pipeline:
      @sb.utils.data_pipeline.takes("semantics")
      @sb.utils.data_pipeline.provides(
          "semantics",
          "semantics_token_list",
          "semantics_tokens_bos",
          "semantics_tokens_eos",
          "semantics_tokens",
      )
      def semantics_pipeline(semantics):
          yield semantics
          semantics_tokens_list = slu_tokenizer.encode_as_ids(semantics)
          yield semantics_tokens_list
          semantics_tokens_bos = torch.LongTensor(
              [hparams["bos_index"]] + (semantics_tokens_list)
          )
          yield semantics_tokens_bos
          semantics_tokens_eos = torch.LongTensor(
              semantics_tokens_list + [hparams["eos_index"]]
          )
          yield semantics_tokens_eos
          semantics_tokens = torch.LongTensor(semantics_tokens_list)
          yield semantics_tokens
  
      sb.dataio.dataset.add_dynamic_item(datasets, semantics_pipeline)
  
      # 4. Set output:
      sb.dataio.dataset.set_output_keys(
          datasets,
          [
              "id",
              "transcript",
              "transcript_tokens",
              "semantics",
              "semantics_tokens_bos",
              "semantics_tokens_eos",
              "semantics_tokens",
          ],
      )
      return (
          train_data,
          valid_data,
          test_data,
          asr_tokenizer,
          slu_tokenizer,
      )
  
file_path: speechbrain/recipes/SLURP/NLU/train.py
imports: import sys
  import torch
  import speechbrain
  from hyperpyyaml import load_hyperpyyaml
  from speechbrain.utils.distributed import run_on_main, if_main_process
  import jsonlines
  import ast
  import pandas
label_desc: Use image node_id 1 for calling a global function with example usage: dataio_prepare(hparams) and returns: train_data,  valid_data,  test_data,  asr_tokenizer,  slu_tokenizer