code: def dataio_prepare(hparams):
      """This function prepares the datasets to be used in the brain class.
      It also defines the data processing pipeline through user-defined functions.
      """
  
      # Define audio pipeline. In this case, we simply read the path contained
      # in the variable wav with the audio reader.
      @sb.utils.data_pipeline.takes("wav")
      @sb.utils.data_pipeline.provides("sig")
      def audio_pipeline(wav):
          """Load the audio signal. This is done on the CPU in the `collate_fn`."""
          sig = sb.dataio.dataio.read_audio(wav)
          return sig
  
      @sb.utils.data_pipeline.takes("wav")
      @sb.utils.data_pipeline.provides("sig")
      def sp_audio_pipeline(wav):
          """Load the audio signal. This is done on the CPU in the `collate_fn`."""
          sig = sb.dataio.dataio.read_audio(wav)
          sig = sig.unsqueeze(0)
          sig = hparams["speed_perturb"](sig)
          sig = sig.squeeze(0)
          return sig
  
      # Define text processing pipeline. We start from the raw text and then
      # encode it using the tokenizer. The tokens with BOS are used for feeding
      # decoder during training, the tokens with EOS for computing the cost function.
      # The tokens without BOS or EOS is for computing CTC loss.
      @sb.utils.data_pipeline.takes("translation_0")
      @sb.utils.data_pipeline.provides(
          "translation_0",
          "tokens_list",
          "tokens_bos",
          "tokens_eos",
          "tokens",
      )
      def one_reference_text_pipeline(translation):
          """Processes the transcriptions to generate proper labels"""
          yield translation
          tokens_list = hparams["tokenizer"].encode_as_ids(translation)
          yield tokens_list
          tokens_bos = torch.LongTensor(
              [hparams["bos_index"]] + (tokens_list)
          )
          yield tokens_bos
          tokens_eos = torch.LongTensor(
              tokens_list + [hparams["eos_index"]]
          )
          yield tokens_eos
          tokens = torch.LongTensor(tokens_list)
          yield tokens
  
      @sb.utils.data_pipeline.takes(
          "translation_0",
          "translation_1",
          "translation_2",
          "translation_3",
      )
      @sb.utils.data_pipeline.provides(
          "translation_0",
          "translation_1",
          "translation_2",
          "translation_3",
          "tokens_list",
          "tokens_bos",
          "tokens_eos",
          "tokens",
      )
      def four_reference_text_pipeline(*translations):
          """Processes the transcriptions to generate proper labels"""
          yield translations[0]
          yield translations[1]
          yield translations[2]
          yield translations[3]
          tokens_list = hparams["tokenizer"].encode_as_ids(
              translations[0]
          )
          yield tokens_list
          tokens_bos = torch.LongTensor(
              [hparams["bos_index"]] + (tokens_list)
          )
          yield tokens_bos
          tokens_eos = torch.LongTensor(
              tokens_list + [hparams["eos_index"]]
          )
          yield tokens_eos
          tokens = torch.LongTensor(tokens_list)
          yield tokens
  
      @sb.utils.data_pipeline.takes("transcription")
      @sb.utils.data_pipeline.provides(
          "transcription",
          "transcription_list",
          "transcription_bos",
          "transcription_eos",
          "transcription_tokens",
      )
      def transcription_text_pipeline(transcription):
          yield transcription
          tokens_list = hparams["tokenizer"].encode_as_ids(
              transcription
          )
          yield tokens_list
          tokens_bos = torch.LongTensor(
              [hparams["bos_index"]] + (tokens_list)
          )
          yield tokens_bos
          tokens_eos = torch.LongTensor(
              tokens_list + [hparams["eos_index"]]
          )
          yield tokens_eos
          tokens = torch.LongTensor(tokens_list)
          yield tokens
  
      datasets = {}
      data_folder = hparams["data_folder"]
      for dataset in ["train", "dev"]:
          json_path = f"{data_folder}/{dataset}/data.json"
          dataset = dataset if dataset == "train" else "valid"
  
          is_use_sp = dataset == "train" and "speed_perturb" in hparams
          audio_pipeline_func = (
              sp_audio_pipeline if is_use_sp else audio_pipeline
          )
  
          datasets[
              dataset
          ] = sb.dataio.dataset.DynamicItemDataset.from_json(
              json_path=json_path,
              replacements={"data_root": data_folder},
              dynamic_items=[
                  audio_pipeline_func,
                  one_reference_text_pipeline,
                  transcription_text_pipeline,
              ],
              output_keys=[
                  "id",
                  "sig",
                  "duration",
                  "translation_0",
                  "tokens_bos",
                  "tokens_eos",
                  "tokens",
                  "transcription",
                  "transcription_list",
                  "transcription_bos",
                  "transcription_eos",
                  "transcription_tokens",
              ],
          )
  
      for dataset in ["dev", "dev2", "test"]:
          json_path = f"{data_folder}/{dataset}/data.json"
          datasets[
              dataset
          ] = sb.dataio.dataset.DynamicItemDataset.from_json(
              json_path=json_path,
              replacements={"data_root": data_folder},
              dynamic_items=[
                  audio_pipeline,
                  four_reference_text_pipeline,
                  transcription_text_pipeline,
              ],
              output_keys=[
                  "id",
                  "sig",
                  "duration",
                  "translation_0",
                  "translation_1",
                  "translation_2",
                  "translation_3",
                  "tokens_bos",
                  "tokens_eos",
                  "tokens",
                  "transcription",
                  "transcription_list",
                  "transcription_bos",
                  "transcription_eos",
                  "transcription_tokens",
              ],
          )
  
      # Sorting training data with ascending order makes the code  much
      # faster  because we minimize zero-padding. In most of the cases, this
      # does not harm the performance.
      if hparams["sorting"] == "ascending":
          # use smaller dataset to debug the model
          if hparams["debug"]:
              datasets["train"] = datasets["train"].filtered_sorted(
                  key_min_value={"duration": 1},
                  key_max_value={"duration": 5},
                  sort_key="duration",
                  reverse=True,
              )
              datasets["valid"] = datasets["valid"].filtered_sorted(
                  key_min_value={"duration": 1},
                  key_max_value={"duration": 5},
                  sort_key="duration",
                  reverse=True,
              )
          else:
              datasets["train"] = datasets["train"].filtered_sorted(
                  sort_key="duration"
              )
              datasets["valid"] = datasets["valid"].filtered_sorted(
                  sort_key="duration"
              )
  
          hparams["train_dataloader_opts"]["shuffle"] = False
          hparams["valid_dataloader_opts"]["shuffle"] = False
      elif hparams["sorting"] == "descending":
          # use smaller dataset to debug the model
          if hparams["debug"]:
              datasets["train"] = datasets["train"].filtered_sorted(
                  key_min_value={"duration": 1},
                  key_max_value={"duration": 5},
                  sort_key="duration",
                  reverse=True,
              )
              datasets["valid"] = datasets["valid"].filtered_sorted(
                  key_min_value={"duration": 1},
                  key_max_value={"duration": 5},
                  sort_key="duration",
                  reverse=True,
              )
          else:
              datasets["train"] = datasets["train"].filtered_sorted(
                  sort_key="duration", reverse=True
              )
              datasets["valid"] = datasets["valid"].filtered_sorted(
                  sort_key="duration", reverse=True
              )
  
          hparams["train_dataloader_opts"]["shuffle"] = False
          hparams["valid_dataloader_opts"]["shuffle"] = False
      elif hparams["sorting"] == "random":
          # use smaller dataset to debug the model
          if hparams["debug"]:
              datasets["train"] = datasets["train"].filtered_sorted(
                  key_min_value={"duration": 3},
                  key_max_value={"duration": 5},
                  sort_key="duration",
              )
              datasets["valid"] = datasets["valid"].filtered_sorted(
                  key_min_value={"duration": 1},
                  key_max_value={"duration": 5},
              )
  
          hparams["train_dataloader_opts"]["shuffle"] = True
      else:
          raise NotImplementedError(
              "sorting must be random, ascending or descending"
          )
  
      return datasets
  
file_path: speechbrain/recipes/Fisher-Callhome-Spanish/ST/transformer/train.py
imports: import sys
  import torch
  import logging
  import speechbrain
  from sacremoses import MosesDetokenizer
  from hyperpyyaml import load_hyperpyyaml
  from speechbrain.utils.distributed import run_on_main
label_desc: Use image node_id 1 for calling a global function with example usage: dataio_prepare(hparams) and returns: datasets, sig, sig