code: def dataio_prepare(hparams):
      """grap all the .txt files for transcripts"""
      logging.info("generating datasets...")
      data_folder = hparams["data_folder"]
      train_transcripts = glob.glob(
          os.path.join(
              data_folder, hparams["train_transcripts_pattern"]
          ),
          recursive=True,
      )
      dev_transcripts = glob.glob(
          os.path.join(data_folder, hparams["dev_transcripts_pattern"]),
          recursive=True,
      )
      test_transcripts = glob.glob(
          os.path.join(
              data_folder, hparams["test_transcripts_pattern"]
          ),
          recursive=True,
      )
  
      """prepare data and generate datasets"""
      datasets = load_dataset(
          "dataset.py",
          lm_corpus_path=hparams["lm_corpus_path"],
          data_files={
              "train": train_transcripts,
              "dev": dev_transcripts,
              "test": test_transcripts,
          },
      )
  
      train_data, valid_data, test_data = (
          datasets["train"],
          datasets["dev"],
          datasets["test"],
      )
  
      """convert huggingface's dataset to DynamicItemDataset via a magical function"""
      train_data = (
          sb.dataio.dataset.DynamicItemDataset.from_arrow_dataset(
              train_data
          )
      )
      valid_data = (
          sb.dataio.dataset.DynamicItemDataset.from_arrow_dataset(
              valid_data
          )
      )
      test_data = (
          sb.dataio.dataset.DynamicItemDataset.from_arrow_dataset(
              test_data
          )
      )
  
      datasets = [train_data, valid_data, test_data]
  
      tokenizer = hparams["tokenizer"]
  
      """Define text pipeline"""
  
      # TODO: implement text augmentations pipelines
      @sb.utils.data_pipeline.takes("text")
      @sb.utils.data_pipeline.provides(
          "text", "tokens_bos", "tokens_eos"
      )
      def text_pipeline(text):
          yield text
          tokens_list = tokenizer.encode_as_ids(text)
          tokens_bos = torch.LongTensor(
              [hparams["bos_index"]] + (tokens_list)
          )
          yield tokens_bos
          tokens_eos = torch.LongTensor(
              tokens_list + [hparams["eos_index"]]
          )
          yield tokens_eos
  
      sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
  
      # 4. Set output:
      sb.dataio.dataset.set_output_keys(
          datasets,
          ["id", "text", "tokens_bos", "tokens_eos"],
      )
      return train_data, valid_data, test_data
  
file_path: speechbrain/recipes/LibriSpeech/LM/train.py
imports: import os
  import sys
  import logging
  import glob
  import torch
  from datasets import load_dataset
  from hyperpyyaml import load_hyperpyyaml
  import speechbrain
  from speechbrain.utils.distributed import run_on_main
label_desc: Use image node_id 1 for calling a global function with example usage: dataio_prepare(hparams) and returns: train_data,  valid_data,  test_data