code: def dataio_prep_shards(hparams):
      # load the meta info json file
      with wds.gopen(hparams["train_meta"], "rb") as f:
          train_meta = json.load(f)
      with wds.gopen(hparams["val_meta"], "rb") as f:
          val_meta = json.load(f)
  
      # define the mapping functions in the data pipeline
      snt_len_sample = int(
          hparams["sample_rate"] * hparams["sentence_len"]
      )
      label_encoder = sb.dataio.encoder.CategoricalEncoder()
      lab_enc_file = os.path.join(
          hparams["save_folder"], "label_encoder.txt"
      )
      label_encoder.load_or_create(
          path=lab_enc_file,
          from_iterables=[train_meta["language_ids"]],
          output_key="lang_id",
      )
      # breakpoint()
  
      def audio_pipeline(sample_dict: Dict, random_chunk=True):
          key = sample_dict["__key__"]
          language_id = sample_dict["language_id"].decode("ascii")
          audio_tensor = sample_dict["audio.pth"]
  
          # determine what part of audio sample to use
          audio_tensor = audio_tensor.squeeze()
  
          if random_chunk:
              if len(audio_tensor) - snt_len_sample - 1 <= 0:
                  start = 0
              else:
                  start = random.randint(
                      0, len(audio_tensor) - snt_len_sample - 1
                  )
  
              stop = start + snt_len_sample
          else:
              start = 0
              stop = len(audio_tensor)
  
          sig = audio_tensor[start:stop]
  
          # determine the language ID of the sample
          lang_id_idx = label_encoder.encode_label(language_id)
  
          return {
              "sig": sig,
              "lang_id_encoded": lang_id_idx,
              "id": key,
          }
  
      train_data = (
          wds.WebDataset(
              hparams["train_shards"],
              cache_dir=hparams["shard_cache_dir"],
          )
          .repeat()
          .shuffle(1000)
          .decode("pil")
          .map(partial(audio_pipeline, random_chunk=True))
      )
      logger.info(
          f"Training data consist of {train_meta['num_data_samples']} samples"
      )
  
      valid_data = (
          wds.WebDataset(
              hparams["val_shards"],
              cache_dir=hparams["shard_cache_dir"],
          )
          .decode("pil")
          .map(partial(audio_pipeline, random_chunk=False))
      )
      logger.info(
          f"Validation data consist of {val_meta['num_data_samples']} samples"
      )
  
      return (
          train_data,
          valid_data,
          train_meta["num_data_samples"],
          val_meta["num_data_samples"],
      )
  
file_path: speechbrain/recipes/VoxLingua107/lang_id/train.py
imports: import os
  import sys
  import random
  from typing import Dict
  import json
  from functools import partial
  import webdataset
  import logging
  import torch
  import speechbrain
  from hyperpyyaml import load_hyperpyyaml
  from speechbrain.dataio.batch import PaddedBatch
label_desc: Use image node_id 1 for calling a global function with example usage: dataio_prep_shards(hparams) and returns: train_data,  valid_data,  train_meta,  val_meta, dict