code: def main_gen(params):
      """Calls gen_audio() to generate the audio signals, verifies that they meet
      the requirements, and writes the files to storage"""
  
      clean_source_files = []
      clean_clipped_files = []
      clean_low_activity_files = []
      noise_source_files = []
      noise_clipped_files = []
      noise_low_activity_files = []
  
      clean_index = 0
      noise_index = 0
  
      # write shards
      train_shards_path = Path(params["train_shard_destination"])
      train_shards_path.mkdir(exist_ok=True, parents=True)
      valid_shards_path = Path(params["valid_shard_destination"])
      valid_shards_path.mkdir(exist_ok=True, parents=True)
  
      all_keys = set()
      train_pattern = str(train_shards_path / "shard") + "-%06d.tar"
      valid_pattern = str(valid_shards_path / "shard") + "-%06d.tar"
      samples_per_shard = params["samples_per_shard"]
  
      # track statistics on data
      train_sample_keys = defaultdict(list)
      valid_sample_keys = defaultdict(list)
  
      # Define the percentage of data to be used for validation
      validation_percentage = 0.05
  
      # Calculate the number of samples for training and validation
      total_samples = (
          params["fileindex_end"] - params["fileindex_start"] + 1
      )
      num_validation_samples = int(
          total_samples * validation_percentage
      )
  
      # Define separate ShardWriters for training and validation data
      train_writer = wds.ShardWriter(
          train_pattern, maxcount=samples_per_shard
      )
      valid_writer = wds.ShardWriter(
          valid_pattern, maxcount=samples_per_shard
      )
  
      # Initialize counters and data lists for statistics
      file_num = params["fileindex_start"]
      train_data_tuples = []
      valid_data_tuples = []
  
      while file_num <= params["fileindex_end"]:
          print(
              "\rFiles synthesized {:4d}/{:4d}".format(
                  file_num, params["fileindex_end"]
              ),
              end="",
          )
          # CLEAN SPEECH GENERATION
          clean, clean_sf, clean_cf, clean_laf, clean_index = gen_audio(
              True, params, clean_index
          )
  
          # add reverb with selected RIR
          rir_index = random.randint(0, len(params["myrir"]) - 1)
  
          my_rir = os.path.normpath(
              os.path.join(params["myrir"][rir_index])
          )
          (fs_rir, samples_rir) = wavfile.read(my_rir)
  
          my_channel = int(params["mychannel"][rir_index])
  
          if samples_rir.ndim == 1:
              samples_rir_ch = np.array(samples_rir)
  
          elif my_channel > 1:
              samples_rir_ch = samples_rir[:, my_channel - 1]
          else:
              samples_rir_ch = samples_rir[:, my_channel - 1]
              # print(samples_rir.shape)
              # print(my_channel)
  
          # REVERB MIXED TO THE CLEAN SPEECH
          clean = add_pyreverb(clean, samples_rir_ch)
  
          # generate noise
          noise, noise_sf, noise_cf, noise_laf, noise_index = gen_audio(
              False, params, noise_index, len(clean)
          )
  
          clean_clipped_files += clean_cf
          clean_low_activity_files += clean_laf
          noise_clipped_files += noise_cf
          noise_low_activity_files += noise_laf
  
          # mix clean speech and noise
          # if specified, use specified SNR value
          if not params["randomize_snr"]:
              snr = params["snr"]
          # use a randomly sampled SNR value between the specified bounds
          else:
              snr = np.random.randint(
                  params["snr_lower"], params["snr_upper"]
              )
  
          # NOISE ADDED TO THE REVERBED SPEECH
          (
              clean_snr,
              noise_snr,
              noisy_snr,
              target_level,
          ) = segmental_snr_mixer(
              params=params, clean=clean, noise=noise, snr=snr
          )
          # Uncomment the below lines if you need segmental SNR and comment the above lines using snr_mixer
          # clean_snr, noise_snr, noisy_snr, target_level = segmental_snr_mixer(params=params,
          #                                                         clean=clean,
          #                                                          noise=noise,
          #                                                         snr=snr)
          # unexpected clipping
          if (
              is_clipped(clean_snr)
              or is_clipped(noise_snr)
              or is_clipped(noisy_snr)
          ):
              print(
                  "\nWarning: File #"
                  + str(file_num)
                  + " has unexpected clipping, "
                  + "returning without writing audio to disk"
              )
              continue
  
          clean_source_files += clean_sf
          noise_source_files += noise_sf
  
          # write resultant audio streams to files
          hyphen = "-"
          clean_source_filenamesonly = [
              i[:-4].split(os.path.sep)[-1] for i in clean_sf
          ]
          clean_files_joined = hyphen.join(clean_source_filenamesonly)[
              :MAXFILELEN
          ]
          noise_source_filenamesonly = [
              i[:-4].split(os.path.sep)[-1] for i in noise_sf
          ]
          noise_files_joined = hyphen.join(noise_source_filenamesonly)[
              :MAXFILELEN
          ]
  
          noisyfilename = (
              clean_files_joined
              + "_"
              + noise_files_joined
              + "_snr"
              + str(snr)
              + "_tl"
              + str(target_level)
              + "_fileid_"
              + str(file_num)
          )
  
          # Period is not allowed in a WebDataset key name
          cleanfilename = "clean_fileid_" + str(file_num)
          cleanfilename = cleanfilename.replace(".", "_")
          noisefilename = "noise_fileid_" + str(file_num)
          noisefilename = noisefilename.replace(".", "_")
  
          file_num += 1
  
          # store statistics
          key = noisyfilename
          key = key.replace(".", "_")
          lang = params["split_name"].split("_")[0]
          t = (key, lang)
  
          # verify key is unique
          assert cleanfilename not in all_keys
          all_keys.add(cleanfilename)
  
          # Split the data between training and validation based on the file number
          if file_num % total_samples <= num_validation_samples:
              # Write to validation set
              valid_sample_keys[lang].append(key)
              valid_data_tuples.append(t)
              sample = {
                  "__key__": key,
                  "noisy_file": key,
                  "clean_file": cleanfilename,
                  "noise_file": noisefilename,
                  "clean_audio.pth": torch.tensor(clean_snr).to(
                      torch.float32
                  ),
                  "noise_audio.pth": torch.tensor(noise_snr).to(
                      torch.float32
                  ),
                  "noisy_audio.pth": torch.tensor(noisy_snr).to(
                      torch.float32
                  ),
              }
              valid_writer.write(sample)
          else:
              # Write to training set
              train_sample_keys[lang].append(key)
              train_data_tuples.append(t)
              sample = {
                  "__key__": key,
                  "noisy_file": key,
                  "clean_file": cleanfilename,
                  "noise_file": noisefilename,
                  "clean_audio.pth": torch.tensor(clean_snr).to(
                      torch.float32
                  ),
                  "noise_audio.pth": torch.tensor(noise_snr).to(
                      torch.float32
                  ),
                  "noisy_audio.pth": torch.tensor(noisy_snr).to(
                      torch.float32
                  ),
              }
              train_writer.write(sample)
  
      train_writer.close()
      valid_writer.close()
  
      # Write meta.json files for both training and validation
      train_meta_dict = {
          "language_id": lang,
          "sample_keys_per_language": train_sample_keys,
          "num_data_samples": len(train_data_tuples),
      }
      valid_meta_dict = {
          "language_id": lang,
          "sample_keys_per_language": valid_sample_keys,
          "num_data_samples": len(valid_data_tuples),
      }
  
      with (train_shards_path / "meta.json").open("w") as f:
          json.dump(train_meta_dict, f, indent=4)
  
      with (valid_shards_path / "meta.json").open("w") as f:
          json.dump(valid_meta_dict, f, indent=4)
  
      return (
          clean_source_files,
          clean_clipped_files,
          clean_low_activity_files,
          noise_source_files,
          noise_clipped_files,
          noise_low_activity_files,
      )
  
imports: import sys
  import os
  from pathlib import Path
  import random
  import time
  import numpy
  from scipy import signal
  from scipy.io import wavfile
  import librosa
  import utils
  from audiolib import segmental_snr_mixer, activitydetector, is_clipped
  import pandas
  import json
  from functools import partial
  from typing import Dict
  from collections import defaultdict
  import speechbrain
  import webdataset
  from hyperpyyaml import load_hyperpyyaml
  import torch
label_desc: Use image node_id 4 for calling a global function with example usage: main_gen(params) and returns: clean_source_files,  clean_clipped_files,  clean_low_activity_files,  noise_source_files,  noise_clipped_files,  noise_low_activity_files