code: def __process_data(
      data_folder: str,
      dst_folder: str,
      num_processes: int = 1,
      rebalance: bool = False,
      class_split: str = "all",
      skip_duration: bool = False,
  ):
      """
      Processes the data and generates the manifests.
  
      Args:
          data_folder: source with wav files and validation / test lists
          dst_folder: where manifest files will be stored
          num_processes: number of processes
          rebalance: rebalance the classes to have same number of samples
          class_split: whether to use all classes as distinct labels, or to use
              10 classes subset and rest of the classes as noise or background
          skip_duration: Bool whether to skip duration computation. Use this only for
              colab notebooks where knowing duration is not necessary for demonstration
      """
  
      os.makedirs(dst_folder, exist_ok=True)
  
      # Used for 10 classes + silence + unknown class setup - Only used when class_split is 'sub'
      class_subset = [
          "yes",
          "no",
          "up",
          "down",
          "left",
          "right",
          "on",
          "off",
          "stop",
          "go",
      ]
  
      pattern = re.compile(r"(.+\/)?(\w+)\/([^_]+)_.+wav")
      all_files = glob.glob(os.path.join(data_folder, "*/*wav"))
  
      # Get files in the validation set
      valset_uids = set()
      with open(
          os.path.join(data_folder, "validation_list.txt")
      ) as fin:
          for line in fin:
              r = re.match(pattern, line)
              if r:
                  valset_uids.add(r.group(3))
  
      # Get files in the test set
      testset_uids = set()
      with open(os.path.join(data_folder, "testing_list.txt")) as fin:
          for line in fin:
              r = re.match(pattern, line)
              if r:
                  testset_uids.add(r.group(3))
  
      logging.info("Validation and test set lists extracted")
  
      filepath_info = __construct_filepaths(
          all_files,
          valset_uids,
          testset_uids,
          class_split,
          class_subset,
          pattern,
      )
      label_count = filepath_info["label_count"]
      label_filepaths = filepath_info["label_filepaths"]
      unknown_val_filepaths = filepath_info["unknown_val_filepaths"]
      unknown_test_filepaths = filepath_info["unknown_test_filepaths"]
      train = filepath_info["train"]
      val = filepath_info["val"]
      test = filepath_info["test"]
  
      logging.info("Prepared filepaths for dataset")
  
      pool = Pool(num_processes)
  
      # Add silence and unknown class label samples
      if class_split == "sub":
          logging.info("Perforiming 10+2 class subsplit")
  
          silence_path = os.path.join(data_folder, "silence")
          os.makedirs(silence_path, exist_ok=True)
  
          silence_stride = 1000  # 0.0625 second stride
          sampling_rate = 16000
          folder = os.path.join(data_folder, "_background_noise_")
  
          silence_files = []
          rng = np.random.RandomState(0)
  
          background_noise_files = [
              os.path.join(folder, x) for x in os.listdir(folder)
          ]
          silence_set_fn = partial(
              __construct_silence_set,
              rng,
              sampling_rate,
              silence_stride,
              data_folder,
          )
          for silence_flist in tqdm(
              pool.imap(
                  silence_set_fn,
                  background_noise_files,
                  __get_mp_chunksize(
                      len(background_noise_files), num_processes
                  ),
              ),
              total=len(background_noise_files),
              desc="Constructing silence set",
          ):
              silence_files.extend(silence_flist)
  
          rng = np.random.RandomState(0)
          rng.shuffle(silence_files)
          logging.info(
              f"Constructed silence set of {len(silence_files)}"
          )
  
          # Create the splits
          rng = np.random.RandomState(0)
          silence_split = 0.1
          unknown_split = 0.1
  
          # train split
          num_total_samples = sum(
              [label_count[cls] for cls in class_subset]
          )
          num_silence_samples = int(
              np.ceil(silence_split * num_total_samples)
          )
  
          # initialize sample
          label_count["silence"] = 0
          label_filepaths["silence"] = []
  
          for silence_id in range(num_silence_samples):
              label_count["silence"] += 1
              label_filepaths["silence"].append(
                  silence_files[silence_id]
              )
  
          train.extend(label_filepaths["silence"])
  
          # Update train unknown set
          unknown_train_samples = label_filepaths["unknown"]
  
          rng.shuffle(unknown_train_samples)
          unknown_size = int(np.ceil(unknown_split * num_total_samples))
  
          label_count["unknown"] = unknown_size
          label_filepaths["unknown"] = unknown_train_samples[
              :unknown_size
          ]
  
          train.extend(label_filepaths["unknown"])
  
          logging.info("Train set prepared")
  
          # val set silence
          num_val_samples = len(val)
          num_silence_samples = int(
              np.ceil(silence_split * num_val_samples)
          )
  
          val_idx = label_count["silence"] + 1
          for silence_id in range(num_silence_samples):
              val.append(silence_files[val_idx + silence_id])
  
          # Update val unknown set
          rng.shuffle(unknown_val_filepaths)
          unknown_size = int(np.ceil(unknown_split * num_val_samples))
  
          val.extend(unknown_val_filepaths[:unknown_size])
  
          logging.info("Validation set prepared")
  
          # test set silence
          num_test_samples = len(test)
          num_silence_samples = int(
              np.ceil(silence_split * num_test_samples)
          )
  
          test_idx = val_idx + num_silence_samples + 1
          for silence_id in range(num_silence_samples):
              test.append(silence_files[test_idx + silence_id])
  
          # Update test unknown set
          rng.shuffle(unknown_test_filepaths)
          unknown_size = int(np.ceil(unknown_split * num_test_samples))
  
          test.extend(unknown_test_filepaths[:unknown_size])
  
          logging.info("Test set prepared")
  
      max_command = None
      max_count = -1
      for command, count in label_count.items():
          if command == "unknown":
              continue
  
          if count > max_count:
              max_count = count
              max_command = command
  
      if rebalance:
          logging.info(
              f"Command with maximum number of samples = {max_command} with {max_count} samples"
          )
          logging.info(
              f"Rebalancing dataset by duplicating classes with less than {max_count} samples..."
          )
  
          rebalance_fn = partial(__rebalance_files, max_count)
          for command, filepaths, num_samples in tqdm(
              pool.imap(
                  rebalance_fn,
                  label_filepaths.items(),
                  __get_mp_chunksize(
                      len(label_filepaths), num_processes
                  ),
              ),
              total=len(label_filepaths),
              desc="Rebalancing dataset",
          ):
              if num_samples < max_count:
                  logging.info(
                      f"Extended class label {command} from {num_samples} samples to {len(filepaths)} samples"
                  )
                  label_filepaths[command] = [
                      (command, filepath) for filepath in filepaths
                  ]
  
          del train
          train = []
          for label, samples in label_filepaths.items():
              train.extend(samples)
  
      manifests = [
          ("train_manifest.json", train),
          ("validation_manifest.json", val),
          ("test_manifest.json", test),
      ]
  
      metadata_fn = partial(__prepare_metadata, skip_duration)
      for manifest_filename, dataset in manifests:
          num_files = len(dataset)
  
          logging.info(
              f"Preparing manifest : {manifest_filename} with #{num_files} files"
          )
  
          manifest = [
              metadata
              for metadata in tqdm(
                  pool.imap(
                      metadata_fn,
                      dataset,
                      __get_mp_chunksize(len(dataset), num_processes),
                  ),
                  total=num_files,
                  desc=f"Preparing {manifest_filename}",
              )
          ]
  
          with open(
              os.path.join(dst_folder, manifest_filename), "w"
          ) as fout:
              for metadata in manifest:
                  fout.write(metadata + "\n")
  
          logging.info(
              f"Finished construction of manifest. Path: {os.path.join(dst_folder, manifest_filename)}"
          )
  
      pool.close()
  
      if skip_duration:
          logging.info(
              f"\n<<NOTE>> Duration computation was skipped for demonstration purposes on Colaboratory.\n"
              f"In order to replicate paper results and properly perform data augmentation, \n"
              f"please recompute the manifest file without the `--skip_duration` flag !\n"
          )
  
imports: import argparse
  import glob
  import json
  import logging
  import os
  import re
  import tarfile
  import urllib.request
  from collections import defaultdict
  from functools import partial
  from multiprocessing import Pool
  from typing import Dict, List, Set, Tuple
  import librosa
  import numpy
  import soundfile
  from tqdm import tqdm
label_desc: Use image node_id 9 for calling a global function with example usage: __process_data(data_folder, dst_folder, num_processes, rebalance, class_split, skip_duration) without return types