code: def _generate_dataset(data_dir, data_set):
      extracted_dir = path.join(data_dir, data_set)
      files = []
      for promts_file in glob(
          os.path.join(extracted_dir + "/*/etc/", "PROMPTS")
      ):
          if path.isdir(os.path.join(promts_file[:-11], "wav")):
              with codecs.open(promts_file, "r", "utf-8") as f:
                  for line in f:
                      id = line.split(" ")[0].split("/")[-1]
                      sentence = " ".join(line.split(" ")[1:])
                      sentence = re.sub(
                          "[^a-z']", " ", sentence.strip().lower()
                      )
                      transcript = ""
                      for token in sentence.split(" "):
                          word = token.strip()
                          if word != "" and word != " ":
                              transcript += word + " "
                      transcript = (
                          unicodedata.normalize(
                              "NFKD", transcript.strip()
                          )
                          .encode("ascii", "ignore")
                          .decode("ascii", "ignore")
                      )
                      wav_file = path.join(
                          promts_file[:-11], "wav/" + id + ".wav"
                      )
                      if gfile.Exists(wav_file):
                          wav_filesize = path.getsize(wav_file)
                          # remove audios that are shorter than 0.5s and longer than 20s.
                          # remove audios that are too short for transcript.
                          if (
                              (wav_filesize / 32000) > 0.5
                              and (wav_filesize / 32000) < 20
                              and transcript != ""
                              and wav_filesize / len(transcript) > 1400
                          ):
                              files.append(
                                  (
                                      os.path.abspath(wav_file),
                                      wav_filesize,
                                      transcript,
                                  )
                              )
  
      return pandas.DataFrame(
          data=files,
          columns=["wav_filename", "wav_filesize", "transcript"],
      )
  
imports: import codecs
  import os
  import re
  import sys
  import tarfile
  import threading
  import unicodedata
  import urllib
  from glob import glob
  from multiprocessing.pool import ThreadPool
  from os import makedirs, path
  import pandas
  from bs4 import BeautifulSoup
  from tensorflow.python.platform import gfile
  from deepspeech_training.util.downloader import maybe_download
label_desc: Use image node_id 4 for calling a global function with example usage: _generate_dataset(data_dir, data_set) and returns: pandas