code: def generate_vocab(data_path, text_vocab_path, min_freq):
      text_vocab = defaultdict(int)
      for parent, _, filenames in os.walk(data_path):
          for name in filenames:
              if name.endswith(".txt"):
                  with open(os.path.join(parent, name), "r") as f:
                      for line in f.readlines():
                          text = re.findall(
                              r"[\w']+|[.,!?;]", line.strip().lower()
                          )
                          for w in text:
                              text_vocab[w] += 1
      text_vocab_file = open(text_vocab_path, "w")
      sorted_text_vocab = sorted(
          text_vocab.items(), key=lambda x: x[1], reverse=True
      )
      text_vocab_file.write("<pad>" + "\t" + str(0) + "\n")
      text_vocab_file.write("<unk>" + "\t" + str(1) + "\n")
      idx = 2
      text_vocab = {"<pad>": 0, "<unk>": 1}
      for word, count in sorted_text_vocab:
          if count >= min_freq:
              text_vocab_file.write(word + "\t" + str(idx) + "\n")
              text_vocab[word] = idx
              idx += 1
      try:
          assert idx == len(text_vocab)
      except Exception as e:
          print("idx:", idx)
          print("vocab:", len(text_vocab))
  
      logging.info("generate vocab: {}".format(idx))
      return text_vocab
  
file_path: delta/egs/iemocap/emo/v1/local/python/generate_vocab.py
imports: import sys
  import os
  import re
  import numpy
  from absl import logging
  from collections import defaultdict
  import pickle
label_desc: Use image node_id 1 for calling a global function with example usage: generate_vocab(data_path, text_vocab_path, min_freq) and returns: text_vocab