code: def get_database(test_datapath, train_datapath, data_type):
      """Get the database by topics"""
  
      assert data_type in [
          "wow_seen",
          "wow_unseen",
          "woi",
      ], "Please input a correct data type!!"
  
      # get test data topic dictionary
      print("> reading test data from %s" % test_datapath)
      test_topics = {}
      with open(test_datapath, "r") as f:
          for i, line in enumerate(f):
              line = line.strip()
              splits = line.split("\t")
              topic = splits[0]
              test_topics[topic] = True
  
      print("> reading data from %s" % train_datapath)
      train_data_by_topic = {}
      dialog_data_by_topic = {}
      dialog_examples = []
      with open(train_datapath, "r") as f:
          for i, line in enumerate(f):
              line = line.strip()
              splits = line.split("\t")
              topic = splits[0]
              turns = splits[1].split(" [SEP] ")[-3:]
              knowledge = splits[2]
              response = splits[3]
              # filtering data samples
              if knowledge == "no_passages_used":
                  # when no knowledge is used
                  continue
              if data_type != "wow_seen" and (
                  "(" in knowledge or ")" in knowledge
              ):
                  # when bracket exists in the knowledge
                  continue
              if data_type != "wow_seen" and topic not in knowledge:
                  # when topic does not exist in the knowledge
                  continue
  
              # get the instance
              last_turn = turns[-1]
              instance = (
                  "( " + last_turn + " ) " + topic + " => " + knowledge
              )
  
              # construct dialog example
              dialog_example = ""
              if data_type != "wow_seen":
                  dialog_example += "( " + topic + " ) "
              for i, turn in enumerate(turns):
                  if i != 0:
                      dialog_example += " "
                  dialog_example += turn
  
              # check overlaps
              if topic in test_topics:
                  if topic not in train_data_by_topic:
                      train_data_by_topic[topic] = [instance]
                  else:
                      train_data_by_topic[topic].append(instance)
  
                  if topic not in dialog_data_by_topic:
                      dialog_data_by_topic[topic] = [dialog_example]
                  else:
                      dialog_data_by_topic[topic].append(dialog_example)
  
              else:
                  # filtering data samples
                  if len(knowledge.split()) > 20:
                      # knowledge is too long
                      continue
                  if (
                      knowledge.startswith("It")
                      or knowledge.startswith("it")
                      or knowledge.startswith("This")
                      or knowledge.startswith("this")
                  ):
                      continue
  
              # append all the data into dialogue examples list
              dialog_examples.append((topic, dialog_example, instance))
  
      return train_data_by_topic, dialog_data_by_topic, dialog_examples
  
file_path: megatron-lm/tasks/msdp/preprocessing.py
imports: import torch
  import argparse
  from nltk import word_tokenize
  from tqdm import tqdm
  import numpy
  import json
label_desc: Use image node_id 4 for calling a global function with example usage: get_database(test_datapath, train_datapath, data_type) and returns: train_data_by_topic,  dialog_data_by_topic,  dialog_examples