code: def main():
      opt = setup_args()
      reddit = opt["subreddit_name"]
      n_slice = opt["slice_id"]
      n_sents = opt["num_selected"]
      n_context = opt["num_context"]
      eli_path = pjoin(opt["datapath"], "eli5")
      slice_json = pjoin(
          eli_path,
          "processed_data/collected_docs/%s/slices/slice_%d.json"
          % (reddit, n_slice),
      )
      if isfile(slice_json):
          print("loading data", reddit, n_slice)
          qa_json = pjoin(
              eli_path, "processed_data/%s_qalist.json" % (reddit,)
          )
          qa_data = dict(json.load(open(qa_json)))
          docs_slice = json.load(open(slice_json))
          unigram_json = pjoin(
              eli_path,
              "pre_computed/%s_unigram_counts.json" % (reddit,),
          )
          word_counts = json.load(open(unigram_json))
          qt_freqs = dict(word_counts["question_title"])
          qt_sum = sum(qt_freqs.values())
          d_freqs = dict(word_counts["document"])
          d_sum = sum(d_freqs.values())
          word_freqs = {
              "title": (qt_freqs, qt_sum),
              "doc": (d_freqs, d_sum),
          }
          print("loaded data")
          processed = []
          st_time = time()
          for i, (k, docs_list) in enumerate(docs_slice):
              if k in qa_data:
                  processed += [
                      make_example(
                          qa_data[k],
                          docs_list,
                          word_freqs,
                          n_sents,
                          n_context,
                      )
                  ]
              if i % 10 == 0:
                  print(i, len(processed), time() - st_time)
          selected_dir = pjoin(
              eli_path,
              "processed_data/selected_%d_%d" % (n_sents, n_context),
          )
          if not isdir(selected_dir):
              os.mkdir(selected_dir)
          reddit_dir = pjoin(
              eli_path,
              "processed_data/selected_%d_%d/%s"
              % (n_sents, n_context, reddit),
          )
          if not isdir(reddit_dir):
              os.mkdir(reddit_dir)
          selected_slice_json = pjoin(
              eli_path,
              "processed_data/selected_%d_%d/%s/selected_slice_%d.json"
              % (n_sents, n_context, reddit, n_slice),
          )
          json.dump(processed, open(selected_slice_json, "w"))
  
file_path: ParlAI/parlai/tasks/eli5/data_creation/select_sentences_tfidf.py
imports: import json
  import os
  from os.path import join
  from os.path import isfile, isdir
  from parlai.core.params import ParlaiParser
  from time import time
  from data_utils import sentence_split, tf_idf_vec, tf_idf_dist
label_desc: Use image node_id 4 for calling a global function with example usage: main() without return types