code: def clean_ngrams_below_threshold(
      args, ngrams_below_threshold, dedup_file, dedup_key
  ):
      start_time = time.time()
      # Now actually filter the dataset
      args.get_ngram_freq_only = False
      # id_prefix = '-'.join(args.tasks[::2])
      id_prefix = "-".join(args.tasks[::1])
  
      # get the range of the size of the ngrams
      ngrams_freq_sorted = compute_ngram_freq_sorted(
          args, ngrams_below_threshold
      )
  
      # Open the large file to process in parallel
      counter = splitted = ignored = split_mt_thld = trimmed_count = 0
      num_workers = args.num_threads
      pool = multiprocessing.Pool(num_workers)
      fin = open(dedup_file, "r", encoding="utf-8")
      free_ngram_clean_partial = partial(
          free_ngram,
          args=args,
          key=dedup_key,
          ngrams=ngrams_below_threshold,
          ngrams_freq_sorted=ngrams_freq_sorted,
      )
      free_ngrams_clean = pool.imap(free_ngram_clean_partial, fin, 500)
  
      out_f = open(args.output, "wb")
  
      for text_buf_ngram_free, trimmed, myjson, _ in free_ngrams_clean:
          counter += 1
          try:
              trimmed_count += trimmed
  
              if len(text_buf_ngram_free) > 1:
                  splitted += 1
              if len(text_buf_ngram_free) == 0:
                  ignored += 1
              # more than 10 splits ignored
              if len(text_buf_ngram_free) > args.splits_count:
                  text_buf_ngram_free = []
                  split_mt_thld += 1
  
              if args.output is not None:
                  if "split_id" in myjson:
                      use_prefix = myjson["split_id"] + "-"
                  else:
                      use_prefix = ""
  
                  for i in range(len(text_buf_ngram_free)):
                      split_id_string = (
                          id_prefix
                          + "-{:010d}".format(int(counter))
                          + "-{:04d}".format(int(i))
                      )
                      myjson[dedup_key] = text_buf_ngram_free[i]
                      myjson["split_id"] = use_prefix + split_id_string
                      outjson = json.dumps(myjson, ensure_ascii=False)
                      # outjson = json.dumps({"text":text_buf_ngram_free[i],
                      #    id_prefix+"_split_id":split_id_string},
                      #    ensure_ascii=False)
                      out_f.write(outjson.encode("utf-8"))
                      out_f.write("\n".encode("utf-8"))
  
              if counter % 1000 == 0:
                  print(
                      " [final]> processed {} documents in {:.2f} seconds ...".format(
                          counter, time.time() - start_time
                      ),
                      flush=True,
                  )
          except Exception as e:
              print("Error:", e)
  
      print(
          " [final]> processed {} documents in {:.2f} seconds ...".format(
              counter, time.time() - start_time
          ),
          flush=True,
      )
  
      print(
          " Total docs {} splitted {} ignored {} splits > theshold {} trimmed"
          " {}".format(
              counter, splitted, ignored, split_mt_thld, trimmed_count
          ),
          flush=True,
      )
  
      pool.close()
      pool.join()
  
      out_f.close()
      fin.close()
  
imports: import argparse
  from functools import partial
  import json
  import multiprocessing
  import nltk
  import pickle
  import re
  import string
  import sys
  import time
label_desc: Use image node_id 12 for calling a global function with example usage: clean_ngrams_below_threshold(args, ngrams_below_threshold, dedup_file, dedup_key) without return types