code: def process_folder(
      data_folder, output_folder, splits_file, remove_newline
  ):
      detokenizer = MosesDetokenizer("en")
      tokenizer = AutoTokenizer("gpt2")
      assert os.path.isdir(data_folder)
      assert os.path.exists(splits_file)
      if not os.path.exists(output_folder):
          os.system(f"mkdir -p {output_folder}")
      if not os.path.exists(os.path.join(output_folder, "train")):
          os.system(f'mkdir -p {os.path.join(output_folder, "train")}')
      if not os.path.exists(os.path.join(output_folder, "test")):
          os.system(f'mkdir -p {os.path.join(output_folder, "test")}')
  
      splits_file_names = [
          line.strip() + ".json" for line in open(splits_file, "r")
      ]
      print(
          f"Found {len(os.listdir(data_folder))} files in the data folder ..."
      )
      print(
          f"Found {len(splits_file_names)} in the splits in the splits file ..."
      )
      print(
          f"Processing {len(splits_file_names)}/{len(os.listdir(data_folder))} files ..."
      )
      pool_args = []
      for idx, file_name in enumerate(splits_file_names):
          print(
              f"Processing file {idx}/{len(splits_file_names)}: {file_name}"
          )
          if not os.path.exists(os.path.join(data_folder, file_name)):
              raise FileNotFoundError(
                  f"Could not find {os.path.join(data_folder, file_name)}"
              )
          if not file_name.endswith(".json"):
              print(
                  f"Skipping {file_name} because it is not a JSON file"
              )
          output_file_name = os.path.join(
              output_folder, file_name.replace(".json", ".jsonl")
          )
          pool_args.append(
              (
                  os.path.join(data_folder, file_name),
                  output_file_name,
                  detokenizer,
                  tokenizer,
                  idx,
                  len(splits_file_names),
                  remove_newline,
              )
          )
  
      write_dataset_to_file(
          os.path.join(data_folder, file_name),
          output_file_name,
          detokenizer,
          tokenizer,
          idx,
          len(splits_file_names),
          remove_newline,
      )
      pool = Pool(42)
      pool.starmap(write_dataset_to_file, pool_args)
  
file_path: NeMo/scripts/nlp_language_modeling/niv2/preprocess_niv2.py
imports: import json
  import os
  import re
  from argparse import ArgumentParser
  from multiprocessing import Pool
  from sacremoses import MosesDetokenizer
  from nemo.collections.common.tokenizers import AutoTokenizer
label_desc: Use image node_id 5 for calling a global function with example usage: process_folder(data_folder, output_folder, splits_file, remove_newline) without return types