code: def write_dataset_to_file(
      file_name,
      output_file_name,
      detokenizer,
      tokenizer,
      idx,
      total_num_files,
      remove_newline,
  ):
      print(
          f"Processing file {idx + 1}/{total_num_files} : {file_name} -> {output_file_name}"
      )
      dataset = json.load(open(file_name, "r"))
      with open(output_file_name, "w") as f:
          instances = dataset["Instances"]
          definitions = dataset["Definition"]
          for definition in definitions:
              if is_empty(definition, tokenizer):
                  continue
              for instance in instances:
                  id = instance["id"]
                  input = instance["input"]
                  outputs = instance["output"]
                  # On rare occasions, the same instance can have multiple outputs. We add all of them as examples.
                  if is_empty(input, tokenizer):
                      continue
                  for output in outputs:
                      if is_empty(output, tokenizer):
                          continue
                      if remove_newline:
                          prompted_input = definition + " " + input
                      else:
                          prompted_input = definition + "\n\n" + input
                      proc_func = (
                          remove_newline_and_detokenize
                          if remove_newline
                          else detokenize
                      )
                      prompted_input = proc_func(
                          prompted_input, detokenizer
                      )
                      output = proc_func(output, detokenizer)
                      instance_object = {
                          "id": id,
                          "input": prompted_input,
                          "output": output,
                      }
                      f.write(json.dumps(instance_object) + "\n")
  
file_path: NeMo/scripts/nlp_language_modeling/niv2/preprocess_niv2.py
imports: import json
  import os
  import re
  from argparse import ArgumentParser
  from multiprocessing import Pool
  from sacremoses import MosesDetokenizer
  from nemo.collections.common.tokenizers import AutoTokenizer
label_desc: Use image node_id 4 for calling a global function with example usage: write_dataset_to_file(file_name, output_file_name, detokenizer, tokenizer, idx, total_num_files, remove_newline) without return types