code: def main():
      args = get_args()
      startup_start = time.time()
  
      print("Opening", args.input)
      fin = open(args.input, "r", encoding="utf-8")
  
      encoder = Encoder(args)
      tokenizer = build_tokenizer(args)
      pool = multiprocessing.Pool(
          args.workers, initializer=encoder.initializer
      )
      encoded_sentences = pool.imap(encoder.encode, fin, 25)
  
      print(f"Vocab size: {tokenizer.vocab_size}")
      print(f"Output prefix: {args.output_prefix}")
      output_bin_file = "{}.bin".format(args.output_prefix)
      output_idx_file = "{}.idx".format(args.output_prefix)
      builder = indexed_dataset.MMapIndexedDatasetBuilder(
          output_bin_file,
          dtype=indexed_dataset.DType.optimal_dtype(
              tokenizer.vocab_size
          ),
      )
  
      startup_end = time.time()
      proc_start = time.time()
      total_bytes_processed = 0
      print("Time to startup:", startup_end - startup_start)
  
      for i, (sentence, bytes_processed) in enumerate(
          encoded_sentences, start=1
      ):
          total_bytes_processed += bytes_processed
          builder.add_item(torch.IntTensor(sentence))
          # documents contain only one sentence.
          builder.end_document()
          if i % args.log_interval == 0:
              current = time.time()
              elapsed = current - proc_start
              mbs = total_bytes_processed / elapsed / 1024 / 1024
              print(
                  f"Processed {i} sentences",
                  f"({i/elapsed} sentences/s, {mbs} MB/s).",
                  file=sys.stderr,
              )
  
      builder.finalize(output_idx_file)
  
file_path: megatron-lm/tools/preprocess_data_nmt.py
imports: import argparse
  import json
  import multiprocessing
  import os
  import sys
  import time
  import torch
  from megatron.tokenizer import build_tokenizer
  from megatron.core.datasets import indexed_dataset
label_desc: Use image node_id 2 for calling a global function with example usage: main() without return types