code: def main(args):
      num_process = min(multiprocessing.cpu_count(), args.num_process)
      _, cfg, tokenizer, _, _ = get_backbone(
          args.model_name, load_backbone=False
      )
  
      fnames = sorted(os.listdir(args.input))
      fnames = [os.path.join(args.input, fname) for fname in fnames]
      if args.shuffle:
          random.shuffle(fnames)
      num_files = len(fnames)
      num_out_files = min(args.num_out_files, num_files)
      splited_files = np.array_split(fnames, num_out_files)
      output_files = [
          os.path.join(
              args.output,
              "owt-pretrain-record-{}.npz".format(str(i).zfill(4)),
          )
          for i in range(num_out_files)
      ]
      print(
          "All preprocessed features will be saved in {} npz files".format(
              num_out_files
          )
      )
      if not os.path.exists(args.output):
          os.makedirs(args.output, exist_ok=True)
      num_process = min(num_process, num_out_files)
      print(
          "Start preprocessing {} text files with {} cores".format(
              num_files, num_process
          )
      )
      process_args = [
          (
              splited_files[i],
              output_files[i],
              tokenizer,
              args.max_seq_length,
              args.short_seq_prob,
          )
          for i in range(num_out_files)
      ]
      start_time = time.time()
      with multiprocessing.Pool(num_process) as pool:
          iter = pool.imap(get_all_features, process_args)
          fea_written = 0
          f_read = 0
          for i, np_features in enumerate(iter):
              elapsed = time.time() - start_time
              fea_written += len(np_features[0])
              f_read += len(splited_files[i])
              print(
                  "Processed {:} files, Elapsed: {:.2f}s, ETA: {:.2f}s, ".format(
                      fea_written,
                      elapsed,
                      (num_files - f_read) / (f_read / elapsed),
                  )
              )
      print("Done processing within {:.2f} seconds".format(elapsed))
  
file_path: gluon-nlp/docs/model_zoo/pretraining/data_preprocessing.py
imports: import os
  import time
  import math
  import random
  import argparse
  import multiprocessing
  import numpy
  from pretraining_utils import get_all_features
  from gluonnlp.models import get_backbone
label_desc: Use image node_id 2 for calling a global function with example usage: main(args) without return types