code: def main():
      args = parse_args()
      set_seed(args.seed)
      get_input_files(args)
  
      def _initializer(function):
          """Initialize state of each process in multiprocessing pool.
  
          The process local state is stored as an attribute of the function
          object, which is specified in Pool(..., initargs=(function, )) and by
          convention refers to the function executed during map.
  
          """
          # TODO gluonnlp shouldn't provide a slow LegacyHuggingFaceTokenizer here...
          _, tokenizer, _, _ = nlp.models.bert.get_pretrained_bert(
              args.model_name, load_backbone=False, load_mlm=False
          )
          function.tokenizer = tokenizer
          function.args = args
          function.vocab = tokenizer.vocab
          function.non_special_ids = tokenizer.vocab[
              tokenizer.vocab.non_special_tokens
          ]
          function.process_idx = 0
  
          tok_type = (
              pa.uint16()
              if len(tokenizer.vocab) <= np.iinfo(np.uint16).max
              else pa.uint32()
          )
          assert len(tokenizer.vocab) <= np.iinfo(np.uint32).max
          length_type = pa.uint16()
          assert args.max_seq_length * 2 <= np.iinfo(np.uint16).max
  
          # pa.large_list instead of pa.list_ to use 64bit offsets
          # See https://issues.apache.org/jira/browse/ARROW-9773
          schema = pa.schema(
              {
                  "quickthought1": pa.large_list(tok_type),
                  "quickthought2": pa.large_list(tok_type),
                  "validlength1": length_type,
                  "validlength2": length_type,
                  "mlmpositions1": pa.large_list(length_type),
                  "mlmpositions2": pa.large_list(length_type),
                  "mlmlabels1": pa.large_list(tok_type),
                  "mlmlabels2": pa.large_list(tok_type),
              }
          )
          function.schema = schema
  
      if args.processes:
          with multiprocessing.Pool(
              initializer=_initializer,
              initargs=(process_file,),
              processes=args.processes,
          ) as pool:
              # pool.map(process_file, args.inputs) with tqdm progress bar
              with tqdm.tqdm(total=len(args.inputs)) as pbar:
                  for i, _ in enumerate(
                      pool.imap_unordered(process_file, args.inputs)
                  ):
                      pbar.update()
      else:
          _initializer(process_file)
          for _ in tqdm.tqdm(
              map(process_file, args.inputs), total=len(args.inputs)
          ):
              pass
  
file_path: gluon-nlp/scripts/pretraining/torch/bert/prepare_quickthought.py
imports: import tqdm
  import argparse
  import time
  import os
  import pathlib
  import random
  import math
  import multiprocessing
  from smart_open import open
  import gluonnlp
  import numpy
  import pyarrow
  import pyarrow.feather
  import pyarrow.compute
label_desc: Use image node_id 6 for calling a global function with example usage: main() without return types