code: def process_file(input_file):
      # Retrieve process-local state
      tokenizer = process_file.tokenizer
      schema = process_file.schema
      args = process_file.args
      vocab = process_file.vocab
      non_special_ids = process_file.non_special_ids
  
      # Process file
      pa_batches = []  # List of PyArrow RecordBatches
      buffers = [
          [] for _ in range(len(schema))
      ]  # Arrays in same order as schema
      with open(input_file, "r", encoding="utf-8") as f:
          document_lines = f.readlines()
      if len(document_lines) < 2:
          print(
              f"Skipped {input_file} as it contains less than 2 sentences."
          )
          return None
  
      # According to the original tensorflow implementation: We *sometimes*
      # (i.e., short_seq_prob == 0.1, 10% of the time) want to use shorter
      # sequences to minimize the mismatch between pre-training and
      # fine-tuning.
      target_seq_length = args.max_seq_length - 2  # [CLS] ... [SEP]
      if random.random() < args.short_seq_prob:
          target_seq_length = random.randint(2, target_seq_length)
  
      toks = tokenizer.encode(document_lines, int)
  
      current_chunk_1, current_chunk_2 = [], []
      current_length_1, current_length_2 = 0, 0
  
      i = 0
      while i < len(toks):  # While there are samples in the file
          segment = toks[i]
          i += 1
          if current_length_1 < target_seq_length:
              current_chunk_1.append(segment)
              current_length_1 += len(segment)
          else:
              current_chunk_2.append(segment)
              current_length_2 += len(segment)
  
          # Reached target length or end of document
          if (
              current_chunk_1
              and current_chunk_2
              and current_length_1 >= target_seq_length
              and current_length_2 >= target_seq_length
          ):
              toks1 = sum(current_chunk_1, [])[:target_seq_length]
              toks2 = sum(current_chunk_2, [])[:target_seq_length]
              current_chunk_1, current_chunk_2 = [], []
              current_length_1, current_length_2 = 0, 0
  
              (
                  toks1,
                  mlmpositions1,
                  mlmlabels1,
              ) = create_masked_lm_predictions(
                  args=args,
                  tokens=toks1,
                  cls_token_id=vocab.cls_id,
                  sep_token_id=vocab.sep_id,
                  mask_token_id=vocab.mask_id,
                  non_special_ids=non_special_ids,
              )
              (
                  toks2,
                  mlmpositions2,
                  mlmlabels2,
              ) = create_masked_lm_predictions(
                  args=args,
                  tokens=toks2,
                  cls_token_id=vocab.cls_id,
                  sep_token_id=vocab.sep_id,
                  mask_token_id=vocab.mask_id,
                  non_special_ids=non_special_ids,
              )
  
              # Arrays in same order as schema
              buffers[0].append(toks1)
              buffers[1].append(toks2)
              buffers[2].append(len(toks1))
              buffers[3].append(len(toks2))
              buffers[4].append(mlmpositions1)
              buffers[5].append(mlmpositions2)
              buffers[6].append(mlmlabels1)
              buffers[7].append(mlmlabels2)
  
              if len(buffers[0]) >= 1000:
                  batch = pa.RecordBatch.from_arrays(
                      buffers, schema=schema
                  )
                  pa_batches.append(batch)
                  buffers = [[] for _ in range(len(schema))]
  
      if len(buffers[0]):
          batch = pa.RecordBatch.from_arrays(buffers, schema=schema)
          pa_batches.append(batch)
          buffers = [[] for _ in range(len(schema))]
  
      batch_tbl = pa.Table.from_batches(pa_batches, schema=schema)
      args.output.mkdir(parents=True, exist_ok=True)
      pa.feather.write_feather(
          batch_tbl,
          args.output
          / f"quickthought_{os.getpid()}_{process_file.process_idx}.feather",
      )
      process_file.process_idx += 1
  
file_path: gluon-nlp/scripts/pretraining/torch/bert/prepare_quickthought.py
imports: import tqdm
  import argparse
  import time
  import os
  import pathlib
  import random
  import math
  import multiprocessing
  from smart_open import open
  import gluonnlp
  import numpy
  import pyarrow
  import pyarrow.feather
  import pyarrow.compute
label_desc: Use image node_id 4 for calling a global function with example usage: process_file(input_file) and returns: None