code: def chunk_iter(self, step=10):
      for corpus_path in self.corpus:
          line_pos = [0]
          with open(corpus_path, "rb") as fcb:
              pos = 0
              for line in fcb:
                  pos += len(line)
                  line_pos.append(pos)
          line_pos = np.array(line_pos, dtype=np.int64)
          line_size = line_pos[1:] - line_pos[:-1]
          num_lines = line_pos.shape[0] - 1
          budget = self.chunk_size
          chunk_start = 0
          cur_chunk_size = 0
          for i in range(0, num_lines, step):
              line_batch_num = min(num_lines - i, step)
              batch_line_size = line_size[
                  i : (i + line_batch_num)
              ].sum()
              budget -= batch_line_size
              cur_chunk_size += batch_line_size
              if budget <= 0 or i + step >= num_lines:
                  yield corpus_path, chunk_start, cur_chunk_size
                  chunk_start += cur_chunk_size
                  budget = self.chunk_size
                  cur_chunk_size = 0
  
file_path: gluon-nlp/docs/model_zoo/processing/apply_subword.py
imports: import os
  import argparse
  import textwrap
  from multiprocessing import Pool
  import numpy
  import time
  from gluonnlp.data import tokenizers
  from gluonnlp.data.tokenizers.huggingface import is_new_version_model_file
label_desc: Use image node_id 2 for calling the ParallelCorpusApplyer obj's underlying member method code with example usage: obj.chunk_iter(step) without return types