code: def tokenize_text(
      data, tokenizer, path, chunk_size=8192, buffer_size=32
  ):
      dataset_len = len(data)
      current_step = 0
      if isinstance(path, str) and os.path.exists(path):
          os.remove(path)
  
      with Parallel(n_jobs=-2, verbose=0) as parallel:
          while True:
              start = current_step * chunk_size
              end = min(
                  (current_step + buffer_size) * chunk_size, dataset_len
              )
  
              tokenized_data = parallel(
                  delayed(tokenize_str)(
                      data[start : start + chunk_size], tokenizer
                  )
                  for start in range(start, end, chunk_size)
              )
  
              # Write dataset
              write_dataset(tokenized_data, path)
              current_step += len(tokenized_data)
              logging.info(
                  f"Finished writing {len(tokenized_data)} chunks to {path}. Current chunk index = {current_step}"
              )
              del tokenized_data
              if end >= dataset_len:
                  break
  
file_path: NeMo/scripts/asr_language_modeling/ngram_lm/kenlm_utils.py
imports: import gzip
  import json
  import os
  import numpy
  import torch
  from joblib import Parallel, delayed
  from tqdm.auto import tqdm
  import nemo.collections.asr
  from nemo.collections.asr.parts.submodules.ctc_beam_decoding import DEFAULT_TOKEN_OFFSET
  from nemo.utils import logging
label_desc: Use image node_id 7 for calling a global function with example usage: tokenize_text(data, tokenizer, path, chunk_size, buffer_size) without return types