code: def __tokenize_text(
      text_paths,
      tokenizer,
      tokenized_cachedir,
      lower_case: bool = False,
      chunk_size=8192,
      write_buffer: int = -1,
  ):
      if write_buffer < 1:
          write_buffer = max(os.cpu_count() - write_buffer, 1)
  
      logging.info(f"Using write chunk buffer of size {write_buffer}")
  
      if not os.path.exists(tokenized_cachedir):
          os.makedirs(tokenized_cachedir)
  
      # global parameters
      global_chunk_idx = 0
      chunk_paths = []
      chunk_lens = []
  
      # buffer parameters
      data_cache = []
      chunk_idx = 0
  
      text_generator = iter(
          __build_dataset_from_text(
              text_paths, lower_case=lower_case, chunk_size=chunk_size
          )
      )
      global_num_lines = 0
      last_batch = False
  
      with joblib.Parallel(n_jobs=-2, verbose=10) as parallel:
          while True:
              try:
                  data, num_lines = next(text_generator)
                  data_cache.append(data)
  
                  global_num_lines = num_lines
              except StopIteration:
                  last_batch = True
  
              # Update counters
              chunk_idx += 1
  
              if (chunk_idx == write_buffer) or last_batch:
                  # write the chunks into disk after parallel tokenization
                  tokenized_data_list = parallel(
                      joblib.delayed(__tokenize_str)(chunk, tokenizer)
                      for chunk in data_cache
                  )
  
                  # Sequential write cache
                  for chunk in tokenized_data_list:
                      fp = os.path.join(
                          tokenized_cachedir,
                          f"chunk_{global_chunk_idx}.npy",
                      )
                      chunk = np.asarray(chunk, dtype=np.int64)
                      np.save(fp, chunk, allow_pickle=False)
  
                      chunk_paths.append(fp)
                      chunk_lens.append(len(chunk))
  
                      global_chunk_idx += 1
  
                  logging.info(
                      f"Wrote a total of {global_chunk_idx} chunks to file..."
                  )
  
                  # reset buffers
                  data_cache.clear()
                  del data_cache
  
                  data_cache = []
                  chunk_idx = 0
  
                  if last_batch:
                      logging.info("Finished tokenizing last chunk")
                      break
  
      logging.info(
          f"Chunking {global_num_lines} rows into {global_num_lines // chunk_size} tasks (each chunk contains {chunk_size} elements)"
      )
      return chunk_paths, chunk_lens
  
file_path: NeMo/scripts/asr_language_modeling/neural_rescorer/create_tarred_transformer_lm_dataset.py
imports: import argparse
  import glob
  import json
  import logging
  import os
  import tarfile
  import joblib
  import numpy
  from tqdm import tqdm
  from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer
label_desc: Use image node_id 3 for calling a global function with example usage: __tokenize_text(text_paths, tokenizer, tokenized_cachedir, lower_case, chunk_size, write_buffer) and returns: chunk_paths,  chunk_lens