code: def process_sentence_chunks(
      ds: MMapRetrievalIndexedDataset,
      tokenizer,
      chunk_size: int,
      stage: int,
      workers: int,
      shard_id: int,
      total_shards: int,
  ):
      """
      This function takes chunked tokens from the retrieval dataset and map it back to text.
      In stage 1, it divides the total work into `total_shards`, and process only at the `shard_id`.
      If the stage is None, it process all the chunks.
      """
      total_chunks = ds.chunks
      start = 0
      threshold = 0
  
      if stage == 1:
          start, total_chunks = calculate_start_end(
              total_chunks=total_chunks,
              total_shards=total_shards,
              shard_id=shard_id,
          )
          logging.info(
              f"shard_id {shard_id}, create index from chunk {start} to {total_chunks}"
          )
  
      with Pool(workers) as p:
          while start < total_chunks:
              if start / total_chunks > threshold:
                  logging.info(
                      f"sentence processing {start / total_chunks} is done"
                  )
                  threshold += 0.1
              slice_id = (start, min(start + chunk_size, total_chunks))
              beg = time.time()
              id_slices = ds.get_chunk(
                  slice(*slice_id), force_no_cont_ids=True
              )
              end = time.time()
              logging.info(f"load {chunk_size} chunks takes {end-beg}")
              start = min(start + chunk_size, total_chunks)
              sentences = p.map(tokenizer.ids_to_text, id_slices)
              end2 = time.time()
              logging.info(
                  f"tokenize {chunk_size} chunks takes {end2-end}"
              )
              queue.put((sentences, slice_id))
      queue.put((None, None))
  
imports: import argparse
  import multiprocessing
  import pathlib
  import sys
  import time
  from multiprocessing import Pool
  import faiss
  import numpy
  import torch
  from numba import njit, prange
  from sentence_transformers import SentenceTransformer
  from nemo.collections.nlp.data.language_modeling.megatron.indexed_retrieval_dataset import KNNIndex, MMapRetrievalIndexedDataset, merge_knn_files
  from nemo.collections.nlp.modules.common.tokenizer_utils import get_nmt_tokenizer
  from nemo.utils import logging
label_desc: Use image node_id 6 for calling a global function with example usage: process_sentence_chunks(ds, tokenizer, chunk_size, stage, workers, shard_id, total_shards) without return types