code: def __write_tarred_tokenized_text_dataset(
      data_root, num_shards, chunk_paths, chunk_lens
  ):
      num_chunks = len(chunk_paths)
  
      if chunk_lens is not None:
          num_text = sum(chunk_lens)
          shard_counts = {
              chunk_id: chunk_len
              for chunk_id, chunk_len in enumerate(chunk_lens)
          }
          compute_metrics = False
  
      else:
          num_text = 0
          shard_counts = {}
          compute_metrics = True
  
      for chunk_id, chunk_path in enumerate(
          tqdm(chunk_paths, desc="Writing chunk ", unit=" chunks")
      ):
          shard_id = chunk_id % num_shards
          metrics = __create_chunk(
              data_root,
              chunk_path,
              shard_id,
              compute_metrics=compute_metrics,
          )
  
          if metrics is not None:
              num_text += metrics[0]
              shard_counts[chunk_id] = metrics[0]
  
      # write metadata
      metadata_path = os.path.join(data_root, "metadata.json")
      with open(metadata_path, "w") as f:
          metadata = {
              "num_chunks": num_chunks,
              "num_text": num_text,
              "shard_count": shard_counts,
          }
          json.dump(metadata, f, indent=4)
          logging.info("Metadata writen..")
  
file_path: NeMo/scripts/asr_language_modeling/neural_rescorer/create_tarred_transformer_lm_dataset.py
imports: import argparse
  import glob
  import json
  import logging
  import os
  import tarfile
  import joblib
  import numpy
  from tqdm import tqdm
  from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer
label_desc: Use image node_id 5 for calling a global function with example usage: __write_tarred_tokenized_text_dataset(data_root, num_shards, chunk_paths, chunk_lens) without return types