code: def __create_chunk(
      data_root, chunk_path, shard_id, compute_metrics=False
  ):
      """Creates a tarball containing the tokenized text chunks."""
      tar = tarfile.open(
          os.path.join(data_root, f"text_{shard_id}.tar"),
          mode="a",
          encoding="utf-8",
      )
  
      # We squash the filename since we do not preserve directory structure of tokenized text in the tarball.
      base, ext = os.path.splitext(chunk_path)
      base = base.replace(os.pathsep, "_")
      # Need the following replacement as long as WebDataset splits on first period
      base = base.replace(".", "_")
      squashed_filename = f"{base}{ext}"
      tar.add(chunk_path, arcname=squashed_filename)
  
      tar.close()
  
      if compute_metrics:
          data = np.load(chunk_path, allow_pickle=False)
          chunk_len = len(data)
          return (chunk_len,)
      else:
          return None
  
file_path: NeMo/scripts/asr_language_modeling/neural_rescorer/create_tarred_transformer_lm_dataset.py
imports: import argparse
  import glob
  import json
  import logging
  import os
  import tarfile
  import joblib
  import numpy
  from tqdm import tqdm
  from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer
label_desc: Use image node_id 4 for calling a global function with example usage: __create_chunk(data_root, chunk_path, shard_id, compute_metrics) and returns: chunk_len, None