code: def main():
      text_path = args.text_path
      data_root = args.data_root
  
      if args.log:
          logging.basicConfig(level=logging.INFO)
  
      tokenized_cachedir = os.path.join(
          data_root, "_tokenized_dataset_cachedir"
      )
  
      if os.path.exists(tokenized_cachedir):
          logging.warning(
              f"Tokenized cache directory {tokenized_cachedir} already potentially contains files."
              f"In such a case, please be aware that the tarfiles will be **appended** instead of overridden!"
          )
  
      if not os.path.exists(data_root):
          os.makedirs(data_root)
  
      chunk_paths = None
      chunk_lens = None
  
      if os.path.exists(tokenized_cachedir):
          paths = glob.glob(os.path.join(tokenized_cachedir, "*.npy"))
          if len(paths) > 0:
              logging.info(
                  "Cached tokenized numpy files found, skipping re-tokenization of dataset"
              )
  
              chunk_paths = paths
              chunk_lens = None
  
      if chunk_paths is None:
          if args.tokenizer_name is None:
              raise ValueError(
                  "`tokenizer_name` name is required when tokenizing the dataset for the first time."
              )
  
          if args.tokenizer_vocab_file is None:
              raise ValueError(
                  "`tokenizer_vocab_file` is required when constructing the tokenized dataset"
              )
  
          tokenizer = get_tokenizer(
              tokenizer_name=args.tokenizer_name,
              tokenizer_model=args.tokenizer_model,
              vocab_file=args.tokenizer_vocab_file,
              special_tokens=args.tokenizer_special_tokens,
          )
  
          logging.info("Built tokenizer")
  
          # tokenize text data into sub-words
          chunk_paths, chunk_lens = __tokenize_text(
              text_paths=text_path,
              tokenizer=tokenizer,
              tokenized_cachedir=tokenized_cachedir,
              lower_case=args.lower_case,
              chunk_size=args.chunk_size,
              write_buffer=args.chunk_write_buffer,
          )
          logging.info(
              f"Tokenized dataset into sub-words and serialized cache at {tokenized_cachedir}"
          )
  
      # Write tarred dataset
      __write_tarred_tokenized_text_dataset(
          data_root,
          num_shards=args.num_shards,
          chunk_paths=chunk_paths,
          chunk_lens=chunk_lens,
      )
  
      logging.info("Done preparing tokenized dataset!")
  
file_path: NeMo/scripts/asr_language_modeling/neural_rescorer/create_tarred_transformer_lm_dataset.py
imports: import argparse
  import glob
  import json
  import logging
  import os
  import tarfile
  import joblib
  import numpy
  from tqdm import tqdm
  from nemo.collections.nlp.modules.common.tokenizer_utils import get_tokenizer
label_desc: Use image node_id 6 for calling a global function with example usage: main() without return types