code: def test_indexed_dataset(args):
      ds = indexed_dataset.make_dataset(args.data, args.dataset_impl)
      tokenizer = build_tokenizer(args)
      print(len(ds.doc_idx))
      print(len(ds))
      print(ds.doc_idx[-1])
      if ds.supports_prefetch:
          # just prefetch the whole thing in test (so assume it is small)
          ds.prefetch(range(len(ds)))
      if args.count > len(ds.doc_idx) - 1:
          args.count = len(ds.doc_idx) - 1
  
      for i in range(args.count):
          start = ds.doc_idx[i]
          end = ds.doc_idx[i + 1]
          ids = ds[start:end]
          print(f"Document {i}:")
          print("--------------")
          for s in ids:
              assert len(s) > 0
              l = s.data.tolist()
              text = tokenizer.detokenize(l)
              print(text)
              print("---")
  
file_path: ColossalAI/examples/tutorial/sequence_parallel/data/datasets/test/test_indexed_dataset.py
imports: import argparse
  import os
  import sys
  from megatron.data import indexed_dataset
  from megatron.tokenizer import build_tokenizer
label_desc: Use image node_id 1 for calling a global function with example usage: test_indexed_dataset(args) without return types