class_docstr: Dataset for iterating chunks.
  
  Requires:
  - List of indexed datasets
  - Chunk index array, with format:
      [dataset_idx, doc_id, start_idx, end_idx, bert_length])
class_docstr_tok: ["Dataset","for","iterating","chunks",".","Requires",":","-","List","of","indexed","datasets","-","Chunk","index","array",",","with","format",":","[","dataset_idx",",","doc_id",",","start_idx",",","end_idx",",","bert_length","]",")"]
code: def __getitem__(self, chunk_id):
      # Chunk start/end indexes.
      indexed_dataset_id, doc_id, token_start_idx, token_end_idx, _ = [
          value.item() for value in self.chunks[chunk_id]
      ]
      chunk_length = token_end_idx - token_start_idx
      indexed_dataset = self.indexed_datasets[indexed_dataset_id]
  
      # Chunk token ids.
      token_ids = indexed_dataset.get(
          doc_id, offset=token_start_idx, length=chunk_length
      )
  
      # Extend chunks to max_chunk_length by padding with EOD tokens.
      if chunk_length != self.max_chunk_length:
          assert (
              chunk_length < self.max_chunk_length
          ), "invalid chunk len."
          token_ids = token_ids.tolist()
          token_ids += [self.eod_token_id] * (
              self.max_chunk_length - chunk_length
          )
  
      return {
          "doc_id": doc_id,
          "text": np.array(token_ids, dtype=np.int64),
      }
  
file_path: megatron-lm/tools/retro/db/dataset.py
imports: import json
  import numpy
  import torch
  from tqdm import tqdm
  from megatron import get_args, print_rank_0
  from tools.retro.external_libs import h5py
  from tools.retro.utils import get_gpt_tokenizer
label_desc: Use image node_id 3 for calling the DBDataset obj's underlying member method code with example usage: obj.__getitem__(chunk_id) and returns: dict