class_docstr: Dataset for iterating chunks.
  
  Requires:
  - List of indexed datasets
  - Chunk index array, with format:
      [dataset_idx, doc_id, start_idx, end_idx, bert_length])
class_docstr_tok: ["Dataset","for","iterating","chunks",".","Requires",":","-","List","of","indexed","datasets","-","Chunk","index","array",",","with","format",":","[","dataset_idx",",","doc_id",",","start_idx",",","end_idx",",","bert_length","]",")"]
docstr: Load the dataset & document ids.
  
  Load the dataset id & document id of each chunk in the database, to
  be used for causality filtering during querying.
docstr_tok: ["Load","the","dataset","&","document","ids",".","Load","the","dataset","id","&","document","id","of","each","chunk","in","the","database",",","to","be","used","for","causality","filtering","during","querying","."]
code: def load_doc_tuples(self):
      """Load the dataset & document ids.
  
      Load the dataset id & document id of each chunk in the database, to
      be used for causality filtering during querying.
      """
      self.doc_tuples = np.zeros(shape=(len(self), 2), dtype="uint32")
      block_size = int(1e6)
      for start_idx in tqdm(range(0, len(self), block_size)):
          end_idx = min(len(self), start_idx + block_size)
          self.doc_tuples[start_idx:end_idx] = self.chunks[
              start_idx:end_idx, :2
          ]
  
file_path: megatron-lm/tools/retro/db/dataset.py
imports: import json
  import numpy
  import torch
  from tqdm import tqdm
  from megatron import get_args, print_rank_0
  from tools.retro.external_libs import h5py
  from tools.retro.utils import get_gpt_tokenizer
label_desc: Use image node_id 4 for calling the DBDataset obj's underlying member method code with example usage: obj.load_doc_tuples() without return types