class_docstr: Builder class for the MMapIndexedDataset class
  
  Args:
      bin_path (str): The path to the data (.bin) file
  
      dtype (Type[numpy.number], optional): The dtype of the index file. Defaults to numpy.int32.
  
      multimodal (bool, optional): Whether the dataset is multimodal. Defaults to False.
class_docstr_tok: ["Builder","class","for","the","MMapIndexedDataset","class","Args",":","bin_path","(","str",")",":","The","path","to","the","data","(",".bin",")","file","dtype","(","Type","[","numpy.number","]",",","optional",")",":","The","dtype","of","the","index","file",".","Defaults","to","numpy.int32",".","multimodal","(","bool",",","optional",")",":","Whether","the","dataset","is","multimodal",".","Defaults","to","False","."]
docstr: Add an entire MMapIndexedDataset to the dataset
  
  Args:
      path_prefix (str): The index (.idx) and data (.bin) prefix
docstr_tok: ["Add","an","entire","MMapIndexedDataset","to","the","dataset","Args",":","path_prefix","(","str",")",":","The","index","(",".idx",")","and","data","(",".bin",")","prefix"]
code: def add_index(self, path_prefix: str) -> None:
      """Add an entire MMapIndexedDataset to the dataset
  
      Args:
          path_prefix (str): The index (.idx) and data (.bin) prefix
      """
      # Concatenate index
      index = _IndexReader(
          get_idx_path(path_prefix), multimodal=self.multimodal
      )
      assert index.dtype == self.dtype
  
      offset = len(self.sequence_lengths)
      self.sequence_lengths.extend(index.sequence_lengths)
      self.document_indices.extend(
          (offset + index.document_indices)[1:]
      )
  
      if self.multimodal:
          self.sequence_modes.extend(index.sequence_modes)
  
      # Concatenate data
      with open(get_bin_path(path_prefix), "rb") as f:
          shutil.copyfileobj(f, self.data_file)
  
imports: import logging
  import os
  import shutil
  import struct
  import time
  from enum import Enum
  from functools import lru_cache
  from itertools import accumulate
  from types import TracebackType
  from typing import List, Optional, Tuple, Type, Union
  import numpy
  import torch
  from megatron.core.datasets.utils import log_single_rank
label_desc: Use image node_id 5 for calling the MMapIndexedDatasetBuilder obj's underlying member method code with example usage: obj.add_index(path_prefix) without return types