class_docstr: Builder class for the MMapIndexedDataset class
  
  Args:
      bin_path (str): The path to the data (.bin) file
  
      dtype (Type[numpy.number], optional): The dtype of the index file. Defaults to numpy.int32.
  
      multimodal (bool, optional): Whether the dataset is multimodal. Defaults to False.
class_docstr_tok: ["Builder","class","for","the","MMapIndexedDataset","class","Args",":","bin_path","(","str",")",":","The","path","to","the","data","(",".bin",")","file","dtype","(","Type","[","numpy.number","]",",","optional",")",":","The","dtype","of","the","index","file",".","Defaults","to","numpy.int32",".","multimodal","(","bool",",","optional",")",":","Whether","the","dataset","is","multimodal",".","Defaults","to","False","."]
docstr: Add an entire document to the dataset
  
  Args:
      tensor (torch.Tensor): The document to add
      lengths (List[int]): The lengths of each item in the document
      modes (Optional[List[int]], optional): The modes for each item in the document.
      Defaults to None.
docstr_tok: ["Add","an","entire","document","to","the","dataset","Args",":","tensor","(","torch.Tensor",")",":","The","document","to","add","lengths","(","List","[","int","]",")",":","The","lengths","of","each","item","in","the","document","modes","(","Optional","[","List","[","int","]","]",",","optional",")",":","The","modes","for","each","item","in","the","document",".","Defaults","to","None","."]
code: def add_document(
      self,
      tensor: torch.Tensor,
      lengths: List[int],
      modes: Optional[List[int]] = None,
  ) -> None:
      """Add an entire document to the dataset
  
      Args:
          tensor (torch.Tensor): The document to add
          lengths (List[int]): The lengths of each item in the document
          modes (Optional[List[int]], optional): The modes for each item in the document.
          Defaults to None.
      """
      np_array = numpy.array(tensor, dtype=self.dtype)
      self.data_file.write(np_array.tobytes(order="C"))
      self.sequence_lengths.extend(lengths)
      self.document_indices.append(len(self.sequence_lengths))
      if self.multimodal:
          self.sequence_modes.extend(
              modes if modes is not None else [0] * lengths
          )
  
imports: import logging
  import os
  import shutil
  import struct
  import time
  from enum import Enum
  from functools import lru_cache
  from itertools import accumulate
  from types import TracebackType
  from typing import List, Optional, Tuple, Type, Union
  import numpy
  import torch
  from megatron.core.datasets.utils import log_single_rank
label_desc: Use image node_id 3 for calling the MMapIndexedDatasetBuilder obj's underlying member method code with example usage: obj.add_document(tensor, lengths, modes) without return types