class_docstr: Conjugating class for a set of MegatronDataset instances
  
  Args:
      datasets (List[MegatronDataset]): The MegatronDataset instances to blend
  
      weights (List[float]): The weights which determines the dataset blend ratios
  
      size (int): The number of samples to draw from the blend
  
      config (BlendedMegatronDatasetConfig): The config object which informs dataset creation
  
  Raises:
      RuntimeError: When the dataset has fewer or more samples than 'size' post-initialization
class_docstr_tok: ["Conjugating","class","for","a","set","of","MegatronDataset","instances","Args",":","datasets","(","List","[","MegatronDataset","]",")",":","The","MegatronDataset","instances","to","blend","weights","(","List","[","float","]",")",":","The","weights","which","determines","the","dataset","blend","ratios","size","(","int",")",":","The","number","of","samples","to","draw","from","the","blend","config","(","BlendedMegatronDatasetConfig",")",":","The","config","object","which","informs","dataset","creation","Raises",":","RuntimeError",":","When","the","dataset","has","fewer","or","more","samples","than","'size","'","post-initialization"]
code: def __init__(
      self,
      datasets: List[MegatronDataset],
      weights: List[float],
      size: int,
      config: BlendedMegatronDatasetConfig,
  ) -> None:
      assert len(datasets) < 32767
      assert len(datasets) == len(weights)
      assert numpy.isclose(sum(weights), 1.0)
      assert all(map(lambda _: type(_) == type(datasets[0]), datasets))
  
      # Alert user to unnecessary blending
      if len(datasets) == 1:
          log_single_rank(
              logger,
              logging.WARNING,
              f"Building a BlendedDataset for a single MegatronDataset",
          )
  
      # Redundant normalization for bitwise identical comparison with Megatron-LM
      weights = normalize(weights)
  
      self.datasets = datasets
      self.weights = weights
      self.size = size
      self.config = config
  
      unique_identifiers = OrderedDict()
      unique_identifiers["class"] = type(self).__name__
      unique_identifiers["datasets"] = [
          dataset.unique_identifiers for dataset in self.datasets
      ]
      unique_identifiers["weights"] = self.weights
      unique_identifiers["size"] = self.size
  
      self.unique_description = json.dumps(unique_identifiers, indent=4)
      self.unique_description_hash = hashlib.md5(
          self.unique_description.encode("utf-8")
      ).hexdigest()
  
      (
          self.dataset_index,
          self.dataset_sample_index,
      ) = self._build_indices()
  
      # Check size
      _ = self[self.size - 1]
      try:
          _ = self[self.size]
          raise RuntimeError(
              f"{type(self).__name__} size is improperly bounded"
          )
      except IndexError:
          log_single_rank(
              logger,
              logging.INFO,
              f"> {type(self).__name__} length: {len(self)}",
          )
  
file_path: megatron-lm/megatron/core/datasets/blended_dataset.py
imports: import hashlib
  import json
  import logging
  import os
  import time
  from collections import OrderedDict
  from typing import Dict, List, Tuple, Union
  import numpy
  import torch
  from megatron.core.datasets.blended_megatron_dataset_config import BlendedMegatronDatasetConfig
  from megatron.core.datasets.megatron_dataset import MegatronDataset
  from megatron.core.datasets.utils import log_single_rank, normalize
label_desc: Use image node_id 1 to create a new BlendedDataset object from inherited base classes: torch.utils.data with example: obj = BlendedDataset(datasets, weights, size, config)