class_docstr: A BlendableDataset implementation that uses less memory than the original implementation.
  Indices are computed algorithmically instead of storing them in memory.
  
  To test call: MemoryEfficientBlendableDataset.test_index_blending()
class_docstr_tok: ["A","BlendableDataset","implementation","that","uses","less","memory","than","the","original","implementation",".","Indices","are","computed","algorithmically","instead","of","storing","them","in","memory",".","To","test","call",":","MemoryEfficientBlendableDataset.test_index_blending","(",")"]
docstr: Visualize indices of blended dataset
docstr_tok: ["Visualize","indices","of","blended","dataset"]
code: def test_index_blending(cls):
      """Visualize indices of blended dataset"""
  
      import matplotlib.pyplot as plt
  
      plt.ion()
  
      class DS(torch.utils.data.Dataset):
          def __init__(self, size, data):
              self.size = size
              self.data = data
  
          def __len__(self):
              return self.size
  
          def __getitem__(self, idx):
              return self.data[idx]
  
      for weight_bins in [10, 100]:
          blend_ds = MemoryEfficientBlendableDataset(
              [DS(10, "a"), DS(10, "b"), DS(10, "c")],
              [0.5, 0.3, 0.2],
              50,
              weight_bins=weight_bins,
          )
  
          ds_sample_idx_list = [
              blend_ds.get_ds_sample_idx(i) for i in range(50)
          ]
          ds_list = list(zip(*ds_sample_idx_list))[0]
          sample_list = list(zip(*ds_sample_idx_list))[1]
  
          plt.figure()
          plt.plot(ds_list, label="ds idx")
          plt.plot(sample_list, label="sample")
          plt.legend()
          plt.grid()
          plt.title(f"weight_bins={weight_bins}")
  
file_path: NeMo/nemo/collections/nlp/data/language_modeling/megatron/blendable_dataset.py
imports: import time
  import numpy
  import torch
  from nemo.utils import logging
  from nemo.utils.app_state import AppState
label_desc: Use image node_id 5 for calling the MemoryEfficientBlendableDataset obj's underlying member method code with example usage: obj.test_index_blending(cls) and returns: self, self