Storing a dictionary with random indices as keys and simulated values as values in hdf5 possibly using pytorch?
UPDATED QUESTION:
Each entry in an nd-array (say Sim_nDArray) correspond to a combination of parameters chosen from 8D search space. I have used Sim_nDArray.ravel() to convert it to 1D equivalent. Since I can not search from ~100 million entries, I decided to choose ~1 million random entries. I have corresponding ~1 million simulated values.
I have been able to simulate and save it. However, it seems that I have not been able to load data properly. I am getting error while overloading &quot;len&quot; during the declaration of the object: &quot;dataset&quot;.
I am planning to use hdf5 to store and read data. Can someone please guide me how to achieve this?
def add_trace(arrInd, arr):
    &quot;&quot;&quot; Add one trace to the dataset, keeping count of the # of traces written &quot;&quot;&quot;
    global ntraces
    dset1[ntraces, :] = arrInd
    dset2[ntraces, :] = arr
    ntraces += 1


def done():
    &quot;&quot;&quot; After all calls to add_trace_2, trim the dataset to size &quot;&quot;&quot;
    dset1.resize((ntraces, 1000))
    dset2.resize((ntraces, 1000))


import torch
from torch.utils.data import Dataset, DataLoader

class Dataset(torch.utils.data.Dataset):
    # Characterizes a dataset for PyTorch
    def __init__(self, dset1, dset2):
        'Initialization'
        self.dset1 = dset1
        self.dset2 = dset2
        self._data_len = len(dset1)

def __len__(self):
    # Denotes the total number of samples
    return len(self._data_len)

def __getitem__(self, index):
    # Generates one sample of data
    # Select sample
    ID = self.dset1[index]
    SimData = self.dset2[index]
    return ID, SimData


# Running the main.
if __name__ == '__main__':
    import h5py
    import numpy as np
    import timeit

    &quot;&quot;&quot; Re-initialize both datasets for the tests &quot;&quot;&quot;
    global data, N, dset1, dset2, ntraces
    N = 1000
    ################ WRITE #############################################################################################
    ## Creating two datasets
    f = h5py.File(&quot;randomDataset2.hdf5&quot;, 'w')
    dset1 = f.create_dataset('dataset1', (5000, 1000), maxshape=(None, 1000), dtype=&quot;float32&quot;, chunks=(1, 1000))
    dset2 = f.create_dataset('dataset2', (5000, 1000), maxshape=(None, 1000),
                             dtype=&quot;float32&quot;)  # DK: why faster if I do not define chunk

    dset1.resize((10001, 1000))  # Allocating extra space
    dset2.resize((10001, 1000))  # Allocating extra space

    ## TEST 1: Less efficient way of writing to hdf5
    ntraces = 0
    start1 = timeit.default_timer()
    for idx in range(N):
        IndxVec1 = np.random.randint(low=0, high=1000, size=1000);
        DataVec1 = np.random.random(1000)
        add_trace(IndxVec1, DataVec1)
    done()

    # All the program statements
    stop1 = timeit.default_timer()
    execution_time = stop1 - start1
    print(&quot;Program Executed in &quot; + str(execution_time))  # It returns time in seconds
    f.close()
    ##################################
    ## READING HDF files
    fr = h5py.File(&quot;randomDataset2.hdf5&quot;, 'r')
    dset10 = fr['dataset1']
    dset20 = fr['dataset2']
    fr.close()

    # Parameters
    params = {'batch_size': 64, 'shuffle': True, 'num_workers': 6}
    max_epochs = 100

    # Generators
    training_set = Dataset(dset10, dset20)
    training_generator = torch.utils.data.DataLoader(training_set, batch_size= 64, shuffle=True, num_workers= 6)

Error:
(PipInConda_DKU) dushyant20@DESKTOP-U96RKFC:/mnt/c/PyImageSearch/Sim_Write_n_Read$ python3 main.
py
Program Executed in 2.265893899995717
Traceback (most recent call last):
  File &quot;main.py&quot;, line 89, in &lt;module&gt;
    training_set = Dataset(dset10, dset20)
  File &quot;main.py&quot;, line 30, in __init__
    self._data_len = len(dset1)
  File &quot;h5py/_objects.pyx&quot;, line 54, in h5py._objects.with_phil.wrapper
  File &quot;h5py/_objects.pyx&quot;, line 55, in h5py._objects.with_phil.wrapper
  File &quot;/home/dushyant20/miniconda3/envs/PipInConda_DKU/lib/python3.8/site-packages/h5py/_hl/dat
aset.py&quot;, line 447, in __len__
    size = self.len()
  File &quot;/home/dushyant20/miniconda3/envs/PipInConda_DKU/lib/python3.8/site-packages/h5py/_hl/dat
aset.py&quot;, line 459, in len
    shape = self.shape
  File &quot;h5py/_objects.pyx&quot;, line 54, in h5py._objects.with_phil.wrapper
  File &quot;h5py/_objects.pyx&quot;, line 55, in h5py._objects.with_phil.wrapper
  File &quot;/home/dushyant20/miniconda3/envs/PipInConda_DKU/lib/python3.8/site-packages/h5py/_hl/dataset.py&quot;, line 286, in shape
    return self.id.shape
  File &quot;h5py/h5d.pyx&quot;, line 132, in h5py.h5d.DatasetID.shape.__get__
  File &quot;h5py/h5d.pyx&quot;, line 133, in h5py.h5d.DatasetID.shape.__get__
  File &quot;h5py/_objects.pyx&quot;, line 54, in h5py._objects.with_phil.wrapper
  File &quot;h5py/_objects.pyx&quot;, line 55, in h5py._objects.with_phil.wrapper
  File &quot;h5py/h5d.pyx&quot;, line 289, in h5py.h5d.DatasetID.get_space
ValueError: Not a dataset (not a dataset)


I don't use PyTorch, so can't comment on that (or run the entire code). Observations:

I noticed 2 methods for Class Dataset are not indented properly:
def __len__(self): and def __getitem__(self, index):. I assume
that's an error from cut-n-paste to your SO post...but you should
double check.
I ran your code (after commenting out the PyTorch stuff), and it runs
to completion and creates randomDataset2.hdf5 with 2 1000x1000
datasets. So, the problem is not in HDF5 file creation.
You are passing h5py datasets to your PyTorch generators. However, you close the HDF5 file BEFORE you call them. So, the data isn't available at that point in time. That could be the problem. Also, do your generators expect NumPy arrays or dataset objects? That could also cause a problem (once you fix the file close issue).

Other observations:

Use with / as : context manager when working with files to avoid
open/close issues.
Recommended practice is to put all imports at the top of the file.
Use this call if you want a NumPy array instead of a h5py dataset: arr10 = fr['dataset1'][:]

Modified code to reflect above is shown below.
I don't know if this will solve your problem...but it might get you pointed in the right direction.
import h5py
import numpy as np
import timeit
import torch
from torch.utils.data import Dataset, DataLoader   
    
def add_trace(arrInd, arr):
    &quot;&quot;&quot; Add one trace to the dataset, keeping count of the # of traces written &quot;&quot;&quot;
    global ntraces
    dset1[ntraces, :] = arrInd
    dset2[ntraces, :] = arr
    ntraces += 1


def done():
    &quot;&quot;&quot; After all calls to add_trace_2, trim the dataset to size &quot;&quot;&quot;
    dset1.resize((ntraces, 1000))
    dset2.resize((ntraces, 1000))

class Dataset(torch.utils.data.Dataset):
    # Characterizes a dataset for PyTorch
    def __init__(self, dset1, dset2):
        'Initialization'
        self.dset1 = dset1
        self.dset2 = dset2
        self._data_len = len(dset1)

    def __len__(self):
        # Denotes the total number of samples
        return len(self._data_len)
    
    def __getitem__(self, index):
        # Generates one sample of data
        # Select sample
        ID = self.dset1[index]
        SimData = self.dset2[index]
        return ID, SimData


# Running the main.
if __name__ == '__main__':

    &quot;&quot;&quot; Re-initialize both datasets for the tests &quot;&quot;&quot;
    global data, N, dset1, dset2, ntraces
    N = 1000
    ################ WRITE #############################################################################################
    ## Creating two datasets
    with h5py.File(&quot;randomDataset2.hdf5&quot;, 'w') as f:
        dset1 = f.create_dataset('dataset1', (5000, 1000), maxshape=(None, 1000), dtype=&quot;float32&quot;, chunks=(1, 1000))
        dset2 = f.create_dataset('dataset2', (5000, 1000), maxshape=(None, 1000),
                                 dtype=&quot;float32&quot;)  # DK: why faster if I do not define chunk
    
        dset1.resize((10001, 1000))  # Allocating extra space
        dset2.resize((10001, 1000))  # Allocating extra space
    
        ## TEST 1: Less efficient way of writing to hdf5
        ntraces = 0
        start1 = timeit.default_timer()
        for idx in range(N):
            IndxVec1 = np.random.randint(low=0, high=1000, size=1000);
            DataVec1 = np.random.random(1000)
            add_trace(IndxVec1, DataVec1)
        done()
    
        # All the program statements
        stop1 = timeit.default_timer()
        execution_time = stop1 - start1
        print(&quot;Program Executed in &quot; + str(execution_time))  # It returns time in seconds

    ##################################
    ## READING HDF files
    with h5py.File(&quot;randomDataset2.hdf5&quot;, 'r') as fr:
        dset10 = fr['dataset1']
        arr10 = fr['dataset1'][:]
        dset20 = fr['dataset2']
        arr20 = fr['dataset2'][:]

    # Parameters
    params = {'batch_size': 64, 'shuffle': True, 'num_workers': 6}
    max_epochs = 100

    # Generators
    training_set = Dataset(dset10, dset20)
    training_generator = torch.utils.data.DataLoader(training_set, batch_size= 64, shuffle=True, num_workers= 6)

