tools/combine_HDFs.py
def main(paths : list) -> None:
    """
    Combine many small HDF files (their paths defined in `paths`) into one large HDF file.
    """
    total_n_subgraphs = get_total_n_subgraphs(paths)
    dims              = get_dims()

    print(f"* Creating HDF file to contain {total_n_subgraphs} subgraphs")
    new_hdf_file = h5py.File(f"data/{dataset}/train.h5", "a")
    new_dataset_nodes = new_hdf_file.create_dataset("nodes",
                                                    (total_n_subgraphs, *dims["nodes"]),
                                                    dtype=np.dtype("int8"))
    new_dataset_edges = new_hdf_file.create_dataset("edges",
                                                    (total_n_subgraphs, *dims["edges"]),
                                                    dtype=np.dtype("int8"))
    new_dataset_APDs  = new_hdf_file.create_dataset("APDs",
                                                    (total_n_subgraphs, *dims["APDs"]),
                                                    dtype=np.dtype("int8"))

    print("* Combining data from smaller HDFs into a new larger HDF.")
    init_index = 0
    for path in paths:
        print("path:", path)
        hdf_file = h5py.File(path, "r")

        nodes = hdf_file.get("nodes")
        edges = hdf_file.get("edges")
        APDs  = hdf_file.get("APDs")

        n_subgraphs = nodes.shape[0]

        new_dataset_nodes[init_index:(init_index + n_subgraphs)] = nodes
        new_dataset_edges[init_index:(init_index + n_subgraphs)] = edges
        new_dataset_APDs[init_index:(init_index + n_subgraphs)]  = APDs

        init_index += n_subgraphs
        hdf_file.close()

    new_hdf_file.close()

    print("* Combining data from respective `train.csv` files into one.")
    csv_list = [f"data/{dataset}_{i}/train.csv" for i in range(1, n_dirs + 1)]

    ts_properties_old = None
    csv_files_processed = 0
    for path in csv_list:
        ts_properties     = load_ts_properties_from_csv(csv_path=path)
        ts_properties_new = {}
        if ts_properties_old:
            for key, value in ts_properties_old.items():
                if type(value) == float:
                    ts_properties_new[key] = (
                        value * csv_files_processed + ts_properties[key]
                    )/(csv_files_processed + 1)
                else:
                    new_list = []
                    for i, value_i in enumerate(value):
                        new_list.append(
                            float(
                                value_i * csv_files_processed + ts_properties[key][i]
                            )/(csv_files_processed + 1)
                        )
                    ts_properties_new[key] = new_list
        else:
            ts_properties_new = ts_properties
        ts_properties_old = ts_properties_new
        csv_files_processed += 1

    write_ts_properties_to_csv(ts_properties_dict=ts_properties_new)

    Combine many small HDF files (their paths defined in `paths`) into one large HDF file.
    
Combine many small HDF files (their paths defined in `paths`) into one large HDF file.
Combine many small HDF files ( their paths defined in ` paths ` ) into one large HDF file .
def main(paths : list) -> None:
    total_n_subgraphs = get_total_n_subgraphs(paths)
    dims              = get_dims()
    print(f"* Creating HDF file to contain {total_n_subgraphs} subgraphs")
    new_hdf_file = h5py.File(f"data/{dataset}/train.h5", "a")
    new_dataset_nodes = new_hdf_file.create_dataset("nodes",
                                                    (total_n_subgraphs, *dims["nodes"]),
                                                    dtype=np.dtype("int8"))
    new_dataset_edges = new_hdf_file.create_dataset("edges",
                                                    (total_n_subgraphs, *dims["edges"]),
                                                    dtype=np.dtype("int8"))
    new_dataset_APDs  = new_hdf_file.create_dataset("APDs",
                                                    (total_n_subgraphs, *dims["APDs"]),
                                                    dtype=np.dtype("int8"))
    print("* Combining data from smaller HDFs into a new larger HDF.")
    init_index = 0
    for path in paths:
        print("path:", path)
        hdf_file = h5py.File(path, "r")
        nodes = hdf_file.get("nodes")
        edges = hdf_file.get("edges")
        APDs  = hdf_file.get("APDs")
        n_subgraphs = nodes.shape[0]
        new_dataset_nodes[init_index:(init_index + n_subgraphs)] = nodes
        new_dataset_edges[init_index:(init_index + n_subgraphs)] = edges
        new_dataset_APDs[init_index:(init_index + n_subgraphs)]  = APDs
        init_index += n_subgraphs
        hdf_file.close()
    new_hdf_file.close()
    print("* Combining data from respective `train.csv` files into one.")
    csv_list = [f"data/{dataset}_{i}/train.csv" for i in range(1, n_dirs + 1)]
    ts_properties_old = None
    csv_files_processed = 0
    for path in csv_list:
        ts_properties     = load_ts_properties_from_csv(csv_path=path)
        ts_properties_new = {}
        if ts_properties_old:
            for key, value in ts_properties_old.items():
                if type(value) == float:
                    ts_properties_new[key] = (
                        value * csv_files_processed + ts_properties[key]
                    )/(csv_files_processed + 1)
                else:
                    new_list = []
                    for i, value_i in enumerate(value):
                        new_list.append(
                            float(
                                value_i * csv_files_processed + ts_properties[key][i]
                            )/(csv_files_processed + 1)
                        )
                    ts_properties_new[key] = new_list
        else:
            ts_properties_new = ts_properties
        ts_properties_old = ts_properties_new
        csv_files_processed += 1
    write_ts_properties_to_csv(ts_properties_dict=ts_properties_new)
def main ( paths : list ) -> None : total_n_subgraphs = get_total_n_subgraphs ( paths ) dims = get_dims ( ) print ( f"* Creating HDF file to contain {total_n_subgraphs} subgraphs" ) new_hdf_file = h5py . File ( f"data/{dataset}/train.h5" , "a" ) new_dataset_nodes = new_hdf_file . create_dataset ( "nodes" , ( total_n_subgraphs , * dims [ "nodes" ] ) , dtype = np . dtype ( "int8" ) ) new_dataset_edges = new_hdf_file . create_dataset ( "edges" , ( total_n_subgraphs , * dims [ "edges" ] ) , dtype = np . dtype ( "int8" ) ) new_dataset_APDs = new_hdf_file . create_dataset ( "APDs" , ( total_n_subgraphs , * dims [ "APDs" ] ) , dtype = np . dtype ( "int8" ) ) print ( "* Combining data from smaller HDFs into a new larger HDF." ) init_index = 0 for path in paths : print ( "path:" , path ) hdf_file = h5py . File ( path , "r" ) nodes = hdf_file . get ( "nodes" ) edges = hdf_file . get ( "edges" ) APDs = hdf_file . get ( "APDs" ) n_subgraphs = nodes . shape [ 0 ] new_dataset_nodes [ init_index : ( init_index + n_subgraphs ) ] = nodes new_dataset_edges [ init_index : ( init_index + n_subgraphs ) ] = edges new_dataset_APDs [ init_index : ( init_index + n_subgraphs ) ] = APDs init_index += n_subgraphs hdf_file . close ( ) new_hdf_file . close ( ) print ( "* Combining data from respective `train.csv` files into one." ) csv_list = [ f"data/{dataset}_{i}/train.csv" for i in range ( 1 , n_dirs + 1 ) ] ts_properties_old = None csv_files_processed = 0 for path in csv_list : ts_properties = load_ts_properties_from_csv ( csv_path = path ) ts_properties_new = { } if ts_properties_old : for key , value in ts_properties_old . items ( ) : if type ( value ) == float : ts_properties_new [ key ] = ( value * csv_files_processed + ts_properties [ key ] ) / ( csv_files_processed + 1 ) else : new_list = [ ] for i , value_i in enumerate ( value ) : new_list . append ( float ( value_i * csv_files_processed + ts_properties [ key ] [ i ] ) / ( csv_files_processed + 1 ) ) ts_properties_new [ key ] = new_list else : ts_properties_new = ts_properties ts_properties_old = ts_properties_new csv_files_processed += 1 write_ts_properties_to_csv ( ts_properties_dict = ts_properties_new )
Combine many small HDF files (their paths defined in `paths`) into one large HDF file.
Combine many small HDF files ( their paths defined in ` paths ` ) into one large HDF file .
"""
    Combine many small HDF files (their paths defined in `paths`) into one large HDF file.
    """
paths
list