verdict.py
def from_hdf5(
        cls,
        filepath,
        load_attributes = True,
        unpack = False,
        unpack_name = 'name',
        look_for_saved_jagged_arrs = True,
        jagged_flag = 'jagged',
        sparse = False,
        create_nonexistent = False,
    ):
        '''Load a HDF5 file as a verdict Dict.

        Args:
            filepath (str):
                Location to load the hdf5 file from.

            load_attributes (boolean):
                If True, load attributes stored in the hdf5 file's .attrs keys
                and return as a separate dictionary.

            unpack (boolean):
                If True and the inner-most groups are combined into a condensed
                DataFrame/array-like format, unpack them into a traditional
                structure.

            unpack_name (str):
                What name is used for the array containing the columns.

            look_for_saved_jagged_arrs (boolean):
                If True, check dataset names to see if the array is a jagged array.

            jagged_flag (str):
                Flag to indicate that this part of the hdf5 file contains part of
                an jagged array-like.

            sparse (boolean):
                If True use h5sparse instead of h5py, allowing reading of sparse matrices
                with reduced file size.

            create_nonexistent (boolean):
                If True if the file does not exist create it.
        '''

        # If using sparse matrices
        if sparse:
            hdf5_module = h5sparse
        else:
            hdf5_module = h5py

        if not os.path.exists( filepath ) and create_nonexistent:
            return Dict({})

        with hdf5_module.File( filepath, 'r' ) as f:

            def recursive_retrieve( current_path, key ):
                '''Function for recursively loading from an hdf5 file.

                Args:
                    current_path (str):
                        Current location in the hdf5 file.

                    key (str):
                        Key to load.
                '''

                # Update path
                current_path = '{}/{}'.format( current_path, key )

                item = f[current_path]

                if isinstance( item, hdf5_module.Group ):

                    group = f[current_path]
                    result = {}

                    # Sometimes the data is saved in a condensed, DataFrame-like,
                    # format. But when we load it we may want it back in the
                    # usual format.
                    if unpack and unpack_name in group.keys():
                        for i_key in group.keys():

                            if i_key == unpack_name:
                                continue

                            i_result = {}
                            ii_items = zip( group[unpack_name][...].astype( str ), group[i_key][...] )
                            for ii_key, ii_item in ii_items:
                                ii_item = if_byte_then_to_str( ii_item )
                                i_result[ii_key] = ii_item

                            result[i_key] = Dict( i_result )

                        return Dict( result )

                    else:
                        for i_key in group.keys():
                            result[i_key] = recursive_retrieve( current_path, i_key )

                        if look_for_saved_jagged_arrs:
                            # Look for saved jagged arrays
                            if i_key[:len(jagged_flag)] == jagged_flag:
                                unpacked = []
                                for i, result_i in enumerate( result ):
                                    unpacked_i = result['{}{}'.format( jagged_flag, i )]
                                    unpacked_i = if_byte_then_to_str( unpacked_i )
                                    unpacked.append( unpacked_i )
                                return unpacked

                        return Dict( result )

                elif isinstance( item, hdf5_module.Dataset ) or isinstance( item, h5py.Dataset ):
                    try:
                        arr = np.array( item[...] )
                    except NotImplementedError:
                        if sparse:
                            arr = item[()]
                        else:
                            raise Exception( 'Not sure how you got here. If sparse is not turned on the above functionality *should* be implemented.' )

                    if look_for_saved_jagged_arrs:
                        if 'jagged saved as filled' in item.attrs:
                            arr = filled_arr_to_jagged_arr(
                                item[...],
                                item.attrs['fill value'],
                            )
                            return arr

                    # Handle 0-length arrays
                    if arr.shape == ():
                        arr = arr[()]

                    arr = if_byte_then_to_str( arr )

                    return arr

            result = {}
            for key in f.keys():
                result[key] = recursive_retrieve( '', key )

            result = Dict( result )
            result.unpack_name = unpack_name

            # For shallow save files
            if unpack and unpack_name in result.keys():
                true_result = {}

                for i_key in result.keys():

                    if i_key == unpack_name:
                        continue

                    i_result = {}
                    ii_items = zip( result[unpack_name].astype( str ), result[i_key] )
                    for ii_key, ii_item in ii_items:
                        i_result[ii_key] = ii_item

                    true_result[i_key] = Dict( i_result )

                result = true_result

            # Load (or don't) attributes and return
            if load_attributes and len( f.attrs.keys() ) > 0:
                attrs = {}
                for key in f.attrs.keys():
                    attrs[key] = f.attrs[key]
                return result, attrs
            else:
                return result
Load a HDF5 file as a verdict Dict.

        Args:
            filepath (str):
                Location to load the hdf5 file from.

            load_attributes (boolean):
                If True, load attributes stored in the hdf5 file's .attrs keys
                and return as a separate dictionary.

            unpack (boolean):
                If True and the inner-most groups are combined into a condensed
                DataFrame/array-like format, unpack them into a traditional
                structure.

            unpack_name (str):
                What name is used for the array containing the columns.

            look_for_saved_jagged_arrs (boolean):
                If True, check dataset names to see if the array is a jagged array.

            jagged_flag (str):
                Flag to indicate that this part of the hdf5 file contains part of
                an jagged array-like.

            sparse (boolean):
                If True use h5sparse instead of h5py, allowing reading of sparse matrices
                with reduced file size.

            create_nonexistent (boolean):
                If True if the file does not exist create it.
        
Load a HDF5 file as a verdict Dict.
Load a HDF5 file as a verdict Dict .
def from_hdf5(
        cls,
        filepath,
        load_attributes = True,
        unpack = False,
        unpack_name = 'name',
        look_for_saved_jagged_arrs = True,
        jagged_flag = 'jagged',
        sparse = False,
        create_nonexistent = False,
    ):
        if sparse:
            hdf5_module = h5sparse
        else:
            hdf5_module = h5py
        if not os.path.exists( filepath ) and create_nonexistent:
            return Dict({})
        with hdf5_module.File( filepath, 'r' ) as f:
            def recursive_retrieve( current_path, key ):
                current_path = '{}/{}'.format( current_path, key )
                item = f[current_path]
                if isinstance( item, hdf5_module.Group ):
                    group = f[current_path]
                    result = {}
                    if unpack and unpack_name in group.keys():
                        for i_key in group.keys():
                            if i_key == unpack_name:
                                continue
                            i_result = {}
                            ii_items = zip( group[unpack_name][...].astype( str ), group[i_key][...] )
                            for ii_key, ii_item in ii_items:
                                ii_item = if_byte_then_to_str( ii_item )
                                i_result[ii_key] = ii_item
                            result[i_key] = Dict( i_result )
                        return Dict( result )
                    else:
                        for i_key in group.keys():
                            result[i_key] = recursive_retrieve( current_path, i_key )
                        if look_for_saved_jagged_arrs:
                            if i_key[:len(jagged_flag)] == jagged_flag:
                                unpacked = []
                                for i, result_i in enumerate( result ):
                                    unpacked_i = result['{}{}'.format( jagged_flag, i )]
                                    unpacked_i = if_byte_then_to_str( unpacked_i )
                                    unpacked.append( unpacked_i )
                                return unpacked
                        return Dict( result )
                elif isinstance( item, hdf5_module.Dataset ) or isinstance( item, h5py.Dataset ):
                    try:
                        arr = np.array( item[...] )
                    except NotImplementedError:
                        if sparse:
                            arr = item[()]
                        else:
                            raise Exception( 'Not sure how you got here. If sparse is not turned on the above functionality *should* be implemented.' )
                    if look_for_saved_jagged_arrs:
                        if 'jagged saved as filled' in item.attrs:
                            arr = filled_arr_to_jagged_arr(
                                item[...],
                                item.attrs['fill value'],
                            )
                            return arr
                    if arr.shape == ():
                        arr = arr[()]
                    arr = if_byte_then_to_str( arr )
                    return arr
            result = {}
            for key in f.keys():
                result[key] = recursive_retrieve( '', key )
            result = Dict( result )
            result.unpack_name = unpack_name
            if unpack and unpack_name in result.keys():
                true_result = {}
                for i_key in result.keys():
                    if i_key == unpack_name:
                        continue
                    i_result = {}
                    ii_items = zip( result[unpack_name].astype( str ), result[i_key] )
                    for ii_key, ii_item in ii_items:
                        i_result[ii_key] = ii_item
                    true_result[i_key] = Dict( i_result )
                result = true_result
            if load_attributes and len( f.attrs.keys() ) > 0:
                attrs = {}
                for key in f.attrs.keys():
                    attrs[key] = f.attrs[key]
                return result, attrs
            else:
                return result
def from_hdf5 ( cls , filepath , load_attributes = True , unpack = False , unpack_name = 'name' , look_for_saved_jagged_arrs = True , jagged_flag = 'jagged' , sparse = False , create_nonexistent = False , ) : if sparse : hdf5_module = h5sparse else : hdf5_module = h5py if not os . path . exists ( filepath ) and create_nonexistent : return Dict ( { } ) with hdf5_module . File ( filepath , 'r' ) as f : def recursive_retrieve ( current_path , key ) : '''Function for recursively loading from an hdf5 file.

                Args:
                    current_path (str):
                        Current location in the hdf5 file.

                    key (str):
                        Key to load.
                ''' current_path = '{}/{}' . format ( current_path , key ) item = f [ current_path ] if isinstance ( item , hdf5_module . Group ) : group = f [ current_path ] result = { } if unpack and unpack_name in group . keys ( ) : for i_key in group . keys ( ) : if i_key == unpack_name : continue i_result = { } ii_items = zip ( group [ unpack_name ] [ ... ] . astype ( str ) , group [ i_key ] [ ... ] ) for ii_key , ii_item in ii_items : ii_item = if_byte_then_to_str ( ii_item ) i_result [ ii_key ] = ii_item result [ i_key ] = Dict ( i_result ) return Dict ( result ) else : for i_key in group . keys ( ) : result [ i_key ] = recursive_retrieve ( current_path , i_key ) if look_for_saved_jagged_arrs : if i_key [ : len ( jagged_flag ) ] == jagged_flag : unpacked = [ ] for i , result_i in enumerate ( result ) : unpacked_i = result [ '{}{}' . format ( jagged_flag , i ) ] unpacked_i = if_byte_then_to_str ( unpacked_i ) unpacked . append ( unpacked_i ) return unpacked return Dict ( result ) elif isinstance ( item , hdf5_module . Dataset ) or isinstance ( item , h5py . Dataset ) : try : arr = np . array ( item [ ... ] ) except NotImplementedError : if sparse : arr = item [ ( ) ] else : raise Exception ( 'Not sure how you got here. If sparse is not turned on the above functionality *should* be implemented.' ) if look_for_saved_jagged_arrs : if 'jagged saved as filled' in item . attrs : arr = filled_arr_to_jagged_arr ( item [ ... ] , item . attrs [ 'fill value' ] , ) return arr if arr . shape == ( ) : arr = arr [ ( ) ] arr = if_byte_then_to_str ( arr ) return arr result = { } for key in f . keys ( ) : result [ key ] = recursive_retrieve ( '' , key ) result = Dict ( result ) result . unpack_name = unpack_name if unpack and unpack_name in result . keys ( ) : true_result = { } for i_key in result . keys ( ) : if i_key == unpack_name : continue i_result = { } ii_items = zip ( result [ unpack_name ] . astype ( str ) , result [ i_key ] ) for ii_key , ii_item in ii_items : i_result [ ii_key ] = ii_item true_result [ i_key ] = Dict ( i_result ) result = true_result if load_attributes and len ( f . attrs . keys ( ) ) > 0 : attrs = { } for key in f . attrs . keys ( ) : attrs [ key ] = f . attrs [ key ] return result , attrs else : return result
Load a HDF5 file as a verdict Dict.
Load a HDF5 file as a verdict Dict .
'''Load a HDF5 file as a verdict Dict.

        Args:
            filepath (str):
                Location to load the hdf5 file from.

            load_attributes (boolean):
                If True, load attributes stored in the hdf5 file's .attrs keys
                and return as a separate dictionary.

            unpack (boolean):
                If True and the inner-most groups are combined into a condensed
                DataFrame/array-like format, unpack them into a traditional
                structure.

            unpack_name (str):
                What name is used for the array containing the columns.

            look_for_saved_jagged_arrs (boolean):
                If True, check dataset names to see if the array is a jagged array.

            jagged_flag (str):
                Flag to indicate that this part of the hdf5 file contains part of
                an jagged array-like.

            sparse (boolean):
                If True use h5sparse instead of h5py, allowing reading of sparse matrices
                with reduced file size.

            create_nonexistent (boolean):
                If True if the file does not exist create it.
        ''' # If using sparse matrices '''Function for recursively loading from an hdf5 file.

                Args:
                    current_path (str):
                        Current location in the hdf5 file.

                    key (str):
                        Key to load.
                ''' # Update path # Sometimes the data is saved in a condensed, DataFrame-like, # format. But when we load it we may want it back in the # usual format. # Look for saved jagged arrays # Handle 0-length arrays # For shallow save files # Load (or don't) attributes and return
cls
filepath
Location to load the hdf5 file from.
Location to load the hdf5 file from .
load_attributes
If True, load attributes stored in the hdf5 file's .attrs keys
and return as a separate dictionary.
If True load attributes stored in the hdf5 file ' s . attrs keys and return as a separate dictionary .
unpack
If True and the inner-most groups are combined into a condensed
DataFrame/array-like format, unpack them into a traditional
structure.
If True and the inner - most groups are combined into a condensed DataFrame / array - like format unpack them into a traditional structure .
unpack_name
None
look_for_saved_jagged_arrs
If True, check dataset names to see if the array is a jagged array.
If True check dataset names to see if the array is a jagged array .
jagged_flag
Flag to indicate that this part of the hdf5 file contains part of
an jagged array-like.
Flag to indicate that this part of the hdf5 file contains part of an jagged array - like .
sparse
If True use h5sparse instead of h5py, allowing reading of sparse matrices
with reduced file size.
If True use h5sparse instead of h5py allowing reading of sparse matrices with reduced file size .
create_nonexistent
If True if the file does not exist create it.
If True if the file does not exist create it .