blimpy/waterfall.py
def __write_to_hdf5_heavy(self, filename_out, *args, **kwargs):
        """ Write data to HDF5 file.

        Args:
            filename_out (str): Name of output file
        """

        block_size = 0

        #Note that a chunk is not a blob!!
        chunk_dim = self.__get_chunk_dimensions()
        blob_dim = self.__get_blob_dimensions(chunk_dim)
        n_blobs = self.container.calc_n_blobs(blob_dim)

        with h5py.File(filename_out, 'w') as h5:

            h5.attrs[b'CLASS'] = b'FILTERBANK'
            h5.attrs[b'VERSION'] = b'1.0'

            if HAS_BITSHUFFLE:
                bs_compression = bitshuffle.h5.H5FILTER
                bs_compression_opts = (block_size, bitshuffle.h5.H5_COMPRESS_LZ4)
            else:
                bs_compression = None
                bs_compression_opts = None
                logger.warning("Warning: bitshuffle not found. No compression applied.")

            dset = h5.create_dataset('data',
                            shape=self.selection_shape,
                            chunks=chunk_dim,
                            compression=bs_compression,
                            compression_opts=bs_compression_opts,
                            dtype=self.data.dtype)

            dset_mask = h5.create_dataset('mask',
                            shape=self.selection_shape,
                            chunks=chunk_dim,
                            compression=bs_compression,
                            compression_opts=bs_compression_opts,
                            dtype='uint8')

            dset.dims[0].label = b"frequency"
            dset.dims[1].label = b"feed_id"
            dset.dims[2].label = b"time"

            dset_mask.dims[0].label = b"frequency"
            dset_mask.dims[1].label = b"feed_id"
            dset_mask.dims[2].label = b"time"

            # Copy over header information as attributes
            for key, value in self.header.items():
                dset.attrs[key] = value

            if blob_dim[self.freq_axis] < self.selection_shape[self.freq_axis]:

                logger.info('Using %i n_blobs to write the data.'% n_blobs)
                for ii in range(0, n_blobs):
                    logger.info('Reading %i of %i' % (ii + 1, n_blobs))

                    bob = self.container.read_blob(blob_dim,n_blob=ii)

                    #-----
                    #Using channels instead of frequency.
                    c_start = self.container.chan_start_idx + ii*blob_dim[self.freq_axis]
                    t_start = self.container.t_start + (c_start/self.selection_shape[self.freq_axis])*blob_dim[self.time_axis]
                    t_stop = t_start + blob_dim[self.time_axis]

                    # Reverse array if frequency axis is flipped
#                     if self.header['foff'] < 0:
#                         c_stop = self.selection_shape[self.freq_axis] - (c_start)%self.selection_shape[self.freq_axis]
#                         c_start = c_stop - blob_dim[self.freq_axis]
#                     else:
                    c_start = (c_start)%self.selection_shape[self.freq_axis]
                    c_stop = c_start + blob_dim[self.freq_axis]
                    #-----

                    logger.debug(t_start,t_stop,c_start,c_stop)

                    dset[t_start:t_stop,0,c_start:c_stop] = bob[:]

            else:

                logger.info('Using %i n_blobs to write the data.'% n_blobs)
                for ii in range(0, n_blobs):
                    logger.info('Reading %i of %i' % (ii + 1, n_blobs))

                    bob = self.container.read_blob(blob_dim,n_blob=ii)
                    t_start = self.container.t_start + ii*blob_dim[self.time_axis]

                    #This prevents issues when the last blob is smaller than the others in time
                    if (ii+1)*blob_dim[self.time_axis] > self.n_ints_in_file:
                        t_stop = self.n_ints_in_file
                    else:
                        t_stop = (ii+1)*blob_dim[self.time_axis]

                    dset[t_start:t_stop] = bob[:]
def __write_to_hdf5_heavy(self, filename_out, *args, **kwargs):
        """ Write data to HDF5 file.

        Args:
            filename_out (str): Name of output file
        """

        block_size = 0

        #Note that a chunk is not a blob!!
        chunk_dim = self.__get_chunk_dimensions()
        blob_dim = self.__get_blob_dimensions(chunk_dim)
        n_blobs = self.container.calc_n_blobs(blob_dim)

        with h5py.File(filename_out, 'w') as h5:

            h5.attrs[b'CLASS'] = b'FILTERBANK'
            h5.attrs[b'VERSION'] = b'1.0'

            if HAS_BITSHUFFLE:
                bs_compression = bitshuffle.h5.H5FILTER
                bs_compression_opts = (block_size, bitshuffle.h5.H5_COMPRESS_LZ4)
            else:
                bs_compression = None
                bs_compression_opts = None
                logger.warning("Warning: bitshuffle not found. No compression applied.")

            dset = h5.create_dataset('data',
                            shape=self.selection_shape,
                            chunks=chunk_dim,
                            compression=bs_compression,
                            compression_opts=bs_compression_opts,
                            dtype=self.data.dtype)

            dset_mask = h5.create_dataset('mask',
                            shape=self.selection_shape,
                            chunks=chunk_dim,
                            compression=bs_compression,
                            compression_opts=bs_compression_opts,
                            dtype='uint8')

            dset.dims[0].label = b"frequency"
            dset.dims[1].label = b"feed_id"
            dset.dims[2].label = b"time"

            dset_mask.dims[0].label = b"frequency"
            dset_mask.dims[1].label = b"feed_id"
            dset_mask.dims[2].label = b"time"

            # Copy over header information as attributes
            for key, value in self.header.items():
                dset.attrs[key] = value

            if blob_dim[self.freq_axis] < self.selection_shape[self.freq_axis]:

                logger.info('Using %i n_blobs to write the data.'% n_blobs)
                for ii in range(0, n_blobs):
                    logger.info('Reading %i of %i' % (ii + 1, n_blobs))

                    bob = self.container.read_blob(blob_dim,n_blob=ii)

                    #-----
                    #Using channels instead of frequency.
                    c_start = self.container.chan_start_idx + ii*blob_dim[self.freq_axis]
                    t_start = self.container.t_start + (c_start/self.selection_shape[self.freq_axis])*blob_dim[self.time_axis]
                    t_stop = t_start + blob_dim[self.time_axis]

                    # Reverse array if frequency axis is flipped
#                     if self.header['foff'] < 0:
#                         c_stop = self.selection_shape[self.freq_axis] - (c_start)%self.selection_shape[self.freq_axis]
#                         c_start = c_stop - blob_dim[self.freq_axis]
#                     else:
                    c_start = (c_start)%self.selection_shape[self.freq_axis]
                    c_stop = c_start + blob_dim[self.freq_axis]
                    #-----

                    logger.debug(t_start,t_stop,c_start,c_stop)

                    dset[t_start:t_stop,0,c_start:c_stop] = bob[:]

            else:

                logger.info('Using %i n_blobs to write the data.'% n_blobs)
                for ii in range(0, n_blobs):
                    logger.info('Reading %i of %i' % (ii + 1, n_blobs))

                    bob = self.container.read_blob(blob_dim,n_blob=ii)
                    t_start = self.container.t_start + ii*blob_dim[self.time_axis]

                    #This prevents issues when the last blob is smaller than the others in time
                    if (ii+1)*blob_dim[self.time_axis] > self.n_ints_in_file:
                        t_stop = self.n_ints_in_file
                    else:
                        t_stop = (ii+1)*blob_dim[self.time_axis]

                    dset[t_start:t_stop] = bob[:]
def __write_to_hdf5_heavy ( self , filename_out , * args , * * kwargs ) : block_size = 0 #Note that a chunk is not a blob!! chunk_dim = self . __get_chunk_dimensions ( ) blob_dim = self . __get_blob_dimensions ( chunk_dim ) n_blobs = self . container . calc_n_blobs ( blob_dim ) with h5py . File ( filename_out , 'w' ) as h5 : h5 . attrs [ b'CLASS' ] = b'FILTERBANK' h5 . attrs [ b'VERSION' ] = b'1.0' if HAS_BITSHUFFLE : bs_compression = bitshuffle . h5 . H5FILTER bs_compression_opts = ( block_size , bitshuffle . h5 . H5_COMPRESS_LZ4 ) else : bs_compression = None bs_compression_opts = None logger . warning ( "Warning: bitshuffle not found. No compression applied." ) dset = h5 . create_dataset ( 'data' , shape = self . selection_shape , chunks = chunk_dim , compression = bs_compression , compression_opts = bs_compression_opts , dtype = self . data . dtype ) dset_mask = h5 . create_dataset ( 'mask' , shape = self . selection_shape , chunks = chunk_dim , compression = bs_compression , compression_opts = bs_compression_opts , dtype = 'uint8' ) dset . dims [ 0 ] . label = b"frequency" dset . dims [ 1 ] . label = b"feed_id" dset . dims [ 2 ] . label = b"time" dset_mask . dims [ 0 ] . label = b"frequency" dset_mask . dims [ 1 ] . label = b"feed_id" dset_mask . dims [ 2 ] . label = b"time" # Copy over header information as attributes for key , value in self . header . items ( ) : dset . attrs [ key ] = value if blob_dim [ self . freq_axis ] < self . selection_shape [ self . freq_axis ] : logger . info ( 'Using %i n_blobs to write the data.' % n_blobs ) for ii in range ( 0 , n_blobs ) : logger . info ( 'Reading %i of %i' % ( ii + 1 , n_blobs ) ) bob = self . container . read_blob ( blob_dim , n_blob = ii ) #----- #Using channels instead of frequency. c_start = self . container . chan_start_idx + ii * blob_dim [ self . freq_axis ] t_start = self . container . t_start + ( c_start / self . selection_shape [ self . freq_axis ] ) * blob_dim [ self . time_axis ] t_stop = t_start + blob_dim [ self . time_axis ] # Reverse array if frequency axis is flipped #                     if self.header['foff'] < 0: #                         c_stop = self.selection_shape[self.freq_axis] - (c_start)%self.selection_shape[self.freq_axis] #                         c_start = c_stop - blob_dim[self.freq_axis] #                     else: c_start = ( c_start ) % self . selection_shape [ self . freq_axis ] c_stop = c_start + blob_dim [ self . freq_axis ] #----- logger . debug ( t_start , t_stop , c_start , c_stop ) dset [ t_start : t_stop , 0 , c_start : c_stop ] = bob [ : ] else : logger . info ( 'Using %i n_blobs to write the data.' % n_blobs ) for ii in range ( 0 , n_blobs ) : logger . info ( 'Reading %i of %i' % ( ii + 1 , n_blobs ) ) bob = self . container . read_blob ( blob_dim , n_blob = ii ) t_start = self . container . t_start + ii * blob_dim [ self . time_axis ] #This prevents issues when the last blob is smaller than the others in time if ( ii + 1 ) * blob_dim [ self . time_axis ] > self . n_ints_in_file : t_stop = self . n_ints_in_file else : t_stop = ( ii + 1 ) * blob_dim [ self . time_axis ] dset [ t_start : t_stop ] = bob [ : ]
Write data to HDF5 file.

        Args:
            filename_out (str): Name of output file
Write data to HDF5 file .