paleo/profilers/flops_profiler.py
def _profile_conv2d_fft(self, layer, tiling=False):
        """Returns the flops of convolution 2d."""

        def _fft_flops(fft_dim, mode='r2c', filter_1d=False):
            # Note this is not an accurate flops count.
            # Pad to the nearest power of 2.
            tile_size = math.sqrt(fft_dim)
            tile_size_2 = _to_pow2(tile_size)
            f = 2 * tile_size_2 * 5 * tile_size_2 * (math.log(tile_size_2) /
                                                     math.log(2))
            if filter_1d:
                f /= 2
            if mode == 'r2c':
                f /= 2
            return f

        def _to_pow2(n):
            return math.pow(2, math.ceil(math.log(n) / math.log(2)))

        filter_1d = False
        filter_size = layer.filters[0] * layer.filters[1]
        if filter_size in [layer.filters[0], layer.filters[1]]:
            # one of the filter dimension is 1.
            filter_1d = True

        if tiling:
            _TILE_SIZE = 32
            h_tiles = (layer.inputs[1] + _TILE_SIZE - 1) // _TILE_SIZE
            w_tiles = (layer.inputs[2] + _TILE_SIZE - 1) // _TILE_SIZE
            fft_size = _TILE_SIZE ** 2
            num_tiles = h_tiles * w_tiles
            self._logger.info('Tile FFT: %d (%dx%d) 1D: %s' %
                              (num_tiles, _TILE_SIZE, _TILE_SIZE, filter_1d))
            tile_size = _TILE_SIZE
        else:
            # Filters and inputs are padded to the same size.
            # padded_h = (layer.inputs[1] + layer._pad_h +
            #             layer.filters[0] // 2)
            # padded_w = (layer.inputs[2] + layer._pad_w +
            #             layer.filters[1] // 2)
            padded_h = max(layer.inputs[1] + layer._pad_h * 2,
                           layer.filters[0])
            padded_w = max(layer.inputs[2] + layer._pad_w * 2,
                           layer.filters[1])
            fft_size = padded_h * padded_w
            num_tiles = 1
            self._logger.debug('FFT size: %dx%d (%dx%d) 1D: %s' %
                               (_to_pow2(padded_h), _to_pow2(padded_w),
                                padded_h, padded_w, filter_1d))
            tile_size = max(padded_h, padded_w)

        # Calculate time for filters separateily.
        comp_time, comm_time = 0, 0
        comp_time_filters, comm_time_filters = 0, 0

        # (1) fft2d r2c.
        inputs_nc = layer.inputs[0] * layer.inputs[3]
        filters_ck = layer.filters[2] * layer.filters[3]
        comp_time += num_tiles * self._estimate_comp_time(
            inputs_nc * _fft_flops(fft_size, filter_1d=filter_1d))
        comp_time_filters += self._estimate_comp_time(filters_ck * _fft_flops(
            fft_size, filter_1d=filter_1d))

        # Read inputs and write transformed inputs.
        comm_time += num_tiles * self._estimate_comm_time(
            inputs_nc * fft_size * _BYTES_FLOAT)
        comm_time += num_tiles * self._estimate_comm_time(
            inputs_nc * fft_size * _BYTES_COMPLEX)
        # Read filters and write transformed filters.
        # Padding time are not considered here.
        comm_time_filters += self._estimate_comm_time(filters_ck * fft_size *
                                                      _BYTES_FLOAT)
        if filter_1d:
            comm_time_filters += self._estimate_comm_time(
                filters_ck * tile_size * _BYTES_COMPLEX)
        else:
            comm_time_filters += self._estimate_comm_time(
                filters_ck * fft_size * _BYTES_COMPLEX)

        # (2) Elementwise multiplication.
        # Complex number: add 2, muliply 4
        comp_time += num_tiles * self._estimate_comp_time(
            4 * layer.inputs[0] * layer.inputs[3] * layer.filters[3] *
            fft_size)

        # Pipe: Writing results while doing FFT for the next set of tiles?
        # Read transformed inputs.
        comm_time += num_tiles * self._estimate_comm_time(
            inputs_nc * fft_size * _BYTES_COMPLEX)
        # Read transformed filters.
        if filter_1d:
            comm_time_filters += num_tiles * self._estimate_comm_time(
                filters_ck * tile_size * _BYTES_COMPLEX)
        else:
            comm_time_filters += num_tiles * self._estimate_comm_time(
                filters_ck * fft_size * _BYTES_COMPLEX)
        # Write results.
        comm_time += num_tiles * self._estimate_comm_time(
            layer.inputs[0] * layer.filters[3] * fft_size * _BYTES_COMPLEX)

        # Assume additional memory needed is only for one tile.
        # FIXME: how to match this number with cuDNN suggestion?
        mem_inputs = inputs_nc * fft_size * _BYTES_COMPLEX
        mem_filters = filters_ck * fft_size * _BYTES_COMPLEX
        mem_outputs = (layer.inputs[0] * layer.filters[3] * fft_size *
                       _BYTES_COMPLEX)
        self._logger.debug('FFT mem: %d MB (%d, %d, %d)' % (
            (mem_inputs + mem_filters + mem_outputs) / 2 ** 20, mem_inputs / 2
            ** 20, mem_filters / 2 ** 20, mem_outputs / 2 ** 20))

        # (3) fft2d c2r on num_tiles.
        comp_time += num_tiles * self._estimate_comp_time(
            layer.inputs[0] * layer.filters[3] * _fft_flops(
                fft_size, 'c2r', filter_1d=filter_1d))

        # Read complex.
        comm_time += self._estimate_comm_time(
            layer.inputs[0] * layer.filters[3] * fft_size * _BYTES_COMPLEX)
        # Write outputs. Ignore clipping time here.
        comm_time += self._estimate_comm_time(
            layer.outputs[0] * layer.outputs[1] * layer.outputs[2] *
            layer.outputs[3] * _BYTES_FLOAT)

        # Do not multiple batch size for filters.
        comp_time += comp_time_filters
        comm_time += comm_time_filters

        self._logger.debug('FFT estimates: %f = %f + %f' %
                           (comp_time + comm_time, comp_time, comm_time))
        return TimeMeasure(comp_time=comp_time, comm_time=comm_time)
Returns the flops of convolution 2d.
Returns the flops of convolution 2d.
Returns the flops of convolution 2d .
def _profile_conv2d_fft(self, layer, tiling=False):
        def _fft_flops(fft_dim, mode='r2c', filter_1d=False):
            tile_size = math.sqrt(fft_dim)
            tile_size_2 = _to_pow2(tile_size)
            f = 2 * tile_size_2 * 5 * tile_size_2 * (math.log(tile_size_2) /
                                                     math.log(2))
            if filter_1d:
                f /= 2
            if mode == 'r2c':
                f /= 2
            return f
        def _to_pow2(n):
            return math.pow(2, math.ceil(math.log(n) / math.log(2)))
        filter_1d = False
        filter_size = layer.filters[0] * layer.filters[1]
        if filter_size in [layer.filters[0], layer.filters[1]]:
            filter_1d = True
        if tiling:
            _TILE_SIZE = 32
            h_tiles = (layer.inputs[1] + _TILE_SIZE - 1) // _TILE_SIZE
            w_tiles = (layer.inputs[2] + _TILE_SIZE - 1) // _TILE_SIZE
            fft_size = _TILE_SIZE ** 2
            num_tiles = h_tiles * w_tiles
            self._logger.info('Tile FFT: %d (%dx%d) 1D: %s' %
                              (num_tiles, _TILE_SIZE, _TILE_SIZE, filter_1d))
            tile_size = _TILE_SIZE
        else:
            padded_h = max(layer.inputs[1] + layer._pad_h * 2,
                           layer.filters[0])
            padded_w = max(layer.inputs[2] + layer._pad_w * 2,
                           layer.filters[1])
            fft_size = padded_h * padded_w
            num_tiles = 1
            self._logger.debug('FFT size: %dx%d (%dx%d) 1D: %s' %
                               (_to_pow2(padded_h), _to_pow2(padded_w),
                                padded_h, padded_w, filter_1d))
            tile_size = max(padded_h, padded_w)
        comp_time, comm_time = 0, 0
        comp_time_filters, comm_time_filters = 0, 0
        inputs_nc = layer.inputs[0] * layer.inputs[3]
        filters_ck = layer.filters[2] * layer.filters[3]
        comp_time += num_tiles * self._estimate_comp_time(
            inputs_nc * _fft_flops(fft_size, filter_1d=filter_1d))
        comp_time_filters += self._estimate_comp_time(filters_ck * _fft_flops(
            fft_size, filter_1d=filter_1d))
        comm_time += num_tiles * self._estimate_comm_time(
            inputs_nc * fft_size * _BYTES_FLOAT)
        comm_time += num_tiles * self._estimate_comm_time(
            inputs_nc * fft_size * _BYTES_COMPLEX)
        comm_time_filters += self._estimate_comm_time(filters_ck * fft_size *
                                                      _BYTES_FLOAT)
        if filter_1d:
            comm_time_filters += self._estimate_comm_time(
                filters_ck * tile_size * _BYTES_COMPLEX)
        else:
            comm_time_filters += self._estimate_comm_time(
                filters_ck * fft_size * _BYTES_COMPLEX)
        comp_time += num_tiles * self._estimate_comp_time(
            4 * layer.inputs[0] * layer.inputs[3] * layer.filters[3] *
            fft_size)
        comm_time += num_tiles * self._estimate_comm_time(
            inputs_nc * fft_size * _BYTES_COMPLEX)
        if filter_1d:
            comm_time_filters += num_tiles * self._estimate_comm_time(
                filters_ck * tile_size * _BYTES_COMPLEX)
        else:
            comm_time_filters += num_tiles * self._estimate_comm_time(
                filters_ck * fft_size * _BYTES_COMPLEX)
        comm_time += num_tiles * self._estimate_comm_time(
            layer.inputs[0] * layer.filters[3] * fft_size * _BYTES_COMPLEX)
        mem_inputs = inputs_nc * fft_size * _BYTES_COMPLEX
        mem_filters = filters_ck * fft_size * _BYTES_COMPLEX
        mem_outputs = (layer.inputs[0] * layer.filters[3] * fft_size *
                       _BYTES_COMPLEX)
        self._logger.debug('FFT mem: %d MB (%d, %d, %d)' % (
            (mem_inputs + mem_filters + mem_outputs) / 2 ** 20, mem_inputs / 2
            ** 20, mem_filters / 2 ** 20, mem_outputs / 2 ** 20))
        comp_time += num_tiles * self._estimate_comp_time(
            layer.inputs[0] * layer.filters[3] * _fft_flops(
                fft_size, 'c2r', filter_1d=filter_1d))
        comm_time += self._estimate_comm_time(
            layer.inputs[0] * layer.filters[3] * fft_size * _BYTES_COMPLEX)
        comm_time += self._estimate_comm_time(
            layer.outputs[0] * layer.outputs[1] * layer.outputs[2] *
            layer.outputs[3] * _BYTES_FLOAT)
        comp_time += comp_time_filters
        comm_time += comm_time_filters
        self._logger.debug('FFT estimates: %f = %f + %f' %
                           (comp_time + comm_time, comp_time, comm_time))
        return TimeMeasure(comp_time=comp_time, comm_time=comm_time)
def _profile_conv2d_fft ( self , layer , tiling = False ) : def _fft_flops ( fft_dim , mode = 'r2c' , filter_1d = False ) : tile_size = math . sqrt ( fft_dim ) tile_size_2 = _to_pow2 ( tile_size ) f = 2 * tile_size_2 * 5 * tile_size_2 * ( math . log ( tile_size_2 ) / math . log ( 2 ) ) if filter_1d : f /= 2 if mode == 'r2c' : f /= 2 return f def _to_pow2 ( n ) : return math . pow ( 2 , math . ceil ( math . log ( n ) / math . log ( 2 ) ) ) filter_1d = False filter_size = layer . filters [ 0 ] * layer . filters [ 1 ] if filter_size in [ layer . filters [ 0 ] , layer . filters [ 1 ] ] : filter_1d = True if tiling : _TILE_SIZE = 32 h_tiles = ( layer . inputs [ 1 ] + _TILE_SIZE - 1 ) // _TILE_SIZE w_tiles = ( layer . inputs [ 2 ] + _TILE_SIZE - 1 ) // _TILE_SIZE fft_size = _TILE_SIZE ** 2 num_tiles = h_tiles * w_tiles self . _logger . info ( 'Tile FFT: %d (%dx%d) 1D: %s' % ( num_tiles , _TILE_SIZE , _TILE_SIZE , filter_1d ) ) tile_size = _TILE_SIZE else : padded_h = max ( layer . inputs [ 1 ] + layer . _pad_h * 2 , layer . filters [ 0 ] ) padded_w = max ( layer . inputs [ 2 ] + layer . _pad_w * 2 , layer . filters [ 1 ] ) fft_size = padded_h * padded_w num_tiles = 1 self . _logger . debug ( 'FFT size: %dx%d (%dx%d) 1D: %s' % ( _to_pow2 ( padded_h ) , _to_pow2 ( padded_w ) , padded_h , padded_w , filter_1d ) ) tile_size = max ( padded_h , padded_w ) comp_time , comm_time = 0 , 0 comp_time_filters , comm_time_filters = 0 , 0 inputs_nc = layer . inputs [ 0 ] * layer . inputs [ 3 ] filters_ck = layer . filters [ 2 ] * layer . filters [ 3 ] comp_time += num_tiles * self . _estimate_comp_time ( inputs_nc * _fft_flops ( fft_size , filter_1d = filter_1d ) ) comp_time_filters += self . _estimate_comp_time ( filters_ck * _fft_flops ( fft_size , filter_1d = filter_1d ) ) comm_time += num_tiles * self . _estimate_comm_time ( inputs_nc * fft_size * _BYTES_FLOAT ) comm_time += num_tiles * self . _estimate_comm_time ( inputs_nc * fft_size * _BYTES_COMPLEX ) comm_time_filters += self . _estimate_comm_time ( filters_ck * fft_size * _BYTES_FLOAT ) if filter_1d : comm_time_filters += self . _estimate_comm_time ( filters_ck * tile_size * _BYTES_COMPLEX ) else : comm_time_filters += self . _estimate_comm_time ( filters_ck * fft_size * _BYTES_COMPLEX ) comp_time += num_tiles * self . _estimate_comp_time ( 4 * layer . inputs [ 0 ] * layer . inputs [ 3 ] * layer . filters [ 3 ] * fft_size ) comm_time += num_tiles * self . _estimate_comm_time ( inputs_nc * fft_size * _BYTES_COMPLEX ) if filter_1d : comm_time_filters += num_tiles * self . _estimate_comm_time ( filters_ck * tile_size * _BYTES_COMPLEX ) else : comm_time_filters += num_tiles * self . _estimate_comm_time ( filters_ck * fft_size * _BYTES_COMPLEX ) comm_time += num_tiles * self . _estimate_comm_time ( layer . inputs [ 0 ] * layer . filters [ 3 ] * fft_size * _BYTES_COMPLEX ) mem_inputs = inputs_nc * fft_size * _BYTES_COMPLEX mem_filters = filters_ck * fft_size * _BYTES_COMPLEX mem_outputs = ( layer . inputs [ 0 ] * layer . filters [ 3 ] * fft_size * _BYTES_COMPLEX ) self . _logger . debug ( 'FFT mem: %d MB (%d, %d, %d)' % ( ( mem_inputs + mem_filters + mem_outputs ) / 2 ** 20 , mem_inputs / 2 ** 20 , mem_filters / 2 ** 20 , mem_outputs / 2 ** 20 ) ) comp_time += num_tiles * self . _estimate_comp_time ( layer . inputs [ 0 ] * layer . filters [ 3 ] * _fft_flops ( fft_size , 'c2r' , filter_1d = filter_1d ) ) comm_time += self . _estimate_comm_time ( layer . inputs [ 0 ] * layer . filters [ 3 ] * fft_size * _BYTES_COMPLEX ) comm_time += self . _estimate_comm_time ( layer . outputs [ 0 ] * layer . outputs [ 1 ] * layer . outputs [ 2 ] * layer . outputs [ 3 ] * _BYTES_FLOAT ) comp_time += comp_time_filters comm_time += comm_time_filters self . _logger . debug ( 'FFT estimates: %f = %f + %f' % ( comp_time + comm_time , comp_time , comm_time ) ) return TimeMeasure ( comp_time = comp_time , comm_time = comm_time )
Returns the flops of convolution 2d.
Returns the flops of convolution 2d .
"""Returns the flops of convolution 2d.""" # Note this is not an accurate flops count. # Pad to the nearest power of 2. # one of the filter dimension is 1. # Filters and inputs are padded to the same size. # padded_h = (layer.inputs[1] + layer._pad_h + #             layer.filters[0] // 2) # padded_w = (layer.inputs[2] + layer._pad_w + #             layer.filters[1] // 2) # Calculate time for filters separateily. # (1) fft2d r2c. # Read inputs and write transformed inputs. # Read filters and write transformed filters. # Padding time are not considered here. # (2) Elementwise multiplication. # Complex number: add 2, muliply 4 # Pipe: Writing results while doing FFT for the next set of tiles? # Read transformed inputs. # Read transformed filters. # Write results. # Assume additional memory needed is only for one tile. # FIXME: how to match this number with cuDNN suggestion? # (3) fft2d c2r on num_tiles. # Read complex. # Write outputs. Ignore clipping time here. # Do not multiple batch size for filters.
self
layer
tiling