Format the kernel modification. (#2016)

Format the kernel modification. (#2016)

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

// Disable specific warnings
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wconditional-uninitialized"
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wsometimes-uninitialized"
#endif

/*#ifdef __AMDGCN__
#undef __AMDGCN__
#endif
*/

#include "batchnorm_functions.h"
#include "reduction_functions.h"

#if(MIO_BN_VARIANT == 0)

#define MIO_BN_SEGTMP (MIO_BN_HW * (MIO_BN_GRP0 / MIO_BN_HW))
#define MIO_BN_SEGMENT ((MIO_BN_SEGTMP > MIO_BN_NHW) ? (MIO_BN_NHW) : (MIO_BN_SEGTMP))
#define MIO_BN_NLOOP ((MIO_BN_NHW + MIO_BN_SEGMENT - 1) / MIO_BN_SEGMENT)
#define MIO_BN_SEGIHW (MIO_BN_SEGMENT / MIO_BN_HW)
#define MIO_BN_NLOOPM (MIO_BN_NLOOP - 1)
#define MIO_BN_SNHW (MIO_BN_NLOOPM * MIO_BN_SEGIHW)

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
MIOpenBatchNormFwdTrainSpatial(const __global _FLOAT* __restrict in,
                               __global _FLOAT* __restrict out,
                               __constant _FLOAT_PREC* __restrict scale,
                               __constant _FLOAT_PREC* __restrict bias,
                               _FLOAT_PREC INHW,
#if(MIO_RUNNING_RESULT == 1)
                               double expAvgFactor,
                               __global _FLOAT_PREC* __restrict resultRunningMean,
                               __global _FLOAT_PREC* __restrict resultRunningVariance,
#endif
                               double epsilon
#if(MIO_SAVE_MEAN_VARIANCE == 1)
                               ,
                               __global _FLOAT_PREC* __restrict resultSaveMean,
                               __global _FLOAT_PREC* __restrict resultSaveInvVariance
#endif
                               )
{

    // SPATIAL
    _FLOAT_ACCUM mean        = (_FLOAT_ACCUM)0.;
    _FLOAT_ACCUM variance    = (_FLOAT_ACCUM)0.;
    _FLOAT_ACCUM invVariance = (_FLOAT_ACCUM)0.;
    _FLOAT_ACCUM pvscale     = (_FLOAT_ACCUM)0.;
    _FLOAT_ACCUM pvbias      = (_FLOAT_ACCUM)0.;
    _FLOAT batchvalues[MIO_BN_NLOOP];
    _FLOAT_ACCUM temp;

    __local _FLOAT_PREC lcl_bias;
    __local _FLOAT_PREC lcl_scale;

    unsigned int index  = 0;
    unsigned int lid    = get_local_id(0);
    unsigned int grpid  = get_group_id(0);
    unsigned int chwid  = grpid * MIO_BN_HW + (lid % MIO_BN_HW);
    unsigned int lidihw = lid / MIO_BN_HW;
    unsigned int nid    = 0;

    if(lid == 0)
    {
        lcl_scale = *(scale + grpid);
        lcl_bias  = *(bias + grpid);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(lid < MIO_BN_SEGMENT)
    {
#if MIOPEN_USE_FP16 == 1
        __attribute__((opencl_unroll_hint(2)))
#endif
        for(unsigned int n = 0; n < MIO_BN_NLOOPM; ++n)
        {
            nid            = n * MIO_BN_SEGIHW + lidihw;
            index          = nid * MIO_BN_CHW + chwid;
            batchvalues[n] = (*(in + index));
            temp           = (_FLOAT_ACCUM)(*(in + index));
            mean += temp;
            variance = mad(temp, temp, variance);
        }
        nid                        = MIO_BN_SNHW + lidihw;
        index                      = nid * MIO_BN_CHW + chwid;
        batchvalues[MIO_BN_NLOOPM] = (index < MIO_BN_NCHW) ? (*(in + index)) : (_FLOAT)0.;
        temp                       = (_FLOAT_ACCUM)batchvalues[MIO_BN_NLOOPM];
        mean += temp;
        variance = mad(temp, temp, variance);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

#ifndef __AMDGCN__
    local _FLOAT_ACCUM lcl_data_x[MIO_BN_LDS_SIZE];
    local _FLOAT_ACCUM lcl_data_y[MIO_BN_LDS_SIZE];
    lds_reduce2(&mean, &variance, (_FLOAT_ACCUM)INHW, lcl_data_x, lcl_data_y, lid);
#else
    local _FLOAT_ACCUM lcl_data_x[MIO_BN_LDSGCN_SIZE];
    local _FLOAT_ACCUM lcl_data_y[MIO_BN_LDSGCN_SIZE];
    gcn_reduce2(&mean, &variance, (_FLOAT_ACCUM)INHW, lcl_data_x, lcl_data_y, lid);
#endif

    variance    = mad(-mean, mean, variance);
    invVariance = rsqrt(variance + (_FLOAT_ACCUM)epsilon);
    pvscale     = (_FLOAT_ACCUM)lcl_scale;
    pvbias      = (_FLOAT_ACCUM)lcl_bias;

    if(lid < MIO_BN_SEGMENT)
    {
        //==== CALC NORM =======================
        _FLOAT_ACCUM inhat = (_FLOAT_ACCUM)0.;

#if MIOPEN_USE_FP16 == 1
        __attribute__((opencl_unroll_hint(2)))
#endif
        for(unsigned int n = 0; n < MIO_BN_NLOOPM; n++)
        { // apply normalization
            inhat      = (batchvalues[n] - mean) * invVariance;
            nid        = n * MIO_BN_SEGIHW + lidihw;
            index      = nid * MIO_BN_CHW + chwid;
            out[index] = (_FLOAT)mad(pvscale, inhat, pvbias);
        } // end for

        // Tail of loop
        inhat = (batchvalues[MIO_BN_NLOOPM] - mean) * invVariance;
        nid   = MIO_BN_SNHW + lidihw;
        index = nid * MIO_BN_CHW + chwid;
        if(index < MIO_BN_NCHW)
            out[index] = (_FLOAT)mad(pvscale, inhat, pvbias);
    }

    if(lid == 0)
    {
#if(MIO_RUNNING_RESULT == 1)
        running_stash(
            resultRunningMean, resultRunningVariance, expAvgFactor, mean, variance, grpid);
#endif

#if(MIO_SAVE_MEAN_VARIANCE == 1)
        saved_stash(resultSaveMean, resultSaveInvVariance, mean, invVariance, grpid);
#endif
    }
} // end spatial norm

#elif(MIO_BN_VARIANT == 1)

//===========

#if(MIO_BN_HW >= 4096)
#define MIO_MAX_READ 3
#else
#define MIO_MAX_READ 2
#endif
#define RD_BLK 1
#define GRPRD (MIO_BN_GRP0 * RD_BLK * 4)
#define MIO_BN_REM4 (MIO_BN_NHW - ((MIO_BN_NHW / GRPRD) * GRPRD))
#define MIO_BN_LESS4 (MIO_BN_NHW - MIO_BN_REM4)
#define MIO_BN_CHUNK4 (MIO_MAX_READ * GRPRD)
#define MIO_BN_REMOUT4 (MIO_BN_NHW - ((MIO_BN_NHW / MIO_BN_CHUNK4) * MIO_BN_CHUNK4))
#define MIO_BN_LESSOUT4 (MIO_BN_NHW - MIO_BN_REMOUT4)
#define MIO_BN_REM (MIO_BN_NHW - ((MIO_BN_NHW / MIO_BN_GRP0) * MIO_BN_GRP0))
#define MIO_BN_LESS (MIO_BN_NHW - MIO_BN_REM)
#define MIO_BN_CHUNK (MIO_MAX_READ * MIO_BN_GRP0)
#define MIO_BN_REMOUT (MIO_BN_NHW - ((MIO_BN_NHW / MIO_BN_CHUNK) * MIO_BN_CHUNK))
#define MIO_BN_LESSOUT (MIO_BN_NHW - MIO_BN_REMOUT)

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
MIOpenBatchNormFwdTrainSpatial(const __global _FLOAT* __restrict in,
                               __global _FLOAT* __restrict out,
                               __constant _FLOAT_PREC* __restrict scale,
                               __constant _FLOAT_PREC* __restrict bias,
                               _FLOAT_PREC INHW,
#if(MIO_RUNNING_RESULT == 1)
                               double expAvgFactor,
                               __global _FLOAT_PREC* __restrict resultRunningMean,
                               __global _FLOAT_PREC* __restrict resultRunningVariance,
#endif
                               double epsilon
#if(MIO_SAVE_MEAN_VARIANCE == 1)
                               ,
                               __global _FLOAT_PREC* __restrict resultSaveMean,
                               __global _FLOAT_PREC* __restrict resultSaveInvVariance
#endif
                               )
{

    // SPATIAL

    _FLOAT_PREC mean        = (_FLOAT_PREC)0.;
    _FLOAT_PREC variance    = (_FLOAT_PREC)0.;
    _FLOAT_PREC invVariance = (_FLOAT_PREC)0.;
    _FLOAT_PREC pvscale, pvbias;

    __local _FLOAT_PREC lcl_bias;
    __local _FLOAT_PREC lcl_scale;

    uint index = 0;
    uint lid   = get_local_id(0);
    uint grpid = get_group_id(0);
    uint chwid = grpid * MIO_BN_HW;
    uint nidx  = 0;
    uint hwidx = 0;

    if(lid == 0)
    {
        lcl_scale = *(scale + grpid);
        lcl_bias  = *(bias + grpid);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

#if(MIO_BN_HW >= 4096)
    _FLOAT4 read4;
    __attribute__((opencl_unroll_hint(2))) for(unsigned int k = lid << 2; k < MIO_BN_LESS4;
                                               k += GRPRD)
    {
        nidx  = k / MIO_BN_HW;
        hwidx = k - (nidx * MIO_BN_HW);
        index = nidx * MIO_BN_CHW + chwid + hwidx;
        read4 = *((const global _FLOAT4*)(in + index));
        mean += (_FLOAT_PREC)read4.x;
        mean += (_FLOAT_PREC)read4.y;
        mean += (_FLOAT_PREC)read4.z;
        mean += (_FLOAT_PREC)read4.w;
        variance = mad((_FLOAT_PREC)read4.x, (_FLOAT_PREC)read4.x, variance);
        variance = mad((_FLOAT_PREC)read4.y, (_FLOAT_PREC)read4.y, variance);
        variance = mad((_FLOAT_PREC)read4.z, (_FLOAT_PREC)read4.z, variance);
        variance = mad((_FLOAT_PREC)read4.w, (_FLOAT_PREC)read4.w, variance);
    }

#if(MIO_BN_REM4)
    unsigned int remkey = (lid << 2) + MIO_BN_LESS4;
    nidx                = remkey / MIO_BN_HW;
    hwidx               = remkey - (nidx * MIO_BN_HW);
    index               = nidx * MIO_BN_CHW + chwid + hwidx;
    if(index < MIO_BN_NCHW)
    {
        read4 = *((const global _FLOAT4*)(in + index));
        mean += (_FLOAT_PREC)read4.x;
        mean += (_FLOAT_PREC)read4.y;
        mean += (_FLOAT_PREC)read4.z;
        mean += (_FLOAT_PREC)read4.w;
        variance = mad((_FLOAT_PREC)read4.x, (_FLOAT_PREC)read4.x, variance);
        variance = mad((_FLOAT_PREC)read4.y, (_FLOAT_PREC)read4.y, variance);
        variance = mad((_FLOAT_PREC)read4.z, (_FLOAT_PREC)read4.z, variance);
        variance = mad((_FLOAT_PREC)read4.w, (_FLOAT_PREC)read4.w, variance);
    }

#endif

#else
    __attribute__((opencl_unroll_hint(4))) for(unsigned int k = lid; k < MIO_BN_LESS;
                                               k += MIO_BN_GRP0)
    {
        nidx            = k / MIO_BN_HW;
        hwidx           = k - (nidx * MIO_BN_HW);
        index           = nidx * MIO_BN_CHW + chwid + hwidx;
        _FLOAT_PREC xin = (_FLOAT_PREC)(*(in + index));
        mean += xin;
        variance = mad(xin, xin, variance);
    }
#if(MIO_BN_REM)
    if(lid < MIO_BN_REM)
    {
        unsigned int remkey = lid + MIO_BN_LESS;
        nidx                = remkey / MIO_BN_HW;
        hwidx               = remkey - (nidx * MIO_BN_HW);
        index               = nidx * MIO_BN_CHW + chwid + hwidx;
        _FLOAT_PREC xin = (index < MIO_BN_NCHW) ? (_FLOAT_PREC)(*(in + index)) : (_FLOAT_PREC)0.;
        mean += xin;
        variance = mad(xin, xin, variance);
    }
#endif
#endif
    barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);

#ifndef __AMDGCN__
    local _FLOAT_ACCUM lcl_data_x[MIO_BN_LDS_SIZE];
    local _FLOAT_ACCUM lcl_data_y[MIO_BN_LDS_SIZE];
    lds_reduce2(&mean, &variance, (_FLOAT_ACCUM)INHW, lcl_data_x, lcl_data_y, lid);
#else
    local _FLOAT_ACCUM lcl_data_x[MIO_BN_LDSGCN_SIZE];
    local _FLOAT_ACCUM lcl_data_y[MIO_BN_LDSGCN_SIZE];
    gcn_reduce2(&mean, &variance, (_FLOAT_ACCUM)INHW, lcl_data_x, lcl_data_y, lid);
#endif

    // REDUCTION COMPLETE ---------------------------
    variance    = mad(-mean, mean, variance);
    invVariance = rsqrt(variance + epsilon);

    pvscale = lcl_scale;
    pvbias  = lcl_bias;

#if(MIO_BN_REM == 0)
    __attribute__((opencl_unroll_hint(2))) for(unsigned int k = lid; k < MIO_BN_LESS;
                                               k += MIO_BN_GRP0)
    {
        nidx  = k / MIO_BN_HW;
        hwidx = k - (nidx * MIO_BN_HW);
        index = nidx * MIO_BN_CHW + chwid + hwidx;
        out[index] =
            (_FLOAT)mad(pvscale, ((_FLOAT_PREC)(*(in + index)) - mean) * invVariance, pvbias);
    } // end for
#else
    _FLOAT_PREC xhat[MIO_MAX_READ];
    __attribute__((opencl_unroll_hint(2))) for(unsigned int k = (MIO_MAX_READ * lid);
                                               k < MIO_BN_LESSOUT;
                                               k += MIO_BN_CHUNK)
    {
        for(unsigned int j = 0; j < MIO_MAX_READ; j++)
        {
            unsigned int l = k + j;
            nidx           = l / MIO_BN_HW;
            hwidx          = l - (nidx * MIO_BN_HW);
            index          = nidx * MIO_BN_CHW + chwid + hwidx;
            xhat[j]        = ((_FLOAT_PREC)(*(in + index)) - mean) * invVariance;
        }
        barrier(CLK_GLOBAL_MEM_FENCE);
        for(unsigned int j = 0; j < MIO_MAX_READ; j++)
        {
            unsigned int l = k + j;
            nidx           = l / MIO_BN_HW;
            hwidx          = l - (nidx * MIO_BN_HW);
            index          = nidx * MIO_BN_CHW + chwid + hwidx;
            *(out + index) = (_FLOAT)mad(pvscale, xhat[j], pvbias);
        }
    } // end for

#if(MIO_BN_REMOUT)
    unsigned int remkeyout = (MIO_MAX_READ * lid) + MIO_BN_LESSOUT;
    for(unsigned int j = 0; j < MIO_MAX_READ; j++)
    {
        unsigned int l  = remkeyout + j;
        nidx            = l / MIO_BN_HW;
        hwidx           = l - (nidx * MIO_BN_HW);
        index           = nidx * MIO_BN_CHW + chwid + hwidx;
        _FLOAT_PREC xin = (index < MIO_BN_NCHW) ? (_FLOAT_PREC)(*(in + index)) : (_FLOAT_PREC)0.;
        xhat[j]         = (xin - mean) * invVariance;
    }
    barrier(CLK_GLOBAL_MEM_FENCE);
    for(unsigned int j = 0; j < MIO_MAX_READ; j++)
    {
        unsigned int l = remkeyout + j;
        nidx           = l / MIO_BN_HW;
        hwidx          = l - (nidx * MIO_BN_HW);
        index          = nidx * MIO_BN_CHW + chwid + hwidx;
        if(index < MIO_BN_NCHW)
        {
            *(out + index) = (_FLOAT)mad(pvscale, xhat[j], pvbias);
        }
    }
#endif
#endif

    if(lid == 0)
    {
#if(MIO_RUNNING_RESULT == 1)
        running_stash(
            resultRunningMean, resultRunningVariance, expAvgFactor, mean, variance, grpid);
#endif

#if(MIO_SAVE_MEAN_VARIANCE == 1)
        saved_stash(resultSaveMean, resultSaveInvVariance, mean, invVariance, grpid);
#endif
    }
} // end spatial norm

#elif(MIO_BN_VARIANT == 2) // MULTI-KERNEL reduction for > 33M elements

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
MIOpenBatchNormFwdTrainSpatialNorm(const __global _FLOAT* __restrict in,
                                   __global _FLOAT* __restrict out,
                                   const __global _FLOAT* __restrict scale,
                                   const __global _FLOAT* __restrict bias)
{

    // SPATIAL
    _FLOAT mean        = (_FLOAT)0.;
    _FLOAT invVariance = (_FLOAT)0.;
    _FLOAT inhat       = (_FLOAT)0.;
    _FLOAT pvt_scale   = (_FLOAT)0.;
    _FLOAT pvt_bias    = (_FLOAT)0.;
    __local _FLOAT lcl_mean, lcl_ivar, lcl_scale, lcl_bias;

    unsigned int ygrp_id = get_group_id(1);
    unsigned int xgid    = get_global_id(0);
    unsigned int ygid    = get_global_id(1);
    unsigned int ygrp_sz = get_local_size(1);
    unsigned int index;
    unsigned int cidx           = xgid * MIO_BN_HW;
    unsigned int meanstashindex = cidx + ygrp_sz * ygrp_id + 1;
    unsigned int varstashindex  = cidx + ygrp_sz * ygrp_id + 3;

    // #4 apply the normalization :: x_hat = (x_i - mean) / sqrt(variance_accum + epsilon)
    if(get_local_id(1) == 0)
    {
        lcl_scale = *(scale + xgid);
        lcl_bias  = *(bias + xgid);
        lcl_mean  = *(out + meanstashindex); // load stashed mean
        lcl_ivar  = *(out + varstashindex);
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    if(ygid < MIO_BN_HW)
    {
        mean        = lcl_mean;
        invVariance = lcl_ivar;
        pvt_scale   = lcl_scale;
        pvt_bias    = lcl_bias;
        __attribute__((opencl_unroll_hint(2))) for(unsigned int n = 0; n < MIO_BN_N; n++)
        { // apply normalization
            index = n * MIO_BN_CHW + cidx + ygid;
            inhat = (*(in + index) - mean) * invVariance;
            // #5 Gamma and Beta adjust :: y_i = gamma*x_hat + beta
            out[index] = mad(pvt_scale, inhat, pvt_bias);
        } // end for(n)
    }     // end if(inImgIndex)
} // end spatial norm

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
MIOpenBatchNormFwdTrainSpatialFinalMeanVariance(
    __global _FLOAT* __restrict meanvarbuff,
    _FLOAT INHW
#if(MIO_RUNNING_RESULT == 1)
    ,
    double expAvgFactor /* input momentum */
    ,
    __global _FLOAT* __restrict resultRunningMean, /*input and output*/
    __global _FLOAT* __restrict resultRunningVariance
#endif
    ,
    double epsilon
#if(MIO_SAVE_MEAN_VARIANCE == 1)
    ,
    __global _FLOAT* __restrict resultSaveMean /*output only*/
    ,
    __global _FLOAT* __restrict resultSaveInvVariance
#endif
    )
{
    _FLOAT variance             = (_FLOAT)0.;
    _FLOAT invVariance          = (_FLOAT)0.;
    _FLOAT mean                 = (_FLOAT)0.;
    unsigned int lid            = get_local_id(1);
    unsigned int ygrp_id        = get_group_id(1);
    unsigned int xgid           = get_global_id(0);
    unsigned int ygrp_sz        = get_local_size(1);
    unsigned int yngrps         = get_num_groups(1);
    unsigned int cidx           = xgid * MIO_BN_HW;
    unsigned int meanstashindex = cidx + ygrp_sz * ygrp_id + 1;
    unsigned int varstashindex  = cidx + ygrp_sz * ygrp_id + 3;
    unsigned int commitID       = 0;

    for(int gn = 0; gn < yngrps; gn++)
    {
        unsigned int offset    = gn * ygrp_sz + lid;
        unsigned int meanindex = cidx + ygrp_sz * offset;
        unsigned int varindex  = cidx + ygrp_sz * offset + 2;
        if(offset < yngrps)
        { // modify to span larger number of groups
            mean += *(meanvarbuff + meanindex);
            variance += *(meanvarbuff + varindex); // load per group variance
        }
    }

#ifndef __AMDGCN__
    local _FLOAT_ACCUM lcl_data_x[MIO_BN_LDS_SIZE];
    local _FLOAT_ACCUM lcl_data_y[MIO_BN_LDS_SIZE];
    lds_reduce2(&mean, &variance, (_FLOAT_ACCUM)INHW, lcl_data_x, lcl_data_y, lid);
#else
    commitID = 64;
    local _FLOAT_ACCUM lcl_data_x[MIO_BN_LDSGCN_SIZE];
    local _FLOAT_ACCUM lcl_data_y[MIO_BN_LDSGCN_SIZE];
    gcn_reduce2(&mean, &variance, (_FLOAT_ACCUM)INHW, lcl_data_x, lcl_data_y, lid);
#endif

    barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);
    variance    = mad(-mean, mean, variance);
    invVariance = rsqrt(variance + epsilon);
    if(lid == commitID)
    {
        meanvarbuff[meanstashindex] = mean;        // stash mean
        meanvarbuff[varstashindex]  = invVariance; // stash mean
    }

    // Save mean and calculate and save running mean
    unsigned int ygid = get_global_id(1);
    if(ygid == commitID)
    {
#if(MIO_RUNNING_RESULT == 1)
        running_stash(resultRunningMean, resultRunningVariance, expAvgFactor, mean, variance, xgid);
#endif

#if(MIO_SAVE_MEAN_VARIANCE == 1)
        saved_stash(resultSaveMean, resultSaveInvVariance, mean, invVariance, xgid);
#endif
    }
}

__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
MIOpenBatchNormFwdTrainSpatialMeanVariance(const __global _FLOAT* __restrict in,
                                           __global _FLOAT* __restrict mvbuff)
{

    unsigned int ylid    = get_local_id(1);
    unsigned int ygrp_id = get_group_id(1);
    unsigned int xgid    = get_global_id(0);
    unsigned int ygid    = get_global_id(1);
    unsigned int ygrp_sz = get_local_size(1);
    unsigned int index;
    unsigned int cidx      = xgid * MIO_BN_HW;
    unsigned int meanindex = cidx + ygrp_sz * ygrp_id;
    unsigned int varindex  = meanindex + 2;
    _FLOAT mean            = (_FLOAT)0.;
    _FLOAT variance        = (_FLOAT)0.;
    _FLOAT value           = (_FLOAT)0.;

    if(ygid < MIO_BN_HW)
    {
        __attribute__((opencl_unroll_hint(2))) for(unsigned int n = 0; n < MIO_BN_N; n++)
        {
            index = n * MIO_BN_CHW + cidx + ygid;
            value = *(in + index);
            mean += value;
            variance = mad(value, value, variance);
        }
    }

#ifndef __AMDGCN__
    local _FLOAT_ACCUM lcl_data_x[MIO_BN_LDS_SIZE];
    local _FLOAT_ACCUM lcl_data_y[MIO_BN_LDS_SIZE];
    lds_reduce2(&mean, &variance, (_FLOAT_ACCUM)1.0, lcl_data_x, lcl_data_y, ylid);
#else
    local _FLOAT_ACCUM lcl_data_x[MIO_BN_LDSGCN_SIZE];
    local _FLOAT_ACCUM lcl_data_y[MIO_BN_LDSGCN_SIZE];
    gcn_reduce2(&mean, &variance, (_FLOAT_ACCUM)1.0, lcl_data_x, lcl_data_y, ylid);
#endif

    if(ylid == 0)
    {
        mvbuff[meanindex] = mean;
        mvbuff[varindex]  = variance;
    }
} // end spatial mean kernel

#elif(MIO_BN_VARIANT == 3)

// This kernel implies the image is greater than a wavefront, but smaller than 257
__attribute__((reqd_work_group_size(MIO_BN_GRP0, MIO_BN_GRP1, MIO_BN_GRP2))) __kernel void
MIOpenBatchNormFwdTrainSpatial(const __global _FLOAT* __restrict in,
                               __global _FLOAT* __restrict out,
                               __constant _FLOAT_PREC* __restrict scale,
                               __constant _FLOAT_PREC* __restrict bias,
                               _FLOAT_PREC INHW,
#if(MIO_RUNNING_RESULT == 1)
                               double expAvgFactor,
                               __global _FLOAT_PREC* __restrict resultRunningMean,
                               __global _FLOAT_PREC* __restrict resultRunningVariance,
#endif
                               double epsilon
#if(MIO_SAVE_MEAN_VARIANCE == 1)
                               ,
                               __global _FLOAT_PREC* __restrict resultSaveMean,
                               __global _FLOAT_PREC* __restrict resultSaveInvVariance
#endif
                               )
{

    // SPATIAL
    _FLOAT_PREC mean        = (_FLOAT_PREC)0.;
    _FLOAT_PREC variance    = (_FLOAT_PREC)0.;
    _FLOAT_PREC invVariance = (_FLOAT_PREC)0.;
    _FLOAT_PREC inhat       = (_FLOAT_PREC)0.;
    _FLOAT_PREC pvscale     = (_FLOAT_PREC)0.;
    _FLOAT_PREC pvbias      = (_FLOAT_PREC)0.;
    _FLOAT_PREC xin         = (_FLOAT_PREC)0.;

    local _FLOAT_PREC lcl_bias;
    local _FLOAT_PREC lcl_scale;

    unsigned int index = 0;
    unsigned int lid   = get_local_id(0);
    unsigned int grpid = get_group_id(0);
    unsigned int cidx  = grpid * MIO_BN_HW;

#if(MIO_BN_N < MIO_BN_MAXN)
    _FLOAT minibatch[MIO_BN_N];
#endif

    if(lid == 0)
    {
        lcl_scale = *(scale + grpid);
        lcl_bias  = *(bias + grpid);
    }

    if(lid < MIO_BN_HW)
    {
        __attribute__((opencl_unroll_hint(2))) for(unsigned int n = 0; n < MIO_BN_N; n++)
        {
            index = n * MIO_BN_CHW + cidx + lid;
            xin   = (_FLOAT_PREC)(*(in + index));
            mean += xin;
            variance     = mad(xin, xin, variance);

#if(MIO_BN_N < MIO_BN_MAXN)
            minibatch[n] = (*(in + index));
#endif
        }
    }
    barrier(CLK_LOCAL_MEM_FENCE | CLK_GLOBAL_MEM_FENCE);

#ifndef __AMDGCN__
    local _FLOAT_ACCUM lcl_data_x[MIO_BN_LDS_SIZE];
    local _FLOAT_ACCUM lcl_data_y[MIO_BN_LDS_SIZE];
    lds_reduce2(&mean, &variance, (_FLOAT_ACCUM)INHW, lcl_data_x, lcl_data_y, lid);
#else
    local _FLOAT_ACCUM lcl_data_x[MIO_BN_LDSGCN_SIZE];
    local _FLOAT_ACCUM lcl_data_y[MIO_BN_LDSGCN_SIZE];
    gcn_reduce2(&mean, &variance, (_FLOAT_ACCUM)INHW, lcl_data_x, lcl_data_y, lid);
#endif

    variance    = mad(-mean, mean, variance);
    invVariance = rsqrt(variance + (_FLOAT_PREC)epsilon);

    if(lid < MIO_BN_HW)
    {
        pvscale = lcl_scale;
        pvbias  = lcl_bias;

        __attribute__((opencl_unroll_hint(2))) for(unsigned int n = 0; n < MIO_BN_N; n++)
        { // apply normalization
            index      = n * MIO_BN_CHW + cidx + lid;
#if(MIO_BN_N < MIO_BN_MAXN)
            inhat      = (minibatch[n] - mean) * invVariance; // (in[index] - mean) * invVariance;
#else
            inhat = ((_FLOAT_PREC)(*(in + index)) - mean) * invVariance;
#endif
            out[index] = (_FLOAT)mad(pvscale, inhat, pvbias);
        } // end for
    }     // end if

    if(lid == 0)
    {
#if(MIO_RUNNING_RESULT == 1)
        running_stash(
            resultRunningMean, resultRunningVariance, expAvgFactor, mean, variance, grpid);
#endif

#if(MIO_SAVE_MEAN_VARIANCE == 1)
        saved_stash(resultSaveMean, resultSaveInvVariance, mean, invVariance, grpid);
#endif
    }
} // end spatial norm

#endif

// Restore warnings
#ifdef __clang__
#pragma clang diagnostic pop
#pragma clang diagnostic pop
#endif


Stored local copy of rnd back to global memory.  This was a bug because the state of rnd is updated when a new random number is generated.

git-svn-id: 602a26e73ae9f2a14e2bb95676f2e137fcc39919@3491 7c0444ca-5550-0410-9008-faecdbe78eb5

#include "LIF_params.h"
#include "cl_random.hcl"
#include "../utils/pv_random.h"

#ifndef PV_USE_OPENCL
#  include <math.h>
#  define EXP expf
#  define CL_KERNEL
#  define CL_MEM_GLOBAL
#  define CL_MEM_LOCAL
#else  /* compiling with OpenCL */
#  define EXP exp
#  define CL_KERNEL       __kernel
#  define CL_MEM_GLOBAL   __global
#  define CL_MEM_LOCAL    __local
#  include "conversions.hcl"
#endif

//
// update the state of a retinal layer (spiking)
//
//    assume called with 1D kernel
//
CL_KERNEL
void LIF_update_state(
    const float time,
    const float dt,

    const int nx,
    const int ny,
    const int nf,
    const int nb,

    CL_MEM_GLOBAL LIF_params * params,
    CL_MEM_GLOBAL uint4 * rnd,
    CL_MEM_GLOBAL float * V,
    CL_MEM_GLOBAL float * Vth,
    CL_MEM_GLOBAL float * G_E,
    CL_MEM_GLOBAL float * G_I,
    CL_MEM_GLOBAL float * G_IB,
    CL_MEM_GLOBAL float * phiExc,
    CL_MEM_GLOBAL float * phiInh,
    CL_MEM_GLOBAL float * phiInhB,
    CL_MEM_GLOBAL float * R,
    CL_MEM_GLOBAL float * activity)
{
   int k;

   const float exp_tauE    = EXP(-dt/params->tauE);
   const float exp_tauI    = EXP(-dt/params->tauI);
   const float exp_tauIB   = EXP(-dt/params->tauIB);
   const float exp_tauVth  = EXP(-dt/params->tauVth);

   const float dt_sec = .001 * dt;   // convert to seconds
   const float exp_tauRate = EXP(-dt/params->tauRate);

#ifndef PV_USE_OPENCL

for (k = 0; k < nx*ny*nf; k++) {
#else
   k = get_global_id(0);
#endif

   int kex = kIndexExtended(k, nx, ny, nf, nb);

   //
   // kernel (nonheader part) begins here
   //

   // local param variables
   float tau, Vrest, VthRest, Vexc, Vinh, VinhB, deltaVth;

   const float GMAX = 10.0;

   // local variables
   float l_activ;

   uint4 l_rnd = rnd[k];

   float l_V   = V[k];
   float l_Vth = Vth[k];

   float l_G_E  = G_E[k];
   float l_G_I  = G_I[k];
   float l_G_IB = G_IB[k];

   float l_phiExc  = phiExc[k];
   float l_phiInh  = phiInh[k];
   float l_phiInhB = phiInhB[k];

   float l_R = R[k];

   // temporary arrays
   float tauInf, VmemInf;

   //
   // start of LIF2_update_exact_linear
   //

   // define local param variables
   //
   tau   = params->tau;
   Vexc  = params->Vexc;
   Vinh  = params->Vinh;
   VinhB = params->VinhB;
   Vrest = params->Vrest;

   VthRest  = params->VthRest;
   deltaVth = params->deltaVth;

   // add noise
   //

#undef CLRANDOM

#ifdef CLRANDOM
   l_rnd = cl_random_get(l_rnd);
   if (cl_random_prob(l_rnd) < dt_sec*params->noiseFreqE) {
      l_rnd = cl_random_get(l_rnd);
      l_phiExc = l_phiExc + params->noiseAmpE*cl_random_prob(l_rnd);
   }

   l_rnd = cl_random_get(l_rnd);
   if (cl_random_prob(l_rnd) < dt_sec*params->noiseFreqI) {
      l_rnd = cl_random_get(l_rnd);
      l_phiInh = l_phiInh + params->noiseAmpI*cl_random_prob(l_rnd);
   }

   l_rnd = cl_random_get(l_rnd);
   if (cl_random_prob(l_rnd) < dt_sec*params->noiseFreqIB) {
      l_rnd = cl_random_get(l_rnd);
      l_phiInhB = l_phiInhB + params->noiseAmpIB*cl_random_prob(l_rnd);
   }
#else
   if (pv_random_prob() < dt_sec*params->noiseFreqE) {
      l_phiExc = l_phiExc + params->noiseAmpE*pv_random_prob();
   }

   if (pv_random_prob() < dt_sec*params->noiseFreqI) {
      l_phiInh = l_phiInh + params->noiseAmpI*pv_random_prob();
   }

   if (pv_random_prob() < dt_sec*params->noiseFreqIB) {
      l_phiInhB = l_phiInhB + params->noiseAmpIB*pv_random_prob();
   }
#endif

   l_G_E  = l_phiExc  + l_G_E *exp_tauE;
   l_G_I  = l_phiInh  + l_G_I *exp_tauI;
   l_G_IB = l_phiInhB + l_G_IB*exp_tauIB;

   l_G_E  = (l_G_E  > GMAX) ? GMAX : l_G_E;
   l_G_I  = (l_G_I  > GMAX) ? GMAX : l_G_I;
   l_G_IB = (l_G_IB > GMAX) ? GMAX : l_G_IB;

   tauInf  = (dt/tau) * (1.0 + l_G_E + l_G_I + l_G_IB);
   VmemInf = (Vrest + l_G_E*Vexc + l_G_I*Vinh + l_G_IB*VinhB)
           / (1.0 + l_G_E + l_G_I + l_G_IB);

   l_V = VmemInf + (l_V - VmemInf)*EXP(-tauInf);

   //
   // start of LIF2_update_finish
   //

   l_Vth = VthRest + (l_Vth - VthRest)*exp_tauVth;

   //
   // start of update_f
   //

   bool fired_flag = (l_V > l_Vth);

   l_activ = fired_flag ? 1.0f             : 0.0f;
   l_V     = fired_flag ? Vrest            : l_V;
   l_Vth   = fired_flag ? l_Vth + deltaVth : l_Vth;
   l_G_IB  = fired_flag ? l_G_IB + 1.0f    : l_G_IB;

   // update average rate
   l_R = l_activ + l_R*exp_tauRate;

   //
   // These actions must be done outside of kernel
   //    1. set activity to 0 in boundary (if needed)
   //    2. update active indices
   //

   // store local variables back to global memory
   //
   rnd[k] = l_rnd;

   activity[kex] = l_activ;

   R[k] = l_R;

   V[k]   = l_V;
   Vth[k] = l_Vth;

   G_E[k]  = l_G_E;
   G_I[k]  = l_G_I;
   G_IB[k] = l_G_IB;

   phiExc[k]  = 0.0f;
   phiInh[k]  = 0.0f;
   phiInhB[k] = 0.0f;

#ifndef PV_USE_OPENCL
   } // loop over k
#endif

}

#include "LIF_params.h"
#include "cl_random.hcl"
#include "../utils/pv_random.h"

#ifndef PV_USE_OPENCL
#  include <math.h>
#  define EXP expf
#  define CL_KERNEL
#  define CL_MEM_GLOBAL
#  define CL_MEM_LOCAL
#else  /* compiling with OpenCL */
#  define EXP exp
#  define CL_KERNEL       __kernel
#  define CL_MEM_GLOBAL   __global
#  define CL_MEM_LOCAL    __local
#  include "conversions.hcl"
#endif

//
// update the state of a retinal layer (spiking)
//
//    assume called with 1D kernel
//
CL_KERNEL
void LIF_update_state(
    const float time,
    const float dt,

    const int nx,
    const int ny,
    const int nf,
    const int nb,

    CL_MEM_GLOBAL LIF_params * params,
    CL_MEM_GLOBAL uint4 * rnd,
    CL_MEM_GLOBAL float * V,
    CL_MEM_GLOBAL float * Vth,
    CL_MEM_GLOBAL float * G_E,
    CL_MEM_GLOBAL float * G_I,
    CL_MEM_GLOBAL float * G_IB,
    CL_MEM_GLOBAL float * phiExc,
    CL_MEM_GLOBAL float * phiInh,
    CL_MEM_GLOBAL float * phiInhB,
    CL_MEM_GLOBAL float * R,
    CL_MEM_GLOBAL float * activity)
{
   int k;

   const float exp_tauE    = EXP(-dt/params->tauE);
   const float exp_tauI    = EXP(-dt/params->tauI);
   const float exp_tauIB   = EXP(-dt/params->tauIB);
   const float exp_tauVth  = EXP(-dt/params->tauVth);

   const float dt_sec = .001 * dt;   // convert to seconds
   const float exp_tauRate = EXP(-dt/params->tauRate);

#ifndef PV_USE_OPENCL

for (k = 0; k < nx*ny*nf; k++) {
#else
   k = get_global_id(0);
#endif

   int kex = kIndexExtended(k, nx, ny, nf, nb);

   //
   // kernel (nonheader part) begins here
   //

   // local param variables
   float tau, Vrest, VthRest, Vexc, Vinh, VinhB, deltaVth;

   const float GMAX = 10.0;

   // local variables
   float l_activ;

   uint4 l_rnd = rnd[k];

   float l_V   = V[k];
   float l_Vth = Vth[k];

   float l_G_E  = G_E[k];
   float l_G_I  = G_I[k];
   float l_G_IB = G_IB[k];

   float l_phiExc  = phiExc[k];
   float l_phiInh  = phiInh[k];
   float l_phiInhB = phiInhB[k];

   float l_R = R[k];

   // temporary arrays
   float tauInf, VmemInf;

   //
   // start of LIF2_update_exact_linear
   //

   // define local param variables
   //
   tau   = params->tau;
   Vexc  = params->Vexc;
   Vinh  = params->Vinh;
   VinhB = params->VinhB;
   Vrest = params->Vrest;

   VthRest  = params->VthRest;
   deltaVth = params->deltaVth;

   // add noise
   //

#undef CLRANDOM

#ifdef CLRANDOM
   l_rnd = cl_random_get(l_rnd);
   if (cl_random_prob(l_rnd) < dt_sec*params->noiseFreqE) {
      l_rnd = cl_random_get(l_rnd);
      l_phiExc = l_phiExc + params->noiseAmpE*cl_random_prob(l_rnd);
   }

   l_rnd = cl_random_get(l_rnd);
   if (cl_random_prob(l_rnd) < dt_sec*params->noiseFreqI) {
      l_rnd = cl_random_get(l_rnd);
      l_phiInh = l_phiInh + params->noiseAmpI*cl_random_prob(l_rnd);
   }

   l_rnd = cl_random_get(l_rnd);
   if (cl_random_prob(l_rnd) < dt_sec*params->noiseFreqIB) {
      l_rnd = cl_random_get(l_rnd);
      l_phiInhB = l_phiInhB + params->noiseAmpIB*cl_random_prob(l_rnd);
   }
#else
   if (pv_random_prob() < dt_sec*params->noiseFreqE) {
      l_phiExc = l_phiExc + params->noiseAmpE*pv_random_prob();
   }

   if (pv_random_prob() < dt_sec*params->noiseFreqI) {
      l_phiInh = l_phiInh + params->noiseAmpI*pv_random_prob();
   }

   if (pv_random_prob() < dt_sec*params->noiseFreqIB) {
      l_phiInhB = l_phiInhB + params->noiseAmpIB*pv_random_prob();
   }
#endif

   l_G_E  = l_phiExc  + l_G_E *exp_tauE;
   l_G_I  = l_phiInh  + l_G_I *exp_tauI;
   l_G_IB = l_phiInhB + l_G_IB*exp_tauIB;

   l_G_E  = (l_G_E  > GMAX) ? GMAX : l_G_E;
   l_G_I  = (l_G_I  > GMAX) ? GMAX : l_G_I;
   l_G_IB = (l_G_IB > GMAX) ? GMAX : l_G_IB;

   tauInf  = (dt/tau) * (1.0 + l_G_E + l_G_I + l_G_IB);
   VmemInf = (Vrest + l_G_E*Vexc + l_G_I*Vinh + l_G_IB*VinhB)
           / (1.0 + l_G_E + l_G_I + l_G_IB);

   l_V = VmemInf + (l_V - VmemInf)*EXP(-tauInf);;

   //
   // start of LIF2_update_finish
   //

   l_Vth = VthRest + (l_Vth - VthRest)*exp_tauVth;

   //
   // start of update_f
   //

   bool fired_flag = (l_V > l_Vth);

   l_activ = fired_flag ? 1.0f             : 0.0f;
   l_V     = fired_flag ? Vrest            : l_V;
   l_Vth   = fired_flag ? l_Vth + deltaVth : l_Vth;
   l_G_IB  = fired_flag ? l_G_IB + 1.0f    : l_G_IB;

   // update average rate
   l_R = l_activ + l_R*exp_tauRate;

   //
   // These actions must be done outside of kernel
   //    1. set activity to 0 in boundary (if needed)
   //    2. update active indices
   //

   // store local variables back to global memory
   //
   activity[kex] = l_activ;

   R[k] = l_R;

   V[k]   = l_V;
   Vth[k] = l_Vth;

   G_E[k]  = l_G_E;
   G_I[k]  = l_G_I;
   G_IB[k] = l_G_IB;

   phiExc[k]  = 0.0f;
   phiInh[k]  = 0.0f;
   phiInhB[k] = 0.0f;

#ifndef PV_USE_OPENCL
   } // loop over k
#endif

}


not sure

not sure

(in-package #:smarkup)

(defclass filtered-object (ch-asdf:object-from-variable)
  ((filters :accessor object-filters :initarg :filters)))

(defmethod perform ((op compile-op) (c filtered-object))
  (call-next-method)
  (setf (symbol-value (ch-asdf::object-symbol c))
        (markup::apply-filters
         (symbol-value (ch-asdf::object-symbol c))
         (object-filters c))))

(defparameter *pdflatex-program* "pdflatex")
(defparameter *pdflatex-program-path*
  (let ((found (sb-ext:find-executable-in-search-path
                *pdflatex-program*)))
    (unless found
      (setf found
            #+darwin "/sw/bin/pdflatex"
            #-darwin "/usr/local/bin/pdflatex"))
    found))

(defclass object-latex-file (ch-asdf:object-from-variable generated-file) ())

(defmethod perform ((op ch-asdf::generate-op) (c object-latex-file))
  (call-next-method)
  (markup::render-as :latex
                     (markup::strip-invisible
                      (symbol-value (ch-asdf::object-symbol c)))
                     (component-pathname c)))

(defmethod perform ((operation compile-op) (c object-latex-file))
  (with-component-directory (c)
    (let ((unix-path (ch-util::unix-name (component-pathname c))))
      (ch-util::run-program *pdflatex-program-path*
                            (list unix-path))
      ;; we have to do this twice to get the references right!
      ;; maybe 3x?
      (ch-util::run-program *pdflatex-program-path*
                            (list unix-path)))))

(defmethod operation-done-p ((o ch-asdf::generate-op) (c object-latex-file))
  (declare (optimize (debug 3)))
  (let ((on-disk-time
         (file-write-date (component-pathname c)))
        (obj (asdf::find-component
              (asdf::component-parent c)
              (asdf::coerce-name (ch-asdf::object-input-object c)))))

    (let ((obj-date (asdf::component-property obj 'ch-asdf::last-loaded)))
      (and on-disk-time
           obj-date
           (>= on-disk-time obj-date)))))

(defclass object-xhtml-file (ch-asdf:object-from-variable source-file) ())

(defmethod perform ((op compile-op) (c object-xhtml-file))
  (call-next-method)
  (markup::render-as :xhtml
                    (markup::strip-invisible
                     (symbol-value (ch-asdf::object-symbol c)))
                    (component-pathname c)))
(defmethod perform ((op load-op) (c object-xhtml-file))
  (call-next-method)
  (ch-util::firefox-open (ch-util::unix-name (component-pathname c))))

(in-package #:smarkup)

(defclass filtered-object (ch-asdf:object-from-variable)
  ((filters :accessor object-filters :initarg :filters)))

(defmethod perform ((op compile-op) (c filtered-object))
  (call-next-method)
  (setf (symbol-value (ch-asdf::object-symbol c))
        (markup::apply-filters
         (symbol-value (ch-asdf::object-symbol c))
         (object-filters c))))

(defparameter *pdflatex-program* "pdflatex")
(defparameter *pdflatex-program-path*
  (let ((found (sb-ext:find-executable-in-search-path
                *pdflatex-program*)))
    (unless found
      (setf found
            #+darwin "/sw/bin/pdflatex"
            #-darwin "/usr/local/bin/pdflatex"))
    found))

(defclass object-latex-file (ch-asdf:object-from-variable generated-file) ())

(defmethod perform ((op ch-asdf::generate-op) (c object-latex-file))
  (call-next-method)
  (markup::render-as :latex
                     (markup::strip-invisible
                      (symbol-value (ch-asdf::object-symbol c)))
                     (component-pathname c)))

(defmethod perform ((operation compile-op) (c object-latex-file))
  (with-component-directory (c)
    (let ((unix-path (ch-util::unix-name (component-pathname c))))
      (ch-util::run-program *pdflatex-program-path*
                            (list unix-path))
      ;; we have to do this twice to get the references right!
      ;; maybe 3x?
      (ch-util::run-program *pdflatex-program-path*
                            (list unix-path)))))

(defmethod operation-done-p ((o ch-asdf::generate-op) (c object-latex-file))
  (let ((on-disk-time
         (file-write-date (component-pathname c)))
        (obj (asdf::find-component
              (asdf::component-parent c)
              (asdf::coerce-name (ch-asdf::object-input-object c)))))

    (let ((obj-date (asdf::component-property obj 'ch-asdf::last-loaded)))
      (and on-disk-time
           obj-date
           (> on-disk-time obj-date)))))

(defclass object-xhtml-file (ch-asdf:object-from-variable source-file) ())

(defmethod perform ((op compile-op) (c object-xhtml-file))
  (call-next-method)
  (markup::render-as :xhtml
                    (markup::strip-invisible
                     (symbol-value (ch-asdf::object-symbol c)))
                    (component-pathname c)))
(defmethod perform ((op load-op) (c object-xhtml-file))
  (call-next-method)
  (ch-util::firefox-open (ch-util::unix-name (component-pathname c))))


Optimize pyrUp_unrolled() by mad function.

It could improve performance when image size is large.
E.g. OCL_PyrUpFixture_PyrUp.PyrUp/18

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Zhang Chunpeng	chunpeng@multicorewareinc.com
//    Dachuan Zhao, dachuan@multicorewareinc.com
//    Yao Wang, yao@multicorewareinc.com
//    Peng Xiao, pengxiao@outlook.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

///////////////////////////////////////////////////////////////////////
////////////////////////  Generic PyrUp  //////////////////////////////
///////////////////////////////////////////////////////////////////////

#ifdef DOUBLE_SUPPORT
#ifdef cl_amd_fp64
#pragma OPENCL EXTENSION cl_amd_fp64:enable
#elif defined (cl_khr_fp64)
#pragma OPENCL EXTENSION cl_khr_fp64:enable
#endif
#endif

#if cn != 3
#define loadpix(addr)  *(__global const T*)(addr)
#define storepix(val, addr)  *(__global T*)(addr) = (val)
#define PIXSIZE ((int)sizeof(T))
#else
#define loadpix(addr)  vload3(0, (__global const T1*)(addr))
#define storepix(val, addr) vstore3((val), 0, (__global T1*)(addr))
#define PIXSIZE ((int)sizeof(T1)*3)
#endif

#define EXTRAPOLATE(x, maxV) min(maxV - 1, (int) abs(x))

#define noconvert

__kernel void pyrUp(__global const uchar * src, int src_step, int src_offset, int src_rows, int src_cols,
                         __global uchar * dst, int dst_step, int dst_offset, int dst_rows, int dst_cols)
{
    const int x = get_global_id(0);
    const int y = get_global_id(1);

    const int tidx = get_local_id(0);
    const int tidy = get_local_id(1);

    __local FT s_srcPatch[LOCAL_SIZE/2 + 2][LOCAL_SIZE/2 + 2];
    __local FT s_dstPatch[LOCAL_SIZE/2 + 2][LOCAL_SIZE];

    __global uchar * dstData = dst + dst_offset;
    __global const uchar * srcData = src + src_offset;

    if( tidx < (LOCAL_SIZE/2 + 2) && tidy < LOCAL_SIZE/2 + 2 )
    {
        int srcx = EXTRAPOLATE(mad24((int)get_group_id(0), LOCAL_SIZE/2, tidx) - 1, src_cols);
        int srcy = EXTRAPOLATE(mad24((int)get_group_id(1), LOCAL_SIZE/2, tidy) - 1, src_rows);

        s_srcPatch[tidy][tidx] = convertToFT(loadpix(srcData + srcy * src_step + srcx * PIXSIZE));
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    FT sum = 0.f;

    const FT co1 = 0.75f;
    const FT co2 = 0.5f;
    const FT co3 = 0.125f;

    const FT coef1 = (tidx & 1) == 0 ? co1 : (FT) 0;
    const FT coef2 = (tidx & 1) == 0 ? co3 : co2;
    const FT coefy1 = (tidy & 1) == 0 ? co1 : (FT) 0;
    const FT coefy2 = (tidy & 1) == 0 ? co3 : co2;

    if(tidy < LOCAL_SIZE/2 + 2)
    {
        sum =     coef2* s_srcPatch[tidy][1 + ((tidx - 1) >> 1)];
        sum = mad(coef1, s_srcPatch[tidy][1 + ((tidx    ) >> 1)], sum);
        sum = mad(coef2, s_srcPatch[tidy][1 + ((tidx + 2) >> 1)], sum);

        s_dstPatch[tidy][tidx] = sum;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    sum =     coefy2* s_dstPatch[1 + ((tidy - 1) >> 1)][tidx];
    sum = mad(coefy1, s_dstPatch[1 + ((tidy    ) >> 1)][tidx], sum);
    sum = mad(coefy2, s_dstPatch[1 + ((tidy + 2) >> 1)][tidx], sum);

    if ((x < dst_cols) && (y < dst_rows))
        storepix(convertToT(sum), dstData + y * dst_step + x * PIXSIZE);
}

__kernel void pyrUp_unrolled(__global const uchar * src, int src_step, int src_offset, int src_rows, int src_cols,
                         __global uchar * dst, int dst_step, int dst_offset, int dst_rows, int dst_cols)
{
    const int lx = 2*get_local_id(0);
    const int ly = 2*get_local_id(1);

    __local FT s_srcPatch[LOCAL_SIZE+2][LOCAL_SIZE+2];
    __local FT s_dstPatch[LOCAL_SIZE+2][2*LOCAL_SIZE];

    __global uchar * dstData = dst + dst_offset;
    __global const uchar * srcData = src + src_offset;

    if( lx < (LOCAL_SIZE+2) && ly < (LOCAL_SIZE+2) )
    {
        int srcx = mad24((int)get_group_id(0), LOCAL_SIZE, lx) - 1;
        int srcy = mad24((int)get_group_id(1), LOCAL_SIZE, ly) - 1;

        int srcx1 = EXTRAPOLATE(srcx, src_cols);
        int srcx2 = EXTRAPOLATE(srcx+1, src_cols);
        int srcy1 = EXTRAPOLATE(srcy, src_rows);
        int srcy2 = EXTRAPOLATE(srcy+1, src_rows);
        s_srcPatch[ly][lx] = convertToFT(loadpix(srcData + srcy1 * src_step + srcx1 * PIXSIZE));
        s_srcPatch[ly+1][lx] = convertToFT(loadpix(srcData + srcy2 * src_step + srcx1 * PIXSIZE));
        s_srcPatch[ly][lx+1] = convertToFT(loadpix(srcData + srcy1 * src_step + srcx2 * PIXSIZE));
        s_srcPatch[ly+1][lx+1] = convertToFT(loadpix(srcData + srcy2 * src_step + srcx2 * PIXSIZE));
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    FT sum;

    const FT co1 = 0.75f;
    const FT co2 = 0.5f;
    const FT co3 = 0.125f;

    // (x,y)
    sum =       co3 * s_srcPatch[1 + (ly >> 1)][1 + ((lx - 2) >> 1)];
    sum = mad(co1, s_srcPatch[1 + (ly >> 1)][1 + ((lx    ) >> 1)], sum);
    sum = mad(co3, s_srcPatch[1 + (ly >> 1)][1 + ((lx + 2) >> 1)], sum);

    s_dstPatch[1 + get_local_id(1)][lx] = sum;

    // (x+1,y)
    sum =       co2 * s_srcPatch[1 + (ly >> 1)][1 + ((lx + 1 - 1) >> 1)];
    sum = mad(co2, s_srcPatch[1 + (ly >> 1)][1 + ((lx + 1 + 1) >> 1)], sum);
    s_dstPatch[1 + get_local_id(1)][lx+1] = sum;

    if (ly < 1)
    {
        // (x,y)
        sum =       co3 * s_srcPatch[0][1 + ((lx - 2) >> 1)];
        sum = mad(co1, s_srcPatch[0][1 + ((lx    ) >> 1)], sum);
        sum = mad(co3, s_srcPatch[0][1 + ((lx + 2) >> 1)], sum);
        s_dstPatch[0][lx] = sum;

        // (x+1,y)
        sum =       co2 * s_srcPatch[0][1 + ((lx + 1 - 1) >> 1)];
        sum = mad(co2, s_srcPatch[0][1 + ((lx + 1 + 1) >> 1)], sum);
        s_dstPatch[0][lx+1] = sum;
    }

    if (ly > 2*LOCAL_SIZE-3)
    {
        // (x,y)
        sum =       co3 * s_srcPatch[LOCAL_SIZE+1][1 + ((lx - 2) >> 1)];
        sum = mad(co1, s_srcPatch[LOCAL_SIZE+1][1 + ((lx    ) >> 1)], sum);
        sum = mad(co3, s_srcPatch[LOCAL_SIZE+1][1 + ((lx + 2) >> 1)], sum);
        s_dstPatch[LOCAL_SIZE+1][lx] = sum;

        // (x+1,y)
        sum =       co2 * s_srcPatch[LOCAL_SIZE+1][1 + ((lx + 1 - 1) >> 1)];
        sum = mad(co2, s_srcPatch[LOCAL_SIZE+1][1 + ((lx + 1 + 1) >> 1)], sum);
        s_dstPatch[LOCAL_SIZE+1][lx+1] = sum;
    }

    barrier(CLK_LOCAL_MEM_FENCE);
    int dst_x = 2*get_global_id(0);
    int dst_y = 2*get_global_id(1);

    if ((dst_x < dst_cols) && (dst_y < dst_rows))
    {
        // (x,y)
        sum =       co3 * s_dstPatch[1 + get_local_id(1) - 1][lx];
        sum = mad(co1, s_dstPatch[1 + get_local_id(1)    ][lx], sum);
        sum = mad(co3, s_dstPatch[1 + get_local_id(1) + 1][lx], sum);
        storepix(convertToT(sum), dstData + dst_y * dst_step + dst_x * PIXSIZE);

        // (x+1,y)
        sum =       co3 * s_dstPatch[1 + get_local_id(1) - 1][lx+1];
        sum = mad(co1, s_dstPatch[1 + get_local_id(1)    ][lx+1], sum);
        sum = mad(co3, s_dstPatch[1 + get_local_id(1) + 1][lx+1], sum);
        storepix(convertToT(sum), dstData + dst_y * dst_step + (dst_x+1) * PIXSIZE);

        // (x,y+1)
        sum =       co2 * s_dstPatch[1 + get_local_id(1)    ][lx];
        sum = mad(co2, s_dstPatch[1 + get_local_id(1) + 1][lx], sum);
        storepix(convertToT(sum), dstData + (dst_y+1) * dst_step + dst_x * PIXSIZE);

        // (x+1,y+1)
        sum =       co2 * s_dstPatch[1 + get_local_id(1)    ][lx+1];
        sum = mad(co2, s_dstPatch[1 + get_local_id(1) + 1][lx+1], sum);
        storepix(convertToT(sum), dstData + (dst_y+1) * dst_step + (dst_x+1) * PIXSIZE);
    }
}

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Zhang Chunpeng	chunpeng@multicorewareinc.com
//    Dachuan Zhao, dachuan@multicorewareinc.com
//    Yao Wang, yao@multicorewareinc.com
//    Peng Xiao, pengxiao@outlook.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

///////////////////////////////////////////////////////////////////////
////////////////////////  Generic PyrUp  //////////////////////////////
///////////////////////////////////////////////////////////////////////

#ifdef DOUBLE_SUPPORT
#ifdef cl_amd_fp64
#pragma OPENCL EXTENSION cl_amd_fp64:enable
#elif defined (cl_khr_fp64)
#pragma OPENCL EXTENSION cl_khr_fp64:enable
#endif
#endif

#if cn != 3
#define loadpix(addr)  *(__global const T*)(addr)
#define storepix(val, addr)  *(__global T*)(addr) = (val)
#define PIXSIZE ((int)sizeof(T))
#else
#define loadpix(addr)  vload3(0, (__global const T1*)(addr))
#define storepix(val, addr) vstore3((val), 0, (__global T1*)(addr))
#define PIXSIZE ((int)sizeof(T1)*3)
#endif

#define EXTRAPOLATE(x, maxV) min(maxV - 1, (int) abs(x))

#define noconvert

__kernel void pyrUp(__global const uchar * src, int src_step, int src_offset, int src_rows, int src_cols,
                         __global uchar * dst, int dst_step, int dst_offset, int dst_rows, int dst_cols)
{
    const int x = get_global_id(0);
    const int y = get_global_id(1);

    const int tidx = get_local_id(0);
    const int tidy = get_local_id(1);

    __local FT s_srcPatch[LOCAL_SIZE/2 + 2][LOCAL_SIZE/2 + 2];
    __local FT s_dstPatch[LOCAL_SIZE/2 + 2][LOCAL_SIZE];

    __global uchar * dstData = dst + dst_offset;
    __global const uchar * srcData = src + src_offset;

    if( tidx < (LOCAL_SIZE/2 + 2) && tidy < LOCAL_SIZE/2 + 2 )
    {
        int srcx = EXTRAPOLATE(mad24((int)get_group_id(0), LOCAL_SIZE/2, tidx) - 1, src_cols);
        int srcy = EXTRAPOLATE(mad24((int)get_group_id(1), LOCAL_SIZE/2, tidy) - 1, src_rows);

        s_srcPatch[tidy][tidx] = convertToFT(loadpix(srcData + srcy * src_step + srcx * PIXSIZE));
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    FT sum = 0.f;

    const FT co1 = 0.75f;
    const FT co2 = 0.5f;
    const FT co3 = 0.125f;

    const FT coef1 = (tidx & 1) == 0 ? co1 : (FT) 0;
    const FT coef2 = (tidx & 1) == 0 ? co3 : co2;
    const FT coefy1 = (tidy & 1) == 0 ? co1 : (FT) 0;
    const FT coefy2 = (tidy & 1) == 0 ? co3 : co2;

    if(tidy < LOCAL_SIZE/2 + 2)
    {
        sum =     coef2* s_srcPatch[tidy][1 + ((tidx - 1) >> 1)];
        sum = mad(coef1, s_srcPatch[tidy][1 + ((tidx    ) >> 1)], sum);
        sum = mad(coef2, s_srcPatch[tidy][1 + ((tidx + 2) >> 1)], sum);

        s_dstPatch[tidy][tidx] = sum;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    sum =     coefy2* s_dstPatch[1 + ((tidy - 1) >> 1)][tidx];
    sum = mad(coefy1, s_dstPatch[1 + ((tidy    ) >> 1)][tidx], sum);
    sum = mad(coefy2, s_dstPatch[1 + ((tidy + 2) >> 1)][tidx], sum);

    if ((x < dst_cols) && (y < dst_rows))
        storepix(convertToT(sum), dstData + y * dst_step + x * PIXSIZE);
}

__kernel void pyrUp_unrolled(__global const uchar * src, int src_step, int src_offset, int src_rows, int src_cols,
                         __global uchar * dst, int dst_step, int dst_offset, int dst_rows, int dst_cols)
{
    const int lx = 2*get_local_id(0);
    const int ly = 2*get_local_id(1);

    __local FT s_srcPatch[LOCAL_SIZE+2][LOCAL_SIZE+2];
    __local FT s_dstPatch[LOCAL_SIZE+2][2*LOCAL_SIZE];

    __global uchar * dstData = dst + dst_offset;
    __global const uchar * srcData = src + src_offset;

    if( lx < (LOCAL_SIZE+2) && ly < (LOCAL_SIZE+2) )
    {
        int srcx = mad24((int)get_group_id(0), LOCAL_SIZE, lx) - 1;
        int srcy = mad24((int)get_group_id(1), LOCAL_SIZE, ly) - 1;

        int srcx1 = EXTRAPOLATE(srcx, src_cols);
        int srcx2 = EXTRAPOLATE(srcx+1, src_cols);
        int srcy1 = EXTRAPOLATE(srcy, src_rows);
        int srcy2 = EXTRAPOLATE(srcy+1, src_rows);
        s_srcPatch[ly][lx] = convertToFT(loadpix(srcData + srcy1 * src_step + srcx1 * PIXSIZE));
        s_srcPatch[ly+1][lx] = convertToFT(loadpix(srcData + srcy2 * src_step + srcx1 * PIXSIZE));
        s_srcPatch[ly][lx+1] = convertToFT(loadpix(srcData + srcy1 * src_step + srcx2 * PIXSIZE));
        s_srcPatch[ly+1][lx+1] = convertToFT(loadpix(srcData + srcy2 * src_step + srcx2 * PIXSIZE));
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    FT sum;

    const FT co1 = 0.75f;
    const FT co2 = 0.5f;
    const FT co3 = 0.125f;

    // (x,y)
    sum =       co3 * s_srcPatch[1 + (ly >> 1)][1 + ((lx - 2) >> 1)];
    sum = sum + co1 * s_srcPatch[1 + (ly >> 1)][1 + ((lx    ) >> 1)];
    sum = sum + co3 * s_srcPatch[1 + (ly >> 1)][1 + ((lx + 2) >> 1)];

    s_dstPatch[1 + get_local_id(1)][lx] = sum;

    // (x+1,y)
    sum =       co2 * s_srcPatch[1 + (ly >> 1)][1 + ((lx + 1 - 1) >> 1)];
    sum = sum + co2 * s_srcPatch[1 + (ly >> 1)][1 + ((lx + 1 + 1) >> 1)];
    s_dstPatch[1 + get_local_id(1)][lx+1] = sum;

    if (ly < 1)
    {
        // (x,y)
        sum =       co3 * s_srcPatch[0][1 + ((lx - 2) >> 1)];
        sum = sum + co1 * s_srcPatch[0][1 + ((lx    ) >> 1)];
        sum = sum + co3 * s_srcPatch[0][1 + ((lx + 2) >> 1)];
        s_dstPatch[0][lx] = sum;

        // (x+1,y)
        sum =       co2 * s_srcPatch[0][1 + ((lx + 1 - 1) >> 1)];
        sum = sum + co2 * s_srcPatch[0][1 + ((lx + 1 + 1) >> 1)];
        s_dstPatch[0][lx+1] = sum;
    }

    if (ly > 2*LOCAL_SIZE-3)
    {
        // (x,y)
        sum =       co3 * s_srcPatch[LOCAL_SIZE+1][1 + ((lx - 2) >> 1)];
        sum = sum + co1 * s_srcPatch[LOCAL_SIZE+1][1 + ((lx    ) >> 1)];
        sum = sum + co3 * s_srcPatch[LOCAL_SIZE+1][1 + ((lx + 2) >> 1)];
        s_dstPatch[LOCAL_SIZE+1][lx] = sum;

        // (x+1,y)
        sum =       co2 * s_srcPatch[LOCAL_SIZE+1][1 + ((lx + 1 - 1) >> 1)];
        sum = sum + co2 * s_srcPatch[LOCAL_SIZE+1][1 + ((lx + 1 + 1) >> 1)];
        s_dstPatch[LOCAL_SIZE+1][lx+1] = sum;
    }

    barrier(CLK_LOCAL_MEM_FENCE);
    int dst_x = 2*get_global_id(0);
    int dst_y = 2*get_global_id(1);

    if ((dst_x < dst_cols) && (dst_y < dst_rows))
    {
        // (x,y)
        sum =       co3 * s_dstPatch[1 + get_local_id(1) - 1][lx];
        sum = sum + co1 * s_dstPatch[1 + get_local_id(1)    ][lx];
        sum = sum + co3 * s_dstPatch[1 + get_local_id(1) + 1][lx];
        storepix(convertToT(sum), dstData + dst_y * dst_step + dst_x * PIXSIZE);

        // (x+1,y)
        sum =       co3 * s_dstPatch[1 + get_local_id(1) - 1][lx+1];
        sum = sum + co1 * s_dstPatch[1 + get_local_id(1)    ][lx+1];
        sum = sum + co3 * s_dstPatch[1 + get_local_id(1) + 1][lx+1];
        storepix(convertToT(sum), dstData + dst_y * dst_step + (dst_x+1) * PIXSIZE);

        // (x,y+1)
        sum =       co2 * s_dstPatch[1 + get_local_id(1)    ][lx];
        sum = sum + co2 * s_dstPatch[1 + get_local_id(1) + 1][lx];
        storepix(convertToT(sum), dstData + (dst_y+1) * dst_step + dst_x * PIXSIZE);

        // (x+1,y+1)
        sum =       co2 * s_dstPatch[1 + get_local_id(1)    ][lx+1];
        sum = sum + co2 * s_dstPatch[1 + get_local_id(1) + 1][lx+1];
        storepix(convertToT(sum), dstData + (dst_y+1) * dst_step + (dst_x+1) * PIXSIZE);
    }
}


Added 3D simplex noise

Added 3D simplex noise


Update nn_fpga.cl

Update nn_fpga.cl

Updated to work with larger datasets

/*
======================================================
 Copyright 2016 Fahad Bin Muslim
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
======================================================
*
* Author:   Fahad Bin Muslim (fahad.muslim@polito.it)
*
* This OpenCL code has a kernel to calculate the distances
* between the query point and all the points in reference data set.
* A second kernel is used to identify the nearest neighbors.
*
*----------------------------------------------------------------------------
*/
#include "params.h"

global float dist[GLOBAL_SIZE]; //on-chip global buffer

// This computes the distance of all points in d_locations from the given
// longitude and latitude, and returns it in the on-chip global buffer
// dist.
__kernel __attribute__ ((reqd_work_group_size(WORK_GROUP_SIZE, 1, 1)))
void distance_calc(__global float2 *d_locations,
                     const float lat,
		     const float lng) {
__attribute__((xcl_pipeline_workitems)) {

  int globalId = get_global_id(0);
  float lat_tmp, lng_tmp, dist_lat, dist_lng;

  // using temporaries for the latitude and longitude
  lat_tmp = d_locations[globalId].x;
  lng_tmp = d_locations[globalId].y;

  dist_lat = lat-lat_tmp;
  dist_lng = lng-lng_tmp;

  //squared euclidean distance calculation
  dist[globalId] = (dist_lat*dist_lat) + (dist_lng*dist_lng);
}
}

// This computes the resultsCount nearest neighbors, one per work item.
__kernel __attribute__ ((reqd_work_group_size(WORK_GROUP_SIZE,1,1)))
void NearestNeighbor (__global float *d_distances,
                      __global int *indices,
                        const int numRecords,
                        const int resultsCount) {
  __local float dmin1;
  int localId = get_local_id(0);
  #ifdef GLOBAL_SIZE
  int numR = GLOBAL_SIZE;
  #else
  int numR = numRecords;
  #endif

  if (localId < resultsCount) {
      float dist1;
      float dmin = MAXFLOAT;
      int count = 0;

      if (localId == 0) {
	  dmin1 = 0.0f;
      }

      __attribute__((xcl_pipeline_loop))
      for (int k = 0; k < numR; k++) {
	  dist1 = dist[k];
	  if (dist1 < dmin && dist1 > dmin1) {
	      dmin = dist1;
	      count = k;
	  }
	  if (k == numR-1) {
	      dmin1 = dmin;
	      indices[localId] = count;
	      d_distances[localId] = dmin;
	  }
      }
  }
}

/*
======================================================
 Copyright 2016 Fahad Bin Muslim
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
       http://www.apache.org/licenses/LICENSE-2.0
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
======================================================
*
* Author:   Fahad Bin Muslim (fahad.muslim@polito.it)
*
* This OpenCL code has a kernel to calculate the distances
* between the query point and all the points in reference data set.
* A second kernel is used to identify the nearest neighbors.
*
*----------------------------------------------------------------------------
*/
#include "params.h"

global float dist[GLOBAL_SIZE]; //on-chip global buffer

// This computes the distance of all points in d_locations from the given
// longitude and latitude, and returns it in the on-chip global buffer
// dist.
__kernel __attribute__ ((reqd_work_group_size(WORK_GROUP_SIZE, 1, 1)))
void distance_calc(__global float2 *d_locations,
                     const float lat,
		     const float lng) {
__attribute__((xcl_pipeline_workitems)) {

  int globalId = get_global_id(0);
  float lat_tmp, lng_tmp, dist_lat, dist_lng;

  // using temporaries for the latitude and longitude
  lat_tmp = d_locations[globalId].x;
  lng_tmp = d_locations[globalId].y;

  dist_lat = lat-lat_tmp;
  dist_lng = lng-lng_tmp;

  //squared euclidean distance calculation
  dist[globalId] = (dist_lat*dist_lat) + (dist_lng*dist_lng);
}
}

// This computes the resultsCount nearest neighbors, one per work item.
__kernel __attribute__ ((reqd_work_group_size(WORK_GROUP_SIZE,1,1)))
void NearestNeighbor (__global float *d_distances,
                      __global int *indices,
                        const int resultsCount) {
  __local float dmin1;
  int localId = get_local_id(0);
  if (localId < resultsCount) {
      float dist1;
      float dmin = MAXFLOAT;
      int count = 0;

      if (localId == 0) {
	  dmin1 = 0.0f;
      }

      __attribute__((xcl_pipeline_loop))
      for (int k = 0; k < WORK_GROUP_SIZE; k++) {
	  dist1 = dist[k];
	  if (dist1 < dmin && dist1 > dmin1) {
	      dmin = dist1;
	      count = k;
	  }
	  if (k == WORK_GROUP_SIZE-1) {
	      dmin1 = dmin;
	      indices[localId] = count;
	      d_distances[localId] = dmin;
	  }
      }
  }
}