Remove unnecesary operations and calculations in loop body.

Signed-off-by: Yan Wang <8ca54580088a7ffe1dd243d1c16c6d1f04463d55@linux.intel.com>

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Nathan, liujun@multicorewareinc.com
//    Peng Xiao, pengxiao@outlook.com
//    Baichuan Su, baichuan@multicorewareinc.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics:enable
#define MAX_FLOAT 3.40282e+038f

#ifndef T
#define T float
#endif

#ifndef BLOCK_SIZE
#define BLOCK_SIZE 16
#endif
#ifndef MAX_DESC_LEN
#define MAX_DESC_LEN 64
#endif

#define BLOCK_SIZE_ODD          (BLOCK_SIZE + 1)
#ifndef SHARED_MEM_SZ
#  if (BLOCK_SIZE < MAX_DESC_LEN)
#    define SHARED_MEM_SZ      (kercn * (BLOCK_SIZE * MAX_DESC_LEN + BLOCK_SIZE * BLOCK_SIZE))
#  else
#    define SHARED_MEM_SZ      (kercn * 2 * BLOCK_SIZE_ODD * BLOCK_SIZE)
#  endif
#endif

#ifndef DIST_TYPE
#define DIST_TYPE 2
#endif

// dirty fix for non-template support
#if (DIST_TYPE == 2) // L1Dist
#   ifdef T_FLOAT
        typedef float result_type;
#       if (8 == kercn)
            typedef float8 value_type;
#           define DIST(x, y) {value_type d = fabs((x) - (y)); result += d.s0 + d.s1 + d.s2 + d.s3 + d.s4 + d.s5 + d.s6 + d.s7;}
#       elif (4 == kercn)
            typedef float4 value_type;
#           define DIST(x, y) {value_type d = fabs((x) - (y)); result += d.s0 + d.s1 + d.s2 + d.s3;}
#       else
            typedef float value_type;
#           define DIST(x, y) result += fabs((x) - (y))
#       endif
#   else
        typedef int result_type;
#       if (8 == kercn)
            typedef int8 value_type;
#           define DIST(x, y) {value_type d = abs((x) - (y)); result += d.s0 + d.s1 + d.s2 + d.s3 + d.s4 + d.s5 + d.s6 + d.s7;}
#       elif (4 == kercn)
            typedef int4 value_type;
#           define DIST(x, y) {value_type d = abs((x) - (y)); result += d.s0 + d.s1 + d.s2 + d.s3;}
#       else
            typedef int  value_type;
#           define DIST(x, y) result += abs((x) - (y))
#       endif
#   endif
#   define DIST_RES(x) (x)
#elif (DIST_TYPE == 4) // L2Dist
    typedef float result_type;
#   if (8 == kercn)
        typedef float8 value_type;
#       define DIST(x, y)   {value_type d = ((x) - (y)); result += dot(d.s0123, d.s0123) + dot(d.s4567, d.s4567);}
#   elif (4 == kercn)
        typedef float4      value_type;
#       define DIST(x, y)   {value_type d = ((x) - (y)); result += dot(d, d);}
#   else
        typedef float       value_type;
#       define DIST(x, y)   {value_type d = ((x) - (y)); result = mad(d, d, result);}
#   endif
#   define DIST_RES(x) sqrt(x)
#elif (DIST_TYPE == 6) // Hamming
#   if (8 == kercn)
        typedef int8 value_type;
#   elif (4 == kercn)
        typedef int4 value_type;
#   else
        typedef int value_type;
#   endif
    typedef int result_type;
#   define DIST(x, y) result += popcount( (x) ^ (y) )
#   define DIST_RES(x) (x)
#endif

inline result_type reduce_block(
    __local value_type *s_query,
    __local value_type *s_train,
    int lidx,
    int lidy
    )
{
    result_type result = 0;
    #pragma unroll
    for (int j = 0 ; j < BLOCK_SIZE ; j++)
    {
        DIST(s_query[lidy * BLOCK_SIZE_ODD + j], s_train[j * BLOCK_SIZE_ODD + lidx]);
    }
    return DIST_RES(result);
}

inline result_type reduce_block_match(
    __local value_type *s_query,
    __local value_type *s_train,
    int lidx,
    int lidy
    )
{
    result_type result = 0;
    #pragma unroll
    for (int j = 0 ; j < BLOCK_SIZE ; j++)
    {
        DIST(s_query[lidy * BLOCK_SIZE_ODD + j], s_train[j * BLOCK_SIZE_ODD + lidx]);
    }
    return result;
}

inline result_type reduce_multi_block(
    __local value_type *s_query,
    __local value_type *s_train,
    int block_index,
    int lidx,
    int lidy
    )
{
    result_type result = 0;
    #pragma unroll
    for (int j = 0 ; j < BLOCK_SIZE ; j++)
    {
        DIST(s_query[lidy * MAX_DESC_LEN + block_index * BLOCK_SIZE + j], s_train[j * BLOCK_SIZE + lidx]);
    }
    return result;
}

__kernel void BruteForceMatch_Match(
    __global T *query,
    __global T *train,
    __global int *bestTrainIdx,
    __global float *bestDistance,
    int query_rows,
    int query_cols,
    int train_rows,
    int train_cols,
    int step
)
{
    const int lidx = get_local_id(0);
    const int lidy = get_local_id(1);
    const int groupidx = get_group_id(0);

    const int queryIdx = mad24(BLOCK_SIZE, groupidx, lidy);
    const int queryOffset = min(queryIdx, query_rows - 1) * step;
    __global TN *query_vec = (__global TN *)(query + queryOffset);
    query_cols /= kercn;

    __local float sharebuffer[SHARED_MEM_SZ];
    __local value_type *s_query = (__local value_type *)sharebuffer;

#if 0 < MAX_DESC_LEN
    __local value_type *s_train = (__local value_type *)sharebuffer + BLOCK_SIZE * MAX_DESC_LEN;
    // load the query into local memory.
    #pragma unroll
    for (int i = 0; i < MAX_DESC_LEN / BLOCK_SIZE; i++)
    {
        const int loadx = mad24(BLOCK_SIZE, i, lidx);
        s_query[mad24(MAX_DESC_LEN, lidy, loadx)] = loadx < query_cols ? query_vec[loadx] : 0;
    }
#else
    __local value_type *s_train = (__local value_type *)sharebuffer + BLOCK_SIZE_ODD * BLOCK_SIZE;
    const int s_query_i = mad24(BLOCK_SIZE_ODD, lidy, lidx);
    const int s_train_i = mad24(BLOCK_SIZE_ODD, lidx, lidy);
#endif

    float myBestDistance = MAX_FLOAT;
    int myBestTrainIdx = -1;

    // loopUnrolledCached to find the best trainIdx and best distance.
    for (int t = 0, endt = (train_rows + BLOCK_SIZE - 1) / BLOCK_SIZE; t < endt; t++)
    {
        result_type result = 0;

        const int trainOffset = min(mad24(BLOCK_SIZE, t, lidy), train_rows - 1) * step;
        __global TN *train_vec = (__global TN *)(train + trainOffset);
#if 0 < MAX_DESC_LEN
        #pragma unroll
        for (int i = 0; i < MAX_DESC_LEN / BLOCK_SIZE; i++)
        {
            //load a BLOCK_SIZE * BLOCK_SIZE block into local train.
            const int loadx = mad24(BLOCK_SIZE, i, lidx);
            s_train[mad24(BLOCK_SIZE, lidx, lidy)] = loadx < train_cols ? train_vec[loadx] : 0;

            //synchronize to make sure each elem for reduceIteration in share memory is written already.
            barrier(CLK_LOCAL_MEM_FENCE);

            result += reduce_multi_block(s_query, s_train, i, lidx, lidy);

            barrier(CLK_LOCAL_MEM_FENCE);
        }
#else
        for (int i = 0, endq = (query_cols + BLOCK_SIZE - 1) / BLOCK_SIZE; i < endq; i++)
        {
            const int loadx = mad24(i, BLOCK_SIZE, lidx);
            //load query and train into local memory
            if (loadx < query_cols)
            {
                s_query[s_query_i] = query_vec[loadx];
                s_train[s_train_i] = train_vec[loadx];
            }
            else
            {
                s_query[s_query_i] = 0;
                s_train[s_train_i] = 0;
            }

            barrier(CLK_LOCAL_MEM_FENCE);

            result += reduce_block_match(s_query, s_train, lidx, lidy);

            barrier(CLK_LOCAL_MEM_FENCE);
        }
#endif
        result = DIST_RES(result);

        const int trainIdx = mad24(BLOCK_SIZE, t, lidx);

        if (queryIdx < query_rows && trainIdx < train_rows && result < myBestDistance /*&& mask(queryIdx, trainIdx)*/)
        {
            myBestDistance = result;
            myBestTrainIdx = trainIdx;
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    __local float *s_distance = (__local float *)sharebuffer;
    __local int *s_trainIdx = (__local int *)(sharebuffer + BLOCK_SIZE_ODD * BLOCK_SIZE);

    //findBestMatch
    s_distance += lidy * BLOCK_SIZE_ODD;
    s_trainIdx += lidy * BLOCK_SIZE_ODD;
    s_distance[lidx] = myBestDistance;
    s_trainIdx[lidx] = myBestTrainIdx;

    barrier(CLK_LOCAL_MEM_FENCE);

    //reduce -- now all reduce implement in each threads.
    #pragma unroll
    for (int k = 0 ; k < BLOCK_SIZE; k++)
    {
        if (myBestDistance > s_distance[k])
        {
            myBestDistance = s_distance[k];
            myBestTrainIdx = s_trainIdx[k];
        }
    }

    if (queryIdx < query_rows && lidx == 0)
    {
        bestTrainIdx[queryIdx] = myBestTrainIdx;
        bestDistance[queryIdx] = myBestDistance;
    }
}

//radius_match
__kernel void BruteForceMatch_RadiusMatch(
    __global T *query,
    __global T *train,
    float maxDistance,
    __global int *bestTrainIdx,
    __global float *bestDistance,
    __global int *nMatches,
    int query_rows,
    int query_cols,
    int train_rows,
    int train_cols,
    int bestTrainIdx_cols,
    int step,
    int ostep
)
{
    const int lidx = get_local_id(0);
    const int lidy = get_local_id(1);
    const int groupidx = get_group_id(0);
    const int groupidy = get_group_id(1);

    const int queryIdx = mad24(BLOCK_SIZE, groupidy, lidy);
    const int queryOffset = min(queryIdx, query_rows - 1) * step;
    __global TN *query_vec = (__global TN *)(query + queryOffset);

    const int trainIdx = mad24(BLOCK_SIZE, groupidx, lidx);
    const int trainOffset = min(mad24(BLOCK_SIZE, groupidx, lidy), train_rows - 1) * step;
    __global TN *train_vec = (__global TN *)(train + trainOffset);

    query_cols /= kercn;

    __local float sharebuffer[SHARED_MEM_SZ];
    __local value_type *s_query = (__local value_type *)sharebuffer;
    __local value_type *s_train = (__local value_type *)sharebuffer + BLOCK_SIZE_ODD * BLOCK_SIZE;

    result_type result = 0;
    const int s_query_i = mad24(BLOCK_SIZE_ODD, lidy, lidx);
    const int s_train_i = mad24(BLOCK_SIZE_ODD, lidx, lidy);
    for (int i = 0 ; i < (query_cols + BLOCK_SIZE - 1) / BLOCK_SIZE ; ++i)
    {
        //load a BLOCK_SIZE * BLOCK_SIZE block into local train.
        const int loadx = mad24(BLOCK_SIZE, i, lidx);

        if (loadx < query_cols)
        {
            s_query[s_query_i] = query_vec[loadx];
            s_train[s_train_i] = train_vec[loadx];
        }
        else
        {
            s_query[s_query_i] = 0;
            s_train[s_train_i] = 0;
        }

        //synchronize to make sure each elem for reduceIteration in share memory is written already.
        barrier(CLK_LOCAL_MEM_FENCE);

        result += reduce_block(s_query, s_train, lidx, lidy);

        barrier(CLK_LOCAL_MEM_FENCE);
    }
    if (queryIdx < query_rows && trainIdx < train_rows && convert_float(result) < maxDistance)
    {
        int ind = atom_inc(nMatches + queryIdx);

        if(ind < bestTrainIdx_cols)
        {
            bestTrainIdx[mad24(queryIdx, ostep, ind)] = trainIdx;
            bestDistance[mad24(queryIdx, ostep, ind)] = result;
        }
    }
}

__kernel void BruteForceMatch_knnMatch(
    __global T *query,
    __global T *train,
    __global int2 *bestTrainIdx,
    __global float2 *bestDistance,
    int query_rows,
    int query_cols,
    int train_rows,
    int train_cols,
    int step
)
{
    const int lidx = get_local_id(0);
    const int lidy = get_local_id(1);
    const int groupidx = get_group_id(0);

    const int queryIdx = mad24(BLOCK_SIZE, groupidx, lidy);
    const int queryOffset = min(queryIdx, query_rows - 1) * step;
    __global TN *query_vec = (__global TN *)(query + queryOffset);
    query_cols /= kercn;

    __local float sharebuffer[SHARED_MEM_SZ];
    __local value_type *s_query = (__local value_type *)sharebuffer;

#if 0 < MAX_DESC_LEN
    __local value_type *s_train = (__local value_type *)sharebuffer + BLOCK_SIZE * MAX_DESC_LEN;
    // load the query into local memory.
    #pragma unroll
    for (int i = 0 ;  i <  MAX_DESC_LEN / BLOCK_SIZE; i ++)
    {
        int loadx = mad24(BLOCK_SIZE, i, lidx);
        s_query[mad24(MAX_DESC_LEN, lidy, loadx)] = loadx < query_cols ? query_vec[loadx] : 0;
    }
#else
    __local value_type *s_train = (__local value_type *)sharebuffer + BLOCK_SIZE_ODD * BLOCK_SIZE;
    const int s_query_i = mad24(BLOCK_SIZE_ODD, lidy, lidx);
    const int s_train_i = mad24(BLOCK_SIZE_ODD, lidx, lidy);
#endif

    float myBestDistance1 = MAX_FLOAT;
    float myBestDistance2 = MAX_FLOAT;
    int myBestTrainIdx1 = -1;
    int myBestTrainIdx2 = -1;

    for (int t = 0, endt = (train_rows + BLOCK_SIZE - 1) / BLOCK_SIZE; t < endt ; t++)
    {
        result_type result = 0;

        int trainOffset = min(mad24(BLOCK_SIZE, t, lidy), train_rows - 1) * step;
        __global TN *train_vec = (__global TN *)(train + trainOffset);
#if 0 < MAX_DESC_LEN
        #pragma unroll
        for (int i = 0 ; i < MAX_DESC_LEN / BLOCK_SIZE ; i++)
        {
            //load a BLOCK_SIZE * BLOCK_SIZE block into local train.
            const int loadx = mad24(BLOCK_SIZE, i, lidx);
            s_train[mad24(BLOCK_SIZE, lidx, lidy)] = loadx < train_cols ? train_vec[loadx] : 0;

            //synchronize to make sure each elem for reduceIteration in share memory is written already.
            barrier(CLK_LOCAL_MEM_FENCE);

            result += reduce_multi_block(s_query, s_train, i, lidx, lidy);

            barrier(CLK_LOCAL_MEM_FENCE);
        }
#else
        for (int i = 0, endq = (query_cols + BLOCK_SIZE -1) / BLOCK_SIZE; i < endq ; i++)
        {
            const int loadx = mad24(BLOCK_SIZE, i, lidx);
            //load query and train into local memory
            if (loadx < query_cols)
            {
                s_query[s_query_i] = query_vec[loadx];
                s_train[s_train_i] = train_vec[loadx];
            }
            else
            {
                s_query[s_query_i] = 0;
                s_train[s_train_i] = 0;
            }

            barrier(CLK_LOCAL_MEM_FENCE);

            result += reduce_block_match(s_query, s_train, lidx, lidy);

            barrier(CLK_LOCAL_MEM_FENCE);
        }
#endif
        result = DIST_RES(result);

        const int trainIdx = mad24(BLOCK_SIZE, t, lidx);

        if (queryIdx < query_rows && trainIdx < train_rows)
        {
            if (result < myBestDistance1)
            {
                myBestDistance2 = myBestDistance1;
                myBestTrainIdx2 = myBestTrainIdx1;
                myBestDistance1 = result;
                myBestTrainIdx1 = trainIdx;
            }
            else if (result < myBestDistance2)
            {
                myBestDistance2 = result;
                myBestTrainIdx2 = trainIdx;
            }
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    __local float *s_distance = (__local float *)sharebuffer;
    __local int *s_trainIdx = (__local int *)(sharebuffer + BLOCK_SIZE_ODD * BLOCK_SIZE);

    // find BestMatch
    s_distance += lidy * BLOCK_SIZE_ODD;
    s_trainIdx += lidy * BLOCK_SIZE_ODD;
    s_distance[lidx] = myBestDistance1;
    s_trainIdx[lidx] = myBestTrainIdx1;

    float bestDistance1 = MAX_FLOAT;
    float bestDistance2 = MAX_FLOAT;
    int bestTrainIdx1 = -1;
    int bestTrainIdx2 = -1;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (lidx == 0)
    {
        for (int i = 0 ; i < BLOCK_SIZE ; i++)
        {
            float val = s_distance[i];
            if (val < bestDistance1)
            {
                bestDistance2 = bestDistance1;
                bestTrainIdx2 = bestTrainIdx1;

                bestDistance1 = val;
                bestTrainIdx1 = s_trainIdx[i];
            }
            else if (val < bestDistance2)
            {
                bestDistance2 = val;
                bestTrainIdx2 = s_trainIdx[i];
            }
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    s_distance[lidx] = myBestDistance2;
    s_trainIdx[lidx] = myBestTrainIdx2;

    barrier(CLK_LOCAL_MEM_FENCE);

    if (lidx == 0)
    {
        for (int i = 0 ; i < BLOCK_SIZE ; i++)
        {
            float val = s_distance[i];

            if (val < bestDistance2)
            {
                bestDistance2 = val;
                bestTrainIdx2 = s_trainIdx[i];
            }
        }
    }

    myBestDistance1 = bestDistance1;
    myBestDistance2 = bestDistance2;

    myBestTrainIdx1 = bestTrainIdx1;
    myBestTrainIdx2 = bestTrainIdx2;

    if (queryIdx < query_rows && lidx == 0)
    {
        bestTrainIdx[queryIdx] = (int2)(myBestTrainIdx1, myBestTrainIdx2);
        bestDistance[queryIdx] = (float2)(myBestDistance1, myBestDistance2);
    }
}

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Multicoreware, Inc., all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Nathan, liujun@multicorewareinc.com
//    Peng Xiao, pengxiao@outlook.com
//    Baichuan Su, baichuan@multicorewareinc.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors "as is" and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

#pragma OPENCL EXTENSION cl_khr_global_int32_base_atomics:enable
#define MAX_FLOAT 3.40282e+038f

#ifndef T
#define T float
#endif

#ifndef BLOCK_SIZE
#define BLOCK_SIZE 16
#endif
#ifndef MAX_DESC_LEN
#define MAX_DESC_LEN 64
#endif

#define BLOCK_SIZE_ODD          (BLOCK_SIZE + 1)
#ifndef SHARED_MEM_SZ
#  if (BLOCK_SIZE < MAX_DESC_LEN)
#    define SHARED_MEM_SZ      (kercn * (BLOCK_SIZE * MAX_DESC_LEN + BLOCK_SIZE * BLOCK_SIZE))
#  else
#    define SHARED_MEM_SZ      (kercn * 2 * BLOCK_SIZE_ODD * BLOCK_SIZE)
#  endif
#endif

#ifndef DIST_TYPE
#define DIST_TYPE 2
#endif

// dirty fix for non-template support
#if (DIST_TYPE == 2) // L1Dist
#   ifdef T_FLOAT
        typedef float result_type;
#       if (8 == kercn)
            typedef float8 value_type;
#           define DIST(x, y) {value_type d = fabs((x) - (y)); result += d.s0 + d.s1 + d.s2 + d.s3 + d.s4 + d.s5 + d.s6 + d.s7;}
#       elif (4 == kercn)
            typedef float4 value_type;
#           define DIST(x, y) {value_type d = fabs((x) - (y)); result += d.s0 + d.s1 + d.s2 + d.s3;}
#       else
            typedef float value_type;
#           define DIST(x, y) result += fabs((x) - (y))
#       endif
#   else
        typedef int result_type;
#       if (8 == kercn)
            typedef int8 value_type;
#           define DIST(x, y) {value_type d = abs((x) - (y)); result += d.s0 + d.s1 + d.s2 + d.s3 + d.s4 + d.s5 + d.s6 + d.s7;}
#       elif (4 == kercn)
            typedef int4 value_type;
#           define DIST(x, y) {value_type d = abs((x) - (y)); result += d.s0 + d.s1 + d.s2 + d.s3;}
#       else
            typedef int  value_type;
#           define DIST(x, y) result += abs((x) - (y))
#       endif
#   endif
#   define DIST_RES(x) (x)
#elif (DIST_TYPE == 4) // L2Dist
    typedef float result_type;
#   if (8 == kercn)
        typedef float8 value_type;
#       define DIST(x, y)   {value_type d = ((x) - (y)); result += dot(d.s0123, d.s0123) + dot(d.s4567, d.s4567);}
#   elif (4 == kercn)
        typedef float4      value_type;
#       define DIST(x, y)   {value_type d = ((x) - (y)); result += dot(d, d);}
#   else
        typedef float       value_type;
#       define DIST(x, y)   {value_type d = ((x) - (y)); result = mad(d, d, result);}
#   endif
#   define DIST_RES(x) sqrt(x)
#elif (DIST_TYPE == 6) // Hamming
#   if (8 == kercn)
        typedef int8 value_type;
#   elif (4 == kercn)
        typedef int4 value_type;
#   else
        typedef int value_type;
#   endif
    typedef int result_type;
#   define DIST(x, y) result += popcount( (x) ^ (y) )
#   define DIST_RES(x) (x)
#endif

inline result_type reduce_block(
    __local value_type *s_query,
    __local value_type *s_train,
    int lidx,
    int lidy
    )
{
    result_type result = 0;
    #pragma unroll
    for (int j = 0 ; j < BLOCK_SIZE ; j++)
    {
        DIST(s_query[lidy * BLOCK_SIZE_ODD + j], s_train[j * BLOCK_SIZE_ODD + lidx]);
    }
    return DIST_RES(result);
}

inline result_type reduce_block_match(
    __local value_type *s_query,
    __local value_type *s_train,
    int lidx,
    int lidy
    )
{
    result_type result = 0;
    #pragma unroll
    for (int j = 0 ; j < BLOCK_SIZE ; j++)
    {
        DIST(s_query[lidy * BLOCK_SIZE_ODD + j], s_train[j * BLOCK_SIZE_ODD + lidx]);
    }
    return result;
}

inline result_type reduce_multi_block(
    __local value_type *s_query,
    __local value_type *s_train,
    int block_index,
    int lidx,
    int lidy
    )
{
    result_type result = 0;
    #pragma unroll
    for (int j = 0 ; j < BLOCK_SIZE ; j++)
    {
        DIST(s_query[lidy * MAX_DESC_LEN + block_index * BLOCK_SIZE + j], s_train[j * BLOCK_SIZE + lidx]);
    }
    return result;
}

__kernel void BruteForceMatch_Match(
    __global T *query,
    __global T *train,
    __global int *bestTrainIdx,
    __global float *bestDistance,
    int query_rows,
    int query_cols,
    int train_rows,
    int train_cols,
    int step
)
{
    const int lidx = get_local_id(0);
    const int lidy = get_local_id(1);
    const int groupidx = get_group_id(0);

    const int queryIdx = mad24(BLOCK_SIZE, groupidx, lidy);
    const int queryOffset = min(queryIdx, query_rows - 1) * step;
    __global TN *query_vec = (__global TN *)(query + queryOffset);
    query_cols /= kercn;

    __local float sharebuffer[SHARED_MEM_SZ];
    __local value_type *s_query = (__local value_type *)sharebuffer;

#if 0 < MAX_DESC_LEN
    __local value_type *s_train = (__local value_type *)sharebuffer + BLOCK_SIZE * MAX_DESC_LEN;
    // load the query into local memory.
    #pragma unroll
    for (int i = 0; i < MAX_DESC_LEN / BLOCK_SIZE; i++)
    {
        const int loadx = mad24(BLOCK_SIZE, i, lidx);
        s_query[mad24(MAX_DESC_LEN, lidy, loadx)] = loadx < query_cols ? query_vec[loadx] : 0;
    }
#else
    __local value_type *s_train = (__local value_type *)sharebuffer + BLOCK_SIZE_ODD * BLOCK_SIZE;
#endif

    float myBestDistance = MAX_FLOAT;
    int myBestTrainIdx = -1;

    // loopUnrolledCached to find the best trainIdx and best distance.
    for (int t = 0, endt = (train_rows + BLOCK_SIZE - 1) / BLOCK_SIZE; t < endt; t++)
    {
        result_type result = 0;

        const int trainOffset = min(mad24(BLOCK_SIZE, t, lidy), train_rows - 1) * step;
        __global TN *train_vec = (__global TN *)(train + trainOffset);
#if 0 < MAX_DESC_LEN
        #pragma unroll
        for (int i = 0; i < MAX_DESC_LEN / BLOCK_SIZE; i++)
        {
            //load a BLOCK_SIZE * BLOCK_SIZE block into local train.
            const int loadx = mad24(BLOCK_SIZE, i, lidx);
            s_train[mad24(BLOCK_SIZE, lidx, lidy)] = loadx < train_cols ? train_vec[loadx] : 0;

            //synchronize to make sure each elem for reduceIteration in share memory is written already.
            barrier(CLK_LOCAL_MEM_FENCE);

            result += reduce_multi_block(s_query, s_train, i, lidx, lidy);

            barrier(CLK_LOCAL_MEM_FENCE);
        }
#else
        for (int i = 0, endq = (query_cols + BLOCK_SIZE - 1) / BLOCK_SIZE; i < endq; i++)
        {
            const int loadx = mad24(i, BLOCK_SIZE, lidx);
            //load query and train into local memory
            s_query[mad24(BLOCK_SIZE_ODD, lidy, lidx)] = 0;
            s_train[mad24(BLOCK_SIZE_ODD, lidx, lidy)] = 0;

            if (loadx < query_cols)
            {
                s_query[mad24(BLOCK_SIZE_ODD, lidy, lidx)] = query_vec[loadx];
                s_train[mad24(BLOCK_SIZE_ODD, lidx, lidy)] = train_vec[loadx];
            }

            barrier(CLK_LOCAL_MEM_FENCE);

            result += reduce_block_match(s_query, s_train, lidx, lidy);

            barrier(CLK_LOCAL_MEM_FENCE);
        }
#endif
        result = DIST_RES(result);

        const int trainIdx = mad24(BLOCK_SIZE, t, lidx);

        if (queryIdx < query_rows && trainIdx < train_rows && result < myBestDistance /*&& mask(queryIdx, trainIdx)*/)
        {
            myBestDistance = result;
            myBestTrainIdx = trainIdx;
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    __local float *s_distance = (__local float *)sharebuffer;
    __local int *s_trainIdx = (__local int *)(sharebuffer + BLOCK_SIZE_ODD * BLOCK_SIZE);

    //findBestMatch
    s_distance += lidy * BLOCK_SIZE_ODD;
    s_trainIdx += lidy * BLOCK_SIZE_ODD;
    s_distance[lidx] = myBestDistance;
    s_trainIdx[lidx] = myBestTrainIdx;

    barrier(CLK_LOCAL_MEM_FENCE);

    //reduce -- now all reduce implement in each threads.
    #pragma unroll
    for (int k = 0 ; k < BLOCK_SIZE; k++)
    {
        if (myBestDistance > s_distance[k])
        {
            myBestDistance = s_distance[k];
            myBestTrainIdx = s_trainIdx[k];
        }
    }

    if (queryIdx < query_rows && lidx == 0)
    {
        bestTrainIdx[queryIdx] = myBestTrainIdx;
        bestDistance[queryIdx] = myBestDistance;
    }
}

//radius_match
__kernel void BruteForceMatch_RadiusMatch(
    __global T *query,
    __global T *train,
    float maxDistance,
    __global int *bestTrainIdx,
    __global float *bestDistance,
    __global int *nMatches,
    int query_rows,
    int query_cols,
    int train_rows,
    int train_cols,
    int bestTrainIdx_cols,
    int step,
    int ostep
)
{
    const int lidx = get_local_id(0);
    const int lidy = get_local_id(1);
    const int groupidx = get_group_id(0);
    const int groupidy = get_group_id(1);

    const int queryIdx = mad24(BLOCK_SIZE, groupidy, lidy);
    const int queryOffset = min(queryIdx, query_rows - 1) * step;
    __global TN *query_vec = (__global TN *)(query + queryOffset);

    const int trainIdx = mad24(BLOCK_SIZE, groupidx, lidx);
    const int trainOffset = min(mad24(BLOCK_SIZE, groupidx, lidy), train_rows - 1) * step;
    __global TN *train_vec = (__global TN *)(train + trainOffset);

    query_cols /= kercn;

    __local float sharebuffer[SHARED_MEM_SZ];
    __local value_type *s_query = (__local value_type *)sharebuffer;
    __local value_type *s_train = (__local value_type *)sharebuffer + BLOCK_SIZE_ODD * BLOCK_SIZE;

    result_type result = 0;
    for (int i = 0 ; i < (query_cols + BLOCK_SIZE - 1) / BLOCK_SIZE ; ++i)
    {
        //load a BLOCK_SIZE * BLOCK_SIZE block into local train.
        const int loadx = mad24(BLOCK_SIZE, i, lidx);

        s_query[mad24(BLOCK_SIZE_ODD, lidy, lidx)] = 0;
        s_train[mad24(BLOCK_SIZE_ODD, lidx, lidy)] = 0;

        if (loadx < query_cols)
        {
            s_query[mad24(BLOCK_SIZE_ODD, lidy, lidx)] = query_vec[loadx];
            s_train[mad24(BLOCK_SIZE_ODD, lidx, lidy)] = train_vec[loadx];
        }

        //synchronize to make sure each elem for reduceIteration in share memory is written already.
        barrier(CLK_LOCAL_MEM_FENCE);

        result += reduce_block(s_query, s_train, lidx, lidy);

        barrier(CLK_LOCAL_MEM_FENCE);
    }
    if (queryIdx < query_rows && trainIdx < train_rows && convert_float(result) < maxDistance)
    {
        int ind = atom_inc(nMatches + queryIdx);

        if(ind < bestTrainIdx_cols)
        {
            bestTrainIdx[mad24(queryIdx, ostep, ind)] = trainIdx;
            bestDistance[mad24(queryIdx, ostep, ind)] = result;
        }
    }
}

__kernel void BruteForceMatch_knnMatch(
    __global T *query,
    __global T *train,
    __global int2 *bestTrainIdx,
    __global float2 *bestDistance,
    int query_rows,
    int query_cols,
    int train_rows,
    int train_cols,
    int step
)
{
    const int lidx = get_local_id(0);
    const int lidy = get_local_id(1);
    const int groupidx = get_group_id(0);

    const int queryIdx = mad24(BLOCK_SIZE, groupidx, lidy);
    const int queryOffset = min(queryIdx, query_rows - 1) * step;
    __global TN *query_vec = (__global TN *)(query + queryOffset);
    query_cols /= kercn;

    __local float sharebuffer[SHARED_MEM_SZ];
    __local value_type *s_query = (__local value_type *)sharebuffer;

#if 0 < MAX_DESC_LEN
    __local value_type *s_train = (__local value_type *)sharebuffer + BLOCK_SIZE * MAX_DESC_LEN;
    // load the query into local memory.
    #pragma unroll
    for (int i = 0 ;  i <  MAX_DESC_LEN / BLOCK_SIZE; i ++)
    {
        int loadx = mad24(BLOCK_SIZE, i, lidx);
        s_query[mad24(MAX_DESC_LEN, lidy, loadx)] = loadx < query_cols ? query_vec[loadx] : 0;
    }
#else
    __local value_type *s_train = (__local value_type *)sharebuffer + BLOCK_SIZE_ODD * BLOCK_SIZE;
#endif

    float myBestDistance1 = MAX_FLOAT;
    float myBestDistance2 = MAX_FLOAT;
    int myBestTrainIdx1 = -1;
    int myBestTrainIdx2 = -1;

    for (int t = 0, endt = (train_rows + BLOCK_SIZE - 1) / BLOCK_SIZE; t < endt ; t++)
    {
        result_type result = 0;

        int trainOffset = min(mad24(BLOCK_SIZE, t, lidy), train_rows - 1) * step;
        __global TN *train_vec = (__global TN *)(train + trainOffset);
#if 0 < MAX_DESC_LEN
        #pragma unroll
        for (int i = 0 ; i < MAX_DESC_LEN / BLOCK_SIZE ; i++)
        {
            //load a BLOCK_SIZE * BLOCK_SIZE block into local train.
            const int loadx = mad24(BLOCK_SIZE, i, lidx);
            s_train[mad24(BLOCK_SIZE, lidx, lidy)] = loadx < train_cols ? train_vec[loadx] : 0;

            //synchronize to make sure each elem for reduceIteration in share memory is written already.
            barrier(CLK_LOCAL_MEM_FENCE);

            result += reduce_multi_block(s_query, s_train, i, lidx, lidy);

            barrier(CLK_LOCAL_MEM_FENCE);
        }
#else
        for (int i = 0, endq = (query_cols + BLOCK_SIZE -1) / BLOCK_SIZE; i < endq ; i++)
        {
            const int loadx = mad24(BLOCK_SIZE, i, lidx);
            //load query and train into local memory
            s_query[mad24(BLOCK_SIZE_ODD, lidy, lidx)] = 0;
            s_train[mad24(BLOCK_SIZE_ODD, lidx, lidy)] = 0;

            if (loadx < query_cols)
            {
                s_query[mad24(BLOCK_SIZE_ODD, lidy, lidx)] = query_vec[loadx];
                s_train[mad24(BLOCK_SIZE_ODD, lidx, lidy)] = train_vec[loadx];
            }

            barrier(CLK_LOCAL_MEM_FENCE);

            result += reduce_block_match(s_query, s_train, lidx, lidy);

            barrier(CLK_LOCAL_MEM_FENCE);
        }
#endif
        result = DIST_RES(result);

        const int trainIdx = mad24(BLOCK_SIZE, t, lidx);

        if (queryIdx < query_rows && trainIdx < train_rows)
        {
            if (result < myBestDistance1)
            {
                myBestDistance2 = myBestDistance1;
                myBestTrainIdx2 = myBestTrainIdx1;
                myBestDistance1 = result;
                myBestTrainIdx1 = trainIdx;
            }
            else if (result < myBestDistance2)
            {
                myBestDistance2 = result;
                myBestTrainIdx2 = trainIdx;
            }
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    __local float *s_distance = (__local float *)sharebuffer;
    __local int *s_trainIdx = (__local int *)(sharebuffer + BLOCK_SIZE_ODD * BLOCK_SIZE);

    // find BestMatch
    s_distance += lidy * BLOCK_SIZE_ODD;
    s_trainIdx += lidy * BLOCK_SIZE_ODD;
    s_distance[lidx] = myBestDistance1;
    s_trainIdx[lidx] = myBestTrainIdx1;

    float bestDistance1 = MAX_FLOAT;
    float bestDistance2 = MAX_FLOAT;
    int bestTrainIdx1 = -1;
    int bestTrainIdx2 = -1;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (lidx == 0)
    {
        for (int i = 0 ; i < BLOCK_SIZE ; i++)
        {
            float val = s_distance[i];
            if (val < bestDistance1)
            {
                bestDistance2 = bestDistance1;
                bestTrainIdx2 = bestTrainIdx1;

                bestDistance1 = val;
                bestTrainIdx1 = s_trainIdx[i];
            }
            else if (val < bestDistance2)
            {
                bestDistance2 = val;
                bestTrainIdx2 = s_trainIdx[i];
            }
        }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    s_distance[lidx] = myBestDistance2;
    s_trainIdx[lidx] = myBestTrainIdx2;

    barrier(CLK_LOCAL_MEM_FENCE);

    if (lidx == 0)
    {
        for (int i = 0 ; i < BLOCK_SIZE ; i++)
        {
            float val = s_distance[i];

            if (val < bestDistance2)
            {
                bestDistance2 = val;
                bestTrainIdx2 = s_trainIdx[i];
            }
        }
    }

    myBestDistance1 = bestDistance1;
    myBestDistance2 = bestDistance2;

    myBestTrainIdx1 = bestTrainIdx1;
    myBestTrainIdx2 = bestTrainIdx2;

    if (queryIdx < query_rows && lidx == 0)
    {
        bestTrainIdx[queryIdx] = (int2)(myBestTrainIdx1, myBestTrainIdx2);
        bestDistance[queryIdx] = (float2)(myBestDistance1, myBestDistance2);
    }
}


Improve mask precision.

Improve mask precision.

__kernel void
downscaledconvolution(
        __write_only image2d_t output_image,
        int output_origin_x,
        int output_origin_y,
        __read_only image2d_t input_image,
        int input_origin_x,
        int input_origin_y)
{
    const float mask[5][5] = {
        3.90625e-03f, 1.56250e-02f, 2.34375e-02f, 1.56250e-02f, 3.90625e-03f,
        1.56250e-02f, 6.25000e-02f, 9.37500e-02f, 6.25000e-02f, 1.56250e-02f,
        2.34375e-02f, 9.37500e-02f, 1.40625e-01f, 9.37500e-02f, 2.34375e-02f,
        1.56250e-02f, 6.25000e-02f, 9.37500e-02f, 6.25000e-02f, 1.56250e-02f,
        3.90625e-03f, 1.56250e-02f, 2.34375e-02f, 1.56250e-02f, 3.90625e-03f,
    };

    const sampler_t sampler =
        CLK_FILTER_NEAREST|CLK_NORMALIZED_COORDS_FALSE|CLK_ADDRESS_CLAMP_TO_EDGE;

    int x_in_output = output_origin_x + get_global_id(0);
    int y_in_output = output_origin_y + get_global_id(1);

    float x_in_input = (float)(input_origin_x + get_global_id(0)*2);
    float y_in_input = (float)(input_origin_y + get_global_id(1)*2);

    float4 c = 0.f;

    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-3.5f, y_in_input-3.5f))) * mask[0][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.5f, y_in_input-3.5f))) * mask[1][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.5f, y_in_input-3.5f))) * mask[2][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.5f, y_in_input-3.5f))) * mask[3][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+4.5f, y_in_input-3.5f))) * mask[4][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-3.5f, y_in_input-1.5f))) * mask[0][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.5f, y_in_input-1.5f))) * mask[1][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.5f, y_in_input-1.5f))) * mask[2][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.5f, y_in_input-1.5f))) * mask[3][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+4.5f, y_in_input-1.5f))) * mask[4][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-3.5f, y_in_input+0.5f))) * mask[0][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.5f, y_in_input+0.5f))) * mask[1][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.5f, y_in_input+0.5f))) * mask[2][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.5f, y_in_input+0.5f))) * mask[3][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+4.5f, y_in_input+0.5f))) * mask[4][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-3.5f, y_in_input+2.5f))) * mask[0][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.5f, y_in_input+2.5f))) * mask[1][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.5f, y_in_input+2.5f))) * mask[2][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.5f, y_in_input+2.5f))) * mask[3][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+4.5f, y_in_input+2.5f))) * mask[4][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-3.5f, y_in_input+4.5f))) * mask[0][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.5f, y_in_input+4.5f))) * mask[1][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.5f, y_in_input+4.5f))) * mask[2][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.5f, y_in_input+4.5f))) * mask[3][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+4.5f, y_in_input+4.5f))) * mask[4][4];

    barrier(CLK_LOCAL_MEM_FENCE);

    write_imageui(output_image, (int2)(x_in_output, y_in_output), convert_int4(c));
}

__kernel void
downscaledconvolution(
        __write_only image2d_t output_image,
        int output_origin_x,
        int output_origin_y,
        __read_only image2d_t input_image,
        int input_origin_x,
        int input_origin_y)
{
    const float mask[5][5] = {
        0.003906, 0.015625, 0.023438, 0.015625, 0.003906,
        0.015625, 0.062500, 0.093750, 0.062500, 0.015625,
        0.023438, 0.093750, 0.140625, 0.093750, 0.023438,
        0.015625, 0.062500, 0.093750, 0.062500, 0.015625,
        0.003906, 0.015625, 0.023438, 0.015625, 0.003906
    };

    const sampler_t sampler =
        CLK_FILTER_NEAREST|CLK_NORMALIZED_COORDS_FALSE|CLK_ADDRESS_CLAMP_TO_EDGE;

    int x_in_output = output_origin_x + get_global_id(0);
    int y_in_output = output_origin_y + get_global_id(1);

    float x_in_input = (float)(input_origin_x + get_global_id(0)*2);
    float y_in_input = (float)(input_origin_y + get_global_id(1)*2);

    float4 c = 0.f;

    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-3.5f, y_in_input-3.5f))) * mask[0][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.5f, y_in_input-3.5f))) * mask[1][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.5f, y_in_input-3.5f))) * mask[2][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.5f, y_in_input-3.5f))) * mask[3][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+4.5f, y_in_input-3.5f))) * mask[4][0];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-3.5f, y_in_input-1.5f))) * mask[0][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.5f, y_in_input-1.5f))) * mask[1][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.5f, y_in_input-1.5f))) * mask[2][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.5f, y_in_input-1.5f))) * mask[3][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+4.5f, y_in_input-1.5f))) * mask[4][1];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-3.5f, y_in_input+0.5f))) * mask[0][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.5f, y_in_input+0.5f))) * mask[1][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.5f, y_in_input+0.5f))) * mask[2][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.5f, y_in_input+0.5f))) * mask[3][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+4.5f, y_in_input+0.5f))) * mask[4][2];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-3.5f, y_in_input+2.5f))) * mask[0][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.5f, y_in_input+2.5f))) * mask[1][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.5f, y_in_input+2.5f))) * mask[2][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.5f, y_in_input+2.5f))) * mask[3][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+4.5f, y_in_input+2.5f))) * mask[4][3];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-3.5f, y_in_input+4.5f))) * mask[0][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input-1.5f, y_in_input+4.5f))) * mask[1][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+0.5f, y_in_input+4.5f))) * mask[2][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+2.5f, y_in_input+4.5f))) * mask[3][4];
    c += convert_float4(read_imageui(input_image, sampler, (float2)(x_in_input+4.5f, y_in_input+4.5f))) * mask[4][4];

    barrier(CLK_LOCAL_MEM_FENCE);

    write_imageui(output_image, (int2)(x_in_output, y_in_output), convert_int4(c));
}


[OpenCL] Fix vector literal test broken in rL367675.

Avoid checking alignment unnecessary that is not portable
among targets.

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@367823 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 -emit-llvm %s -o - -O0 | FileCheck %s
// RUN: %clang_cc1 -emit-llvm %s -o - -cl-std=clc++ -O0 | FileCheck %s

typedef __attribute__((ext_vector_type(2))) int int2;
typedef __attribute__((ext_vector_type(3))) int int3;
typedef __attribute__((ext_vector_type(4)))  int int4;
typedef __attribute__((ext_vector_type(8)))  int int8;
typedef __attribute__((ext_vector_type(4))) float float4;

__constant const int4 c1 = (int4)(1, 2, ((int2)(3)));
// CHECK: constant <4 x i32> <i32 1, i32 2, i32 3, i32 3>

__constant const int4 c2 = (int4)(1, 2, ((int2)(3, 4)));
// CHECK: constant <4 x i32> <i32 1, i32 2, i32 3, i32 4>

void vector_literals_valid() {
  //CHECK: insertelement <4 x i32> <i32 1, i32 2, i32 undef, i32 undef>, i32 %{{.+}}, i32 2
  //CHECK: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 3
  int4 a_1_1_1_1 = (int4)(1, 2, c1.s2, c2.s3);

  //CHECK: store <2 x i32> <i32 1, i32 2>, <2 x i32>*
  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  //CHECK: insertelement <4 x i32> %{{.+}}, i32 3, i32 2
  //CHECK: insertelement <4 x i32> %{{.+}}, i32 4, i32 3
  int4 a_2_1_1 = (int4)((int2)(1, 2), 3, 4);

  //CHECK: store <2 x i32> <i32 2, i32 3>, <2 x i32>*
  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  //CHECK: shufflevector <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>, <4 x i32> %{{.+}}, <4 x i32> <i32 0, i32 4, i32 5, i32 undef>
  //CHECK: insertelement <4 x i32> %{{.+}}, i32 4, i32 3
  int4 a_1_2_1 = (int4)(1, (int2)(2, 3), 4);

  //CHECK: store <2 x i32> <i32 3, i32 4>, <2 x i32>*
  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  //CHECK: shufflevector <4 x i32> <i32 1, i32 2, i32 undef, i32 undef>, <4 x i32> %{{.+}}, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
  int4 a_1_1_2 = (int4)(1, 2, (int2)(3, 4));

  //CHECK: store <2 x i32> <i32 1, i32 2>, <2 x i32>*
  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> <i32 3, i32 3, i32 undef, i32 undef>, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
  int4 a_2_2 = (int4)((int2)(1, 2), (int2)(3));

  //CHECK: store <4 x i32> <i32 2, i32 3, i32 4, i32 undef>, <4 x i32>*
  //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
  //CHECK: shufflevector <3 x i32> %{{.+}}, <3 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
  //CHECK: shufflevector <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>, <4 x i32> %{{.+}}, <4 x i32> <i32 0, i32 4, i32 5, i32 6>
  int4 a_1_3 = (int4)(1, (int3)(2, 3, 4));

  //CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32>* %a
  int4 a = (int4)(1);

  //CHECK: load <4 x i32>, <4 x i32>* %a
  //CHECK: shufflevector <4 x i32> %{{[0-9]+}}, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  //CHECK: shufflevector <8 x i32> <i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <8 x i32> %{{.+}}, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef>
  //CHECK: load <4 x i32>, <4 x i32>* %a
  //CHECK: shufflevector <4 x i32> %{{[0-9]+}}, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
  //CHECK: shufflevector <8 x i32> %{{.+}}, <8 x i32> %{{.+}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
  int8 b = (int8)(1, 2, a.xy, a);

  //CHECK: store <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, <4 x float>* %V2
  float4 V2 = (float4)(1);
}

// RUN: %clang_cc1 -emit-llvm %s -o - -O0 | FileCheck %s
// RUN: %clang_cc1 -emit-llvm %s -o - -cl-std=clc++ -O0 | FileCheck %s

typedef __attribute__((ext_vector_type(2))) int int2;
typedef __attribute__((ext_vector_type(3))) int int3;
typedef __attribute__((ext_vector_type(4)))  int int4;
typedef __attribute__((ext_vector_type(8)))  int int8;
typedef __attribute__((ext_vector_type(4))) float float4;

__constant const int4 c1 = (int4)(1, 2, ((int2)(3)));
// CHECK: constant <4 x i32> <i32 1, i32 2, i32 3, i32 3>

__constant const int4 c2 = (int4)(1, 2, ((int2)(3, 4)));
// CHECK: constant <4 x i32> <i32 1, i32 2, i32 3, i32 4>

void vector_literals_valid() {
  //CHECK: insertelement <4 x i32> <i32 1, i32 2, i32 undef, i32 undef>, i32 %{{.+}}, i32 2
  //CHECK: insertelement <4 x i32> %{{.+}}, i32 %{{.+}}, i32 3
  int4 a_1_1_1_1 = (int4)(1, 2, c1.s2, c2.s3);

  //CHECK: store <2 x i32> <i32 1, i32 2>, <2 x i32>*
  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  //CHECK: insertelement <4 x i32> %{{.+}}, i32 3, i32 2
  //CHECK: insertelement <4 x i32> %{{.+}}, i32 4, i32 3
  int4 a_2_1_1 = (int4)((int2)(1, 2), 3, 4);

  //CHECK: store <2 x i32> <i32 2, i32 3>, <2 x i32>*
  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  //CHECK: shufflevector <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>, <4 x i32> %{{.+}}, <4 x i32> <i32 0, i32 4, i32 5, i32 undef>
  //CHECK: insertelement <4 x i32> %{{.+}}, i32 4, i32 3
  int4 a_1_2_1 = (int4)(1, (int2)(2, 3), 4);

  //CHECK: store <2 x i32> <i32 3, i32 4>, <2 x i32>*
  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  //CHECK: shufflevector <4 x i32> <i32 1, i32 2, i32 undef, i32 undef>, <4 x i32> %{{.+}}, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
  int4 a_1_1_2 = (int4)(1, 2, (int2)(3, 4));

  //CHECK: store <2 x i32> <i32 1, i32 2>, <2 x i32>*
  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> undef, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
  //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> <i32 3, i32 3, i32 undef, i32 undef>, <4 x i32> <i32 0, i32 1, i32 4, i32 5>
  int4 a_2_2 = (int4)((int2)(1, 2), (int2)(3));

  //CHECK: store <4 x i32> <i32 2, i32 3, i32 4, i32 undef>, <4 x i32>*
  //CHECK: shufflevector <4 x i32> %{{.+}}, <4 x i32> undef, <3 x i32> <i32 0, i32 1, i32 2>
  //CHECK: shufflevector <3 x i32> %{{.+}}, <3 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 undef>
  //CHECK: shufflevector <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>, <4 x i32> %{{.+}}, <4 x i32> <i32 0, i32 4, i32 5, i32 6>
  int4 a_1_3 = (int4)(1, (int3)(2, 3, 4));

  //CHECK: store <4 x i32> <i32 1, i32 1, i32 1, i32 1>, <4 x i32>* %a
  int4 a = (int4)(1);

  //CHECK: load <4 x i32>, <4 x i32>* %a, align 16
  //CHECK: shufflevector <4 x i32> %{{[0-9]+}}, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
  //CHECK: shufflevector <2 x i32> %{{[0-9]+}}, <2 x i32> undef, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
  //CHECK: shufflevector <8 x i32> <i32 1, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>, <8 x i32> %{{.+}}, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 undef, i32 undef, i32 undef, i32 undef>
  //CHECK: load <4 x i32>, <4 x i32>* %a, align 16
  //CHECK: shufflevector <4 x i32> %{{[0-9]+}}, <4 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
  //CHECK: shufflevector <8 x i32> %{{.+}}, <8 x i32> %{{.+}}, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
  int8 b = (int8)(1, 2, a.xy, a);

  //CHECK: store <4 x float> <float 1.000000e+00, float 1.000000e+00, float 1.000000e+00, float 1.000000e+00>, <4 x float>* %V2
  float4 V2 = (float4)(1);
}


formatting

formatting

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

// calculating the size of the area for weights prefetch

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24(u, d);
    return (r);
}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{

    _FLOAT weights[MLO_N_LCL_OUT_MAPS][MLO_N_LCL_IN_MAPS];

    uint gbl_id0  = get_global_id(0);
    uint batch_id = gbl_id0 / MLO_MAP_SZ4; // batch
    uint pos      = gbl_id0 % MLO_MAP_SZ4;

    uint out_grp_block = get_group_id(1); // block of outputs for the entire group
    uint out_id        = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off = batch_id * MLO_IN_BATCH_STRIDE + pos * MLO_READ_UNIT;

    uint wei_off = out_id *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
    _FLOAT dat[MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            accum[o][i] = 0;
        }
    }

    for(uint ci = 0; ci < MLO_CLOOP0; ++ci,
             // move input offset
                                      gbl_in_off += MLO_N_LCL_IN_MAPS * MLO_IN_CHANNEL_STRIDE,

             // move weights offset
                                      wei_off += MLO_N_LCL_IN_MAPS *
#if MLO_DIR_FORWARD == 1
                                                 MLO_WEI_CHANNEL_STRIDE
#else
                                                 MLO_WEI_BSTRIDE
#endif
        )
    {
        // read weights

        for(uint o = 0, wei_off1 = wei_off; o < MLO_N_LCL_OUT_MAPS; ++o,
                 wei_off1 +=
#if MLO_DIR_FORWARD == 1
                                                                    MLO_WEI_BSTRIDE
#else
                                                                    MLO_WEI_CHANNEL_STRIDE
#endif
            )
        {
            for(uint c = 0, wei_off2 = wei_off1; c < MLO_N_LCL_IN_MAPS; ++c,
                     wei_off2 +=
#if MLO_DIR_FORWARD == 1
                                                                        MLO_WEI_CHANNEL_STRIDE
#else
                                                                        MLO_WEI_BSTRIDE
#endif
                )
            {
                weights[o][c] = wei_ptr[wei_off2];
#if DBG_OUT_OF_RNGE
                if(wei_off2 >= MLO_N_INPUTS * MLO_N_OUTPUTS)
                {
                    printf("K:oor: weights\n");
                }
#endif
            }
        }

        // convolve with all weights
        // read data
        for(uint j = 0, gbl_in_off1 = gbl_in_off; j < MLO_N_LCL_IN_MAPS;
            ++j, gbl_in_off1 += MLO_IN_CHANNEL_STRIDE)
        {
            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                dat[j][i] = in_ptr[gbl_in_off1 + i];
#if DBG_OUT_OF_RNGE
                if(gbl_in_off1 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                {
                    printf("K:oor: inputs\n");
                }
#endif
            }
        }

        // convolve
        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    accum[o][i] += dat[c][i] * weights[o][c];
                }
            }
        }
    }

    uint gbl_out_off =
        batch_id * MLO_OUT_BATCH_STRIDE + pos * MLO_READ_UNIT + out_id * MLO_OUT_CHANNEL_STRIDE;
    for(uint o = 0, gbl_out_off1 = gbl_out_off; o < MLO_N_LCL_OUT_MAPS;
        ++o, gbl_out_off1 += MLO_OUT_CHANNEL_STRIDE)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            out_ptr[gbl_out_off1 + i] = accum[o][i];
        }
    }
}

/************************************************************************
stride and padding
*************************************************************************/
__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1pquv(const __global _FLOAT* __restrict in_ptr,
                  __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
                  const __global _FLOAT* __restrict bias,
#endif
                  __global _FLOAT* __restrict out_ptr,
                  UNUSED _FLOAT dummy_val // nothing
                  )
{

    _FLOAT weights[MLO_N_LCL_OUT_MAPS][MLO_N_LCL_IN_MAPS];

    uint gbl_id0 = get_global_id(0);

    uint batch_id  = gbl_id0 / MLO_MAP_SZ4; // batch
    uint pos       = gbl_id0 % MLO_MAP_SZ4;
    uint pos_out_y = pos / MLO_OUT_WIDTH4;
    uint pos_out_x = pos % MLO_OUT_WIDTH4;

#if MLO_DIR_FORWARD == 1
    uint pos_in_y = pos_out_y * MLO_FILTER_STRIDE1;
    uint pos_in_x = pos_out_x * MLO_FILTER_STRIDE0;
#else
    uint pos_in_y = pos_out_y; /// MLO_FILTER_STRIDE1;   - divided already
    uint pos_in_x = pos_out_x; // MLO_FILTER_STRIDE0;  - divided already
#endif

    uint out_grp_block = get_group_id(1); // block of outputs for the entire group
    uint out_id        = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off =
        batch_id * MLO_IN_BATCH_STRIDE + pos_in_y * MLO_IN_STRIDE + pos_in_x * MLO_READ_UNIT;
    //	bool vis = (pos_in_y < MLO_IN_HEIGHT);
    //	gbl_in_off = (vis) ? gbl_in_off : 0;

    uint wei_off = out_id *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
    _FLOAT dat[MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            accum[o][i] = 0;
        }
    }

    const __global _FLOAT* i_ptr = &in_ptr[gbl_in_off];
    __constant _FLOAT* w_ptr     = &wei_ptr[wei_off];
    for(uint ci = 0; ci < MLO_CLOOP0; ++ci)
    {

        // convolve with all weights
        // read data

        for(uint j = 0; j < MLO_N_LCL_IN_MAPS; ++j)
        {
            uint i = 0;
#if MLO_READ_UNIT > 1
            for(; i < MLO_READ_UNIT - 1; ++i)
            {
                uint off = i
#if MLO_DIR_FORWARD == 1
                           * MLO_FILTER_STRIDE0
#endif
                    ;
                dat[j][i] = i_ptr[off];
            }
#endif

            for(; i < MLO_READ_UNIT; ++i)
            {
                //				vis &= (pos_in_x + i*MLO_FILTER_STRIDE0 <
                // MLO_IN_WIDTH);
                uint off = i
#if MLO_DIR_FORWARD == 1
                           * MLO_FILTER_STRIDE0
#endif
                    ;
                //				off = (vis) ? off : 0;
                _FLOAT val = i_ptr[off];
                dat[j][i]  = val;
                //              dat[j][i] = (vis)? dat[j][i] : 0;
            }

            i_ptr += MLO_IN_CHANNEL_STRIDE;
        }
        // read weights
        __constant _FLOAT* w_ptr0 = w_ptr;

        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {

            __constant _FLOAT* w_ptr1 = w_ptr0;

            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                weights[o][c] = *w_ptr1;
                w_ptr1 +=
#if MLO_DIR_FORWARD == 1
                    MLO_WEI_CHANNEL_STRIDE
#else
                    MLO_WEI_BSTRIDE
#endif
                    ;
            }

            w_ptr0 +=
#if MLO_DIR_FORWARD == 1
                MLO_WEI_BSTRIDE
#else
                MLO_WEI_CHANNEL_STRIDE
#endif
                ;
        }

        w_ptr += MLO_N_LCL_IN_MAPS *
#if MLO_DIR_FORWARD == 1
                 MLO_WEI_CHANNEL_STRIDE
#else
                 MLO_WEI_BSTRIDE
#endif
            ;
        // convolve
        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    accum[o][i] += dat[c][i] * weights[o][c];
#if 0
				if (get_global_id(0) == 4 && get_global_id(1) ==0 && o == 0 && i == 0)
				{
					printf((__constant char *)"K:c: %f %f %f %f\n",
					accum[o][i],
					dat[c][i] * weights[o][c],
					dat[c][i],
					weights[o][c]
					);
				}
#endif
                }
            }
        }
    }

    uint out_y = pos_out_y
#if MLO_DIR_FORWARD == 0
                 * MLO_FILTER_STRIDE1
#endif
        ;
    uint out_x = pos_out_x
#if MLO_DIR_FORWARD == 0
                 * MLO_FILTER_STRIDE0
#endif
        ;

    uint gbl_out_off = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE +
                       out_y * MLO_OUT_STRIDE + out_x * MLO_READ_UNIT;

    for(uint o = 0, gbl_out_off1 = gbl_out_off; o < MLO_N_LCL_OUT_MAPS;
        ++o, gbl_out_off1 += MLO_OUT_CHANNEL_STRIDE)
    {

        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            uint out_off = gbl_out_off1 +
                           i
#if MLO_DIR_FORWARD == 0
                               * MLO_FILTER_STRIDE0
#endif
                ;
            out_ptr[out_off] = accum[o][i];

#if MLO_DIR_FORWARD == 0
            for(uint s = 1; s < MLO_FILTER_STRIDE0; ++s)
            {
#if MLO_HORIZ_ALIGNED == 0
                if(out_x + s < MLO_OUT_WIDTH)
#endif
                {
                    out_ptr[out_off + s] = 0;
                }
            }
#endif
        }

#if MLO_DIR_FORWARD == 0
        for(uint j = 1; j < MLO_FILTER_STRIDE1; ++j)
        {
#if MLO_VERT_ALIGNED == 0
            if(out_y + j < MLO_OUT_HEIGHT)
#endif
            {
                uint out_off = gbl_out_off1 + j * MLO_OUT_STRIDE;
                for(uint s = 0; s < MLO_READ_UNIT * MLO_FILTER_STRIDE0; ++s)
                {
#if MLO_HORIZ_ALIGNED == 0
                    if(out_x + s < MLO_OUT_WIDTH)
#endif
                    {
                        out_ptr[out_off + s] = 0;
                    }
                }
            }
        }
#endif
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define _FLOAT float
#define _FLOAT2 float2
#define _FLOAT4 float4
#define _FLOAT8 float8

#ifndef FLT_MAX
#define FLT_MAX 3.402823466e+38F /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

// calculating the size of the area for weights prefetch

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = (uint)((float)v * (1.0f / (float)d) + 0.00001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24(u, d);
    return (r);
}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{

    _FLOAT weights[MLO_N_LCL_OUT_MAPS][MLO_N_LCL_IN_MAPS];

    uint gbl_id0  = get_global_id(0);
    uint batch_id = gbl_id0 / MLO_MAP_SZ4; // batch
    uint pos      = gbl_id0 % MLO_MAP_SZ4;

    uint out_grp_block = get_group_id(1); // block of outputs for the entire group
    uint out_id        = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off = batch_id * MLO_IN_BATCH_STRIDE + pos * MLO_READ_UNIT;

    uint wei_off = out_id *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
    _FLOAT dat[MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            accum[o][i] = 0;
        }
    }

    for(uint ci = 0; ci < MLO_CLOOP0; ++ci,
             // move input offset
                                      gbl_in_off += MLO_N_LCL_IN_MAPS * MLO_IN_CHANNEL_STRIDE,

             // move weights offset
                                      wei_off += MLO_N_LCL_IN_MAPS *
#if MLO_DIR_FORWARD == 1
                                                 MLO_WEI_CHANNEL_STRIDE
#else
                                                 MLO_WEI_BSTRIDE
#endif
        )
    {
        // read weights

        for(uint o = 0, wei_off1 = wei_off; o < MLO_N_LCL_OUT_MAPS; ++o,
                 wei_off1 +=
#if MLO_DIR_FORWARD == 1
                                                                    MLO_WEI_BSTRIDE
#else
                                                                    MLO_WEI_CHANNEL_STRIDE
#endif
            )
        {
            for(uint c = 0, wei_off2 = wei_off1; c < MLO_N_LCL_IN_MAPS; ++c,
                     wei_off2 +=
#if MLO_DIR_FORWARD == 1
                                                                        MLO_WEI_CHANNEL_STRIDE
#else
                                                                        MLO_WEI_BSTRIDE
#endif
                )
            {
                weights[o][c] = wei_ptr[wei_off2];
#if DBG_OUT_OF_RNGE
				if ( wei_off2 >= MLO_N_INPUTS*MLO_N_OUTPUTS)
				{
					printf("K:oor: weights\n");
				}
#endif
            }
        }

        // convolve with all weights
        // read data
        for(uint j = 0, gbl_in_off1 = gbl_in_off; j < MLO_N_LCL_IN_MAPS;
            ++j, gbl_in_off1 += MLO_IN_CHANNEL_STRIDE)
        {
            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                dat[j][i] = in_ptr[gbl_in_off1 + i];
#if DBG_OUT_OF_RNGE
				if ( gbl_in_off1 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
				{
					printf("K:oor: inputs\n"
					);
				}
#endif
            }
        }

        // convolve
        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    accum[o][i] += dat[c][i] * weights[o][c];
                }
            }
        }
    }

    uint gbl_out_off =
        batch_id * MLO_OUT_BATCH_STRIDE + pos * MLO_READ_UNIT + out_id * MLO_OUT_CHANNEL_STRIDE;
    for(uint o = 0, gbl_out_off1 = gbl_out_off; o < MLO_N_LCL_OUT_MAPS;
        ++o, gbl_out_off1 += MLO_OUT_CHANNEL_STRIDE)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            out_ptr[gbl_out_off1 + i] = accum[o][i];
        }
    }
}

/************************************************************************
stride and padding
*************************************************************************/
__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1pquv(const __global _FLOAT* __restrict in_ptr,
                  __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
                  const __global _FLOAT* __restrict bias,
#endif
                  __global _FLOAT* __restrict out_ptr,
                  UNUSED _FLOAT dummy_val // nothing
                  )
{

    _FLOAT weights[MLO_N_LCL_OUT_MAPS][MLO_N_LCL_IN_MAPS];

    uint gbl_id0 = get_global_id(0);

    uint batch_id  = gbl_id0 / MLO_MAP_SZ4; // batch
    uint pos       = gbl_id0 % MLO_MAP_SZ4;
    uint pos_out_y = pos / MLO_OUT_WIDTH4;
    uint pos_out_x = pos % MLO_OUT_WIDTH4;

#if MLO_DIR_FORWARD == 1
    uint pos_in_y = pos_out_y * MLO_FILTER_STRIDE1;
    uint pos_in_x = pos_out_x * MLO_FILTER_STRIDE0;
#else
    uint pos_in_y = pos_out_y; /// MLO_FILTER_STRIDE1;   - divided already
    uint pos_in_x = pos_out_x; // MLO_FILTER_STRIDE0;  - divided already
#endif

    uint out_grp_block = get_group_id(1); // block of outputs for the entire group
    uint out_id        = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off =
        batch_id * MLO_IN_BATCH_STRIDE + pos_in_y * MLO_IN_STRIDE + pos_in_x * MLO_READ_UNIT;
    //	bool vis = (pos_in_y < MLO_IN_HEIGHT);
    //	gbl_in_off = (vis) ? gbl_in_off : 0;

    uint wei_off = out_id *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
    _FLOAT dat[MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            accum[o][i] = 0;
        }
    }

    const __global _FLOAT* i_ptr = &in_ptr[gbl_in_off];
    __constant _FLOAT* w_ptr     = &wei_ptr[wei_off];
    for(uint ci = 0; ci < MLO_CLOOP0; ++ci)
    {

        // convolve with all weights
        // read data

        for(uint j = 0; j < MLO_N_LCL_IN_MAPS; ++j)
        {
            uint i = 0;
#if MLO_READ_UNIT > 1
            for(; i < MLO_READ_UNIT - 1; ++i)
            {
                uint off = i
#if MLO_DIR_FORWARD == 1
                           * MLO_FILTER_STRIDE0
#endif
                    ;
                dat[j][i] = i_ptr[off];
            }
#endif

            for(; i < MLO_READ_UNIT; ++i)
            {
                //				vis &= (pos_in_x + i*MLO_FILTER_STRIDE0 <
                // MLO_IN_WIDTH);
                uint off = i
#if MLO_DIR_FORWARD == 1
                           * MLO_FILTER_STRIDE0
#endif
                    ;
                //				off = (vis) ? off : 0;
                _FLOAT val = i_ptr[off];
                dat[j][i]  = val;
                //              dat[j][i] = (vis)? dat[j][i] : 0;
            }

            i_ptr += MLO_IN_CHANNEL_STRIDE;
        }
        // read weights
        __constant _FLOAT* w_ptr0 = w_ptr;

        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {

            __constant _FLOAT* w_ptr1 = w_ptr0;

            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                weights[o][c] = *w_ptr1;
                w_ptr1 +=
#if MLO_DIR_FORWARD == 1
                    MLO_WEI_CHANNEL_STRIDE
#else
                    MLO_WEI_BSTRIDE
#endif
                    ;
            }

            w_ptr0 +=
#if MLO_DIR_FORWARD == 1
                MLO_WEI_BSTRIDE
#else
                MLO_WEI_CHANNEL_STRIDE
#endif
                ;
        }

        w_ptr += MLO_N_LCL_IN_MAPS *
#if MLO_DIR_FORWARD == 1
                 MLO_WEI_CHANNEL_STRIDE
#else
                 MLO_WEI_BSTRIDE
#endif
            ;
        // convolve
        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    accum[o][i] += dat[c][i] * weights[o][c];
#if 0
				if (get_global_id(0) == 4 && get_global_id(1) ==0 && o == 0 && i == 0)
				{
					printf((__constant char *)"K:c: %f %f %f %f\n",
					accum[o][i],
					dat[c][i] * weights[o][c],
					dat[c][i],
					weights[o][c]
					);
				}
#endif
                }
            }
        }
    }

    uint out_y = pos_out_y
#if MLO_DIR_FORWARD == 0
                 * MLO_FILTER_STRIDE1
#endif
        ;
    uint out_x = pos_out_x
#if MLO_DIR_FORWARD == 0
                 * MLO_FILTER_STRIDE0
#endif
        ;

    uint gbl_out_off = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE +
                       out_y * MLO_OUT_STRIDE + out_x * MLO_READ_UNIT;

    for(uint o = 0, gbl_out_off1 = gbl_out_off; o < MLO_N_LCL_OUT_MAPS;
        ++o, gbl_out_off1 += MLO_OUT_CHANNEL_STRIDE)
    {

        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            uint out_off = gbl_out_off1 +
                           i
#if MLO_DIR_FORWARD == 0
                               * MLO_FILTER_STRIDE0
#endif
                ;
            out_ptr[out_off] = accum[o][i];

#if MLO_DIR_FORWARD == 0
            for(uint s = 1; s < MLO_FILTER_STRIDE0; ++s)
            {
#if MLO_HORIZ_ALIGNED == 0
                if(out_x + s < MLO_OUT_WIDTH)
#endif
                {
                    out_ptr[out_off + s] = 0;
                }
            }
#endif
        }

#if MLO_DIR_FORWARD == 0
        for(uint j = 1; j < MLO_FILTER_STRIDE1; ++j)
        {
#if MLO_VERT_ALIGNED == 0
            if(out_y + j < MLO_OUT_HEIGHT)
#endif
            {
                uint out_off = gbl_out_off1 + j * MLO_OUT_STRIDE;
                for(uint s = 0; s < MLO_READ_UNIT * MLO_FILTER_STRIDE0; ++s)
                {
#if MLO_HORIZ_ALIGNED == 0
                    if(out_x + s < MLO_OUT_WIDTH)
#endif
                    {
                        out_ptr[out_off + s] = 0;
                    }
                }
            }
        }
#endif
    }
}