Fixed OpenCL SIFT bug causing segmentation faults on Intel

/*******************************************************
 * Copyright (c) 2015, ArrayFire
 * All rights reserved.
 *
 * This file is distributed under 3-clause BSD license.
 * The complete license agreement can be obtained at:
 * http://arrayfire.com/licenses/BSD-3-Clause
 ********************************************************/

// The source code contained in this file is based on the original code by
// Rob Hess. Please note that SIFT is an algorithm patented and protected
// by US law, before using this code or any binary forms generated from it,
// verify that you have permission to do so. The original license by Rob Hess
// can be read below:
//
// Copyright (c) 2006-2012, Rob Hess <rob@iqengines.com>
// All rights reserved.
//
// The following patent has been issued for methods embodied in this
// software: "Method and apparatus for identifying scale invariant features
// in an image and use of same for locating an object in an image," David
// G. Lowe, US Patent 6,711,293 (March 23, 2004). Provisional application
// filed March 8, 1999. Asignee: The University of British Columbia. For
// further details, contact David Lowe (lowe@cs.ubc.ca) or the
// University-Industry Liaison Office of the University of British
// Columbia.
//
// Note that restrictions imposed by this patent (and possibly others)
// exist independently of and may be in conflict with the freedoms granted
// in this license, which refers to copyright of the program, not patents
// for any methods that it implements.  Both copyright and patent law must
// be obeyed to legally use and redistribute this program and it is not the
// purpose of this license to induce you to infringe any patents or other
// property right claims or to contest validity of any such claims.  If you
// redistribute or use the program, then this license merely protects you
// from committing copyright infringement.  It does not protect you from
// committing patent infringement.  So, before you do anything with this
// program, make sure that you have permission to do so not merely in terms
// of copyright, but also in terms of patent law.
//
// Please note that this license is not to be understood as a guarantee
// either.  If you use the program according to this license, but in
// conflict with patent law, it does not mean that the licensor will refund
// you for any losses that you incur if you are sued for your patent
// infringement.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//     * Redistributions of source code must retain the above copyright and
//       patent notices, this list of conditions and the following
//       disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in
//       the documentation and/or other materials provided with the
//       distribution.
//     * Neither the name of Oregon State University nor the names of its
//       contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// width of border in which to ignore keypoints
#define IMG_BORDER 5

// maximum steps of keypoint interpolation before failure
#define MAX_INTERP_STEPS 5

// default number of bins in histogram for orientation assignment
#define ORI_HIST_BINS 36

// determines gaussian sigma for orientation assignment
#define ORI_SIG_FCTR 1.5f

// determines the radius of the region used in orientation assignment
#define ORI_RADIUS (3 * ORI_SIG_FCTR)

// number of passes of orientation histogram smoothing
#define SMOOTH_ORI_PASSES 2

// orientation magnitude relative to max that results in new feature
#define ORI_PEAK_RATIO 0.8f

// determines the size of a single descriptor orientation histogram
#define DESCR_SCL_FCTR 3.f

// threshold on magnitude of elements of descriptor vector
#define DESCR_MAG_THR 0.2f

// factor used to convert floating-point descriptor to unsigned char
#define INT_DESCR_FCTR 512.f

#define PI_VAL 3.14159265358979323846f

void gaussianElimination(float* A, float* b, float* x, const int n)
{
    // forward elimination
    for (int i = 0; i < n-1; i++) {
        for (int j = i+1; j < n; j++) {
            float s = A[j*n+i] / A[i*n+i];

            //for (int k = i+1; k < n; k++)
            for (int k = i; k < n; k++)
                A[j*n+k] -= s * A[i*n+k];

            b[j] -= s * b[i];
        }
    }

    for (int i = 0; i < n; i++)
        x[i] = 0;

    // backward substitution
    float sum = 0;
    for (int i = 0; i <= n-2; i++) {
        sum = b[i];
        for (int j = i+1; j < n; j++)
            sum -= A[i*n+j] * x[j];
        x[i] = sum / A[i*n+i];
    }
}

inline void fatomic_add(volatile __local float *source, const float operand) {
    union {
        unsigned int intVal;
        float floatVal;
    } newVal;
    union {
        unsigned int intVal;
        float floatVal;
    } prevVal;
    do {
        prevVal.floatVal = *source;
        newVal.floatVal = prevVal.floatVal + operand;
    } while (atomic_cmpxchg((volatile __local unsigned int *)source, prevVal.intVal, newVal.intVal) != prevVal.intVal);
}

inline void normalizeDesc(
    __local float* desc,
    __local float* accum,
    const int histlen,
    int lid_x,
    int lid_y,
    int lsz_x)
{
    for (int i = lid_x; i < histlen; i += lsz_x)
        accum[i] = desc[lid_y*histlen+i]*desc[lid_y*histlen+i];
    barrier(CLK_LOCAL_MEM_FENCE);

    float sum = 0.0f;
    for (int i = 0; i < histlen; i++)
        sum += desc[lid_y*histlen+i]*desc[lid_y*histlen+i];
    barrier(CLK_LOCAL_MEM_FENCE);

    if (lid_x < 64)
        accum[lid_x] += accum[lid_x+64];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 32)
        accum[lid_x] += accum[lid_x+32];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 16)
        accum[lid_x] += accum[lid_x+16];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 8)
        accum[lid_x] += accum[lid_x+8];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 4)
        accum[lid_x] += accum[lid_x+4];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 2)
        accum[lid_x] += accum[lid_x+2];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 1)
        accum[lid_x] += accum[lid_x+1];
    barrier(CLK_LOCAL_MEM_FENCE);

    float len_sq = accum[0];
    float len_inv = 1.0f / sqrt(len_sq);

    for (int i = lid_x; i < histlen; i += lsz_x) {
        desc[lid_y*histlen+i] *= len_inv;
    }
    barrier(CLK_LOCAL_MEM_FENCE);
}

__kernel void sub(
    __global T* out,
    __global const T* in,
    unsigned nel,
    unsigned n_layers)
{
    unsigned i = get_global_id(0);

    for (unsigned l = 0; l < n_layers; l++)
        out[l*nel + i] = in[l*nel + i] - in[(l+1)*nel + i];
}

#define LCPTR(Y, X) (l_center[(Y) * l_i + (X)])
#define LPPTR(Y, X) (l_prev[(Y) * l_i + (X)])
#define LNPTR(Y, X) (l_next[(Y) * l_i + (X)])

// Determines whether a pixel is a scale-space extremum by comparing it to its
// 3x3x3 pixel neighborhood.
__kernel void detectExtrema(
    __global float* x_out,
    __global float* y_out,
    __global unsigned* layer_out,
    __global unsigned* counter,
    __global const T* dog,
    KParam iDoG,
    const unsigned max_feat,
    const float threshold,
    __local float* l_mem)
{
    const int dim0 = iDoG.dims[0];
    const int dim1 = iDoG.dims[1];
    const int imel = iDoG.dims[0]*iDoG.dims[1];

    const int lid_i = get_local_id(0);
    const int lid_j = get_local_id(1);
    const int lsz_i = get_local_size(0);
    const int lsz_j = get_local_size(1);
    const int i = get_group_id(0) * lsz_i + lid_i+IMG_BORDER;
    const int j = get_group_id(1) * lsz_j + lid_j+IMG_BORDER;

    // One pixel border for each side
    const int l_i = lsz_i+2;
    const int l_j = lsz_j+2;

    __local float* l_prev   = l_mem;
    __local float* l_center = l_mem + l_i * l_j;
    __local float* l_next   = l_mem + l_i * l_j * 2;

    const int x = lid_i+1;
    const int y = lid_j+1;

    for (int l = 1; l < iDoG.dims[2]-1; l++) {
        const int l_i_half = l_i/2;
        const int l_j_half = l_j/2;
        if (lid_i < l_i_half && lid_j < l_j_half && i < dim0-IMG_BORDER+1 && j < dim1-IMG_BORDER+1) {
                l_next  [lid_j*l_i + lid_i] = (float)dog[(l+1)*imel+(j-1)*dim0+i-1];
                l_center[lid_j*l_i + lid_i] = (float)dog[(l  )*imel+(j-1)*dim0+i-1];
                l_prev  [lid_j*l_i + lid_i] = (float)dog[(l-1)*imel+(j-1)*dim0+i-1];

                l_next  [lid_j*l_i + lid_i+l_i_half] = (float)dog[(l+1)*imel+(j-1)*dim0+i-1+l_i_half];
                l_center[lid_j*l_i + lid_i+l_i_half] = (float)dog[(l  )*imel+(j-1)*dim0+i-1+l_i_half];
                l_prev  [lid_j*l_i + lid_i+l_i_half] = (float)dog[(l-1)*imel+(j-1)*dim0+i-1+l_i_half];

                l_next  [(lid_j+l_j_half)*l_i + lid_i] = (float)dog[(l+1)*imel+(j-1+l_j_half)*dim0+i-1];
                l_center[(lid_j+l_j_half)*l_i + lid_i] = (float)dog[(l  )*imel+(j-1+l_j_half)*dim0+i-1];
                l_prev  [(lid_j+l_j_half)*l_i + lid_i] = (float)dog[(l-1)*imel+(j-1+l_j_half)*dim0+i-1];

                l_next  [(lid_j+l_j_half)*l_i + lid_i+l_i_half] = (float)dog[(l+1)*imel+(j-1+l_j_half)*dim0+i-1+l_i_half];
                l_center[(lid_j+l_j_half)*l_i + lid_i+l_i_half] = (float)dog[(l  )*imel+(j-1+l_j_half)*dim0+i-1+l_i_half];
                l_prev  [(lid_j+l_j_half)*l_i + lid_i+l_i_half] = (float)dog[(l-1)*imel+(j-1+l_j_half)*dim0+i-1+l_i_half];
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        if (i < dim0-IMG_BORDER && j < dim1-IMG_BORDER) {
            const int l_i_half = l_i/2;
            float p = l_center[y*l_i + x];

            if (fabs((float)p) > threshold &&
                ((p > 0 && p > LCPTR(y-1, x-1) && p > LCPTR(y-1, x) &&
                  p > LCPTR(y-1, x+1) && p > LCPTR(y, x-1) && p > LCPTR(y,   x+1)  &&
                  p > LCPTR(y+1, x-1) && p > LCPTR(y+1, x) && p > LCPTR(y+1, x+1)  &&
                  p > LPPTR(y-1, x-1) && p > LPPTR(y-1, x) && p > LPPTR(y-1, x+1)  &&
                  p > LPPTR(y,   x-1) && p > LPPTR(y  , x) && p > LPPTR(y,   x+1)  &&
                  p > LPPTR(y+1, x-1) && p > LPPTR(y+1, x) && p > LPPTR(y+1, x+1)  &&
                  p > LNPTR(y-1, x-1) && p > LNPTR(y-1, x) && p > LNPTR(y-1, x+1)  &&
                  p > LNPTR(y,   x-1) && p > LNPTR(y  , x) && p > LNPTR(y,   x+1)  &&
                  p > LNPTR(y+1, x-1) && p > LNPTR(y+1, x) && p > LNPTR(y+1, x+1)) ||
                 (p < 0 && p < LCPTR(y-1, x-1) && p < LCPTR(y-1, x) &&
                  p < LCPTR(y-1, x+1) && p < LCPTR(y, x-1) && p < LCPTR(y,   x+1)  &&
                  p < LCPTR(y+1, x-1) && p < LCPTR(y+1, x) && p < LCPTR(y+1, x+1)  &&
                  p < LPPTR(y-1, x-1) && p < LPPTR(y-1, x) && p < LPPTR(y-1, x+1)  &&
                  p < LPPTR(y,   x-1) && p < LPPTR(y  , x) && p < LPPTR(y,   x+1)  &&
                  p < LPPTR(y+1, x-1) && p < LPPTR(y+1, x) && p < LPPTR(y+1, x+1)  &&
                  p < LNPTR(y-1, x-1) && p < LNPTR(y-1, x) && p < LNPTR(y-1, x+1)  &&
                  p < LNPTR(y,   x-1) && p < LNPTR(y  , x) && p < LNPTR(y,   x+1)  &&
                  p < LNPTR(y+1, x-1) && p < LNPTR(y+1, x) && p < LNPTR(y+1, x+1)))) {

                unsigned idx = atomic_inc(counter);
                if (idx < max_feat)
                {
                    x_out[idx] = (float)j;
                    y_out[idx] = (float)i;
                    layer_out[idx] = l;
                }
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}

#undef LCPTR
#undef LPPTR
#undef LNPTR
#define CPTR(Y, X) (center[(Y) * dim0 + (X)])
#define PPTR(Y, X) (prev[(Y) * dim0 + (X)])
#define NPTR(Y, X) (next[(Y) * dim0 + (X)])

// Interpolates a scale-space extremum's location and scale to subpixel
// accuracy to form an image feature. Rejects features with low contrast.
// Based on Section 4 of Lowe's paper.
__kernel void interpolateExtrema(
    __global float* x_out,
    __global float* y_out,
    __global unsigned* layer_out,
    __global float* response_out,
    __global float* size_out,
    __global unsigned* counter,
    __global const float* x_in,
    __global const float* y_in,
    __global const unsigned* layer_in,
    const unsigned extrema_feat,
    __global const T* dog_octave,
    KParam iDoG,
    const unsigned max_feat,
    const unsigned octave,
    const unsigned n_layers,
    const float contrast_thr,
    const float edge_thr,
    const float sigma,
    const float img_scale)
{
    const unsigned f = get_global_id(0);

    if (f < extrema_feat)
    {
        const float first_deriv_scale = img_scale*0.5f;
        const float second_deriv_scale = img_scale;
        const float cross_deriv_scale = img_scale*0.25f;

        float xl = 0, xy = 0, xx = 0, contr = 0;
        int i = 0;

        unsigned x = x_in[f];
        unsigned y = y_in[f];
        unsigned layer = layer_in[f];

        const int dim0 = iDoG.dims[0];
        const int dim1 = iDoG.dims[1];
        const int imel = dim0 * dim1;

        __global const T* prev   = dog_octave + (int)((layer-1)*imel);
        __global const T* center = dog_octave + (int)((layer  )*imel);
        __global const T* next   = dog_octave + (int)((layer+1)*imel);

        for(i = 0; i < MAX_INTERP_STEPS; i++) {
            float dD[3] = {(float)(CPTR(x+1, y) - CPTR(x-1, y)) * first_deriv_scale,
                           (float)(CPTR(x, y+1) - CPTR(x, y-1)) * first_deriv_scale,
                           (float)(NPTR(x, y)   - PPTR(x, y))   * first_deriv_scale};

            float d2  = CPTR(x, y) * 2.f;
            float dxx = (CPTR(x+1, y) + CPTR(x-1, y) - d2) * second_deriv_scale;
            float dyy = (CPTR(x, y+1) + CPTR(x, y-1) - d2) * second_deriv_scale;
            float dss = (NPTR(x, y  ) + PPTR(x, y  ) - d2) * second_deriv_scale;
            float dxy = (CPTR(x+1, y+1) - CPTR(x-1, y+1) -
                         CPTR(x+1, y-1) + CPTR(x-1, y-1)) * cross_deriv_scale;
            float dxs = (NPTR(x+1, y) - NPTR(x-1, y) -
                         PPTR(x+1, y) + PPTR(x-1, y)) * cross_deriv_scale;
            float dys = (NPTR(x, y+1) - NPTR(x-1, y-1) -
                         PPTR(x, y-1) + PPTR(x-1, y-1)) * cross_deriv_scale;

            float H[9] = {dxx, dxy, dxs,
                          dxy, dyy, dys,
                          dxs, dys, dss};

            float X[3];
            gaussianElimination(H, dD, X, 3);

            xl = -X[2];
            xy = -X[1];
            xx = -X[0];

            if(fabs(xl) < 0.5f && fabs(xy) < 0.5f && fabs(xx) < 0.5f)
                break;

            x += round(xx);
            y += round(xy);
            layer += round(xl);

            if(layer < 1 || layer > n_layers ||
               x < IMG_BORDER || x >= dim1 - IMG_BORDER ||
               y < IMG_BORDER || y >= dim0 - IMG_BORDER)
                return;
        }

        // ensure convergence of interpolation
        if (i >= MAX_INTERP_STEPS)
            return;

        float dD[3] = {(float)(CPTR(x+1, y) - CPTR(x-1, y)) * first_deriv_scale,
                       (float)(CPTR(x, y+1) - CPTR(x, y-1)) * first_deriv_scale,
                       (float)(NPTR(x, y)   - PPTR(x, y))   * first_deriv_scale};
        float X[3] = {xx, xy, xl};

        float P = dD[0]*X[0] + dD[1]*X[1] + dD[2]*X[2];

        contr = center[x*dim0+y]*img_scale + P * 0.5f;
        if(fabs(contr) < (contrast_thr / n_layers))
            return;

        // principal curvatures are computed using the trace and det of Hessian
        float d2  = CPTR(x, y) * 2.f;
        float dxx = (CPTR(x+1, y) + CPTR(x-1, y) - d2) * second_deriv_scale;
        float dyy = (CPTR(x, y+1) + CPTR(x, y-1) - d2) * second_deriv_scale;
        float dxy = (CPTR(x+1, y+1) - CPTR(x-1, y+1) -
                     CPTR(x+1, y-1) + CPTR(x-1, y-1)) * cross_deriv_scale;

        float tr = dxx + dyy;
        float det = dxx * dyy - dxy * dxy;

        // add FLT_EPSILON for double-precision compatibility
        if (det <= 0 || tr*tr*edge_thr >= (edge_thr + 1)*(edge_thr + 1)*det+FLT_EPSILON)
            return;

        unsigned ridx = atomic_inc(counter);

        if (ridx < max_feat)
        {
            x_out[ridx] = (x + xx) * (1 << octave);
            y_out[ridx] = (y + xy) * (1 << octave);
            layer_out[ridx] = layer;
            response_out[ridx] = fabs(contr);
            size_out[ridx] = sigma*pow(2.f, octave + (layer + xl) / n_layers);
        }
    }
}

#undef CPTR
#undef PPTR
#undef NPTR

// Remove duplicate keypoints
__kernel void removeDuplicates(
    __global float* x_out,
    __global float* y_out,
    __global unsigned* layer_out,
    __global float* response_out,
    __global float* size_out,
    __global unsigned* counter,
    __global const float* x_in,
    __global const float* y_in,
    __global const unsigned* layer_in,
    __global const float* response_in,
    __global const float* size_in,
    const unsigned total_feat)
{
    const unsigned f = get_global_id(0);

    if (f < total_feat) {
        const float prec_fctr = 1e4f;

        bool cond = (f < total_feat-1)
                    ? !(round(x_in[f]*prec_fctr) == round(x_in[f+1]*prec_fctr) &&
                        round(y_in[f]*prec_fctr) == round(y_in[f+1]*prec_fctr) &&
                        layer_in[f] == layer_in[f+1] &&
                        round(response_in[f]*prec_fctr) == round(response_in[f+1]*prec_fctr) &&
                        round(size_in[f]*prec_fctr) == round(size_in[f+1]*prec_fctr))
                    : true;

        if (cond) {
            unsigned idx = atomic_inc(counter);

            x_out[idx] = x_in[f];
            y_out[idx] = y_in[f];
            layer_out[idx] = layer_in[f];
            response_out[idx] = response_in[f];
            size_out[idx] = size_in[f];
        }
    }

}

#define IPTR(Y, X) (img[(Y) * dim0 + X])

// Computes a canonical orientation for each image feature in an array.  Based
// on Section 5 of Lowe's paper.  This function adds features to the array when
// there is more than one dominant orientation at a given feature location.
__kernel void calcOrientation(
    __global float* x_out,
    __global float* y_out,
    __global unsigned* layer_out,
    __global float* response_out,
    __global float* size_out,
    __global float* ori_out,
    __global unsigned* counter,
    __global const float* x_in,
    __global const float* y_in,
    __global const unsigned* layer_in,
    __global const float* response_in,
    __global const float* size_in,
    const unsigned total_feat,
    __global const T* gauss_octave,
    KParam iGauss,
    const unsigned max_feat,
    const unsigned octave,
    const int double_input,
    __local float* l_mem)
{
    const int lid_x = get_local_id(0);
    const int lid_y = get_local_id(1);
    const int lsz_x = get_local_size(0);

    const unsigned f = get_global_id(1);

    const int n = ORI_HIST_BINS;

    __local float* hist = l_mem;
    __local float* temphist = l_mem + n*8;

    // Initialize temporary histogram
    for (int i = lid_x; i < n; i += lsz_x) {
        hist[lid_y*n + i] = 0.f;
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    float real_x, real_y, response, size;
    unsigned layer;

    if (f < total_feat) {
        // Load keypoint information
        real_x = x_in[f];
        real_y = y_in[f];
        layer = layer_in[f];
        response = response_in[f];
        size = size_in[f];

        const int pt_x = (int)round(real_x / (1 << octave));
        const int pt_y = (int)round(real_y / (1 << octave));

        // Calculate auxiliary parameters
        const float scl_octv = size*0.5f / (1 << octave);
        const int radius = (int)round(ORI_RADIUS * scl_octv);
        const float sigma = ORI_SIG_FCTR * scl_octv;
        const int len = (radius*2+1);
        const float exp_denom = 2.f * sigma * sigma;

        const int dim0 = iGauss.dims[0];
        const int dim1 = iGauss.dims[1];

        // Calculate layer offset
        const int layer_offset = layer * dim0 * dim1;
        __global const T* img = gauss_octave + layer_offset;

        // Calculate orientation histogram
        for (int l = lid_x; l < len*len; l += lsz_x) {
            int i = l / len - radius;
            int j = l % len - radius;

            int y = pt_y + i;
            int x = pt_x + j;
            if (y < 1 || y >= dim0 - 1 ||
                x < 1 || x >= dim1 - 1)
                continue;

            float dx = (float)(IPTR(x+1, y) - IPTR(x-1, y));
            float dy = (float)(IPTR(x, y-1) - IPTR(x, y+1));

            float mag = sqrt(dx*dx+dy*dy);
            float ori = atan2(dy,dx);
            float w = exp(-(i*i + j*j)/exp_denom);

            int bin = round(n*(ori+PI_VAL)/(2.f*PI_VAL));
            bin = bin < n ? bin : 0;
            bin = (bin < 0) ? 0 : (bin >= n) ? n-1 : bin;

            fatomic_add(&hist[lid_y*n+bin], w*mag);
        }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    for (int i = 0; i < SMOOTH_ORI_PASSES; i++) {
        for (int j = lid_x; j < n; j += lsz_x) {
            temphist[lid_y*n+j] = hist[lid_y*n+j];
        }
        barrier(CLK_LOCAL_MEM_FENCE);
        for (int j = lid_x; j < n; j += lsz_x) {
            float prev = (j == 0) ? temphist[lid_y*n+n-1] : temphist[lid_y*n+j-1];
            float next = (j+1 == n) ? temphist[lid_y*n] : temphist[lid_y*n+j+1];
            hist[lid_y*n+j] = 0.25f * prev + 0.5f * temphist[lid_y*n+j] + 0.25f * next;
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }

    for (int i = lid_x; i < n; i += lsz_x)
        temphist[lid_y*n+i] = hist[lid_y*n+i];
    barrier(CLK_LOCAL_MEM_FENCE);

    if (lid_x < 16)
        temphist[lid_y*n+lid_x] = fmax(hist[lid_y*n+lid_x], hist[lid_y*n+lid_x+16]);
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 8)
        temphist[lid_y*n+lid_x] = fmax(temphist[lid_y*n+lid_x], temphist[lid_y*n+lid_x+8]);
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 4) {
        temphist[lid_y*n+lid_x] = fmax(temphist[lid_y*n+lid_x], hist[lid_y*n+lid_x+32]);
        temphist[lid_y*n+lid_x] = fmax(temphist[lid_y*n+lid_x], temphist[lid_y*n+lid_x+4]);
    }
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 2)
        temphist[lid_y*n+lid_x] = fmax(temphist[lid_y*n+lid_x], temphist[lid_y*n+lid_x+2]);
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 1)
        temphist[lid_y*n+lid_x] = fmax(temphist[lid_y*n+lid_x], temphist[lid_y*n+lid_x+1]);
    barrier(CLK_LOCAL_MEM_FENCE);
    float omax = temphist[lid_y*n];

    if (f < total_feat) {
        float mag_thr = (float)(omax * ORI_PEAK_RATIO);
        int l, r;
        float bin;
        for (int j = lid_x; j < n; j+=lsz_x) {
            l = (j == 0) ? n - 1 : j - 1;
            r = (j + 1) % n;
            if (hist[lid_y*n+j] > hist[lid_y*n+l] &&
                hist[lid_y*n+j] > hist[lid_y*n+r] &&
                hist[lid_y*n+j] >= mag_thr) {
                unsigned idx = atomic_inc(counter);

                if (idx < max_feat) {
                    float bin = j + 0.5f * (hist[lid_y*n+l] - hist[lid_y*n+r]) /
                                (hist[lid_y*n+l] - 2.0f*hist[lid_y*n+j] + hist[lid_y*n+r]);
                    bin = (bin < 0.0f) ? bin + n : (bin >= n) ? bin - n : bin;
                    float ori = 360.f - ((360.f/n) * bin);

                    float new_real_x = real_x;
                    float new_real_y = real_y;
                    float new_size = size;

                    if (double_input != 0) {
                        float scale = 0.5f;
                        new_real_x *= scale;
                        new_real_y *= scale;
                        new_size *= scale;
                    }

                    x_out[idx] = new_real_x;
                    y_out[idx] = new_real_y;
                    layer_out[idx] = layer;
                    response_out[idx] = response;
                    size_out[idx] = new_size;
                    ori_out[idx] = ori;
                }
            }
        }
    }
}

// Computes feature descriptors for features in an array.  Based on Section 6
// of Lowe's paper.
__kernel void computeDescriptor(
    __global float* desc_out,
    const unsigned desc_len,
    const unsigned histsz,
    __global const float* x_in,
    __global const float* y_in,
    __global const unsigned* layer_in,
    __global const float* response_in,
    __global const float* size_in,
    __global const float* ori_in,
    const unsigned total_feat,
    __global const T* gauss_octave,
    KParam iGauss,
    const int d,
    const int n,
    const float scale,
    const float sigma,
    const int n_layers,
    __local float* l_mem)
{
    const int lid_x = get_local_id(0);
    const int lid_y = get_local_id(1);
    const int lsz_x = get_local_size(0);

    const int f = get_global_id(1);

    __local float* desc = l_mem;
    __local float* accum = l_mem + desc_len * histsz;

    if (f < total_feat) {
        const unsigned layer = layer_in[f];
        float ori = (360.f - ori_in[f]) * PI_VAL / 180.f;
        ori = (ori > PI_VAL) ? ori - PI_VAL*2 : ori;
        const float size = size_in[f];
        const int fx = round(x_in[f] * scale);
        const int fy = round(y_in[f] * scale);

        // Points img to correct Gaussian pyramid layer
        const int dim0 = iGauss.dims[0];
        const int dim1 = iGauss.dims[1];
        __global const T* img = gauss_octave + (layer * dim0 * dim1);

        float cos_t = cos(ori);
        float sin_t = sin(ori);
        float bins_per_rad = n / (PI_VAL * 2.f);
        float exp_denom = d * d * 0.5f;
        float hist_width = DESCR_SCL_FCTR * sigma * pow(2.f, layer/n_layers);
        int radius = hist_width * sqrt(2.f) * (d + 1.f) * 0.5f + 0.5f;

        int len = radius*2+1;
        const int histlen = d*d*n;
        const int hist_off = (lid_x % histsz) * desc_len;

        for (int i = lid_x; i < histlen*histsz; i += lsz_x)
            desc[lid_y*histlen+i] = 0.f;
        barrier(CLK_LOCAL_MEM_FENCE);

        // Calculate orientation histogram
        for (int l = lid_x; l < len*len; l += lsz_x) {
            int i = l / len - radius;
            int j = l % len - radius;

            int y = fy + i;
            int x = fx + j;

            float x_rot = (j * cos_t - i * sin_t) / hist_width;
            float y_rot = (j * sin_t + i * cos_t) / hist_width;
            float xbin = x_rot + d/2 - 0.5f;
            float ybin = y_rot + d/2 - 0.5f;

            if (ybin > -1.0f && ybin < d && xbin > -1.0f && xbin < d &&
                y > 0 && y < dim0 - 1 && x > 0 && x < dim1 - 1) {
                float dx = (float)(IPTR(x+1, y) - IPTR(x-1, y));
                float dy = (float)(IPTR(x, y-1) - IPTR(x, y+1));

                float grad_mag = sqrt(dx*dx + dy*dy);
                float grad_ori = atan2(dy, dx) - ori;
                while (grad_ori < 0.0f)
                    grad_ori += PI_VAL*2;
                while (grad_ori >= PI_VAL*2)
                    grad_ori -= PI_VAL*2;

                float w = exp(-(x_rot*x_rot + y_rot*y_rot) / exp_denom);
                float obin = grad_ori * bins_per_rad;
                float mag = grad_mag*w;

                int x0 = floor(xbin);
                int y0 = floor(ybin);
                int o0 = floor(obin);
                xbin -= x0;
                ybin -= y0;
                obin -= o0;

                for (int yl = 0; yl <= 1; yl++) {
                    int yb = y0 + yl;
                    if (yb >= 0 && yb < d) {
	                    float v_y = mag * ((yl == 0) ? 1.0f - ybin : ybin);
	                    for (int xl = 0; xl <= 1; xl++) {
	                        int xb = x0 + xl;
	                        if (xb >= 0 && xb < d) {
		                        float v_x = v_y * ((xl == 0) ? 1.0f - xbin : xbin);
		                        for (int ol = 0; ol <= 1; ol++) {
		                            int ob = (o0 + ol) % n;
		                            float v_o = v_x * ((ol == 0) ? 1.0f - obin : obin);
		                            fatomic_add(&desc[hist_off + lid_y*desc_len + (yb*d + xb)*n + ob], v_o);
		                        }
		                    }
	                    }
	                }
                }
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        // Combine histograms (reduces previous atomicAdd overhead)
        for (int l = lid_x; l < desc_len*4; l += lsz_x)
            desc[l] += desc[l+4*desc_len];
        barrier(CLK_LOCAL_MEM_FENCE);
        for (int l = lid_x; l < desc_len*2; l += lsz_x)
            desc[l    ] += desc[l+2*desc_len];
        barrier(CLK_LOCAL_MEM_FENCE);
        for (int l = lid_x; l < desc_len; l += lsz_x)
            desc[l] += desc[l+desc_len];
        barrier(CLK_LOCAL_MEM_FENCE);

        normalizeDesc(desc, accum, histlen, lid_x, lid_y, lsz_x);

        for (int i = lid_x; i < d*d*n; i += lsz_x)
            desc[lid_y*desc_len+i] = min(desc[lid_y*desc_len+i], DESCR_MAG_THR);
        barrier(CLK_LOCAL_MEM_FENCE);

        normalizeDesc(desc, accum, histlen, lid_x, lid_y, lsz_x);

        // Calculate final descriptor values
        for (int k = lid_x; k < d*d*n; k += lsz_x) {
            desc_out[f*desc_len+k] = round(min(255.f, desc[lid_y*desc_len+k] * INT_DESCR_FCTR));
        }
    }
}

#undef IPTR

/*******************************************************
 * Copyright (c) 2015, ArrayFire
 * All rights reserved.
 *
 * This file is distributed under 3-clause BSD license.
 * The complete license agreement can be obtained at:
 * http://arrayfire.com/licenses/BSD-3-Clause
 ********************************************************/

// The source code contained in this file is based on the original code by
// Rob Hess. Please note that SIFT is an algorithm patented and protected
// by US law, before using this code or any binary forms generated from it,
// verify that you have permission to do so. The original license by Rob Hess
// can be read below:
//
// Copyright (c) 2006-2012, Rob Hess <rob@iqengines.com>
// All rights reserved.
//
// The following patent has been issued for methods embodied in this
// software: "Method and apparatus for identifying scale invariant features
// in an image and use of same for locating an object in an image," David
// G. Lowe, US Patent 6,711,293 (March 23, 2004). Provisional application
// filed March 8, 1999. Asignee: The University of British Columbia. For
// further details, contact David Lowe (lowe@cs.ubc.ca) or the
// University-Industry Liaison Office of the University of British
// Columbia.
//
// Note that restrictions imposed by this patent (and possibly others)
// exist independently of and may be in conflict with the freedoms granted
// in this license, which refers to copyright of the program, not patents
// for any methods that it implements.  Both copyright and patent law must
// be obeyed to legally use and redistribute this program and it is not the
// purpose of this license to induce you to infringe any patents or other
// property right claims or to contest validity of any such claims.  If you
// redistribute or use the program, then this license merely protects you
// from committing copyright infringement.  It does not protect you from
// committing patent infringement.  So, before you do anything with this
// program, make sure that you have permission to do so not merely in terms
// of copyright, but also in terms of patent law.
//
// Please note that this license is not to be understood as a guarantee
// either.  If you use the program according to this license, but in
// conflict with patent law, it does not mean that the licensor will refund
// you for any losses that you incur if you are sued for your patent
// infringement.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//     * Redistributions of source code must retain the above copyright and
//       patent notices, this list of conditions and the following
//       disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in
//       the documentation and/or other materials provided with the
//       distribution.
//     * Neither the name of Oregon State University nor the names of its
//       contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// width of border in which to ignore keypoints
#define IMG_BORDER 5

// maximum steps of keypoint interpolation before failure
#define MAX_INTERP_STEPS 5

// default number of bins in histogram for orientation assignment
#define ORI_HIST_BINS 36

// determines gaussian sigma for orientation assignment
#define ORI_SIG_FCTR 1.5f

// determines the radius of the region used in orientation assignment
#define ORI_RADIUS (3 * ORI_SIG_FCTR)

// number of passes of orientation histogram smoothing
#define SMOOTH_ORI_PASSES 2

// orientation magnitude relative to max that results in new feature
#define ORI_PEAK_RATIO 0.8f

// determines the size of a single descriptor orientation histogram
#define DESCR_SCL_FCTR 3.f

// threshold on magnitude of elements of descriptor vector
#define DESCR_MAG_THR 0.2f

// factor used to convert floating-point descriptor to unsigned char
#define INT_DESCR_FCTR 512.f

#define PI_VAL 3.14159265358979323846f

void gaussianElimination(float* A, float* b, float* x, const int n)
{
    // forward elimination
    for (int i = 0; i < n-1; i++) {
        for (int j = i+1; j < n; j++) {
            float s = A[j*n+i] / A[i*n+i];

            //for (int k = i+1; k < n; k++)
            for (int k = i; k < n; k++)
                A[j*n+k] -= s * A[i*n+k];

            b[j] -= s * b[i];
        }
    }

    for (int i = 0; i < n; i++)
        x[i] = 0;

    // backward substitution
    float sum = 0;
    for (int i = 0; i <= n-2; i++) {
        sum = b[i];
        for (int j = i+1; j < n; j++)
            sum -= A[i*n+j] * x[j];
        x[i] = sum / A[i*n+i];
    }
}

inline void fatomic_add(volatile __local float *source, const float operand) {
    union {
        unsigned int intVal;
        float floatVal;
    } newVal;
    union {
        unsigned int intVal;
        float floatVal;
    } prevVal;
    do {
        prevVal.floatVal = *source;
        newVal.floatVal = prevVal.floatVal + operand;
    } while (atomic_cmpxchg((volatile __local unsigned int *)source, prevVal.intVal, newVal.intVal) != prevVal.intVal);
}

inline void normalizeDesc(
    __local float* desc,
    __local float* accum,
    const int histlen,
    int lid_x,
    int lid_y,
    int lsz_x)
{
    for (int i = lid_x; i < histlen; i += lsz_x)
        accum[i] = desc[lid_y*histlen+i]*desc[lid_y*histlen+i];
    barrier(CLK_LOCAL_MEM_FENCE);

    float sum = 0.0f;
    for (int i = 0; i < histlen; i++)
        sum += desc[lid_y*histlen+i]*desc[lid_y*histlen+i];
    barrier(CLK_LOCAL_MEM_FENCE);

    if (lid_x < 64)
        accum[lid_x] += accum[lid_x+64];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 32)
        accum[lid_x] += accum[lid_x+32];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 16)
        accum[lid_x] += accum[lid_x+16];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 8)
        accum[lid_x] += accum[lid_x+8];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 4)
        accum[lid_x] += accum[lid_x+4];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 2)
        accum[lid_x] += accum[lid_x+2];
    barrier(CLK_LOCAL_MEM_FENCE);
    if (lid_x < 1)
        accum[lid_x] += accum[lid_x+1];
    barrier(CLK_LOCAL_MEM_FENCE);

    float len_sq = accum[0];
    float len_inv = 1.0f / sqrt(len_sq);

    for (int i = lid_x; i < histlen; i += lsz_x) {
        desc[lid_y*histlen+i] *= len_inv;
    }
    barrier(CLK_LOCAL_MEM_FENCE);
}

__kernel void sub(
    __global T* out,
    __global const T* in,
    unsigned nel,
    unsigned n_layers)
{
    unsigned i = get_global_id(0);

    for (unsigned l = 0; l < n_layers; l++)
        out[l*nel + i] = in[l*nel + i] - in[(l+1)*nel + i];
}

#define LCPTR(Y, X) (l_center[(Y) * l_i + (X)])
#define LPPTR(Y, X) (l_prev[(Y) * l_i + (X)])
#define LNPTR(Y, X) (l_next[(Y) * l_i + (X)])

// Determines whether a pixel is a scale-space extremum by comparing it to its
// 3x3x3 pixel neighborhood.
__kernel void detectExtrema(
    __global float* x_out,
    __global float* y_out,
    __global unsigned* layer_out,
    __global unsigned* counter,
    __global const T* dog,
    KParam iDoG,
    const unsigned max_feat,
    const float threshold,
    __local float* l_mem)
{
    const int dim0 = iDoG.dims[0];
    const int dim1 = iDoG.dims[1];
    const int imel = iDoG.dims[0]*iDoG.dims[1];

    const int lid_i = get_local_id(0);
    const int lid_j = get_local_id(1);
    const int lsz_i = get_local_size(0);
    const int lsz_j = get_local_size(1);
    const int i = get_group_id(0) * lsz_i + lid_i+IMG_BORDER;
    const int j = get_group_id(1) * lsz_j + lid_j+IMG_BORDER;

    // One pixel border for each side
    const int l_i = lsz_i+2;
    const int l_j = lsz_j+2;

    __local float* l_prev   = l_mem;
    __local float* l_center = l_mem + l_i * l_j;
    __local float* l_next   = l_mem + l_i * l_j * 2;

    const int x = lid_i+1;
    const int y = lid_j+1;

    for (int l = 1; l < iDoG.dims[2]-1; l++) {
        const int l_i_half = l_i/2;
        const int l_j_half = l_j/2;
        if (lid_i < l_i_half && lid_j < l_j_half && i < dim0-IMG_BORDER+1 && j < dim1-IMG_BORDER+1) {
                l_next  [lid_j*l_i + lid_i] = (float)dog[(l+1)*imel+(j-1)*dim0+i-1];
                l_center[lid_j*l_i + lid_i] = (float)dog[(l  )*imel+(j-1)*dim0+i-1];
                l_prev  [lid_j*l_i + lid_i] = (float)dog[(l-1)*imel+(j-1)*dim0+i-1];

                l_next  [lid_j*l_i + lid_i+l_i_half] = (float)dog[(l+1)*imel+(j-1)*dim0+i-1+l_i_half];
                l_center[lid_j*l_i + lid_i+l_i_half] = (float)dog[(l  )*imel+(j-1)*dim0+i-1+l_i_half];
                l_prev  [lid_j*l_i + lid_i+l_i_half] = (float)dog[(l-1)*imel+(j-1)*dim0+i-1+l_i_half];

                l_next  [(lid_j+l_j_half)*l_i + lid_i] = (float)dog[(l+1)*imel+(j-1+l_j_half)*dim0+i-1];
                l_center[(lid_j+l_j_half)*l_i + lid_i] = (float)dog[(l  )*imel+(j-1+l_j_half)*dim0+i-1];
                l_prev  [(lid_j+l_j_half)*l_i + lid_i] = (float)dog[(l-1)*imel+(j-1+l_j_half)*dim0+i-1];

                l_next  [(lid_j+l_j_half)*l_i + lid_i+l_i_half] = (float)dog[(l+1)*imel+(j-1+l_j_half)*dim0+i-1+l_i_half];
                l_center[(lid_j+l_j_half)*l_i + lid_i+l_i_half] = (float)dog[(l  )*imel+(j-1+l_j_half)*dim0+i-1+l_i_half];
                l_prev  [(lid_j+l_j_half)*l_i + lid_i+l_i_half] = (float)dog[(l-1)*imel+(j-1+l_j_half)*dim0+i-1+l_i_half];
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        if (i < dim0-IMG_BORDER && j < dim1-IMG_BORDER) {
            const int l_i_half = l_i/2;
            float p = l_center[y*l_i + x];

            if (fabs((float)p) > threshold &&
                ((p > 0 && p > LCPTR(y-1, x-1) && p > LCPTR(y-1, x) &&
                  p > LCPTR(y-1, x+1) && p > LCPTR(y, x-1) && p > LCPTR(y,   x+1)  &&
                  p > LCPTR(y+1, x-1) && p > LCPTR(y+1, x) && p > LCPTR(y+1, x+1)  &&
                  p > LPPTR(y-1, x-1) && p > LPPTR(y-1, x) && p > LPPTR(y-1, x+1)  &&
                  p > LPPTR(y,   x-1) && p > LPPTR(y  , x) && p > LPPTR(y,   x+1)  &&
                  p > LPPTR(y+1, x-1) && p > LPPTR(y+1, x) && p > LPPTR(y+1, x+1)  &&
                  p > LNPTR(y-1, x-1) && p > LNPTR(y-1, x) && p > LNPTR(y-1, x+1)  &&
                  p > LNPTR(y,   x-1) && p > LNPTR(y  , x) && p > LNPTR(y,   x+1)  &&
                  p > LNPTR(y+1, x-1) && p > LNPTR(y+1, x) && p > LNPTR(y+1, x+1)) ||
                 (p < 0 && p < LCPTR(y-1, x-1) && p < LCPTR(y-1, x) &&
                  p < LCPTR(y-1, x+1) && p < LCPTR(y, x-1) && p < LCPTR(y,   x+1)  &&
                  p < LCPTR(y+1, x-1) && p < LCPTR(y+1, x) && p < LCPTR(y+1, x+1)  &&
                  p < LPPTR(y-1, x-1) && p < LPPTR(y-1, x) && p < LPPTR(y-1, x+1)  &&
                  p < LPPTR(y,   x-1) && p < LPPTR(y  , x) && p < LPPTR(y,   x+1)  &&
                  p < LPPTR(y+1, x-1) && p < LPPTR(y+1, x) && p < LPPTR(y+1, x+1)  &&
                  p < LNPTR(y-1, x-1) && p < LNPTR(y-1, x) && p < LNPTR(y-1, x+1)  &&
                  p < LNPTR(y,   x-1) && p < LNPTR(y  , x) && p < LNPTR(y,   x+1)  &&
                  p < LNPTR(y+1, x-1) && p < LNPTR(y+1, x) && p < LNPTR(y+1, x+1)))) {

                unsigned idx = atomic_inc(counter);
                if (idx < max_feat)
                {
                    x_out[idx] = (float)j;
                    y_out[idx] = (float)i;
                    layer_out[idx] = l;
                }
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);
    }
}

#undef LCPTR
#undef LPPTR
#undef LNPTR
#define CPTR(Y, X) (center[(Y) * dim0 + (X)])
#define PPTR(Y, X) (prev[(Y) * dim0 + (X)])
#define NPTR(Y, X) (next[(Y) * dim0 + (X)])

// Interpolates a scale-space extremum's location and scale to subpixel
// accuracy to form an image feature. Rejects features with low contrast.
// Based on Section 4 of Lowe's paper.
__kernel void interpolateExtrema(
    __global float* x_out,
    __global float* y_out,
    __global unsigned* layer_out,
    __global float* response_out,
    __global float* size_out,
    __global unsigned* counter,
    __global const float* x_in,
    __global const float* y_in,
    __global const unsigned* layer_in,
    const unsigned extrema_feat,
    __global const T* dog_octave,
    KParam iDoG,
    const unsigned max_feat,
    const unsigned octave,
    const unsigned n_layers,
    const float contrast_thr,
    const float edge_thr,
    const float sigma,
    const float img_scale)
{
    const unsigned f = get_global_id(0);

    if (f < extrema_feat)
    {
        const float first_deriv_scale = img_scale*0.5f;
        const float second_deriv_scale = img_scale;
        const float cross_deriv_scale = img_scale*0.25f;

        float xl = 0, xy = 0, xx = 0, contr = 0;
        int i = 0;

        unsigned x = x_in[f];
        unsigned y = y_in[f];
        unsigned layer = layer_in[f];

        const int dim0 = iDoG.dims[0];
        const int dim1 = iDoG.dims[1];
        const int imel = dim0 * dim1;

        __global const T* prev   = dog_octave + (int)((layer-1)*imel);
        __global const T* center = dog_octave + (int)((layer  )*imel);
        __global const T* next   = dog_octave + (int)((layer+1)*imel);

        for(i = 0; i < MAX_INTERP_STEPS; i++) {
            float dD[3] = {(float)(CPTR(x+1, y) - CPTR(x-1, y)) * first_deriv_scale,
                           (float)(CPTR(x, y+1) - CPTR(x, y-1)) * first_deriv_scale,
                           (float)(NPTR(x, y)   - PPTR(x, y))   * first_deriv_scale};

            float d2  = CPTR(x, y) * 2.f;
            float dxx = (CPTR(x+1, y) + CPTR(x-1, y) - d2) * second_deriv_scale;
            float dyy = (CPTR(x, y+1) + CPTR(x, y-1) - d2) * second_deriv_scale;
            float dss = (NPTR(x, y  ) + PPTR(x, y  ) - d2) * second_deriv_scale;
            float dxy = (CPTR(x+1, y+1) - CPTR(x-1, y+1) -
                         CPTR(x+1, y-1) + CPTR(x-1, y-1)) * cross_deriv_scale;
            float dxs = (NPTR(x+1, y) - NPTR(x-1, y) -
                         PPTR(x+1, y) + PPTR(x-1, y)) * cross_deriv_scale;
            float dys = (NPTR(x, y+1) - NPTR(x-1, y-1) -
                         PPTR(x, y-1) + PPTR(x-1, y-1)) * cross_deriv_scale;

            float H[9] = {dxx, dxy, dxs,
                          dxy, dyy, dys,
                          dxs, dys, dss};

            float X[3];
            gaussianElimination(H, dD, X, 3);

            xl = -X[2];
            xy = -X[1];
            xx = -X[0];

            if(fabs(xl) < 0.5f && fabs(xy) < 0.5f && fabs(xx) < 0.5f)
                break;

            x += round(xx);
            y += round(xy);
            layer += round(xl);

            if(layer < 1 || layer > n_layers ||
               x < IMG_BORDER || x >= dim1 - IMG_BORDER ||
               y < IMG_BORDER || y >= dim0 - IMG_BORDER)
                return;
        }

        // ensure convergence of interpolation
        if (i >= MAX_INTERP_STEPS)
            return;

        float dD[3] = {(float)(CPTR(x+1, y) - CPTR(x-1, y)) * first_deriv_scale,
                       (float)(CPTR(x, y+1) - CPTR(x, y-1)) * first_deriv_scale,
                       (float)(NPTR(x, y)   - PPTR(x, y))   * first_deriv_scale};
        float X[3] = {xx, xy, xl};

        float P = dD[0]*X[0] + dD[1]*X[1] + dD[2]*X[2];

        contr = center[x*dim0+y]*img_scale + P * 0.5f;
        if(fabs(contr) < (contrast_thr / n_layers))
            return;

        // principal curvatures are computed using the trace and det of Hessian
        float d2  = CPTR(x, y) * 2.f;
        float dxx = (CPTR(x+1, y) + CPTR(x-1, y) - d2) * second_deriv_scale;
        float dyy = (CPTR(x, y+1) + CPTR(x, y-1) - d2) * second_deriv_scale;
        float dxy = (CPTR(x+1, y+1) - CPTR(x-1, y+1) -
                     CPTR(x+1, y-1) + CPTR(x-1, y-1)) * cross_deriv_scale;

        float tr = dxx + dyy;
        float det = dxx * dyy - dxy * dxy;

        // add FLT_EPSILON for double-precision compatibility
        if (det <= 0 || tr*tr*edge_thr >= (edge_thr + 1)*(edge_thr + 1)*det+FLT_EPSILON)
            return;

        unsigned ridx = atomic_inc(counter);

        if (ridx < max_feat)
        {
            x_out[ridx] = (x + xx) * (1 << octave);
            y_out[ridx] = (y + xy) * (1 << octave);
            layer_out[ridx] = layer;
            response_out[ridx] = fabs(contr);
            size_out[ridx] = sigma*pow(2.f, octave + (layer + xl) / n_layers);
        }
    }
}

#undef CPTR
#undef PPTR
#undef NPTR

// Remove duplicate keypoints
__kernel void removeDuplicates(
    __global float* x_out,
    __global float* y_out,
    __global unsigned* layer_out,
    __global float* response_out,
    __global float* size_out,
    __global unsigned* counter,
    __global const float* x_in,
    __global const float* y_in,
    __global const unsigned* layer_in,
    __global const float* response_in,
    __global const float* size_in,
    const unsigned total_feat)
{
    const unsigned f = get_global_id(0);

    if (f < total_feat) {
        const float prec_fctr = 1e4f;

        bool cond = (f < total_feat-1)
                    ? !(round(x_in[f]*prec_fctr) == round(x_in[f+1]*prec_fctr) &&
                        round(y_in[f]*prec_fctr) == round(y_in[f+1]*prec_fctr) &&
                        layer_in[f] == layer_in[f+1] &&
                        round(response_in[f]*prec_fctr) == round(response_in[f+1]*prec_fctr) &&
                        round(size_in[f]*prec_fctr) == round(size_in[f+1]*prec_fctr))
                    : true;

        if (cond) {
            unsigned idx = atomic_inc(counter);

            x_out[idx] = x_in[f];
            y_out[idx] = y_in[f];
            layer_out[idx] = layer_in[f];
            response_out[idx] = response_in[f];
            size_out[idx] = size_in[f];
        }
    }

}

#define IPTR(Y, X) (img[(Y) * dim0 + X])

// Computes a canonical orientation for each image feature in an array.  Based
// on Section 5 of Lowe's paper.  This function adds features to the array when
// there is more than one dominant orientation at a given feature location.
__kernel void calcOrientation(
    __global float* x_out,
    __global float* y_out,
    __global unsigned* layer_out,
    __global float* response_out,
    __global float* size_out,
    __global float* ori_out,
    __global unsigned* counter,
    __global const float* x_in,
    __global const float* y_in,
    __global const unsigned* layer_in,
    __global const float* response_in,
    __global const float* size_in,
    const unsigned total_feat,
    __global const T* gauss_octave,
    KParam iGauss,
    const unsigned max_feat,
    const unsigned octave,
    const int double_input,
    __local float* l_mem)
{
    const int lid_x = get_local_id(0);
    const int lid_y = get_local_id(1);
    const int lsz_x = get_local_size(0);

    const unsigned f = get_global_id(1);

    const int n = ORI_HIST_BINS;

    __local float* hist = l_mem;
    __local float* temphist = l_mem + n*8;

    if (f < total_feat) {
        // Load keypoint information
        const float real_x = x_in[f];
        const float real_y = y_in[f];
        const unsigned layer = layer_in[f];
        const float response = response_in[f];
        const float size = size_in[f];

        const int pt_x = (int)round(real_x / (1 << octave));
        const int pt_y = (int)round(real_y / (1 << octave));

        // Calculate auxiliary parameters
        const float scl_octv = size*0.5f / (1 << octave);
        const int radius = (int)round(ORI_RADIUS * scl_octv);
        const float sigma = ORI_SIG_FCTR * scl_octv;
        const int len = (radius*2+1);
        const float exp_denom = 2.f * sigma * sigma;

        // Initialize temporary histogram
        for (int i = lid_x; i < n; i += lsz_x) {
            hist[lid_y*n + i] = 0.f;
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        const int dim0 = iGauss.dims[0];
        const int dim1 = iGauss.dims[1];

        // Calculate layer offset
        const int layer_offset = layer * dim0 * dim1;
        __global const T* img = gauss_octave + layer_offset;

        // Calculate orientation histogram
        for (int l = lid_x; l < len*len; l += lsz_x) {
            int i = l / len - radius;
            int j = l % len - radius;

            int y = pt_y + i;
            int x = pt_x + j;
            if (y < 1 || y >= dim0 - 1 ||
                x < 1 || x >= dim1 - 1)
                continue;

            float dx = (float)(IPTR(x+1, y) - IPTR(x-1, y));
            float dy = (float)(IPTR(x, y-1) - IPTR(x, y+1));

            float mag = sqrt(dx*dx+dy*dy);
            float ori = atan2(dy,dx);
            float w = exp(-(i*i + j*j)/exp_denom);

            int bin = round(n*(ori+PI_VAL)/(2.f*PI_VAL));
            bin = bin < n ? bin : 0;
            bin = (bin < 0) ? 0 : (bin >= n) ? n-1 : bin;

            fatomic_add(&hist[lid_y*n+bin], w*mag);
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        for (int i = 0; i < SMOOTH_ORI_PASSES; i++) {
            for (int j = lid_x; j < n; j += lsz_x) {
                temphist[lid_y*n+j] = hist[lid_y*n+j];
            }
            barrier(CLK_LOCAL_MEM_FENCE);
            for (int j = lid_x; j < n; j += lsz_x) {
                float prev = (j == 0) ? temphist[lid_y*n+n-1] : temphist[lid_y*n+j-1];
                float next = (j+1 == n) ? temphist[lid_y*n] : temphist[lid_y*n+j+1];
                hist[lid_y*n+j] = 0.25f * prev + 0.5f * temphist[lid_y*n+j] + 0.25f * next;
            }
            barrier(CLK_LOCAL_MEM_FENCE);
        }

        for (int i = lid_x; i < n; i += lsz_x)
            temphist[lid_y*n+i] = hist[lid_y*n+i];
        barrier(CLK_LOCAL_MEM_FENCE);

        if (lid_x < 16)
            temphist[lid_y*n+lid_x] = fmax(hist[lid_y*n+lid_x], hist[lid_y*n+lid_x+16]);
        barrier(CLK_LOCAL_MEM_FENCE);
        if (lid_x < 8)
            temphist[lid_y*n+lid_x] = fmax(temphist[lid_y*n+lid_x], temphist[lid_y*n+lid_x+8]);
        barrier(CLK_LOCAL_MEM_FENCE);
        if (lid_x < 4) {
            temphist[lid_y*n+lid_x] = fmax(temphist[lid_y*n+lid_x], hist[lid_y*n+lid_x+32]);
            temphist[lid_y*n+lid_x] = fmax(temphist[lid_y*n+lid_x], temphist[lid_y*n+lid_x+4]);
        }
        barrier(CLK_LOCAL_MEM_FENCE);
        if (lid_x < 2)
            temphist[lid_y*n+lid_x] = fmax(temphist[lid_y*n+lid_x], temphist[lid_y*n+lid_x+2]);
        barrier(CLK_LOCAL_MEM_FENCE);
        if (lid_x < 1)
            temphist[lid_y*n+lid_x] = fmax(temphist[lid_y*n+lid_x], temphist[lid_y*n+lid_x+1]);
        barrier(CLK_LOCAL_MEM_FENCE);
        float omax = temphist[lid_y*n];

        float mag_thr = (float)(omax * ORI_PEAK_RATIO);
        int l, r;
        float bin;
        for (int j = lid_x; j < n; j+=lsz_x) {
            l = (j == 0) ? n - 1 : j - 1;
            r = (j + 1) % n;
            if (hist[lid_y*n+j] > hist[lid_y*n+l] &&
                hist[lid_y*n+j] > hist[lid_y*n+r] &&
                hist[lid_y*n+j] >= mag_thr) {
                unsigned idx = atomic_inc(counter);

                if (idx < max_feat) {
                    float bin = j + 0.5f * (hist[lid_y*n+l] - hist[lid_y*n+r]) /
                                (hist[lid_y*n+l] - 2.0f*hist[lid_y*n+j] + hist[lid_y*n+r]);
                    bin = (bin < 0.0f) ? bin + n : (bin >= n) ? bin - n : bin;
                    float ori = 360.f - ((360.f/n) * bin);

                    float new_real_x = real_x;
                    float new_real_y = real_y;
                    float new_size = size;

                    if (double_input != 0) {
                        float scale = 0.5f;
                        new_real_x *= scale;
                        new_real_y *= scale;
                        new_size *= scale;
                    }

                    x_out[idx] = new_real_x;
                    y_out[idx] = new_real_y;
                    layer_out[idx] = layer;
                    response_out[idx] = response;
                    size_out[idx] = new_size;
                    ori_out[idx] = ori;
                }
            }
        }
    }
}

// Computes feature descriptors for features in an array.  Based on Section 6
// of Lowe's paper.
__kernel void computeDescriptor(
    __global float* desc_out,
    const unsigned desc_len,
    const unsigned histsz,
    __global const float* x_in,
    __global const float* y_in,
    __global const unsigned* layer_in,
    __global const float* response_in,
    __global const float* size_in,
    __global const float* ori_in,
    const unsigned total_feat,
    __global const T* gauss_octave,
    KParam iGauss,
    const int d,
    const int n,
    const float scale,
    const float sigma,
    const int n_layers,
    __local float* l_mem)
{
    const int lid_x = get_local_id(0);
    const int lid_y = get_local_id(1);
    const int lsz_x = get_local_size(0);

    const int f = get_global_id(1);

    __local float* desc = l_mem;
    __local float* accum = l_mem + desc_len * histsz;

    if (f < total_feat) {
        const unsigned layer = layer_in[f];
        float ori = (360.f - ori_in[f]) * PI_VAL / 180.f;
        ori = (ori > PI_VAL) ? ori - PI_VAL*2 : ori;
        const float size = size_in[f];
        const int fx = round(x_in[f] * scale);
        const int fy = round(y_in[f] * scale);

        // Points img to correct Gaussian pyramid layer
        const int dim0 = iGauss.dims[0];
        const int dim1 = iGauss.dims[1];
        __global const T* img = gauss_octave + (layer * dim0 * dim1);

        float cos_t = cos(ori);
        float sin_t = sin(ori);
        float bins_per_rad = n / (PI_VAL * 2.f);
        float exp_denom = d * d * 0.5f;
        float hist_width = DESCR_SCL_FCTR * sigma * pow(2.f, layer/n_layers);
        int radius = hist_width * sqrt(2.f) * (d + 1.f) * 0.5f + 0.5f;

        int len = radius*2+1;
        const int histlen = d*d*n;
        const int hist_off = (lid_x % histsz) * desc_len;

        for (int i = lid_x; i < histlen*histsz; i += lsz_x)
            desc[lid_y*histlen+i] = 0.f;
        barrier(CLK_LOCAL_MEM_FENCE);

        // Calculate orientation histogram
        for (int l = lid_x; l < len*len; l += lsz_x) {
            int i = l / len - radius;
            int j = l % len - radius;

            int y = fy + i;
            int x = fx + j;

            float x_rot = (j * cos_t - i * sin_t) / hist_width;
            float y_rot = (j * sin_t + i * cos_t) / hist_width;
            float xbin = x_rot + d/2 - 0.5f;
            float ybin = y_rot + d/2 - 0.5f;

            if (ybin > -1.0f && ybin < d && xbin > -1.0f && xbin < d &&
                y > 0 && y < dim0 - 1 && x > 0 && x < dim1 - 1) {
                float dx = (float)(IPTR(x+1, y) - IPTR(x-1, y));
                float dy = (float)(IPTR(x, y-1) - IPTR(x, y+1));

                float grad_mag = sqrt(dx*dx + dy*dy);
                float grad_ori = atan2(dy, dx) - ori;
                while (grad_ori < 0.0f)
                    grad_ori += PI_VAL*2;
                while (grad_ori >= PI_VAL*2)
                    grad_ori -= PI_VAL*2;

                float w = exp(-(x_rot*x_rot + y_rot*y_rot) / exp_denom);
                float obin = grad_ori * bins_per_rad;
                float mag = grad_mag*w;

                int x0 = floor(xbin);
                int y0 = floor(ybin);
                int o0 = floor(obin);
                xbin -= x0;
                ybin -= y0;
                obin -= o0;

                for (int yl = 0; yl <= 1; yl++) {
                    int yb = y0 + yl;
                    if (yb >= 0 && yb < d) {
	                    float v_y = mag * ((yl == 0) ? 1.0f - ybin : ybin);
	                    for (int xl = 0; xl <= 1; xl++) {
	                        int xb = x0 + xl;
	                        if (xb >= 0 && xb < d) {
		                        float v_x = v_y * ((xl == 0) ? 1.0f - xbin : xbin);
		                        for (int ol = 0; ol <= 1; ol++) {
		                            int ob = (o0 + ol) % n;
		                            float v_o = v_x * ((ol == 0) ? 1.0f - obin : obin);
		                            fatomic_add(&desc[hist_off + lid_y*desc_len + (yb*d + xb)*n + ob], v_o);
		                        }
		                    }
	                    }
	                }
                }
            }
        }
        barrier(CLK_LOCAL_MEM_FENCE);

        // Combine histograms (reduces previous atomicAdd overhead)
        for (int l = lid_x; l < desc_len*4; l += lsz_x)
            desc[l] += desc[l+4*desc_len];
        barrier(CLK_LOCAL_MEM_FENCE);
        for (int l = lid_x; l < desc_len*2; l += lsz_x)
            desc[l    ] += desc[l+2*desc_len];
        barrier(CLK_LOCAL_MEM_FENCE);
        for (int l = lid_x; l < desc_len; l += lsz_x)
            desc[l] += desc[l+desc_len];
        barrier(CLK_LOCAL_MEM_FENCE);

        normalizeDesc(desc, accum, histlen, lid_x, lid_y, lsz_x);

        for (int i = lid_x; i < d*d*n; i += lsz_x)
            desc[lid_y*desc_len+i] = min(desc[lid_y*desc_len+i], DESCR_MAG_THR);
        barrier(CLK_LOCAL_MEM_FENCE);

        normalizeDesc(desc, accum, histlen, lid_x, lid_y, lsz_x);

        // Calculate final descriptor values
        for (int k = lid_x; k < d*d*n; k += lsz_x) {
            desc_out[f*desc_len+k] = round(min(255.f, desc[lid_y*desc_len+k] * INT_DESCR_FCTR));
        }
    }
}

#undef IPTR


fixed zero-filing in upsampler

fixed zero-filing in upsampler

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = (uint)((float)v * (1.f / (float)d) + 0.0000000001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24(u, d);
    return (r);
}

#define MLO_OUT_CHANNEL_STRIDE_ALIGNED (MLO_OUT_CHANNEL_STRIDE / MLO_WRITE_UNIT)
#define MLO_OUT_STRIDE_ALIGNED (MLO_OUT_STRIDE / MLO_WRITE_UNIT)

#define MLO_IN_CHANNEL_STRIDE_ALIGNED (MLO_IN0_CHANNEL_STRIDE / MLO_WRITE_UNIT)
#define MLO_IN_STRIDE_ALIGNED (MLO_IN0_STRIDE / MLO_WRITE_UNIT)

__attribute__((reqd_work_group_size(MLO_GRP0_SZ0, MLO_GRP0_SZ1, MLO_GRP0_SZ2))) __kernel void
SubSample(const __global _FLOAT* __restrict in, __global _FLOAT* __restrict out)
{
    uint stack_pos = get_global_id(0);
    uint batch_id  = get_global_id(1);
    uint map_id    = iDiv(stack_pos, MLO_OUT_CHANNEL_STRIDE_ALIGNED);
    uint pix_pos   = iMod(stack_pos, map_id, MLO_OUT_CHANNEL_STRIDE_ALIGNED);
    uint out_y     = iDiv(pix_pos, MLO_OUT_STRIDE_ALIGNED);
    uint out_x     = iMod(pix_pos, out_y, MLO_OUT_STRIDE_ALIGNED) * MLO_WRITE_UNIT;

    uint out_off = batch_id * MLO_IN_BATCH_STRIDE + stack_pos * MLO_WRITE_UNIT;
    uint in_y    = out_y * MLO_FILTER0_STRIDE1;
    uint in_x    = out_x * MLO_FILTER0_STRIDE0;
    uint in_off  = batch_id * MLO_IN0_BATCH_STRIDE + map_id * MLO_IN0_CHANNEL_STRIDE +
                  in_y * MLO_IN0_STRIDE + in_x;

    const __global _FLOAT* in_ptr = &in[in_off];
    __global _FLOAT* out_ptr      = &out[out_off];

    for(uint i = 0; i < MLO_WRITE_UNIT; ++i, in_ptr += MLO_FILTER0_STRIDE0, out_ptr++)
    {
        *out_ptr = *in_ptr;
    }
}

#ifdef UPSAMPLE

__attribute__((reqd_work_group_size(MLO_GRP0_SZ0, MLO_GRP0_SZ1, MLO_GRP0_SZ2))) __kernel void
UpSample(const __global _FLOAT* __restrict in, __global _FLOAT* __restrict out)
{
    uint stack_pos = get_global_id(0);
    uint batch_id  = get_global_id(1);
    uint map_id    = iDiv(stack_pos, MLO_IN_CHANNEL_STRIDE_ALIGNED);
    uint pix_pos   = iMod(stack_pos, map_id, MLO_IN_CHANNEL_STRIDE_ALIGNED);
    uint in_y      = iDiv(pix_pos, MLO_IN_STRIDE_ALIGNED);
    uint in_x      = iMod(pix_pos, in_y, MLO_IN_STRIDE_ALIGNED) * MLO_WRITE_UNIT;

    uint in_off  = batch_id * MLO_IN_BATCH_STRIDE + stack_pos * MLO_WRITE_UNIT;
    uint out_y   = in_y * MLO_FILTER0_STRIDE1;
    uint out_x   = in_x * MLO_FILTER0_STRIDE0;
    uint out_off = batch_id * MLO_OUT_BATCH_STRIDE + map_id * MLO_OUT_CHANNEL_STRIDE +
                   out_y * MLO_OUT_STRIDE + out_x;
    uint out_off_1 = batch_id * MLO_OUT_BATCH_STRIDE + map_id * MLO_OUT_CHANNEL_STRIDE +
                     (out_y + 1) * MLO_OUT_STRIDE + out_x;

    const __global _FLOAT* in_ptr = &in[in_off];
    __global _FLOAT* out_ptr      = &out[out_off];
    __global _FLOAT* out_ptr_1    = &out[out_off_1];

    for(uint i = 0; i < MLO_WRITE_UNIT;
        ++i, in_ptr++, out_ptr += MLO_FILTER0_STRIDE0, out_ptr_1 += MLO_FILTER0_STRIDE0)
    {
        *out_ptr         = *in_ptr;
        *(out_ptr + 1)   = 0;
        *out_ptr_1       = 0;
        *(out_ptr_1 + 1) = 0;
    }
}
#endif

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

__attribute__((always_inline)) uint iDiv(uint v, uint d)
{
    uint r = (uint)((float)v * (1.f / (float)d) + 0.0000000001f);
    return (r);
}

__attribute__((always_inline)) uint iMod(uint v, uint u, uint d)
{
    uint r = v - mul24(u, d);
    return (r);
}

#define MLO_OUT_CHANNEL_STRIDE_ALIGNED (MLO_OUT_CHANNEL_STRIDE / MLO_WRITE_UNIT)
#define MLO_OUT_STRIDE_ALIGNED (MLO_OUT_STRIDE / MLO_WRITE_UNIT)

#define MLO_IN_CHANNEL_STRIDE_ALIGNED (MLO_IN0_CHANNEL_STRIDE / MLO_WRITE_UNIT)
#define MLO_IN_STRIDE_ALIGNED (MLO_IN0_STRIDE / MLO_WRITE_UNIT)

__attribute__((reqd_work_group_size(MLO_GRP0_SZ0, MLO_GRP0_SZ1, MLO_GRP0_SZ2))) __kernel void
SubSample(const __global _FLOAT* __restrict in, __global _FLOAT* __restrict out)
{
    uint stack_pos = get_global_id(0);
    uint batch_id  = get_global_id(1);
    uint map_id    = iDiv(stack_pos, MLO_OUT_CHANNEL_STRIDE_ALIGNED);
    uint pix_pos   = iMod(stack_pos, map_id, MLO_OUT_CHANNEL_STRIDE_ALIGNED);
    uint out_y     = iDiv(pix_pos, MLO_OUT_STRIDE_ALIGNED);
    uint out_x     = iMod(pix_pos, out_y, MLO_OUT_STRIDE_ALIGNED) * MLO_WRITE_UNIT;

    uint out_off = batch_id * MLO_IN_BATCH_STRIDE + stack_pos * MLO_WRITE_UNIT;
    uint in_y    = out_y * MLO_FILTER0_STRIDE1;
    uint in_x    = out_x * MLO_FILTER0_STRIDE0;
    uint in_off  = batch_id * MLO_IN0_BATCH_STRIDE + map_id * MLO_IN0_CHANNEL_STRIDE +
                  in_y * MLO_IN0_STRIDE + in_x;

    const __global _FLOAT* in_ptr = &in[in_off];
    __global _FLOAT* out_ptr      = &out[out_off];

    for(uint i = 0; i < MLO_WRITE_UNIT; ++i, in_ptr += MLO_FILTER0_STRIDE0, out_ptr++)
    {
        *out_ptr = *in_ptr;
    }
}

#ifdef UPSAMPLE

__attribute__((reqd_work_group_size(MLO_GRP0_SZ0, MLO_GRP0_SZ1, MLO_GRP0_SZ2))) __kernel void
UpSample(const __global _FLOAT* __restrict in, __global _FLOAT* __restrict out)
{
    uint stack_pos = get_global_id(0);
    uint batch_id  = get_global_id(1);
    uint map_id    = iDiv(stack_pos, MLO_IN_CHANNEL_STRIDE_ALIGNED);
    uint pix_pos   = iMod(stack_pos, map_id, MLO_IN_CHANNEL_STRIDE_ALIGNED);
    uint in_y      = iDiv(pix_pos, MLO_IN_STRIDE_ALIGNED);
    uint in_x      = iMod(pix_pos, in_y, MLO_IN_STRIDE_ALIGNED) * MLO_WRITE_UNIT;

    uint in_off  = batch_id * MLO_IN_BATCH_STRIDE + stack_pos * MLO_WRITE_UNIT;
    uint out_y   = in_y * MLO_FILTER0_STRIDE1;
    uint out_x   = in_x * MLO_FILTER0_STRIDE0;
    uint out_off = batch_id * MLO_OUT_BATCH_STRIDE + map_id * MLO_OUT_CHANNEL_STRIDE +
                   out_y * MLO_OUT_STRIDE + out_x;
    uint out_off_1 = batch_id * MLO_OUT_BATCH_STRIDE + map_id * MLO_OUT_CHANNEL_STRIDE +
                     (out_y + 1) * MLO_OUT_STRIDE + out_x;

    const __global _FLOAT* in_ptr = &in[in_off];
    __global _FLOAT* out_ptr      = &out[out_off];
    __global _FLOAT* out_ptr_1    = &out[out_off_1];

    for(uint i = 0; i < MLO_WRITE_UNIT; ++i, in_ptr++, out_ptr += MLO_FILTER0_STRIDE0)
    {
        *out_ptr         = *in_ptr;
        *(out_ptr + 1)   = 0;
        *out_ptr_1       = 0;
        *(out_ptr_1 + 1) = 0;
    }
}
#endif


clean up cl script

clean up cl script

#! /usr/bin/ecl -shell
;; note
(setq no "Lorem ipsum dolor sit amet, consectetur adipiscing
elit. Nullam orci sapien, mattis id ornare sed,
congue nec odio. Praesent aliquam lectus sit amet
ligula tempus, at luctus arcu accumsan.")

;; titles will make as many frames
(setq t1 "aleph")
(setq ti '("brave" "chi" "doom" "echo" "ferous" "ghost" "h"
           "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t"
           "u" "v" "w" "x" "y" "z" "aa" "ab" "ac" "ad" "ae"))

;; triangle size
(setq a 740) (setq -a (- 0 a))
(setq b 640) (setq -b (- 0 b))
(setq c 370) (setq -c (- 0 c))
;; listst of changes
(setq ys (list a b c 0 -c -b -a -b -c 0 c b))
(setq zs (list 0 -c -b -a -b -c 0 c b a b c))
;; init iterative y and z axis values and start
;; count
(setq zy 0) (setq zz 0) (setq cn 1)
;; first slide
(setq sl "~%### ~a~%~%~a~%~%")

(format t sl t1 no)

;; rest of the slides
(dolist (d ti)
  (progn
    (setq zy (+ zy (nth (mod cn 12) ys)))
    (setq zz (+ zz (nth (mod cn 12) zs)))
    (setq za (- 1 (* cn .03)))
    (format t (concatenate 'string "---~%=data-y='~d' "
                           "data-z='~d' data-rotate-x='~d' "
                           "data-scale='~4,2F'~%" sl)
            (* zy za) (* zz za) (* cn -30) za d no)
    (setq cn (+ 1 cn))))

#! /usr/bin/ecl -shell
(setq no "Lorem ipsum dolor sit amet, consectetur adipiscing
elit. Nullam orci sapien, mattis id ornare sed,
congue nec odio. Praesent aliquam lectus sit amet
ligula tempus, at luctus arcu accumsan.")

;; (setq no "")
(setq t1 "aleph")
(setq ti '("brave" "chi" "doom" "echo" "ferous" "ghost" "h"
           "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t"
           "u" "v" "w" "x" "y" "z" "aa" "ab" "ac" "ad" "ae"))

(setq a 740) (setq -a (- 0 a))
(setq b 640) (setq -b (- 0 b))
(setq c 370) (setq -c (- 0 c))
(setq ys (list a b c 0 -c -b -a -b -c 0 c b))
(setq zs (list 0 -c -b -a -b -c 0 c b a b c))
(setq zy 0) (setq zz 0) (setq cn 1)
;; (setq cn 0)
(setq sl "~%### ~a~%~%~a~%~%")

(format t sl t1 no)

(dolist (d ti)
  (progn
    ;; (if (eq cn 2) (return))
    (setq zy (+ zy (nth (mod cn 12) ys)))
    (setq zz (+ zz (nth (mod cn 12) zs)))
    (setq za (- 1 (* cn .03)))
    (format t (concatenate 'string "---~%=data-y='~d' "
                           "data-z='~d' data-rotate-x='~d' "
                           "data-scale='~4,2F'~%" sl)
            (* zy za) (* zz za) (* cn -30) za d no)
    ;; (setq br (format nil "---~%"))
    (setq cn (+ 1 cn))))
    ;; (if (< cn 11)
    ;;     (setq cn (+ 1 cn))
    ;;     (setq cn 0))))


[OpenCL] Minor improvements in default header testing

Differential Revision: https://reviews.llvm.org/D59544

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@356479 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 -O0 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -verify | FileCheck %s
// RUN: %clang_cc1 -O0 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -verify -cl-std=CL1.1 | FileCheck %s
// RUN: %clang_cc1 -O0 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -verify -cl-std=CL1.2 | FileCheck %s
// RUN: %clang_cc1 -O0 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -verify -cl-std=c++ | FileCheck %s --check-prefix=CHECK20

// Test including the default header as a module.
// The module should be compiled only once and loaded from cache afterwards.
// Change the directory mode to read only to make sure no new modules are created.
// Check time report to make sure module is used.
// Check that some builtins occur in the generated IR when called.

// ===
// Clear current directory.
// RUN: rm -rf %t
// RUN: mkdir -p %t

// ===
// Compile for OpenCL 1.0 for the first time. A module should be generated.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: chmod u-w %t/opencl_c.pcm

// ===
// Compile for OpenCL 1.0 for the second time. The module should not be re-created.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: chmod u+w %t/opencl_c.pcm
// RUN: mv %t/opencl_c.pcm %t/1_0.pcm

// ===
// Compile for OpenCL 2.0 for the first time. The module should change.
// RUN: %clang_cc1 -triple spir-unknown-unknown -O0 -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: not diff %t/1_0.pcm %t/opencl_c.pcm
// RUN: chmod u-w %t/opencl_c.pcm

// ===
// Compile for OpenCL 2.0 for the second time. The module should not change.
// RUN: %clang_cc1 -triple spir-unknown-unknown -O0 -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s

// Check cached module works for different OpenCL versions.
// RUN: rm -rf %t
// RUN: mkdir -p %t
// RUN: %clang_cc1 -triple spir64-unknown-unknown -emit-llvm -o - -cl-std=CL1.2 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple amdgcn--amdhsa -O0 -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: chmod u-w %t
// RUN: %clang_cc1 -triple spir64-unknown-unknown -emit-llvm -o - -cl-std=CL1.2 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple amdgcn--amdhsa -O0 -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: chmod u+w %t

// Verify that called builtins occur in the generated IR.

// CHECK-NOT: intel_sub_group_avc_mce_get_default_inter_base_multi_reference_penalty
// CHECK-NOT: ndrange_t
// CHECK20: ndrange_t
// CHECK: _Z16convert_char_rtec
// CHECK-NOT: _Z3ctzc
// CHECK20: _Z3ctzc
// CHECK20: _Z16convert_char_rtec
char f(char x) {
// Check functionality from OpenCL 2.0 onwards
#if defined(__OPENCL_CPP_VERSION__) || (__OPENCL_C_VERSION__ == CL_VERSION_2_0)
  ndrange_t t;
  x = ctz(x);
#endif //__OPENCL_C_VERSION__
  return convert_char_rte(x);
}

// Verify that a builtin using a write_only image3d_t type is available
// from OpenCL 2.0 onwards.

// CHECK20: _Z12write_imagef14ocl_image3d_wo
#if defined(__OPENCL_CPP_VERSION__) || (__OPENCL_C_VERSION__ >= CL_VERSION_2_0)
void test_image3dwo(write_only image3d_t img) {
  write_imagef(img, (0), (0.0f));
}
#endif //__OPENCL_C_VERSION__

// Verify that non-builtin cl_intel_planar_yuv extension is defined from
// OpenCL 1.2 onwards.
#if defined(__OPENCL_CPP_VERSION__) || (__OPENCL_C_VERSION__ >= CL_VERSION_1_2)
// expected-no-diagnostics
#ifndef cl_intel_planar_yuv
#error "Missing cl_intel_planar_yuv define"
#endif
#else //__OPENCL_C_VERSION__
// expected-warning@+2{{unknown OpenCL extension 'cl_intel_planar_yuv' - ignoring}}
#endif //__OPENCL_C_VERSION__
#pragma OPENCL EXTENSION cl_intel_planar_yuv : enable

// CHECK-MOD: Reading modules

// RUN: %clang_cc1 -O0 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -verify | FileCheck %s
// RUN: %clang_cc1 -O0 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -verify -cl-std=CL1.1 | FileCheck %s
// RUN: %clang_cc1 -O0 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -verify -cl-std=CL1.2 | FileCheck %s
// RUN: %clang_cc1 -O0 -triple spir-unknown-unknown -internal-isystem ../../lib/Headers -include opencl-c.h -emit-llvm -o - %s -verify -cl-std=c++ | FileCheck %s --check-prefix=CHECK20

// Test including the default header as a module.
// The module should be compiled only once and loaded from cache afterwards.
// Change the directory mode to read only to make sure no new modules are created.
// Check time report to make sure module is used.
// Check that some builtins occur in the generated IR when called.

// ===
// Clear current directory.
// RUN: rm -rf %t
// RUN: mkdir -p %t

// ===
// Compile for OpenCL 1.0 for the first time. A module should be generated.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: chmod u-w %t/opencl_c.pcm

// ===
// Compile for OpenCL 1.0 for the second time. The module should not be re-created.
// RUN: %clang_cc1 -triple spir-unknown-unknown -emit-llvm -o - -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: chmod u+w %t/opencl_c.pcm
// RUN: mv %t/opencl_c.pcm %t/1_0.pcm

// ===
// Compile for OpenCL 2.0 for the first time. The module should change.
// RUN: %clang_cc1 -triple spir-unknown-unknown -O0 -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: not diff %t/1_0.pcm %t/opencl_c.pcm
// RUN: chmod u-w %t/opencl_c.pcm

// ===
// Compile for OpenCL 2.0 for the second time. The module should not change.
// RUN: %clang_cc1 -triple spir-unknown-unknown -O0 -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -fdisable-module-hash -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s

// Check cached module works for different OpenCL versions.
// RUN: rm -rf %t
// RUN: mkdir -p %t
// RUN: %clang_cc1 -triple spir64-unknown-unknown -emit-llvm -o - -cl-std=CL1.2 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple amdgcn--amdhsa -O0 -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: chmod u-w %t
// RUN: %clang_cc1 -triple spir64-unknown-unknown -emit-llvm -o - -cl-std=CL1.2 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK --check-prefix=CHECK-MOD %s
// RUN: %clang_cc1 -triple amdgcn--amdhsa -O0 -emit-llvm -o - -cl-std=CL2.0 -finclude-default-header -fmodules -fimplicit-module-maps -fmodules-cache-path=%t -ftime-report %s 2>&1 | FileCheck --check-prefix=CHECK20 --check-prefix=CHECK-MOD %s
// RUN: chmod u+w %t

// Verify that called builtins occur in the generated IR.

// CHECK-NOT: intel_sub_group_avc_mce_get_default_inter_base_multi_reference_penalty
// CHECK-NOT: ndrange_t
// CHECK20: ndrange_t
// CHECK: _Z16convert_char_rtec
// CHECK-NOT: _Z3ctzc
// CHECK20: _Z3ctzc
// CHECK20-NOT: _Z16convert_char_rtec
char f(char x) {
#if !defined(__OPENCL_CPP_VERSION__) && (__OPENCL_C_VERSION__ != CL_VERSION_2_0)
  return convert_char_rte(x);

#else //__OPENCL_C_VERSION__
  ndrange_t t;
  return ctz(x);
#endif //__OPENCL_C_VERSION__
}

// Verify that a builtin using a write_only image3d_t type is available
// from OpenCL 2.0 onwards.

// CHECK20: _Z12write_imagef14ocl_image3d_wo
#if defined(__OPENCL_CPP_VERSION__) || (__OPENCL_C_VERSION__ >= CL_VERSION_2_0)
void test_image3dwo(write_only image3d_t img) {
  write_imagef(img, (0), (0.0f));
}
#endif //__OPENCL_C_VERSION__

// Verify that non-builtin cl_intel_planar_yuv extension is defined from
// OpenCL 1.2 onwards.
#if defined(__OPENCL_CPP_VERSION__) || (__OPENCL_C_VERSION__ >= CL_VERSION_1_2)
// expected-no-diagnostics
#ifndef cl_intel_planar_yuv
#error "Missing cl_intel_planar_yuv define"
#endif
#else //__OPENCL_C_VERSION__
// expected-warning@+2{{unknown OpenCL extension 'cl_intel_planar_yuv' - ignoring}}
#endif //__OPENCL_C_VERSION__
#pragma OPENCL EXTENSION cl_intel_planar_yuv : enable

// CHECK-MOD: Reading modules


Added to spearman2 kernel.

Added to spearman2 kernel.

More helper functions added to future kernel for new spearman GPU code.

/* BASIC PLAN FOR NEW SPEARMAN KERNEL..

Prune lists... ignoring infinities
IF pruned list is big enough... (else just make result NaN)
use bitonic sort with for loop to do 2 required sorts for spearman
make summation of rank differences
compute the spearman coefficient

MEMORY REQUIREMENTS...
will need TWO float arrays of size n
one int array of size n
where n is the number of samples per gene
in turn, this will be multilpied by the total number of threads
so size = ( 2*sizeof(float) + sizeof(int) )*thread_size

*/

// Fetch and build array of expressions for both genes, skipping any expressions that are missing
// for either gene. Also builds ordered rank list used for spearman algorithm.
//
// @param indexA Index into expression list for gene A.
// @param indexB Index into expression list for gene B.
// @param size Size of lists for both genes.
// @param listA New array of expressions for gene A that this function builds.
// @param listB New array of expressions for gene B that this function builds.
// @param rankList New array that is initialized to start at 1 and increment by one for each
// successive element.
// @param expressions Array of all expressions for all genes to generate gene lists from.
// @return Returns size of newly generated arrays which excludes any missing expression values.
int fetchLists(int indexA, int indexB, int size, __global float* listA, __global float* listB
               , __global int* rankList, __global float* expressions)
{
   // initialize counters
   int i;
   int j = 0;
   int newSize;

   // go through expression list with given indexes, generating new lists from it
   for (i = 0; i < size ;++i)
   {
      if ( !isnan(expressions[indexA+i]) && !isnan(expressions[indexB+i]) )
      {
         // if both expressions exist add expressions to new lists, add next rank, and increment
         listA[j] = expressions[indexA+i];
         listB[j] = expressions[indexB+i];
         rankList[j] = ++j;
      }
   }

   // set new size of generated lists and set unused end of lists to infinity or zero
   newSize = j;
   for (i = j; i < size;++i)
   {
      listA[i] = INFINITY;
      listB[i] = INFINITY;
      rankList[i] = 0;
   }

   // return new size for generated lists
   return newSize;
}

// Swap two floating point values.
//
// @param a First floating point value.
// @param b Second floating point value.
void swapF(__global float* a, __global float* b)
{
   float c = *a;
   *a = *b;
   *b = c;
}

// Sort a given list using the bitonic algorithm along with rearranging a second list with the same
// operations that are done to sort the sorted list.
//
// @param size Size of lists. It MUST be a power of 2.
// @param sortList The list to be sorted.
// @param extraList The extra list that will be rearranged identical to the sorted list.
void bitonicSortFF(int size, __global float* sortList, __global float* extraList)
{
   // initialize all variables
   int bsize = size/2;
   int ob,ib,i,dir,a,b,t;
   float pivot;

   // bitonic algorithm, starting with an outer block of 2 and working up to total size of list
   for (ob = 2; ob <= size ;ob *= 2)
   {
      for (ib = ob; ib >= 2 ;ib /= 2)
      {
         for (i = 0; i < bsize ;++i)
         {
            dir = -((i/(ob/2))&0x1);
            t = ib/2;
            a = (i/t)*ib+(i%t);
            b = a+t;
            if ( ( ( sortList[a] > sortList[b] ) && !dir )
                 || ( ( sortList[a] < sortList[b] ) && dir ) )
            {
               swapF(&sortList[a],&sortList[b]);
               swapF(&extraList[a],&extraList[b]);
            }
         }
      }
   }
}

// Swap two integer values.
//
// @param a First integer value.
// @param b Second integer value.
void swapI(__global int* a, __global int* b)
{
   int c = *a;
   *a = *b;
   *b = c;
}

// Sort a given list using the bitonic algorithm along with rearranging a second list with the same
// operations that are done to sort the sorted list.
//
// @param size Size of lists. It MUST be a power of 2.
// @param sortList The list to be sorted.
// @param extraList The extra list that will be rearranged identical to the sorted list.
void bitonicSortFI(int size, __global float* sortList, __global int* extraList)
{
   // initialize all variables
   int bsize = size/2;
   int ob,ib,i,dir,a,b,t;
   float pivot;

   // bitonic algorithm, starting with an outer block of 2 and working up to total size of list
   for (ob = 2; ob <= size ;ob *= 2)
   {
      for (ib = ob; ib >= 2 ;ib /= 2)
      {
         for (i = 0; i < bsize ;++i)
         {
            dir = -((i/(ob/2))&0x1);
            t = ib/2;
            a = (i/t)*ib+(i%t);
            b = a+t;
            if ( ( ( sortList[a] > sortList[b] ) && !dir )
                 || ( ( sortList[a] < sortList[b] ) && dir ) )
            {
               swapF(&sortList[a],&sortList[b]);
               swapI(&extraList[a],&extraList[b]);
            }
         }
      }
   }
}

/* BASIC PLAN FOR NEW SPEARMAN KERNEL..

Prune lists... ignoring infinities
IF pruned list is big enough... (else just make result NaN)
use bitonic sort with for loop to do 2 required sorts for spearman
make summation of rank differences
compute the spearman coefficient

MEMORY REQUIREMENTS...
will need TWO float arrays of size n
one int array of size n
where n is the number of samples per gene
in turn, this will be multilpied by the total number of threads
so size = ( 2*sizeof(float) + sizeof(int) )*thread_size

*/

// Fetch and build array of expressions for both genes, skipping any expressions that are missing
// for either gene.
//
// @param indexA Index into expression list for gene A.
// @param indexB Index into expression list for gene B.
// @param size Size of lists for both genes.
// @param listA New array of expressions for gene A that this function builds.
// @param listB New array of expressions for gene B that this function builds.
// @param expressions Array of all expressions for all genes to generate gene lists from.
// @return Returns size of newly generated arrays which excludes any missing expression values.
int fetchLists(int indexA, int indexB, int size, __global float* listA, __global float* listB
               , __global float* expressions)
{
   // initialize counters
   int i;
   int j = 0;
   int newSize;

   // go through expression list with given indexes, generating new lists from it
   for (i = 0; i < size ;++i)
   {
      if ( !isnan(expressions[indexA+i]) && !isnan(expressions[indexB+i]) )
      {
         // if both expressions exist add expressions to new lists and increment
         listA[j] = expressions[indexA+i];
         listB[j] = expressions[indexB+i];
         ++j;
      }
   }

   // set new size of generated lists and set unused end of lists to infinity
   newSize = j;
   for (i = j; i < size;++i)
   {
      listA[i] = INFINITY;
      listB[i] = INFINITY;
   }

   // return new size for generated lists
   return newSize;
}