Removed duplicated variale in cl kernel

Removed duplicated variale in cl kernel

/** Dilation of img_input by mask. Result is stored in img_output.

  */

// SHAPE img_shape (img_input->vglShape->asVglClShape())
// ARRAY convolution_window [window_size_x*window_size_y]
// SCALAR window_size_x
// SCALAR window_size_y

#include "vglClShape.h"

__kernel void vglClBinDilate(__read_only image2d_t img_input,
                             __write_only image2d_t img_output,
                             __constant float* convolution_window,
                             int window_size_x,
                             int window_size_y,
                             __constant VglClShape* img_shape)
{
    int2 coords = (int2)(get_global_id(0), get_global_id(1));
    const sampler_t smp = CLK_NORMALIZED_COORDS_FALSE | //Natural coordinates
                          CLK_ADDRESS_CLAMP_TO_EDGE |   //Clamp to next edge
                          CLK_FILTER_NEAREST;           //Don't interpolate

    int w_r = floor((float)window_size_x / 2.0f);
    int h_r = floor((float)window_size_y / 2.0f);
    int w_img = img_shape->shape[VGL_SHAPE_WIDTH];
    int h_img = img_shape->shape[VGL_SHAPE_HEIGHT];
    int pad = ((w_r / 8) + 1) * 8; // Avoids negative remainder.
    int i_l;
    uint4 pmax;  // In erosion, replace pmax with pmin
    uint4 result = 0;
    for (int bit = 0; bit < 8; bit++)
    {
      pmax = 0;  // In erosion, replace 0 with 1
      i_l = 0;
      for(int i_w = -h_r; i_w <= h_r && pmax.x == 0; i_w++)
      {
        for(int j_w = -w_r; j_w <= w_r && pmax.x == 0; j_w++)
        {
          int i_img = coords.y - i_w;                // In erosion, replace - with +
          int j_img = 8 * coords.x + 7 - bit - j_w;  // In erosion, replace - j_w with + j_w
          i_img = clamp(i_img, 0, h_img-1);
          j_img = clamp(j_img, 0, w_img-1);

          uint4 p = read_imageui(img_input, smp, (int2)((j_img) / 8, i_img));
          unsigned int result_bit;
          result_bit = p.x & (1 << ((pad + bit + j_w) % 8));  // In erosion, replace + j_w with - j_w

          if (!(convolution_window[i_l] == 0))
            if (result_bit > 0)  // In erosion, replace > with ==
              pmax.x = 1;

          i_l++;
        }
      }
      result += pmax.x << bit;
    }
    write_imageui(img_output, coords, result);
}

/** Dilation of img_input by mask. Result is stored in img_output.

  */

// SHAPE img_shape (img_input->vglShape->asVglClShape())
// ARRAY convolution_window [window_size_x*window_size_y]
// SCALAR window_size_x
// SCALAR window_size_y

#include "vglClShape.h"

__kernel void vglClBinDilate(__read_only image2d_t img_input,
                             __write_only image2d_t img_output,
                             __constant float* convolution_window,
                             int window_size_x,
                             int window_size_y,
                             __constant VglClShape* img_shape)
{
    int2 coords = (int2)(get_global_id(0), get_global_id(1));
    const sampler_t smp = CLK_NORMALIZED_COORDS_FALSE | //Natural coordinates
                          CLK_ADDRESS_CLAMP_TO_EDGE |   //Clamp to next edge
                          CLK_FILTER_NEAREST;           //Don't interpolate
    int w_r;
    int w_r = floor((float)window_size_x / 2.0f);
    int h_r = floor((float)window_size_y / 2.0f);
    int w_img = img_shape->shape[VGL_SHAPE_WIDTH];
    int h_img = img_shape->shape[VGL_SHAPE_HEIGHT];
    int pad = ((w_r / 8) + 1) * 8; // Avoids negative remainder.
    int i_l;
    uint4 pmax;  // In erosion, replace pmax with pmin
    uint4 result = 0;
    for (int bit = 0; bit < 8; bit++)
    {
      pmax = 0;  // In erosion, replace 0 with 1
      i_l = 0;
      for(int i_w = -h_r; i_w <= h_r && pmax.x == 0; i_w++)
      {
        for(int j_w = -w_r; j_w <= w_r && pmax.x == 0; j_w++)
        {
          int i_img = coords.y - i_w;                // In erosion, replace - with +
          int j_img = 8 * coords.x + 7 - bit - j_w;  // In erosion, replace - j_w with + j_w
          i_img = clamp(i_img, 0, h_img-1);
          j_img = clamp(j_img, 0, w_img-1);

          uint4 p = read_imageui(img_input, smp, (int2)((j_img) / 8, i_img));
          unsigned int result_bit;
          result_bit = p.x & (1 << ((pad + bit + j_w) % 8));  // In erosion, replace + j_w with - j_w

          if (!(convolution_window[i_l] == 0))
            if (result_bit > 0)  // In erosion, replace > with ==
              pmax.x = 1;

          i_l++;
        }
      }
      result += pmax.x << bit;
    }
    write_imageui(img_output, coords, result);
}


updated reference kernel

updated reference kernel

kernel void calculate(__global bool* var2_global, int var2_rows, int var2_view, __global double* var3_global, int var3_rows, int var3_view, __global double* var4_global, int var4_rows, int var4_view, __global double* var5_global, int var5_rows, int var5_view){
int i = get_global_id(0);
int j = get_global_id(1);
bool var2 = 0; if (!((!contains_nonzero(var2_view, LOWER) && j < i) || (!contains_nonzero(var2_view, UPPER) && j > i))) {var2 = var2_global[i + var2_rows * j];}
double var3 = 0; if (!((!contains_nonzero(var3_view, LOWER) && j < i) || (!contains_nonzero(var3_view, UPPER) && j > i))) {var3 = var3_global[i + var3_rows * j];}
double var4 = 0; if (!((!contains_nonzero(var4_view, LOWER) && j < i) || (!contains_nonzero(var4_view, UPPER) && j > i))) {var4 = var4_global[i + var4_rows * j];}
double var1 = var2 ? var3 : var4;
var5_global[i + var5_rows * j] = var1;
}

kernel void calculate(__global bool* var1_global, int var1_rows, int var1_view, __global double* var2_global, int var2_rows, int var2_view, __global double* var3_global, int var3_rows, int var3_view, __global double* var5_global, int var5_rows, int var5_view){
int i = get_global_id(0);
int j = get_global_id(1);
bool var1 = 0; if (!((!contains_nonzero(var1_view, LOWER) && j < i) || (!contains_nonzero(var1_view, UPPER) && j > i))) {var1 = var1_global[i + var1_rows * j];}
double var2 = 0; if (!((!contains_nonzero(var2_view, LOWER) && j < i) || (!contains_nonzero(var2_view, UPPER) && j > i))) {var2 = var2_global[i + var2_rows * j];}
double var3 = 0; if (!((!contains_nonzero(var3_view, LOWER) && j < i) || (!contains_nonzero(var3_view, UPPER) && j > i))) {var3 = var3_global[i + var3_rows * j];}
double var4 = var1 ? var2 : var3;
var5_global[i + var5_rows * j] = var4;
}


Formatting

Formatting

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

#define UNUSED __attribute__((__unused__))

#define DBG_OUT_OF_RNGE 0

// calculating the size of the area for weights prefetch

#include "math_ops.h"

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{

    _FLOAT weights[MLO_N_LCL_OUT_MAPS][MLO_N_LCL_IN_MAPS];

    uint gbl_id0  = get_global_id(0);
    uint batch_id = gbl_id0 / MLO_MAP_SZ4; // batch
    uint pos      = gbl_id0 % MLO_MAP_SZ4;

    uint out_grp_block = get_group_id(1); // block of outputs for the entire group
    uint out_id        = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off = batch_id * MLO_IN_BATCH_STRIDE + pos * MLO_READ_UNIT;

    uint wei_off = out_id *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT] = {{(_FLOAT)(0)}};
    _FLOAT dat[MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            accum[o][i] = (_FLOAT)(0);
        }
    }

    for(uint ci = 0; ci < MLO_CLOOP0; ++ci,
             // move input offset
                                      gbl_in_off += MLO_N_LCL_IN_MAPS * MLO_IN_CHANNEL_STRIDE,

             // move weights offset
                                      wei_off += MLO_N_LCL_IN_MAPS *
#if MLO_DIR_FORWARD == 1
                                                 MLO_WEI_CHANNEL_STRIDE
#else
                                                 MLO_WEI_BSTRIDE
#endif
        )
    {
        // read weights

        __constant _FLOAT* w1 = wei_ptr + wei_off;

        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o,
                 w1 +=
#if MLO_DIR_FORWARD == 1
                                                MLO_WEI_BSTRIDE
#else
                                                MLO_WEI_CHANNEL_STRIDE
#endif
            )
        {
            __constant _FLOAT* w2 = w1;
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c,
                     w2 +=
#if MLO_DIR_FORWARD == 1
                                                   MLO_WEI_CHANNEL_STRIDE
#else
                                                   MLO_WEI_BSTRIDE
#endif
                )
            {
                weights[o][c] = *w2;
#if DBG_OUT_OF_RNGE
                if(wei_off2 >= MLO_N_INPUTS * MLO_N_OUTPUTS)
                {
                    printf("K:oor: weights\n");
                }
#endif
            }
        }

        // convolve with all weights
        // read data
        // Shader compiler will use      GLOAL_LOAD_DWORD's OFFSET for *(ptr+index) access
        // Shader compiler will not use  GLOAL_LOAD_DWORD's OFFSET for ptr[index] access

        __global const _FLOAT* p = in_ptr + gbl_in_off;

        for(uint j = 0; j < MLO_N_LCL_IN_MAPS; j++)
        {
            for(uint i = 0; i < MLO_READ_UNIT; ++i)
            {
                dat[j][i] = *(p + i);
#if DBG_OUT_OF_RNGE
                if(gbl_in_off1 + i >= MLO_IN_BATCH_STRIDE * MLO_BATCH_SZ)
                {
                    printf("K:oor: inputs\n");
                }
#endif
            }
            p += MLO_IN_CHANNEL_STRIDE;
        }

        // convolve
        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            _FLOAT acc[MLO_READ_UNIT] = {(_FLOAT)(0)};
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                _FLOAT we = weights[o][c];
                _FLOAT* d = &dat[c][0];
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    acc[i] += d[i] * we;
                }
            }
            for(uint i = 0; i < MLO_READ_UNIT; ++i)
                accum[o][i] += acc[i];
        }
    }

    uint gbl_out_off =
        batch_id * MLO_OUT_BATCH_STRIDE + pos * MLO_READ_UNIT + out_id * MLO_OUT_CHANNEL_STRIDE;
    __global _FLOAT* q = out_ptr + gbl_out_off;

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o, q += MLO_OUT_CHANNEL_STRIDE)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            *(q + i) = accum[o][i];
        }
    }
}

/************************************************************************
stride and padding
*************************************************************************/
__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1pquv(const __global _FLOAT* __restrict in_ptr,
                  __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
                  const __global _FLOAT* __restrict bias,
#endif
                  __global _FLOAT* __restrict out_ptr,
                  UNUSED _FLOAT dummy_val // nothing
                  )
{

    _FLOAT weights[MLO_N_LCL_OUT_MAPS][MLO_N_LCL_IN_MAPS];

    uint gbl_id0 = get_global_id(0);

    uint batch_id  = gbl_id0 / MLO_MAP_SZ4; // batch
    uint pos       = gbl_id0 % MLO_MAP_SZ4;
    uint pos_out_y = pos / MLO_OUT_WIDTH4;
    uint pos_out_x = pos % MLO_OUT_WIDTH4;

#if MLO_DIR_FORWARD == 1
    uint pos_in_y = pos_out_y * MLO_FILTER_STRIDE1;
    uint pos_in_x = pos_out_x * MLO_FILTER_STRIDE0;
#else
    uint pos_in_y = pos_out_y; /// MLO_FILTER_STRIDE1;   - divided already
    uint pos_in_x = pos_out_x; // MLO_FILTER_STRIDE0;  - divided already
#endif

    uint out_grp_block = get_group_id(1); // block of outputs for the entire group
    uint out_id        = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off =
        batch_id * MLO_IN_BATCH_STRIDE + pos_in_y * MLO_IN_STRIDE + pos_in_x * MLO_READ_UNIT;
    //	bool vis = (pos_in_y < MLO_IN_HEIGHT);
    //	gbl_in_off = (vis) ? gbl_in_off : 0;

    uint wei_off = out_id *
#if MLO_DIR_FORWARD == 1
                   MLO_WEI_BSTRIDE
#else
                   MLO_WEI_CHANNEL_STRIDE
#endif
        ;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS][MLO_READ_UNIT];
    _FLOAT dat[MLO_N_LCL_IN_MAPS][MLO_READ_UNIT];

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            accum[o][i] = (_FLOAT)(0);
        }
    }

    const __global _FLOAT* i_ptr = in_ptr + gbl_in_off;
    __constant _FLOAT* w_ptr     = wei_ptr + wei_off;
    for(uint ci = 0; ci < MLO_CLOOP0; ++ci)
    {

        // convolve with all weights
        // read data

        for(uint j = 0; j < MLO_N_LCL_IN_MAPS; ++j)
        {
            uint i = 0;
#if MLO_READ_UNIT > 1
            for(; i < MLO_READ_UNIT - 1; ++i)
            {
                uint off = i
#if MLO_DIR_FORWARD == 1
                           * MLO_FILTER_STRIDE0
#endif
                    ;
                dat[j][i] = *(i_ptr + off);
            }
#endif

            for(; i < MLO_READ_UNIT; ++i)
            {
                //				vis &= (pos_in_x + i*MLO_FILTER_STRIDE0 <
                // MLO_IN_WIDTH);
                uint off = i
#if MLO_DIR_FORWARD == 1
                           * MLO_FILTER_STRIDE0
#endif
                    ;
                //				off = (vis) ? off : 0;
                _FLOAT val = *(i_ptr + off);
                dat[j][i]  = val;
                //              dat[j][i] = (vis)? dat[j][i] : (_FLOAT)(0);
            }

            i_ptr += MLO_IN_CHANNEL_STRIDE;
        }
        // read weights
        __constant _FLOAT* w_ptr0 = w_ptr;

        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {

            __constant _FLOAT* w_ptr1 = w_ptr0;

            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                weights[o][c] = *w_ptr1;
                w_ptr1 +=
#if MLO_DIR_FORWARD == 1
                    MLO_WEI_CHANNEL_STRIDE
#else
                    MLO_WEI_BSTRIDE
#endif
                    ;
            }

            w_ptr0 +=
#if MLO_DIR_FORWARD == 1
                MLO_WEI_BSTRIDE
#else
                MLO_WEI_CHANNEL_STRIDE
#endif
                ;
        }

        w_ptr += MLO_N_LCL_IN_MAPS *
#if MLO_DIR_FORWARD == 1
                 MLO_WEI_CHANNEL_STRIDE
#else
                 MLO_WEI_BSTRIDE
#endif
            ;
        // convolve
        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            for(uint c = 0; c < MLO_N_LCL_IN_MAPS; ++c)
            {
                for(uint i = 0; i < MLO_READ_UNIT; ++i)
                {
                    accum[o][i] += dat[c][i] * weights[o][c];
#if 0
                    if (pos_out_y == 2 && pos_out_x == 0)
                    {
                        printf((__constant char *)"K:c: %f %f %f %f\n",
                        accum[o][i],
                        dat[c][i] * weights[o][c],
                        dat[c][i],
                        weights[o][c]
                        );
                    }
#endif
                }
            }
        }
    }

    uint out_y = pos_out_y
#if MLO_DIR_FORWARD == 0
                 * MLO_FILTER_STRIDE1
#endif
        ;
    uint out_x = pos_out_x
#if MLO_DIR_FORWARD == 0
                 * MLO_FILTER_STRIDE0
#endif
        ;

    uint gbl_out_off = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE +
                       out_y * MLO_OUT_STRIDE + out_x * MLO_READ_UNIT;

    __global _FLOAT* q = out_ptr + gbl_out_off;

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o, q += MLO_OUT_CHANNEL_STRIDE)
    {

        for(uint i = 0; i < MLO_READ_UNIT; ++i)
        {
            __global _FLOAT* q1 = q;
            q1 += i
#if MLO_DIR_FORWARD == 0

                  * MLO_FILTER_STRIDE0
#endif
                ;
            *q1 = accum[o][i];

#if MLO_DIR_FORWARD == 0
            for(uint s = 1; s < MLO_FILTER_STRIDE0; ++s)
            {
#if MLO_HORIZ_ALIGNED == 0
                if(out_x + s < MLO_OUT_WIDTH)
#endif
                {
                    *(q1 + s) = (_FLOAT)(0);
                }
            }
#endif
        }

#if MLO_DIR_FORWARD == 0
        __global _FLOAT* q2 = q;
        for(uint j = 1; j < MLO_FILTER_STRIDE1; ++j)
        {
            q2 += MLO_OUT_STRIDE;
#if MLO_VERT_ALIGNED == 0
            if(out_y + j < MLO_OUT_HEIGHT)
#endif
            {

                for(uint s = 0; s < MLO_READ_UNIT * MLO_FILTER_STRIDE0; ++s)
                {
#if MLO_HORIZ_ALIGNED == 0
                    if(out_x + s < MLO_OUT_WIDTH)
#endif
                    {
                        *(q2 + s) = (_FLOAT)(0);
                    }
                }
            }
        }
#endif
    }
}


Loop range to decrease stack usage.

Loop range to decrease stack usage.

; PI Common LISP example.

; Abstract getting command line arguments.
; In sbcl the interpreter is an extra argument.

; For CLISP: clisp pi.cl <n>
; For SBCL: sbcl --script pi.cl <n>
; For GCL: gcl -f pi.cl <n>
; For ECL: ecl -shell pi.cl <n>

(defun getargs ()
  (or
   #+CLISP *args*
   #+SBCL (cdr *posix-argv*)
   #+GCL (cdr si::*command-args*)
   #+ECL (cdr(cdr(cdr si::*command-args*)))
   nil))

(defun range (b n j)
  (loop while (< b n)
    do (setf b (+ b j))
    collect (- b j)
  )
)

(defun pc (c) (/ (apply #'+ (map 'list #'(lambda (x) (/ 4 (+ 1 (* x x))))((lambda (n) (range (/ 0.5 n) 1 (/ 1 n))) c))) c))

(defvar n 5000000)

(if (> (list-length(getargs)) 0)
  (setq n (parse-integer (car (getargs))))
)

(format t "Calculating PI using~%")
(format t "  ~a slices~%" n)
(format t "  1 process~%")

; get-internal-real-time gets time from some arbitrary, implementation
; dependent starting point in some fraction of a second.
(defvar start (get-internal-real-time))

(defvar p (pc n))

; get-internal-real-time is in some platform specific unit which we can find in
; internal-time-units-per-second
(defvar stop (get-internal-real-time))

(defvar runt (/ (- stop start) internal-time-units-per-second))

(format t "Obtained value of PI: ~a~%" p)

; We have to print the time with ~f to force a float output.
(format t "Time taken: ~f seconds~%" runt)

; PI Common LISP example.

; Abstract getting command line arguments.
; In sbcl the interpreter is an extra argument.

; For CLISP: clisp pi.cl <n>
; For SBCL: sbcl --script pi.cl <n>
; For GCL: gcl -f pi.cl <n>
; For ECL: ecl -shell pi.cl <n>

(defun getargs ()
  (or
   #+CLISP *args*
   #+SBCL (cdr *posix-argv*)
   #+GCL (cdr si::*command-args*)
   #+ECL (cdr(cdr(cdr si::*command-args*)))
   nil))

(defun range (b n j)
  (if (>= b n)
    '()
    (cons b (range (+ b j) n j))
  )
)

(defun pc (c) (/ (apply #'+ (map 'list #'(lambda (x) (/ 4 (+ 1 (* x x))))((lambda (n) (range (/ 0.5 n) 1 (/ 1 n))) c))) c))

(defvar n 5000000)

(if (> (list-length(getargs)) 0)
  (setq n (parse-integer (car (getargs))))
)

(format t "Calculating PI using~%")
(format t "  ~a slices~%" n)
(format t "  1 process~%")

; get-internal-real-time gets time from some arbitrary, implementation
; dependent starting point in some fraction of a second.
(defvar start (get-internal-real-time))

(defvar p (pc n))

; get-internal-real-time is in some platform specific unit which we can find in
; internal-time-units-per-second
(defvar stop (get-internal-real-time))

(defvar runt (/ (- stop start) internal-time-units-per-second))

(format t "Obtained value of PI: ~a~%" p)

; We have to print the time with ~f to force a float output.
(format t "Time taken: ~f seconds~%" runt)


fixed out-of-bounds reads/writes

fixed out-of-bounds reads/writes

/* Steps to compute softmax:
 * 1. Compute the max per channel.
 * 2. Subtract the max from each value in the channel.
 * 3. Compute the exponent of all the values.
 * 4. Compute the sum of the vales per channel.
 * 5. Normalize based on the sum.
 *
 * We use CSR-{Vector / Stream} apprach to pick an algorithm depending on the
 * number of channels each workgroup has to work with.
 * J. L. Greathouse, M. Daga, Efficient sparse matrix-vector multiplication
 * on GPUs using the CSR storage format, in: Proc. Int'l Conf. High Performance
 * Computing, Networking, Storage and Analysis (SC'14)
*/

kernel void SoftmaxForward(global float *y, const int c, const int grid_size, const int spatial_dim)
{
#if NUM_BATCH == 1 // CSR-Vector like appraoch

	/* Entire workgroup works on one spatial_dim.
	 * We use logarthmic reductions to compute max and sum per channel.
	 * This approach reads in the same data thrice from DRAM but is still better
	 * than launching three different kernels.
	 * The workgroup begins by computing the nth image and s (spatial_dim) it
	 * is working on and iterates over the entire grid until finished.
 	 */

	local float l_helper[256];

	int gid = get_group_id(0);
	int lid = get_local_id(0);

	// Total number of workgroups launched can be less than the gridsize, hence iterate over.
	for(gid = get_group_id(0); gid < grid_size; gid += get_num_groups(0)) {

		int n = gid / spatial_dim; // nth image
		int s = gid % spatial_dim; // spatial dimension (h*w)

		l_helper[lid] = -FLT_MAX;

		float t_helper = -FLT_MAX; // thread_local helper var

		// Compute max per channel
		// Iterate over all the channels one thread is supposed to loop over
		// and compute max
		for(int i = lid; i < c; i += get_local_size(0)) {
			t_helper = max(y[mad24(n,c, i)*spatial_dim + s], t_helper);
		}

		// Now we have to compute the max from 256 values (one per each thread)
		l_helper[lid] = t_helper;
		barrier(CLK_LOCAL_MEM_FENCE);

		// Logarithmic reduction to compute the max.
		for(int i = (get_local_size(0)>>1); i > 0; i >>= 1) {
			if(lid < i) {
				l_helper[lid] = max(l_helper[lid], l_helper[lid+i]);
			}
			barrier(CLK_LOCAL_MEM_FENCE);
		}

		float channel_max = l_helper[0];
		t_helper = 0.;

		// Subtract channel_max from each value
		for(int i = lid; i < c; i += get_local_size(0)) {
			float value = y[mad24(n,c,i)*spatial_dim + s];

			// Compute exponent of each value
			// Then sum all the values touched by this thread
			t_helper += exp(value - channel_max);
		}

		l_helper[lid] = t_helper;
		barrier(CLK_LOCAL_MEM_FENCE);

		// Compute sum of 256 values (one for each thread)
		// Logarithmic reduction to compute the sum
		for(int i = (get_local_size(0)>>1); i > 0; i >>= 1) {
			if(lid < i) {
				l_helper[lid] += l_helper[lid+i];
			}
			barrier(CLK_LOCAL_MEM_FENCE);
		}

		float channel_sum = l_helper[0];

		// Normalize each value in the channel by the channel_sum
		for(int i = lid; i < c; i += get_local_size(0)) {
			float value = y[mad24(n,c, i)*spatial_dim + s];

			// Subtracting max again because we do not write the output of
			// value-max to DRAM above. Doing a subtraction again is much
			// faster than writing uncoalesced to DRAM
			value = exp(value - channel_max);

			y[mad24(n,c,i)*spatial_dim + s] = value/channel_sum;
		}
	}

#else // CSR-Stream like approach

	/* Each workgroup is computing the softmax for NUM_BATCH spatial_dims ala CSR-Stream.
	 * The number of threads computing softmax for one batch is BATCH_SIZE.
	 * The number of values each thread works on is U_BATCH_SIZE (read micro batch size).
	 * Each batch in the workgroup works on its nth image and s (spatial_dim).
	 * E.g. a 256 thread workgroup with c=31 has 8 batches and a batchsize of 32.
	 * The number of workgroups launched are exactly the number as required
	 * hence, there is no for-loop.
	*/

	local float l_helper[256];

	int gid = get_group_id(0);
	int lid = get_local_id(0);

	// ID of the thread within the batch
	int batch_lid = lid & (BATCH_SIZE-1);
	int batch = lid / BATCH_SIZE;

	// Batch specific n and s
	int batch_n = (NUM_BATCH*gid + batch) / spatial_dim;
	int batch_s = (NUM_BATCH*gid + batch) % spatial_dim;

	l_helper[lid] = -FLT_MAX;

	float t_helper = -FLT_MAX; // thread_local helper var

	// stores all the values touched by one thread so that we do not have load
	// again as the CSR-Vector approach
	float value[U_BATCH_SIZE];
	for(int i = 0; i < U_BATCH_SIZE; i++) {
		value[i] = -FLT_MAX;
	}

	// Compute max per channel
	for(int i = batch_lid; i < c; i += BATCH_SIZE) {
		if(mad24(batch_n,c, i)*spatial_dim + batch_s < c*grid_size)
			value[i / BATCH_SIZE ] = y[mad24(batch_n,c, i)*spatial_dim + batch_s];
		t_helper = max(value[i / BATCH_SIZE], t_helper);
	}

	// Now we have to compute the max from 256 values (one per each thread)
	l_helper[lid] = t_helper;
	barrier(CLK_LOCAL_MEM_FENCE);

	// Logarithmic reduction to compute the max.
	for(int i = (BATCH_SIZE >> 1); i > 0; i >>= 1) {
		if(batch_lid < i) {
			l_helper[lid] = max(l_helper[lid], l_helper[lid+i]);
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	}

	float channel_max = l_helper[batch * BATCH_SIZE];
	t_helper = 0.;

	// Subtract channel_max from each value
	for(int i = batch_lid; i < c; i += BATCH_SIZE) {

		// Compute exponent of each value
		// Then sum all the values touched by this thread
		t_helper += exp(value[i / BATCH_SIZE] - channel_max);
	}

	l_helper[lid] = t_helper;
	barrier(CLK_LOCAL_MEM_FENCE);

	// Compute sum of 256 values (one for each thread)
	// Logarithmic reduction to compute the sum
	for(int i = (BATCH_SIZE >> 1); i > 0; i >>= 1) {
		if(batch_lid < i) {
			l_helper[lid] += l_helper[lid+i];
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	}

	float channel_sum = l_helper[batch * BATCH_SIZE];

	// Normalize each value in the channel by the channel_sum
	for(int i = batch_lid; i < c; i += BATCH_SIZE) {
		value[i / BATCH_SIZE] = exp(value[i / BATCH_SIZE] - channel_max);

		if(mad24(batch_n,c, i)*spatial_dim + batch_s < c*grid_size)
			y[mad24(batch_n,c,i)*spatial_dim + batch_s] = value[i / BATCH_SIZE]/channel_sum;
	}

#endif // CSR-Vector vs CSR-Stream
}

kernel void SoftmaxBackward(global float *y, global float *dx, const int c, const int grid_size, const int spatial_dim)
{

#if NUM_BATCH == 1 // CSR-Vector like appraoch
	local float l_helper[256];

	int gid = get_group_id(0);
	int lid = get_local_id(0);

	// Total number of workgroups launched can be less than the gridsize, hence iterate over.
	for(gid = get_group_id(0); gid < grid_size; gid += get_num_groups(0)) {

		int n = gid / spatial_dim; // nth image
		int s = gid % spatial_dim; // spatial dimension (h*w)

		float channel_dot = 0.f; // thread_local helper var

		// Compute dot product per channel
		// Iterate over all the channels one thread is supposed to loop over
		// and compute dot-product
		for(int i = lid; i < c; i += get_local_size(0)) {
			channel_dot += (y[mad24(n,c, i)*spatial_dim + s] * dx[mad24(n,c, i)*spatial_dim + s]);
		}

		// Now we have to compute the sum from 256 values (one per each thread)
		l_helper[lid] = channel_dot;
		barrier(CLK_LOCAL_MEM_FENCE);

		// Logarithmic reduction to compute the sum.
		for(int i = (get_local_size(0)>>1); i > 0; i >>= 1) {
			if(lid < i) {
				l_helper[lid] += l_helper[lid+i];
			}
			barrier(CLK_LOCAL_MEM_FENCE);
		}

		channel_dot = l_helper[0];

		// Subtract and element-wise multiplication
		for(int i = lid; i < c; i += get_local_size(0)) {
			float value = dx[mad24(n,c,i)*spatial_dim + s] - channel_dot;

			dx[mad24(n,c,i)*spatial_dim + s] = y[mad24(n,c,i)*spatial_dim + s] * value;
		}
	}

#else

	local float l_helper[256];

	int gid = get_group_id(0);
	int lid = get_local_id(0);

	// ID of the thread within the batch
	int batch_lid = lid & (BATCH_SIZE-1);
	int batch = lid / BATCH_SIZE;

	// Batch specific n and s
	int batch_n = (NUM_BATCH*gid + batch) / spatial_dim;
	int batch_s = (NUM_BATCH*gid + batch) % spatial_dim;

	float channel_dot = 0.f; // thread_local helper var

	// stores all the values touched by one thread so that we do not have load
	// again as the CSR-Vector approach
	float y_value[U_BATCH_SIZE];
	float dx_value[U_BATCH_SIZE];

	// Compute dot product per channel
	for(int i = batch_lid; i < c; i += BATCH_SIZE) {
		if(mad24(batch_n,c, i)*spatial_dim + batch_s < c*grid_size) {
			y_value[i / BATCH_SIZE ] = y[mad24(batch_n,c, i)*spatial_dim + batch_s];
			dx_value[i / BATCH_SIZE ] = dx[mad24(batch_n,c, i)*spatial_dim + batch_s];
		}
		channel_dot += y_value[i / BATCH_SIZE] * dx_value[i / BATCH_SIZE] ;
	}

	// Now we have to compute the sum from 256 values (one per each thread)
	l_helper[lid] = channel_dot;
	barrier(CLK_LOCAL_MEM_FENCE);

	// Logarithmic reduction to compute the sum.
	for(int i = (BATCH_SIZE >> 1); i > 0; i >>= 1) {
		if(batch_lid < i) {
			l_helper[lid] += l_helper[lid+i];
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	}

	channel_dot = l_helper[batch * BATCH_SIZE];

	// Subtract and element-wise multiplication
	for(int i = batch_lid; i < c; i += BATCH_SIZE) {
		dx_value[i / BATCH_SIZE ] -= channel_dot;

		if(mad24(batch_n,c, i)*spatial_dim + batch_s < c*grid_size)
			dx[mad24(batch_n,c, i)*spatial_dim + batch_s] = y_value[i / BATCH_SIZE] * dx_value[i / BATCH_SIZE];
	}

#endif // CSR-Vector vs CSR-Stream
}

/* Steps to compute softmax:
 * 1. Compute the max per channel.
 * 2. Subtract the max from each value in the channel.
 * 3. Compute the exponent of all the values.
 * 4. Compute the sum of the vales per channel.
 * 5. Normalize based on the sum.
 *
 * We use CSR-{Vector / Stream} apprach to pick an algorithm depending on the
 * number of channels each workgroup has to work with.
 * J. L. Greathouse, M. Daga, Efficient sparse matrix-vector multiplication
 * on GPUs using the CSR storage format, in: Proc. Int'l Conf. High Performance
 * Computing, Networking, Storage and Analysis (SC'14)
*/

kernel void SoftmaxForward(global float *y, const int c, const int grid_size, const int spatial_dim)
{
#if NUM_BATCH == 1 // CSR-Vector like appraoch

	/* Entire workgroup works on one spatial_dim.
	 * We use logarthmic reductions to compute max and sum per channel.
	 * This approach reads in the same data thrice from DRAM but is still better
	 * than launching three different kernels.
	 * The workgroup begins by computing the nth image and s (spatial_dim) it
	 * is working on and iterates over the entire grid until finished.
 	 */

	local float l_helper[256];

	int gid = get_group_id(0);
	int lid = get_local_id(0);

	// Total number of workgroups launched can be less than the gridsize, hence iterate over.
	for(gid = get_group_id(0); gid < grid_size; gid += get_num_groups(0)) {

		int n = gid / spatial_dim; // nth image
		int s = gid % spatial_dim; // spatial dimension (h*w)

		l_helper[lid] = -FLT_MAX;

		float t_helper = -FLT_MAX; // thread_local helper var

		// Compute max per channel
		// Iterate over all the channels one thread is supposed to loop over
		// and compute max
		for(int i = lid; i < c; i += get_local_size(0)) {
			t_helper = max(y[mad24(n,c, i)*spatial_dim + s], t_helper);
		}

		// Now we have to compute the max from 256 values (one per each thread)
		l_helper[lid] = t_helper;
		barrier(CLK_LOCAL_MEM_FENCE);

		// Logarithmic reduction to compute the max.
		for(int i = (get_local_size(0)>>1); i > 0; i >>= 1) {
			if(lid < i) {
				l_helper[lid] = max(l_helper[lid], l_helper[lid+i]);
			}
			barrier(CLK_LOCAL_MEM_FENCE);
		}

		float channel_max = l_helper[0];
		t_helper = 0.;

		// Subtract channel_max from each value
		for(int i = lid; i < c; i += get_local_size(0)) {
			float value = y[mad24(n,c,i)*spatial_dim + s];

			// Compute exponent of each value
			// Then sum all the values touched by this thread
			t_helper += exp(value - channel_max);
		}

		l_helper[lid] = t_helper;
		barrier(CLK_LOCAL_MEM_FENCE);

		// Compute sum of 256 values (one for each thread)
		// Logarithmic reduction to compute the sum
		for(int i = (get_local_size(0)>>1); i > 0; i >>= 1) {
			if(lid < i) {
				l_helper[lid] += l_helper[lid+i];
			}
			barrier(CLK_LOCAL_MEM_FENCE);
		}

		float channel_sum = l_helper[0];

		// Normalize each value in the channel by the channel_sum
		for(int i = lid; i < c; i += get_local_size(0)) {
			float value = y[mad24(n,c, i)*spatial_dim + s];

			// Subtracting max again because we do not write the output of
			// value-max to DRAM above. Doing a subtraction again is much
			// faster than writing uncoalesced to DRAM
			value = exp(value - channel_max);

			y[mad24(n,c,i)*spatial_dim + s] = value/channel_sum;
		}
	}

#else // CSR-Stream like approach

	/* Each workgroup is computing the softmax for NUM_BATCH spatial_dims ala CSR-Stream.
	 * The number of threads computing softmax for one batch is BATCH_SIZE.
	 * The number of values each thread works on is U_BATCH_SIZE (read micro batch size).
	 * Each batch in the workgroup works on its nth image and s (spatial_dim).
	 * E.g. a 256 thread workgroup with c=31 has 8 batches and a batchsize of 32.
	 * The number of workgroups launched are exactly the number as required
	 * hence, there is no for-loop.
	*/

	local float l_helper[256];

	int gid = get_group_id(0);
	int lid = get_local_id(0);

	// ID of the thread within the batch
	int batch_lid = lid & (BATCH_SIZE-1);
	int batch = lid / BATCH_SIZE;

	// Batch specific n and s
	int batch_n = (NUM_BATCH*gid + batch) / spatial_dim;
	int batch_s = (NUM_BATCH*gid + batch) % spatial_dim;

	l_helper[lid] = -FLT_MAX;

	float t_helper = -FLT_MAX; // thread_local helper var

	// stores all the values touched by one thread so that we do not have load
	// again as the CSR-Vector approach
	float value[U_BATCH_SIZE];
	for(int i = 0; i < U_BATCH_SIZE; i++) {
		value[i] = -FLT_MAX;
	}

	// Compute max per channel
	for(int i = batch_lid; i < c; i += BATCH_SIZE) {
		value[i / BATCH_SIZE ] = y[mad24(batch_n,c, i)*spatial_dim + batch_s];
		t_helper = max(value[i / BATCH_SIZE], t_helper);
	}

	// Now we have to compute the max from 256 values (one per each thread)
	l_helper[lid] = t_helper;
	barrier(CLK_LOCAL_MEM_FENCE);

	// Logarithmic reduction to compute the max.
	for(int i = (BATCH_SIZE >> 1); i > 0; i >>= 1) {
		if(batch_lid < i) {
			l_helper[lid] = max(l_helper[lid], l_helper[lid+i]);
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	}

	float channel_max = l_helper[batch * BATCH_SIZE];
	t_helper = 0.;

	// Subtract channel_max from each value
	for(int i = batch_lid; i < c; i += BATCH_SIZE) {

		// Compute exponent of each value
		// Then sum all the values touched by this thread
		t_helper += exp(value[i / BATCH_SIZE] - channel_max);
	}

	l_helper[lid] = t_helper;
	barrier(CLK_LOCAL_MEM_FENCE);

	// Compute sum of 256 values (one for each thread)
	// Logarithmic reduction to compute the sum
	for(int i = (BATCH_SIZE >> 1); i > 0; i >>= 1) {
		if(batch_lid < i) {
			l_helper[lid] += l_helper[lid+i];
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	}

	float channel_sum = l_helper[batch * BATCH_SIZE];

	// Normalize each value in the channel by the channel_sum
	for(int i = batch_lid; i < c; i += BATCH_SIZE) {
		value[i / BATCH_SIZE] = exp(value[i / BATCH_SIZE] - channel_max);

		y[mad24(batch_n,c,i)*spatial_dim + batch_s] = value[i / BATCH_SIZE]/channel_sum;
	}

#endif // CSR-Vector vs CSR-Stream
}

kernel void SoftmaxBackward(global float *y, global float *dx, const int c, const int grid_size, const int spatial_dim)
{

#if NUM_BATCH == 1 // CSR-Vector like appraoch
	local float l_helper[256];

	int gid = get_group_id(0);
	int lid = get_local_id(0);

	// Total number of workgroups launched can be less than the gridsize, hence iterate over.
	for(gid = get_group_id(0); gid < grid_size; gid += get_num_groups(0)) {

		int n = gid / spatial_dim; // nth image
		int s = gid % spatial_dim; // spatial dimension (h*w)

		float channel_dot = 0.f; // thread_local helper var

		// Compute dot product per channel
		// Iterate over all the channels one thread is supposed to loop over
		// and compute dot-product
		for(int i = lid; i < c; i += get_local_size(0)) {
			channel_dot += (y[mad24(n,c, i)*spatial_dim + s] * dx[mad24(n,c, i)*spatial_dim + s]);
		}

		// Now we have to compute the sum from 256 values (one per each thread)
		l_helper[lid] = channel_dot;
		barrier(CLK_LOCAL_MEM_FENCE);

		// Logarithmic reduction to compute the sum.
		for(int i = (get_local_size(0)>>1); i > 0; i >>= 1) {
			if(lid < i) {
				l_helper[lid] += l_helper[lid+i];
			}
			barrier(CLK_LOCAL_MEM_FENCE);
		}

		channel_dot = l_helper[0];

		// Subtract and element-wise multiplication
		for(int i = lid; i < c; i += get_local_size(0)) {
			float value = dx[mad24(n,c,i)*spatial_dim + s] - channel_dot;

			dx[mad24(n,c,i)*spatial_dim + s] = y[mad24(n,c,i)*spatial_dim + s] * value;
		}
	}

#else

	local float l_helper[256];

	int gid = get_group_id(0);
	int lid = get_local_id(0);

	// ID of the thread within the batch
	int batch_lid = lid & (BATCH_SIZE-1);
	int batch = lid / BATCH_SIZE;

	// Batch specific n and s
	int batch_n = (NUM_BATCH*gid + batch) / spatial_dim;
	int batch_s = (NUM_BATCH*gid + batch) % spatial_dim;

	float channel_dot = 0.f; // thread_local helper var

	// stores all the values touched by one thread so that we do not have load
	// again as the CSR-Vector approach
	float y_value[U_BATCH_SIZE];
	float dx_value[U_BATCH_SIZE];

	// Compute dot product per channel
	for(int i = batch_lid; i < c; i += BATCH_SIZE) {
		y_value[i / BATCH_SIZE ] = y[mad24(batch_n,c, i)*spatial_dim + batch_s];
		dx_value[i / BATCH_SIZE ] = dx[mad24(batch_n,c, i)*spatial_dim + batch_s];
		channel_dot += y_value[i / BATCH_SIZE] * dx_value[i / BATCH_SIZE] ;
	}

	// Now we have to compute the sum from 256 values (one per each thread)
	l_helper[lid] = channel_dot;
	barrier(CLK_LOCAL_MEM_FENCE);

	// Logarithmic reduction to compute the sum.
	for(int i = (BATCH_SIZE >> 1); i > 0; i >>= 1) {
		if(batch_lid < i) {
			l_helper[lid] += l_helper[lid+i];
		}
		barrier(CLK_LOCAL_MEM_FENCE);
	}

	channel_dot = l_helper[batch * BATCH_SIZE];

	// Subtract and element-wise multiplication
	for(int i = batch_lid; i < c; i += BATCH_SIZE) {
		dx_value[i / BATCH_SIZE ] -= channel_dot;

		dx[mad24(batch_n,c, i)*spatial_dim + batch_s] = y_value[i / BATCH_SIZE] * dx_value[i / BATCH_SIZE];
	}

#endif // CSR-Vector vs CSR-Stream
}


sixtracklib/opencl: Refactor the OpenCL kernel; API and interface changes

- The OpenCL top-level Kernel function has been split into two parts to
  allow the remapping of the memory addresses in the serialized blocks and the
  use of the remapped memory during tracking to happen in two separate
  kernel functions.

  Since there is still some ambiguity if the remapping works correct in all
  possible contexts, this should allow better testing and more insight into
  any remaining problems

- to the same end, also a kernel parameter carrying a success flag has been
  introduced which allows the kernel function to communicate in a more detailed
  way with the host side and to report some errors back more specifically.

#ifndef SIXTRACKLIB_OPENCL_TRACK_PARTICLES_KERNEL_CL__
#define SIXTRACKLIB_OPENCL_TRACK_PARTICLES_KERNEL_CL__

#if defined( _GPUCODE )

void __kernel Track_remap_serialized_blocks_buffer(
     __global unsigned char* restrict  particles_data_buffer,
     __global unsigned char* restrict beam_elements_data_buffer,
     __global unsigned char* restrict elem_by_elem_data_buffer,
     __global long int*      restrict ptr_success_flag )
{
    size_t const global_id   = get_global_id( 0 );
    size_t const global_size = get_global_size( 0 );

    size_t const gid_to_remap_particles = 0;

    size_t const gid_to_remap_beam_elements = ( global_size > 1u )
        ? 1u : gid_to_remap_particles;

    size_t const gid_to_remap_elem_by_elem = ( global_size > 2u )
        ? 2u : gid_to_remap_beam_elements;

    long int  success_flag = 0;

    if( global_id <= gid_to_remap_elem_by_elem )
    {
        int ret = 0;

        NS(Blocks) elem_by_elem_buffer;
        NS(Blocks_preset)( &elem_by_elem_buffer );

        if( global_id == gid_to_remap_particles )
        {
            NS(Blocks) particles_buffer;
            NS(Blocks_preset)( &particles_buffer );

            if( 0 != NS(Blocks_unserialize)( &particles_buffer,
                        particles_data_buffer ) )
            {
                ret = -1;
                success_flag |= -1;
            }
        }

        if( ( ret == 0 ) && ( global_id == gid_to_remap_beam_elements ) )
        {
            NS(Blocks) beam_elements;
            NS(Blocks_preset)( &beam_elements );

            if( 0 != NS(Blocks_unserialize)( &beam_elements,
                        beam_elements_data_buffer ) )
            {
                ret = -1;
                success_flag = -2;
            }
        }

        if( ( ret == 0 ) && ( global_id == gid_to_remap_elem_by_elem ) )
        {
            __global unsigned long const* header =
                    ( __global unsigned long const* )elem_by_elem_data_buffer;

            if( header != 0 )
            {
                NS(Blocks) elem_by_elem_buffer;
                NS(Blocks_preset)( &elem_by_elem_buffer );

                if( ( 0 != NS(Blocks_unserialize)( &elem_by_elem_buffer ) ) &&
                    ( *header != ( unsigned long )0 ) )
                {
                    ret = -1;
                    success_flag = -4;
                }
            }
        }

        if( ( success_flag != 0 ) && ( ptr_success_flag  != 0 ) )
        {
            *ptr_success_flag |= success_flag;
        }
    }

    return;
}

void __kernel Track_particles_kernel_opencl(
    unsigned long const num_of_turns,
    __global unsigned char* restrict particles_data_buffer,
    __global unsigned char* restrict beam_elements_data_buffer,
    __global unsigned char* restrict elem_by_elem_data_buffer,
    __global long int*      restrict ptr_success_flag )
{
    typedef __global unsigned long const* g_ulong_cptr_t;
    typedef __global NS(BlockInfo)*       g_info_ptr_t;
    typedef __global NS(Particles)*       g_particles_ptr_t;

    size_t const global_id   = get_global_id( 0 );
    size_t const global_size = get_global_size( 0 );

    NS(block_size_t) num_particle_blocks              = 0u;
    NS(block_size_t) num_beam_elements                = 0u;
    NS(block_size_t) num_elem_by_elem_blocks          = 0u;
    NS(block_size_t) num_elem_by_elem_blocks_per_turn = 0u;
    NS(block_size_t) num_required_elem_by_elem_blocks = 0u;
    NS(block_size_t) num_of_particles                 = 0u;

    NS(Blocks) particles_buffer;
    NS(Blocks) beam_elements;
    NS(Blocks) elem_by_elem_buffer;

    NS(Particles) particles;

    long int success_flag = 0;
    bool use_elem_by_elem_buffer = false;

    /* --------------------------------------------------------------------- */
    /* *****  SECTION FOR DEFUSING THE HEISENBUG *** */
    /* This printf section seems to defuse the Heisenbug on the AMDGPU
     * implementation available to the author -> YMMV */

    /*
    if( global_id == 0u )
    {
        g_particles_ptr_t particles_header =
            ( g_particles_ptr_t )particles_data_buffer;

        printf( "before unserialization: \r\n" );
        printf( "global_id     = %u\r\n", global_id );

        printf( "particles_header[ 0 ]   = %16x at %18x\r\n",
                 particles_header[ 0 ],
                 ( uintptr_t )( particles_data_buffer +  0 ) );

        printf( "particles_header[ 1 ]   = %16x at %18x\r\n",
                 particles_header[ 1 ],
                 ( uintptr_t )( particles_data_buffer +  8 ) );

        printf( "particles_header[ 2 ]   = %16x at %18x\r\n",
                 particles_header[ 2 ],
                ( uintptr_t )( particles_data_buffer + 16 ) );

        printf( "particles_header[ 3 ]   = %16x at %18x\r\n",
                 particles_header[ 3 ],
                ( uintptr_t )( particles_data_buffer + 32 ) );
    }

    barrier( CLK_GLOBAL_MEM_FENCE );
    */

    /* *****  END OF SECTION FOR DEFUSING THE HEISENBUG *** */
    /* --------------------------------------------------------------------- */

    NS(Blocks_preset)( &particles_buffer );
    NS(Blocks_preset)( &beam_elements );

    if( 0 == NS(Blocks_unserialize_without_remapping)(
                &particles_buffer, particles_data_buffer ) )
    {
        num_particle_blocks = NS(Blocks_get_num_of_blocks)( &particles_buffer );

        if( num_particle_blocks == 1u )
        {
            g_particles_ptr_t ptr_particles  = 0;
            g_info_ptr_t  ptr_particles_info =
                NS(Blocks_get_block_infos_begin)( &particles_buffer );

            NS(BlockInfo) particles_info;

            if( ptr_particles_info != 0 )
            {
                particles_info = *ptr_particles_info;
            }
            else
            {
                NS(BlockInfo_preset)( &particles_info );
            }

            ptr_particles = NS(Blocks_get_particles)( &particles_info );

            if( ptr_particles != 0 )
            {
                particles = *ptr_particles;
            }
            else
            {
                NS(Particles_preset)( &particles );
            }

            num_of_particles = NS(Particles_get_num_particles)( &particles );
        }
    }
    else
    {
        NS(Blocks_preset)( &particles_buffer );
        NS(Particles_preset)( &particles );
        success_flag |= -1;
    }

    if( 0 == NS(Blocks_unserialize_without_remapping)(
                &beam_elements, beam_elements_data_buffer ) )
    {
        num_beam_elements = NS(Blocks_get_num_of_blocks)( &beam_elements );
    }
    else
    {
        NS(Blocks_preset)( &beam_elements );
        success_flag |= -2;
    }

    if( elem_by_elem_data_buffer != 0 )
    {
        g_ulong_cptr_t elem_by_elem_header =
            ( g_ulong_cptr_t )elem_by_elem_data_buffer;

        num_elem_by_elem_blocks_per_turn =
            num_beam_elements * num_particle_blocks;

        num_required_elem_by_elem_blocks =
            num_of_turns * num_elem_by_elem_blocks_per_turn;

        NS(Blocks_preset)( &elem_by_elem_buffer );

        if( 0 == NS(Blocks_unserialize_without_remapping)(
                    &elem_by_elem_buffer, elem_by_elem_data_buffer ) )
        {
            num_elem_by_elem_blocks =
                NS(Blocks_get_num_of_blocks)( &elem_by_elem_buffer );

            if( ( required_num_elem_by_elem > 0u ) &&
                ( num_elem_by_elem_blocks >= required_num_elem_by_elem ) )
            {
                use_elem_by_elem_buffer = true;
            }
        }
        else if( elem_by_elem_header[ 0 ] != ( unsigned long )0u )
        {
            success_flag |= -4;
        }
    }

    if( ( success_flag        == 0  ) && ( num_of_turns     != 0u ) &&
        ( num_beam_elements   != 0u ) && ( num_of_particles != 0u ) &&
        ( num_particle_blocks == 1u ) && ( global_id < num_of_particles ) )
    {
        int ret = 0;
        unsigned long ii = 0u;

        if( !use_elem_by_elem_buffer )
        {
            for( ; ii < num_of_turns ; ++ii )
            {
                ret |= NS(Track_beam_elements_particle)(
                        &particles, global_id, &beam_elements, 0 );
            }

            if( ret != 0 ) success_flag |= -8;
        }
        else
        {
            SIXTRL_GLOBAL_DEC NS(BlockInfo)* io_info_it =
                NS(Blocks_get_block_infos_begin)( &elem_by_elem_buffer );

            for( ; ii < num_of_turns ; ++ii,
                    io_info_it = io_info_it + num_beam_elements )
            {
                ret |= NS(Track_beam_elements_particle)(
                        &particles, global_id, &beam_elements, io_info_it );

                if( io_info_it != 0 )
                {
                    io_info_it = io_info_it + num_elem_by_elem_blocks_per_turn;
                }
            }

            if( ret != 0 ) success_flag |= -16;
        }
    }

    if( ( success_flag != 0 ) && ( ptr_success_flag != 0 ) )
    {
        *ptr_success_flag |= success_flag;
    }

    return;
}

#endif /* defined( _GPUCODE ) */

#endif /* SIXTRACKLIB_OPENCL_TRACK_PARTICLES_KERNEL_CL__ */

/* end: sixtracklib/opencl/track_particles_kernel.cl */

#ifndef SIXTRACKLIB_OPENCL_TRACK_PARTICLES_KERNEL_CL__
#define SIXTRACKLIB_OPENCL_TRACK_PARTICLES_KERNEL_CL__

#if defined( _GPUCODE )

void __kernel Track_particles_kernel_opencl(
    unsigned long const num_of_turns,
    __global unsigned char* SIXTRL_RESTRICT particles_data_buffer,
    __global unsigned char* SIXTRL_RESTRICT beam_elements_data_buffer,
    __global unsigned char* SIXTRL_RESTRICT elem_by_elem_data_buffer )
{
    size_t const global_id   = get_global_id( 0 );
    size_t const local_id    = get_local_id( 0 );
    size_t const group_id    = get_group_id( 0 );
    size_t const group_size  = get_num_groups( 0 );
    size_t const local_size  = get_local_size( 0 );
    size_t const global_size = get_global_size( 0 );

    size_t const gid_to_remap_particles = 0;
    size_t const gid_to_remap_beam_elements =
        ( global_size > 1 ) ? 1 : gid_to_remap_particles;

    size_t const gid_to_remap_elem_by_elem  = ( global_size > 2 )
        ? 2 : gid_to_remap_beam_elements;

    int use_elem_by_elem_buffer = 0;

    NS(block_size_t) num_particle_blocks     = 0;
    NS(block_size_t) num_beam_elements       = 0;
    NS(block_size_t) num_elem_by_elem_blocks = 0;

    NS(Blocks) particles_buffer;
    NS(Blocks) beam_elements;
    NS(Blocks) elem_by_elem_buffer;

    NS(BlockInfo) particles_info;
    NS(Particles) particles;

    __global NS(BlockInfo)* ptr_particles_info = 0;
    __global NS(Particles)* ptr_particles = 0;

    int ret = 0;

    /* *****  SECTION FOR DEFUSING THE HEISENBUG *** */
    /* This printf section seems to defuse the Heisenbug on the AMDGPU
     * implementation available to the author -> YMMV */

    /*
    if( global_id == gid_to_remap_particles )
    {
        __global ulong const* header = ( __global ulong const* )particles_data_buffer;

        printf( "before unserialization: \r\n" );
        printf( "global_id     = %u\r\n", global_id );
        printf( "header[ 0 ]   = %16x at %18x\r\n", header[ 0 ],
            ( uintptr_t )( particles_data_buffer +  0 ) );

        printf( "header[ 1 ]   = %16x at %18x\r\n", header[ 1 ],
            ( uintptr_t )( particles_data_buffer +  8 ) );

        printf( "header[ 2 ]   = %16x at %18x\r\n", header[ 2 ],
            ( uintptr_t )( particles_data_buffer + 16 ) );

        printf( "header[ 3 ]   = %16x at %18x\r\n", header[ 3 ],
            ( uintptr_t )( particles_data_buffer + 32 ) );
    }

    barrier( CLK_GLOBAL_MEM_FENCE );
    */

    /* *****  END OF SECTION FOR DEFUSING THE HEISENBUG *** */

    NS(Blocks_preset)( &particles_buffer );
    NS(Blocks_preset)( &beam_elements );
    NS(Blocks_preset)( &elem_by_elem_buffer );

    if( global_id == gid_to_remap_particles )
    {
        ret  = NS(Blocks_unserialize)( &particles_buffer, particles_data_buffer );
    }

    if( ( ret == 0 ) && ( global_id == gid_to_remap_beam_elements ) )
    {
        ret |= NS(Blocks_unserialize)( &beam_elements, beam_elements_data_buffer );
    }

    if( ( ret == 0 ) && ( global_id == gid_to_remap_elem_by_elem ) )
    {
        ret |= NS(Blocks_unserialize)(
            &elem_by_elem_buffer, elem_by_elem_data_buffer );

        if( ret == 0 )
        {
            use_elem_by_elem_buffer = 1;
        }
        else
        {
            __global unsigned long const* header =
                ( __global unsigned long const* )elem_by_elem_data_buffer;

            ret = ( ( header[ 0 ] == ( unsigned long )0u ) &&
                    ( header[ 1 ] == ( unsigned long )0u ) &&
                    ( header[ 2 ] == ( unsigned long )0u ) &&
                    ( header[ 3 ] == ( unsigned long )0u ) ) ? 0 : -1;
        }
    }

    /* All pointer offsets should be compensated by now -> so there should be
     * no more race conditions during unserialization as the content of the
     * data buffers should not be altered any more -> check with oclgrind! */

    barrier( CLK_GLOBAL_MEM_FENCE );

    if( ret == 0 )
    {
        if( global_id != gid_to_remap_particles )
        {
            ret = NS(Blocks_unserialize)(
                &particles_buffer, particles_data_buffer );
        }

        num_particle_blocks = NS(Blocks_get_num_of_blocks)( &particles_buffer );

        if( ( ret == 0 ) && ( num_particle_blocks == 1u ) )
        {
            ptr_particles_info = NS(Blocks_get_block_infos_begin)(
                &particles_buffer );

            if( ptr_particles_info != 0 )
            {
                particles_info = *ptr_particles_info;
            }
            else
            {
                NS(BlockInfo_preset)( &particles_info );
                ret = -1;
            }

            ptr_particles = NS(Blocks_get_particles)( &particles_info );

            if( ptr_particles != 0 )
            {
                particles = *ptr_particles;
            }
            else
            {
                NS(Particles_preset)( &particles );
                ret = -1;
            }
        }

        if( ( ret == 0 ) && ( global_id != gid_to_remap_beam_elements ) )
        {
            ret |= NS(Blocks_unserialize)(
                &beam_elements, beam_elements_data_buffer );
        }

        num_beam_elements = NS(Blocks_get_num_of_blocks)( &beam_elements );

        if( ( ret == 0 ) && ( use_elem_by_elem_buffer == 1 ) )
        {
            NS(block_size_t) const required_num_elem_by_elem =
                    num_of_turns * num_beam_elements * num_particle_blocks;

            num_elem_by_elem_blocks = NS(Blocks_get_num_of_blocks)( &elem_by_elem_buffer );

            if( global_id != gid_to_remap_elem_by_elem )
            {
                ret = NS(Blocks_unserialize)(
                    &elem_by_elem_buffer, elem_by_elem_data_buffer );
            }

            num_elem_by_elem_blocks = NS(Blocks_get_num_of_blocks)(
                &elem_by_elem_buffer );

            if( num_elem_by_elem_blocks < required_num_elem_by_elem )
            {
                use_elem_by_elem_buffer = 0;
            }
        }
    }

    barrier( CLK_GLOBAL_MEM_FENCE );

    /* Now all threads should have a properly unserialized instance of the
     * data structures */

    if( global_id == 0 )
    {
        printf( "ret                 = %d\r\n", ret );
        printf( "num_beam_elements   = %u\r\n", num_beam_elements );
        printf( "ptr_particles       = %x\r\n", ptr_particles );
        printf( "num_of_turns        = %u\r\n", num_of_turns );
        printf( "num_particle_blocks = %u\r\n", num_particle_blocks );
        printf( "num_particles       = %u\r\n",
            NS(Particles_get_num_particles)( &particles ) );
    }

    if( ( ret == 0 ) && ( num_beam_elements != 0u ) &&
        ( ptr_particles != 0 ) && ( num_of_turns != 0 ) &&
        ( num_particle_blocks == 1u ) &&
        ( global_id < NS(Particles_get_num_particles)( &particles ) ) )
    {
        NS(block_size_t) const required_num_elem_by_elem =
            num_of_turns * num_beam_elements;

        if( use_elem_by_elem_buffer != 1 )
        {
            unsigned long ii = 0;

            for( ; ii < num_of_turns ; ++ii )
            {
                ret |= NS(Track_beam_elements_particle)(
                        &particles, global_id, &beam_elements, 0 );
            }
        }
        else
        {
            unsigned long ii = 0;

            SIXTRL_GLOBAL_DEC NS(BlockInfo)* io_info_it =
                NS(Blocks_get_block_infos_begin)( &elem_by_elem_buffer );

            for( ; ii < num_of_turns ; ++ii,
                    io_info_it = io_info_it + num_beam_elements )
            {
                ret |= NS(Track_beam_elements_particle)(
                        &particles, global_id, &beam_elements, io_info_it );
            }
        }
    }

    return;
}

#endif /* defined( _GPUCODE ) */

#endif /* SIXTRACKLIB_OPENCL_TRACK_PARTICLES_KERNEL_CL__ */

/* end: sixtracklib/opencl/track_particles_kernel.cl */


fixed plan et tout

fixed plan et tout

#define WINX 1920
#define WINY 1080
#define BACKCOLOR 0x00000000
#define SCOLOR 0x000000FF
#define PCOLOR 0x0000FF00
#define SSCOLOR 0x00FF0000
#define PUTE ((__global unsigned int *)output)[id]

float3			sub_vect(float3 v1, float3 v2)
{
	float3		res;

	res.x = v1.x - v2.x;
	res.y = v1.y - v2.y;
	res.z = v1.z - v2.z;
	return (res);
}

float3			add_vect(float3 v1, float3 v2)
{
	float3		res;

	res.x = v1.x + v2.x;
	res.y = v1.y + v2.y;
	res.z = v1.z + v2.z;
	return (res);
}

float			norme_vect(float3 v)
{
	return (sqrt(v.x * v.x + v.y * v.y + v.z * v.z));
}

float3			normalize_vect(float3 v)
{
	float3		res;
	int			id = 1 / sqrt(v.x * v.x + v.y * v.y + v.z * v.z);

	res.x = v.x * id;
	res.y = v.y * id;
	res.z = v.z * id;
	return (res);
}

float			dot_vect(float3 v1, float3 v2)
{
	return (v1.x * v2.x + v1.y * v2.y + v1.z * v2.z);
}

float3			mult_fvect(float m, float3 v)
{
	float3		res;

	res.x = m * v.x;
	res.y = m * v.y;
	res.z = m * v.z;
	return (res);
}

float3			get_abc(float radius, float3 ray, float3 origin)
{
	float3		abc;

	abc.x = dot_vect(ray, ray);
	abc.y = 2 * dot_vect(ray, origin);
	abc.z = dot_vect(origin, origin) - (radius * radius);
	return (abc);
}

float			inter_sphere(float radius, float3 ray, float3 cam_origin, float3 sphere_origin)
{
	float3		abc;
	float		d;
	float		res1;
	float		res2;

	cam_origin.x -= sphere_origin.x;
	cam_origin.y -= sphere_origin.y;
	cam_origin.z -= sphere_origin.z;
	abc = get_abc(radius, ray, cam_origin);
	d = (abc.y * abc.y) - (4 * (abc.x * abc.z));
	if (d < 0)
		return (1000000000);
	if (d == 0)
		return ((-abc[1]) / (2 * abc[0]));
	res1 = (((-abc[1]) + sqrt(d)) / (2 * abc[0]));
	res2 = (((-abc[1]) - sqrt(d)) / (2 * abc[0]));
	if (res1 < res2)
		return(res1);
	return(res2);
}

float			inter_plan(float3 plan_origin, float3 plan_normale, float3 ray, float3 cam_origin)
{
	float		t;

	t = dot_vect(ray, plan_normale);

	t = dot_vect(sub_vect(plan_origin, cam_origin), plan_normale) / t;
	if (t < 0.0000001)
		return (1000000000);
	return (t);
}

float			modh_vect(int x)
{
	return ((float)((float)x - (WINX / 2.0)) / (WINX / 2.0));
}

float			modv_vect(int y)
{
	return ((float)(((float)y - (WINY / 2.0)) / (WINY / 2.0) * WINY / WINX));
}

float3			get_ray(float3 n, float3 v, float3 h, int x, int y)
{
	float3		res;

	h = mult_fvect(modh_vect(x), h);
	v = mult_fvect(modv_vect(y), v);
	res = add_vect(n, v);
	res = add_vect(res, h);
	return (res);
}

__kernel void	ray_trace(__global char *output, float mvx, float mvy, float mvz)
{
	int		id = get_global_id(0);
	int		x = id % WINX;
	int		y = id / WINX;

	// CAM
	float3	cam_origin;
	cam_origin.x = 0 + mvx;
	cam_origin.y = 0 + mvy;
	cam_origin.z = 0 + mvz;
	float3	hor;
	hor.x = 0.6;
	hor.y = 0;
	hor.z = 0;
	float3	vert;
	vert.x = 0;
	vert.y = 0.6;
	vert.z = 0;
	float3	cam_dir;
	cam_dir.x = 0;
	cam_dir.y = 0;
	cam_dir.z = 1;

	// SPHERE
	float3	boule_origin;
	boule_origin.x = 0;
	boule_origin.y = 10;
	boule_origin.z = 200;
	float	radius = 50;

	// SPHERE2
	float3	boule_origin2;
	boule_origin2.x = 0;
	boule_origin2.y = -10;
	boule_origin2.z = 50;
	float	radius2 = 10;

	// PLANE
	float3	plan_origin;
	plan_origin.x = 0;
	plan_origin.y = 0;
	plan_origin.z = 300;
	float3	plan_normale;
	plan_normale.x = 0;
	plan_normale.y = -1;
	plan_normale.z = 0;

	float plan;
	float sphere;
	float sphere2;

	float3 ray = get_ray(cam_dir, vert, hor, x ,y);
	if (( plan = inter_plan(plan_origin, plan_normale, ray, cam_origin)) < 0)
		plan = 1000000000;
	if (( sphere= inter_sphere(radius, ray, cam_origin, boule_origin)) < 0)
		sphere = 1000000000;
	if ((sphere2 = inter_sphere(radius2, ray, cam_origin, boule_origin2)) < 0)
		sphere2 = 1000000000;
//	if (id > (WINX * WINY / 3) && id < (WINX * WINY / 3 * 2))
//	printf("plan = %f sphere = %f\n", plan, sphere);
	if (sphere == 1000000000 && plan == 1000000000 && sphere2 == 1000000000)
		PUTE = BACKCOLOR;
	else if (sphere < plan && sphere < sphere2)
		PUTE = SCOLOR;
	else if (sphere2 < sphere && sphere2 < plan)
		PUTE = SSCOLOR;
	else
		PUTE = PCOLOR;
}

#define WINX 1920
#define WINY 1080
#define BACKCOLOR 0x00000000
#define SCOLOR 0x000000FF
#define PCOLOR 0x00FF0000
#define SSCOLOR 0x0000FF00
#define PUTE ((__global unsigned int *)output)[id]

float3			sub_vect(float3 v1, float3 v2)
{
	float3		res;

	res.x = v1.x - v2.x;
	res.y = v1.y - v2.y;
	res.z = v1.z - v2.z;
	return (res);
}

float3			add_vect(float3 v1, float3 v2)
{
	float3		res;

	res.x = v1.x + v2.x;
	res.y = v1.y + v2.y;
	res.z = v1.z + v2.z;
	return (res);
}

float			norme_vect(float3 v)
{
	return (sqrt(v.x * v.x + v.y * v.y + v.z * v.z));
}

float3			normalize_vect(float3 v)
{
	float3		res;
	int			id = 1 / sqrt(v.x * v.x + v.y * v.y + v.z * v.z);

	res.x = v.x * id;
	res.y = v.y * id;
	res.z = v.z * id;
	return (res);
}

float			dot_vect(float3 v1, float3 v2)
{
	return (v1.x * v2.x + v1.y * v2.y + v1.z * v2.z);
}

float3			mult_fvect(float m, float3 v)
{
	float3		res;

	res.x = m * v.x;
	res.y = m * v.y;
	res.z = m * v.z;
	return (res);
}

float3			get_abc(float radius, float3 ray, float3 origin)
{
	float3		abc;

	abc.x = dot_vect(ray, ray);
	abc.y = 2 * dot_vect(ray, origin);
	abc.z = dot_vect(origin, origin) - (radius * radius);
	return (abc);
}

float			inter_sphere(float radius, float3 ray, float3 cam_origin, float3 sphere_origin)
{
	float3		abc;
	float		d;
	float		res1;
	float		res2;

	cam_origin.x -= sphere_origin.x;
	cam_origin.y -= sphere_origin.y;
	cam_origin.z -= sphere_origin.z;
	abc = get_abc(radius, ray, cam_origin);
	d = (abc.y * abc.y) - (4 * (abc.x * abc.z));
	if (d < 0)
		return (1000000000);
	if (d == 0)
		return (-abc[1] / (2 * abc[0]));
	res1 = -(abc[1] + sqrt(d));
	res2 = -(abc[1] - sqrt(d));
	if (res1 < res2)
		return(res1);
	return(res2);
}

float			inter_plan(float3 plan_origin, float3 plan_normale, float3 ray, float3 cam_origin)
{
	float		t;

	t = dot_vect(ray, plan_normale);
	if (t < 0.0000001)
		return (1000000000);
	t = dot_vect(sub_vect(plan_origin, cam_origin), plan_normale) / t;
	if (t < 0)
		return (1000000000);
	return (t);
}

float			modh_vect(int x)
{
	return ((float)((float)x - (WINX / 2.0)) / (WINX / 2.0));
}

float			modv_vect(int y)
{
	return ((float)(((float)y - (WINY / 2.0)) / (WINY / 2.0) * WINY / WINX));
}

float3			get_ray(float3 n, float3 v, float3 h, int x, int y)
{
	float3		res;

	h = mult_fvect(modh_vect(x), h);
	v = mult_fvect(modv_vect(y), v);
	res = add_vect(n, v);
	res = add_vect(res, h);
	return (res);
}

__kernel void	ray_trace(__global char *output, float mvx, float mvy, float mvz)
{
	int		id = get_global_id(0);
	int		x = id % WINX;
	int		y = id / WINX;

	// CAM
	float3	cam_origin;
	cam_origin.x = -20 + mvx;
	cam_origin.y = 0 + mvy;
	cam_origin.z = 50 + mvz;
	float3	hor;
	hor.x = 0.6;
	hor.y = 0;
	hor.z = 0;
	float3	vert;
	vert.x = 0;
	vert.y = 0.6;
	vert.z = 0;
	float3	cam_dir;
	cam_dir.x = 0;
	cam_dir.y = 0;
	cam_dir.z = 1;

	// SPHERE
	float3	boule_origin;
	boule_origin.x = 0;
	boule_origin.y = 10;
	boule_origin.z = 200;
	float	radius = 50;

	// SPHERE2
	float3	boule_origin2;
	boule_origin2.x = 0;
	boule_origin2.y = -10;
	boule_origin2.z = 50;
	float	radius2 = 10;

	// PLANE
	float3	plan_origin;
	plan_origin.x = 0;
	plan_origin.y = 0;
	plan_origin.z = 300;
	float3	plan_normale;
	plan_normale.x = 0;
	plan_normale.y = 0;
	plan_normale.z = 1;

	float plan;
	float sphere;
	float sphere2;

	float3 ray = get_ray(cam_dir, vert, hor, x ,y);
	if (( plan = inter_plan(plan_origin, plan_normale, ray, cam_origin)) < 0)
		plan = 1000000000;
	if (( sphere= inter_sphere(radius, ray, cam_origin, boule_origin)) < 0)
		sphere = 1000000000;
	if ((sphere2 = inter_sphere(radius2, ray, cam_origin, boule_origin2)) < 0)
		sphere2 = 1000000000;
//	if (id > (WINX * WINY / 3) && id < (WINX * WINY / 3 * 2))
//	printf("plan = %f sphere = %f\n", plan, sphere);
	if (sphere == 1000000000 && plan == 1000000000 && sphere2 == 1000000000)
		PUTE = BACKCOLOR;
	else if (sphere < plan && sphere < sphere2)
		PUTE = SCOLOR;
	else if (plan < sphere && plan < sphere2)
		PUTE = PCOLOR;
	else
		PUTE = SSCOLOR;
}


Another fix for orphan semicolon in opencl sources. (#1514)

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)
#define _INT_MASK_GLOBAL uchar
#define _INT_MASK_LOCAL uchar

#define UNUSED __attribute__((__unused__))

#define MLO_POOLING_OP_AVE 0
#define MLO_POOLING_OP_MAX 1
#define MLO_POOLING_OP_STC 2
#define MLO_POOLING_OP_AVE_INCLUSIVE 3

#define MLO_POOLING_GROUP_SZ2 1

#ifndef MLO_POOLING_OP_ID
#define MLO_POOLING_OP_ID 0
#endif
// max
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
#define MLO_POOLING_OP(A, B) (fmax((A), (B)))
#elif(MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE) || \
    (MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE)
#define MLO_POOLING_OP(A, B) ((A) + (B))
#endif

/*********************************************************************************

**********************************************************************************/

#define MLO_BOT_DATA_SZ0 \
    (MLO_POOLING_N_HORIZ_OUT_PIX * MLO_POOLING_STRIDE0 + MLO_POOLING_KERNEL_SZ0 - 1)
#define MLO_BOT_DATA_SZ1 \
    (MLO_POOLING_N_VERT_OUT_PIX * MLO_POOLING_STRIDE1 + MLO_POOLING_KERNEL_SZ1 - 1)

__attribute__((reqd_work_group_size(MLO_POOLING_GROUP_SZ0,
                                    MLO_POOLING_GROUP_SZ1,
                                    MLO_POOLING_GROUP_SZ2))) __kernel void
mloPoolingG(const __global _FLOAT* bot,
            __global _FLOAT* top,
#if !defined(MLO_POOLING_DO_BACKWARD) || MLO_POOLING_OP_ID != MLO_POOLING_OP_MAX
            UNUSED
#endif
                __global _INT_MASK_GLOBAL* mask)
{

    uint x       = get_group_id(0) * MLO_POOLING_GROUP_SZ0 * MLO_POOLING_N_HORIZ_OUT_PIX;
    uint y       = get_group_id(1) * MLO_POOLING_GROUP_SZ1 * MLO_POOLING_N_VERT_OUT_PIX;
    uint lcl_id0 = get_local_id(0);
    uint lcl_id1 = get_local_id(1);
    //		int lcl_id = (lcl_id1 << MLO_POOLING_GROUP_LG2SZ0) + lcl_id0;
    uint ob      = get_global_id(2); // output * batch_sz
    uint b       = ob / MLO_POOLING_N_OUTPUTS;
    uint o       = ob - b * MLO_POOLING_N_OUTPUTS;
    uint bot_x   = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX) * MLO_POOLING_STRIDE0;
    uint bot_y   = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX) * MLO_POOLING_STRIDE1;
    uint bot_off = b * MLO_POOLING_BOT_BATCH_STRIDE + o * MLO_POOLING_BOT_CHANNEL_STRIDE;

    _FLOAT bot_data[MLO_BOT_DATA_SZ1][MLO_BOT_DATA_SZ0];
    _FLOAT res[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
    _INT_MASK_LOCAL mask_private[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#endif
    for(int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
    {
        for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
        {
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
            res[k][l] = (_FLOAT)(-MAX_VAL);
#elif(MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE) || \
    (MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE)
            res[k][l] = (_FLOAT)(0);
#endif
        }
    }

    for(uint j = 0; j < MLO_BOT_DATA_SZ1; ++j)
    {
        int run_y = (int)bot_y + j - MLO_POOLING_PAD1;

        for(uint i = 0; i < MLO_BOT_DATA_SZ0; ++i)
        {
            int run_x        = (int)bot_x + i - MLO_POOLING_PAD0;
            uint bot_gbl_off = bot_off + (uint)run_y * MLO_POOLING_BOT_STRIDE + (uint)run_x;
            bool vis         = ((run_y >= 0 && run_y < MLO_POOLING_BOT_HEIGHT) &&
                        (run_x >= 0 && run_x < MLO_POOLING_BOT_WIDTH))
                           ? true
                           : false;
            bot_data[j][i] = (vis) ? bot[bot_gbl_off] :
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
                                   (_FLOAT)(-MAX_VAL)
#elif(MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE) || \
    (MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE)
                                   (_FLOAT)(0)
#endif
                ;
        }
    }

#pragma unroll
    for(uint k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
    {
#if(MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE) || (MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE)
        uint y_dst = y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX + k;
        int hstart = (int)y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
        int hend   = min((hstart + MLO_POOLING_KERNEL_SZ1), (int)(MLO_POOLING_BOT_HEIGHT));
        hstart     = max(hstart, 0);
#endif
        for(uint l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
        {

#if(MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE) || (MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE)
            uint x_dst = x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX + l;
            int wstart = (int)x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
            int wend   = min((wstart + MLO_POOLING_KERNEL_SZ0), (int)(MLO_POOLING_BOT_WIDTH));
            wstart     = max(wstart, 0);
            uint pool_size =
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE
                MLO_POOLING_KERNEL_SZ0 * MLO_POOLING_KERNEL_SZ1;
            (void)wend;
            (void)hend;
#else
                (hend - hstart) * (wend - wstart);
#endif
            pool_size = (pool_size == 0) ? 1 : pool_size;
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
            mask_private[k][l] = 0xFF;
#endif

            for(uint j = 0; j < MLO_POOLING_KERNEL_SZ1; j++)
            {
                for(uint i = 0; i < MLO_POOLING_KERNEL_SZ0; i++)
                {

                    _FLOAT bot_val =
                        bot_data[j + k * MLO_POOLING_STRIDE1][i + l * MLO_POOLING_STRIDE0];

#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
                    if(bot_val > res[k][l])
                    {
                        res[k][l]          = bot_val;
                        mask_private[k][l] = i + MLO_POOLING_KERNEL_SZ0 * j;
                    }
#else
                    res[k][l] = MLO_POOLING_OP(res[k][l], bot_val);
#endif
                }
            }

#if(MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE) || (MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE)
            res[k][l] *= (_FLOAT)1.f / (_FLOAT)pool_size;
#endif
        }
    }

    uint top_y   = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX);
    uint top_x   = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX);
    uint top_off = b * MLO_POOLING_TOP_BATCH_STRIDE + o * MLO_POOLING_TOP_CHANNEL_STRIDE +
                   top_y * MLO_POOLING_TOP_STRIDE + top_x;
    for(uint k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
    {
        for(uint l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
        {
            if(top_y + k < MLO_POOLING_TOP_HEIGHT && top_x + l < MLO_POOLING_TOP_WIDTH)
            {
                top[top_off + k * MLO_POOLING_TOP_STRIDE + l] = res[k][l];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
                mask[top_off + k * MLO_POOLING_TOP_STRIDE + l] = mask_private[k][l];
#endif
            }
        }
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/
#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)
#define _INT_MASK_GLOBAL uchar
#define _INT_MASK_LOCAL uchar

#define UNUSED __attribute__((__unused__))

#define MLO_POOLING_OP_AVE 0
#define MLO_POOLING_OP_MAX 1
#define MLO_POOLING_OP_STC 2
#define MLO_POOLING_OP_AVE_INCLUSIVE 3

#define MLO_POOLING_GROUP_SZ2 1

#ifndef MLO_POOLING_OP_ID
#define MLO_POOLING_OP_ID 0
#endif
// max
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
#define MLO_POOLING_OP(A, B) fmax(A, B);
#elif(MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE) || \
    (MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE)
#define MLO_POOLING_OP(A, B) (A + B);
#endif

/*********************************************************************************

**********************************************************************************/

#define MLO_BOT_DATA_SZ0 \
    (MLO_POOLING_N_HORIZ_OUT_PIX * MLO_POOLING_STRIDE0 + MLO_POOLING_KERNEL_SZ0 - 1)
#define MLO_BOT_DATA_SZ1 \
    (MLO_POOLING_N_VERT_OUT_PIX * MLO_POOLING_STRIDE1 + MLO_POOLING_KERNEL_SZ1 - 1)

__attribute__((reqd_work_group_size(MLO_POOLING_GROUP_SZ0,
                                    MLO_POOLING_GROUP_SZ1,
                                    MLO_POOLING_GROUP_SZ2))) __kernel void
mloPoolingG(const __global _FLOAT* bot,
            __global _FLOAT* top,
#if !defined(MLO_POOLING_DO_BACKWARD) || MLO_POOLING_OP_ID != MLO_POOLING_OP_MAX
            UNUSED
#endif
                __global _INT_MASK_GLOBAL* mask)
{

    uint x       = get_group_id(0) * MLO_POOLING_GROUP_SZ0 * MLO_POOLING_N_HORIZ_OUT_PIX;
    uint y       = get_group_id(1) * MLO_POOLING_GROUP_SZ1 * MLO_POOLING_N_VERT_OUT_PIX;
    uint lcl_id0 = get_local_id(0);
    uint lcl_id1 = get_local_id(1);
    //		int lcl_id = (lcl_id1 << MLO_POOLING_GROUP_LG2SZ0) + lcl_id0;
    uint ob      = get_global_id(2); // output * batch_sz
    uint b       = ob / MLO_POOLING_N_OUTPUTS;
    uint o       = ob - b * MLO_POOLING_N_OUTPUTS;
    uint bot_x   = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX) * MLO_POOLING_STRIDE0;
    uint bot_y   = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX) * MLO_POOLING_STRIDE1;
    uint bot_off = b * MLO_POOLING_BOT_BATCH_STRIDE + o * MLO_POOLING_BOT_CHANNEL_STRIDE;

    _FLOAT bot_data[MLO_BOT_DATA_SZ1][MLO_BOT_DATA_SZ0];
    _FLOAT res[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
    _INT_MASK_LOCAL mask_private[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#endif
    for(int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
    {
        for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
        {
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
            res[k][l] = (_FLOAT)(-MAX_VAL);
#elif(MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE) || \
    (MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE)
            res[k][l] = (_FLOAT)(0);
#endif
        }
    }

    for(uint j = 0; j < MLO_BOT_DATA_SZ1; ++j)
    {
        int run_y = (int)bot_y + j - MLO_POOLING_PAD1;

        for(uint i = 0; i < MLO_BOT_DATA_SZ0; ++i)
        {
            int run_x        = (int)bot_x + i - MLO_POOLING_PAD0;
            uint bot_gbl_off = bot_off + (uint)run_y * MLO_POOLING_BOT_STRIDE + (uint)run_x;
            bool vis         = ((run_y >= 0 && run_y < MLO_POOLING_BOT_HEIGHT) &&
                        (run_x >= 0 && run_x < MLO_POOLING_BOT_WIDTH))
                           ? true
                           : false;
            bot_data[j][i] = (vis) ? bot[bot_gbl_off] :
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
                                   (_FLOAT)(-MAX_VAL)
#elif(MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE) || \
    (MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE)
                                   (_FLOAT)(0)
#endif
                ;
        }
    }

#pragma unroll
    for(uint k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
    {
#if(MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE) || (MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE)
        uint y_dst = y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX + k;
        int hstart = (int)y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
        int hend   = min((hstart + MLO_POOLING_KERNEL_SZ1), (int)(MLO_POOLING_BOT_HEIGHT));
        hstart     = max(hstart, 0);
#endif
        for(uint l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
        {

#if(MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE) || (MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE)
            uint x_dst = x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX + l;
            int wstart = (int)x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
            int wend   = min((wstart + MLO_POOLING_KERNEL_SZ0), (int)(MLO_POOLING_BOT_WIDTH));
            wstart     = max(wstart, 0);
            uint pool_size =
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE
                MLO_POOLING_KERNEL_SZ0 * MLO_POOLING_KERNEL_SZ1;
            (void)wend;
            (void)hend;
#else
                (hend - hstart) * (wend - wstart);
#endif
            pool_size = (pool_size == 0) ? 1 : pool_size;
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
            mask_private[k][l] = 0xFF;
#endif

            for(uint j = 0; j < MLO_POOLING_KERNEL_SZ1; j++)
            {
                for(uint i = 0; i < MLO_POOLING_KERNEL_SZ0; i++)
                {

                    _FLOAT bot_val =
                        bot_data[j + k * MLO_POOLING_STRIDE1][i + l * MLO_POOLING_STRIDE0];

#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
                    if(bot_val > res[k][l])
                    {
                        res[k][l]          = bot_val;
                        mask_private[k][l] = i + MLO_POOLING_KERNEL_SZ0 * j;
                    }
#else
                    res[k][l] = MLO_POOLING_OP(res[k][l], bot_val);
#endif
                }
            }

#if(MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE) || (MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE_INCLUSIVE)
            res[k][l] *= (_FLOAT)1.f / (_FLOAT)pool_size;
#endif
        }
    }

    uint top_y   = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX);
    uint top_x   = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX);
    uint top_off = b * MLO_POOLING_TOP_BATCH_STRIDE + o * MLO_POOLING_TOP_CHANNEL_STRIDE +
                   top_y * MLO_POOLING_TOP_STRIDE + top_x;
    for(uint k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
    {
        for(uint l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
        {
            if(top_y + k < MLO_POOLING_TOP_HEIGHT && top_x + l < MLO_POOLING_TOP_WIDTH)
            {
                top[top_off + k * MLO_POOLING_TOP_STRIDE + l] = res[k][l];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
                mask[top_off + k * MLO_POOLING_TOP_STRIDE + l] = mask_private[k][l];
#endif
            }
        }
    }
}