Kernel working 3-may sin compact

Kernel working 3-may sin compact

__kernel void getSuperK_M(
   __global uint* R2M_G, // Matrix of reads
   __global uint* counters, // Matrix of temporal output
   __global uint* TMP, // Matrix of temporal output
   const unsigned int nr, // Number of reads
   const unsigned int r, // Read size
   const unsigned int k, // Kmer size
   const unsigned int m // m-mer size
   )
{
   uint x, y, yl, xl, p, start, offset, nmk, nm, min, a, b, c, idt, ts, nt, lsd, nkr, tmp, ls, d = 0;
   bool flag;

   x = get_global_id(0);
   y = get_global_id(1);
   xl = get_local_id(0);
   yl = get_local_id(1);
   ls = get_local_size(0);

  __local uint R2M_L[180]; // Vector of a read and super k-mers (32 bits) , len = lenght of reads
  __local uint mR_10[180-4+1]; // Vector containing mmers, len = lenght of reads - minimizer size + 1
  __local uint counter; // Size of mmers
  __local uint MT[9]; // Position of minimizer in each tile
  __local uint PMT[9]; // Current minimizer in each tile

  if (x==0) {
    counter = 0;
  }

   // Global to local
   ts = ls;
   nt = (r-1)/ts + 1; // nt: Number of tiles or number of sub-reads per read
   if ((x<ts) && (x<r) && (y<nr)){
      for (int i=0; i<nt; i++) { // coalesced
        p=ts*i+x;
        if (p > r-1) {
          break;
        }
        R2M_L[p]=R2M_G[y*r+p];
      }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    // FUNCTION: GetM-mers
    // Read to decimal representation of m-mers
    lsd = ls / m;
    nm = r - m + 1; // nm: Number of m-mers per read
    ts = ((nm-1)/lsd) + 1; // ts: Tile size
    nt = ((nm-1)/ts) + 1 ; // nt: Number of tiles or number of sub-reads per read
    a = 63; // a:Mask , same as: 2^(2m-2)-1  m=4

    // rewrited from here
    if ((x < m*nt) && (y<nr)) {
      // Cómputo en pllo del primer m-mer de cada tile
      idt = x/m;
      start = ts*idt;
      offset = x % m;
      p = (start) + (offset);
      mR_10[start] = 0;
      b = R2M_L [p];
      atomic_or(&mR_10[start], (b<<(m-(x%m)-1)*2));
    }
    barrier(CLK_LOCAL_MEM_FENCE);

      // Cómputo en serie del resto de m-mers de cada tile

      if ((x%m == 0) && (x < m*nt) && (y<nr)) {
        c = mR_10[start];
        for (int z=0; z<(ts-1); z++) {
          if (start+m+z > r-1) {
            break;
          }
          c = (c & a) << 2;
          b = R2M_L[start+m+z];
          c = c | b;
          mR_10[start+1+z]=c;
        }
      }
    barrier(CLK_LOCAL_MEM_FENCE);

   // Clear R2M_L
   ts = ls;  // ts: Tile size, work group size
   nt = (r-1)/ts + 1; // nt: Number of tiles
   if ((x<ts) && (x<r) && (y<nr)){
      for (int i=0; i<nt; i++)  // coalesced
        {
          p=ts*i+x;
          if (p > r-1)
            { break; }
          R2M_L[p] = 0;
        }
    }

   nt = ((nm-1)/ts) + 1; // nt: Number of tiles or number of sub-reads per read

   // Extract MR_10 debuggin
   if ((x<ts) && (y<nr)){
     for (int i=0; i<nt; i++) { // coalesced
       p=ts*i+x;
       if (p > nm-1) {
         break;
       }
       TMP[y*nm+p]=mR_10[p];
     }
   }

    // FUNCTION: GetSK_M
    // m-mers to minimizers
    nmk = k-m+1; // nm: Number of m-mers per k-mers
    nkr = r - k + 1;
    lsd = ls/nmk;
    ts = (nkr-1)/lsd + 1; // ts: Tile size
    nt = (nkr-1)/ts + 1; // nt: Number of tiles

    if (x < nt){
      MT[x] = 0xFFFFFFFF;
      PMT[x] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

  if ((x<(nt*nmk )) && (y<nr)){
  //if ((x>=(5*nmk)) && (y<nr) && (x<(6*nmk))){
      idt = x/nmk;
      start = ts*idt;
      offset = x%nmk;
      p = start + offset;
      // Cómputo en pllo del min del primer k-mer de cada tile
      b = mR_10[p];
      atomic_min(&MT[idt],b); // Atomic min - se obtiene el min del primer k-mer de cada tile
}

barrier(CLK_LOCAL_MEM_FENCE);

 if ((x < (nt*nmk)) && (y<nr)) {
   if (b == MT[idt]) {
    -// if (idt == 0){
        atomic_max(&PMT[idt],p);
 //      } else {
  //      PMT[idt] = atomic_min(&PMT[idt],p);
    //  };
   }
 }
barrier(CLK_LOCAL_MEM_FENCE);

for (int z=0; z < ts ; z++){ // Cómputo en serie del min del resto de k-mers de cada tile

    if ((start + nmk + z) > (r -1)){
      break;
    }

    if ((x%nmk - z%nmk) == 0)
    {
            b = mR_10[start + nmk + z]; // Se lee el último m-mer del k-mer a evaluar
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if ((x < (nt*nmk)) && (y<nr))
    {
      flag = false;
      if (PMT[idt] != (start + z))
      {
        flag = true;
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if ( (x < (nt*nmk)) && (y<nr) ) {
        if (flag){
          if ((x%nmk - z%nmk) == 0){
            if (b < MT[idt]){
              // Atomic OR - end of super k-mers
              atomic_or(&R2M_L[PMT[idt]], (start + z + k - 1));

              PMT[idt] = start+nmk+z;

              MT[idt] = b;
              // Atomic OR - start of super k-mers
              atomic_or(&R2M_L[PMT[idt]], ((start + z + 1) << 12));
            }
          }
        }
      }
      if ((x < (nt*nmk)) && (y<nr))  {
        if (!flag) {
          if (x%nmk == 0){
            // Atomic OR - end of super k-mer
            atomic_or(&R2M_L[PMT[idt]], (start + z + k - 1));

          }
          MT[idt] = 0xFFFFFFFF;
        }
      }
        barrier(CLK_LOCAL_MEM_FENCE);

        if ((x < (nt*nmk)) && (y<nr))
      	{
          if (!flag)
          {
              atomic_min(&MT[idt], b);
          }
        }
          barrier(CLK_LOCAL_MEM_FENCE);

          if ((x < (nt*nmk)) && (y<nr))
          {
    			     if (!flag)
               {
    	            if (b == MT[idt])
                    {
                      atomic_max(&PMT[idt],p);
                    }
               }
    		  }

        barrier(CLK_LOCAL_MEM_FENCE);
      if ((x < (nt*nmk)) && (y<nr) )
        {
  			if (!flag)
          {
          if (p == PMT[idt]) {
            if (x%nmk <= z%nmk){
              PMT[idt] = start + ((z/nmk)+1)*nmk + (x%nmk);
            } else {
              PMT[idt] = start + (z/nmk)*nmk +(x%nmk);
            }
            // Atomic OR - start of super k-mers
            atomic_or(&R2M_L[PMT[idt]], ((start + z + 1) << 12));
          }

         }
       }
     }
    barrier(CLK_LOCAL_MEM_FENCE);

      if (x == (nt*nmk - 1)){
        // Atomic OR - end of super k-mer
        atomic_or(&R2M_L[PMT[idt]], (r-1));
      }
      barrier(CLK_LOCAL_MEM_FENCE);

    // Compact output
/*
    lsd = ls/m; // lsd: Local space divisions
    nm=r-m+1; // nm: Number of m-mers per read
    ts=(nm-1)/lsd + 1; // ts: Tile size
    nt=(nm-1)/ts + 1; // // nt: Number of tiles
    if ((x<ts) && (x<nm) && (y<nr)) {
   		for (int i=0; i<nt; i++) {
        p=ts*i+x;
        if (p > nm-1)
          { break; }
        b = R2M_L[p];
        if (b != 0)
        {
    	     c = atom_inc(&counter); // Atomic increment
           a = ((b & 0x00000FFF) - ((b >> 12) & 0x00000FFF)) & 0x000000FF; // a is the size
           R2M_L[c] =  ((mR_10[p] << 20) & 0xFFF00000) | ((b >> 4) & 0x000FFF00) | a;
        }
      }
    }
    barrier(CLK_LOCAL_MEM_FENCE);
*/

    //   //FUNCTION: Local2Global
    //   //Reads in local memory to Read in global memory
    ts = ls;  // ts: Tile size, work group size
    nt = (r-1)/ts + 1; // nt: Number of tiles

    if ((x<ts) && (x<r) && (y<nr)) {
      for (int i=0; i<nt; i++){  // coalesced
        p=ts*i+x;
        if (p > nm-1)
          {
            R2M_G[y*r+p] = 0;
            break;
          }
        R2M_G[y*r+p] = R2M_L[p];
      }
    }
   // Extract counters
    if ((x == 0) && (y<nr))
     {
             counters[y] = counter;
      }
}

__kernel void getSuperK_M(
   __global uint* R2M_G, // Matrix of reads
   __global uint* counters, // Matrix of temporal output
   __global uint* TMP, // Matrix of temporal output
   const unsigned int nr, // Number of reads
   const unsigned int r, // Read size
   const unsigned int k, // Kmer size
   const unsigned int m // m-mer size
   )
{
   uint x, y, yl, xl, p, start, offset, nmk, nm, min, a, b, c, idt, ts, nt, lsd, nkr, tmp, ls, d = 0;
   bool flag;

   x = get_global_id(0);
   y = get_global_id(1);
   xl = get_local_id(0);
   yl = get_local_id(1);
   ls = get_local_size(0);

  __local uint R2M_L[180]; // Vector of a read and super k-mers (32 bits) , len = lenght of reads
  __local uint mR_10[180-4+1]; // Vector containing mmers, len = lenght of reads - minimizer size + 1
  __local uint counter; // Size of mmers
  __local uint MT[9]; // Position of minimizer in each tile
  __local uint PMT[9]; // Current minimizer in each tile

  if (x==0) {
    counter = 0;
  }

   // Global to local
   ts = ls;
   nt = (r-1)/ts + 1; // nt: Number of tiles or number of sub-reads per read
   if ((x<ts) && (x<r) && (y<nr)){
      for (int i=0; i<nt; i++) { // coalesced
        p=ts*i+x;
        if (p > r-1) {
          break;
        }
        R2M_L[p]=R2M_G[y*r+p];
      }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    // FUNCTION: GetM-mers
    // Read to decimal representation of m-mers
    lsd = ls / m;
    nm = r - m + 1; // nm: Number of m-mers per read
    ts = ((nm-1)/lsd) + 1; // ts: Tile size
    nt = ((nm-1)/ts) + 1 ; // nt: Number of tiles or number of sub-reads per read
    a = 63; // a:Mask , same as: 2^(2m-2)-1  m=4

    // rewrited from here
    if ((x < m*nt) && (y<nr)) {
      // Cómputo en pllo del primer m-mer de cada tile
      idt = x/m;
      start = ts*idt;
      offset = x % m;
      p = (start) + (offset);
      mR_10[start] = 0;
      b = R2M_L [p];
      atomic_or(&mR_10[start], (b<<(m-(x%m)-1)*2));
    }
    barrier(CLK_LOCAL_MEM_FENCE);

      // Cómputo en serie del resto de m-mers de cada tile

      if ((x%m == 0) && (x < m*nt) && (y<nr)) {
        c = mR_10[start];
        for (int z=0; z<(ts-1); z++) {
          if (start+m+z > r-1) {
            break;
          }
          c = (c & a) << 2;
          b = R2M_L[start+m+z];
          c = c | b;
          mR_10[start+1+z]=c;
        }
      }
    barrier(CLK_LOCAL_MEM_FENCE);

   // Clear R2M_L
   ts = ls;  // ts: Tile size, work group size
   nt = (r-1)/ts + 1; // nt: Number of tiles
   if ((x<ts) && (x<r) && (y<nr)){
      for (int i=0; i<nt; i++)  // coalesced
        {
          p=ts*i+x;
          if (p > r-1)
            { break; }
          R2M_L[p] = 0;
        }
    }

   nt = ((nm-1)/ts) + 1; // nt: Number of tiles or number of sub-reads per read

   // Extract MR_10 debuggin
   if ((x<ts) && (y<nr)){
     for (int i=0; i<nt; i++) { // coalesced
       p=ts*i+x;
       if (p > nm-1) {
         break;
       }
       TMP[y*nm+p]=mR_10[p];
     }
   }

    // FUNCTION: GetSK_M
    // m-mers to minimizers
    nmk = k-m+1; // nm: Number of m-mers per k-mers
    nkr = r - k + 1;
    lsd = ls/nmk;
    ts = (nkr-1)/lsd + 1; // ts: Tile size
    nt = (nkr-1)/ts + 1; // nt: Number of tiles

    if (x < nt){
      MT[x] = 0xFFFFFFFF;
      PMT[x] = 0;
    }

    barrier(CLK_LOCAL_MEM_FENCE);

  if ((x<(nt*nmk )) && (y<nr)){
  //if ((x>=(5*nmk)) && (y<nr) && (x<(6*nmk))){
      idt = x/nmk;
      start = ts*idt;
      offset = x%nmk;
      p = start + offset;
      // Cómputo en pllo del min del primer k-mer de cada tile
      b = mR_10[p];
      atomic_min(&MT[idt],b); // Atomic min - se obtiene el min del primer k-mer de cada tile
}

barrier(CLK_LOCAL_MEM_FENCE);

 if ((x < (nt*nmk)) && (y<nr)) {
   if (b == MT[idt]) {
     if (idt == 0){
        PMT[idt] = atomic_max(&PMT[idt],p);
      } else {
        PMT[idt] = atomic_min(&PMT[idt],p);
      };
   }
 }
barrier(CLK_LOCAL_MEM_FENCE);

for (int z=0; z < ts ; z++){ // Cómputo en serie del min del resto de k-mers de cada tile

    if ((start + nmk + z) > (r -1)){
      break;
    }

    if ((x%nmk - z%nmk) == 0)
    {
            b = mR_10[start + nmk + z]; // Se lee el último m-mer del k-mer a evaluar
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if ((x < (nt*nmk)) && (y<nr))
    {
      flag = false;
      if (PMT[idt] != (start + z))
      {
        flag = true;
      }
    }

    barrier(CLK_LOCAL_MEM_FENCE);

    if ( (x < (nt*nmk)) && (y<nr) ) {
        if (flag){
          if ((x%nmk - z%nmk) == 0){
            if (b < MT[idt]){
              // Atomic OR - end of super k-mers
              atomic_or(&R2M_L[PMT[idt]], (start + z + k - 1));

              PMT[idt] = start+nmk+z;

              MT[idt] = b;
              // Atomic OR - start of super k-mers
              atomic_or(&R2M_L[PMT[idt]], ((start + z + 1) << 12));
            }
          }
        }
      }
      if ((x < (nt*nmk)) && (y<nr))  {
        if (!flag) {
          if (x%nmk == 0){
            // Atomic OR - end of super k-mer
            atomic_or(&R2M_L[PMT[idt]], (start + z + k - 1));

          }
          MT[idt] = 0xFFFFFFFF;
        }
      }
        barrier(CLK_LOCAL_MEM_FENCE);

        if ((x < (nt*nmk)) && (y<nr))
      	{
          if (!flag)
          {
              atomic_min(&MT[idt], b);
          }
        }
          barrier(CLK_LOCAL_MEM_FENCE);

          if ((x < (nt*nmk)) && (y<nr))
          {
    			     if (!flag)
               {
    	            if (b == MT[idt])
                    {
                      atomic_min(&PMT[idt],p);
                    }
               }
    		  }

        barrier(CLK_LOCAL_MEM_FENCE);
      if ((x < (nt*nmk)) && (y<nr) )
        {
  			if (!flag)
          {
          if (p == PMT[idt]) {
            if (x%nmk <= z%nmk){
              PMT[idt] = start + ((z/nmk)+1)*nmk + (x%nmk);
            } else {
              PMT[idt] = start + (z/nmk)*nmk +(x%nmk);
            }
            // Atomic OR - start of super k-mers
            atomic_or(&R2M_L[PMT[idt]], ((start + z + 1) << 12));
          }

         }
       }
     }
    barrier(CLK_LOCAL_MEM_FENCE);

      if (x == (nt*nmk - 1)){
        // Atomic OR - end of super k-mer
        atomic_or(&R2M_L[PMT[idt]], (r-1));
      }
      barrier(CLK_LOCAL_MEM_FENCE);

    // Compact output
    lsd = ls/m; // lsd: Local space divisions
    nm=r-m+1; // nm: Number of m-mers per read
    ts=(nm-1)/lsd + 1; // ts: Tile size
    nt=(nm-1)/ts + 1; // // nt: Number of tiles
    if ((x<ts) && (x<nm) && (y<nr)) {
   		for (int i=0; i<nt; i++) {
        p=ts*i+x;
        if (p > nm-1)
          { break; }
        b = R2M_L[p];
        if (b != 0)
        {
    	     c = atom_inc(&counter); // Atomic increment
           a = ((b & 0x00000FFF) - ((b >> 12) & 0x00000FFF)) & 0x000000FF; // a is the size
           R2M_L[c] =  ((mR_10[p] << 20) & 0xFFF00000) | ((b >> 4) & 0x000FFF00) | a;
        }
      }
    }
    barrier(CLK_LOCAL_MEM_FENCE);

    //   //FUNCTION: Local2Global
    //   //Reads in local memory to Read in global memory

    ts = ls;  // ts: Tile size, work group size
    nt = (r-1)/ts + 1; // nt: Number of tiles

    if ((x<ts) && (x<r) && (y<nr)) {
      for (int i=0; i<nt; i++){  // coalesced
        p=ts*i+x;
        if (p > nm-1)
          {
            R2M_G[y*r+p] = 0;
            break;
          }
        R2M_G[y*r+p] = R2M_L[p];
      }
    }
   // Extract counters
    if ((x == 0) && (y<nr))
     {
             counters[y] = counter;
      }
}


Move x_dst and y_dst variables inside MLO_POOLING_OP_AVE ifdef

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8
#define _INT_MASK_GLOBAL		uchar
#define _INT_MASK_LOCAL			uchar

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define MLO_POOLING_OP_AVE			0
#define MLO_POOLING_OP_MAX			1
#define MLO_POOLING_OP_STC			2

#define MLO_POOLING_GROUP_SZ2 1

#ifndef MLO_POOLING_OP_ID
#define MLO_POOLING_OP_ID 0
#endif
// max
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
#define MLO_POOLING_OP(A,B) fmax(A, B);
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
#define MLO_POOLING_OP(A,B) (A + B);
#endif

#define MLO_POOLING_LCL_DATA_WIDTH (MLO_POOLING_GROUP_SZ0 *MLO_POOLING_N_HORIZ_OUT_PIX *MLO_POOLING_STRIDE0 + MLO_POOLING_KERNEL_SZ0 - 1)
#define MLO_POOLING_LCL_DATA_HEIGHT (MLO_POOLING_GROUP_SZ1 *MLO_POOLING_N_VERT_OUT_PIX *MLO_POOLING_STRIDE1 + MLO_POOLING_KERNEL_SZ1 - 1)

__attribute__((reqd_work_group_size(MLO_POOLING_GROUP_SZ0,MLO_POOLING_GROUP_SZ1,MLO_POOLING_GROUP_SZ2)))
__kernel void mloPooling(
       const __global _FLOAT * bot,
       __global _FLOAT * top,
#if !defined(MLO_POOLING_DO_BACKWARD) || MLO_POOLING_OP_ID != MLO_POOLING_OP_MAX
       UNUSED
#endif
	   __global _INT_MASK_GLOBAL * mask
	   )
{
		__local _FLOAT bot_data[MLO_POOLING_LCL_DATA_WIDTH * MLO_POOLING_LCL_DATA_HEIGHT];

		int x = get_group_id(0) * MLO_POOLING_GROUP_SZ0 * MLO_POOLING_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_POOLING_GROUP_SZ1 * MLO_POOLING_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
//		int lcl_id = (lcl_id1 << MLO_POOLING_GROUP_LG2SZ0) + lcl_id0;
		int ob = get_global_id(2); // output * batch_sz
		int b = ob / MLO_POOLING_N_OUTPUTS;
		int o = ob - b * MLO_POOLING_N_OUTPUTS;
		int bot_x = x*MLO_POOLING_STRIDE0;
		int bot_y = y*MLO_POOLING_STRIDE1;
		int bot_off = b * MLO_POOLING_BOT_BATCH_STRIDE + o * MLO_POOLING_BOT_CHANNEL_STRIDE;

		_FLOAT res[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
		_INT_MASK_LOCAL mask_private[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#endif
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				res[k][l] =  -FLT_MAX;
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] = 0;
#endif
			}
		}

// load tile
		for( int b_j = lcl_id1; b_j < MLO_POOLING_LCL_DATA_HEIGHT; b_j += MLO_POOLING_GROUP_SZ1)
		{
			int bot_y_act = bot_y + b_j  - MLO_POOLING_PAD1;

			bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_POOLING_BOT_HEIGHT);

			int bot_y_off = bot_y_act * MLO_POOLING_BOT_STRIDE;

			int lcl_off_v = b_j * MLO_POOLING_LCL_DATA_WIDTH;

			for(int b_i = lcl_id0; b_i < MLO_POOLING_LCL_DATA_WIDTH; b_i += MLO_POOLING_GROUP_SZ0)
			{

				int bot_x_act = bot_x + b_i - MLO_POOLING_PAD0;

				bool invisibleX = (bot_x_act < 0) || (bot_x_act >= MLO_POOLING_BOT_WIDTH);
				bool invisible = invisibleX || invisibleY;

				int bot_idx = invisible ? 0 : (bot_off + bot_y_off + bot_x_act);
				_FLOAT bot_val = bot[bot_idx];

				bot_val = (invisible) ?

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				-FLT_MAX
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				0
#endif
				 : bot_val;
				bot_data[lcl_off_v + b_i] = bot_val;

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int lcl_y = lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX * MLO_POOLING_STRIDE1;
		int lcl_x = lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX * MLO_POOLING_STRIDE0;
		int lcl_off = lcl_y * MLO_POOLING_LCL_DATA_WIDTH + lcl_x;

		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
			int y_dst = y+ lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX + k;
			int hstart = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
			int hend = min(hstart + MLO_POOLING_KERNEL_SZ1, MLO_POOLING_BOT_HEIGHT + MLO_POOLING_PAD1);
#endif
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				int	x_dst = x+ lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX + l;
				int wstart = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
				int wend = min(wstart + MLO_POOLING_KERNEL_SZ0, MLO_POOLING_BOT_WIDTH + MLO_POOLING_PAD0);
				int pool_size = (hend - hstart) * (wend - wstart);
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				mask_private[k][l] = 0xFF;
#endif

				for( int j = 0; j < MLO_POOLING_KERNEL_SZ1; j++)
				{
					for(int i = 0; i < MLO_POOLING_KERNEL_SZ0; i++)
					{

						_FLOAT bot_val =  bot_data[lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i)];
#if 0
						if (y_dst == 0 && x_dst == 6)
						{

							printf("k: %d %f %f\n",
								lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i),
								res[k][l],
								bot_val
								);
						}
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
						if (bot_val > res[k][l])
						{
							res[k][l] = bot_val;
							mask_private[k][l] = i + MLO_POOLING_KERNEL_SZ0 * j;
						}
#else
						res[k][l] = MLO_POOLING_OP(res[k][l],bot_val);
#endif
					}
				}

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] *= (_FLOAT)1.f/ (_FLOAT)pool_size;
#endif
			}
		}

		int top_y = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX);
		int top_x = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX);
		int top_off = b * MLO_POOLING_TOP_BATCH_STRIDE + o * MLO_POOLING_TOP_CHANNEL_STRIDE + top_y * MLO_POOLING_TOP_STRIDE + top_x;
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
				if (top_y + k < MLO_POOLING_TOP_HEIGHT && top_x + l < MLO_POOLING_TOP_WIDTH)
				{
					top[top_off + k * MLO_POOLING_TOP_STRIDE +l] = res[k][l];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
					mask[top_off + k * MLO_POOLING_TOP_STRIDE + l] = mask_private[k][l];
#endif
				}
			}
		}

}

/*
 * Copyright (c) 2016 AMD Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and/or associated documentation files (the
 * "Materials"), to deal in the Materials without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Materials, and to
 * permit persons to whom the Materials are furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Materials.
 *
 * THE MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * MATERIALS OR THE USE OR OTHER DEALINGS IN THE MATERIALS.
 */

#define _FLOAT					float
#define _FLOAT2					float2
#define _FLOAT4					float4
#define _FLOAT8					float8
#define _INT_MASK_GLOBAL		uchar
#define _INT_MASK_LOCAL			uchar

#ifndef FLT_MAX
#define FLT_MAX         3.402823466e+38F        /* max value */
#endif

#define UNUSED __attribute__((__unused__))

#define MLO_POOLING_OP_AVE			0
#define MLO_POOLING_OP_MAX			1
#define MLO_POOLING_OP_STC			2

#define MLO_POOLING_GROUP_SZ2 1

#ifndef MLO_POOLING_OP_ID
#define MLO_POOLING_OP_ID 0
#endif
// max
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
#define MLO_POOLING_OP(A,B) fmax(A, B);
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
#define MLO_POOLING_OP(A,B) (A + B);
#endif

#define MLO_POOLING_LCL_DATA_WIDTH (MLO_POOLING_GROUP_SZ0 *MLO_POOLING_N_HORIZ_OUT_PIX *MLO_POOLING_STRIDE0 + MLO_POOLING_KERNEL_SZ0 - 1)
#define MLO_POOLING_LCL_DATA_HEIGHT (MLO_POOLING_GROUP_SZ1 *MLO_POOLING_N_VERT_OUT_PIX *MLO_POOLING_STRIDE1 + MLO_POOLING_KERNEL_SZ1 - 1)

__attribute__((reqd_work_group_size(MLO_POOLING_GROUP_SZ0,MLO_POOLING_GROUP_SZ1,MLO_POOLING_GROUP_SZ2)))
__kernel void mloPooling(
       const __global _FLOAT * bot,
       __global _FLOAT * top,
#if !defined(MLO_POOLING_DO_BACKWARD) || MLO_POOLING_OP_ID != MLO_POOLING_OP_MAX
       UNUSED
#endif
	   __global _INT_MASK_GLOBAL * mask
	   )
{
		__local _FLOAT bot_data[MLO_POOLING_LCL_DATA_WIDTH * MLO_POOLING_LCL_DATA_HEIGHT];

		int x = get_group_id(0) * MLO_POOLING_GROUP_SZ0 * MLO_POOLING_N_HORIZ_OUT_PIX;
		int y = get_group_id(1) * MLO_POOLING_GROUP_SZ1 * MLO_POOLING_N_VERT_OUT_PIX;
		int lcl_id0 = get_local_id(0);
		int lcl_id1 = get_local_id(1);
//		int lcl_id = (lcl_id1 << MLO_POOLING_GROUP_LG2SZ0) + lcl_id0;
		int ob = get_global_id(2); // output * batch_sz
		int b = ob / MLO_POOLING_N_OUTPUTS;
		int o = ob - b * MLO_POOLING_N_OUTPUTS;
		int bot_x = x*MLO_POOLING_STRIDE0;
		int bot_y = y*MLO_POOLING_STRIDE1;
		int bot_off = b * MLO_POOLING_BOT_BATCH_STRIDE + o * MLO_POOLING_BOT_CHANNEL_STRIDE;

		_FLOAT res[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
		_INT_MASK_LOCAL mask_private[MLO_POOLING_N_VERT_OUT_PIX][MLO_POOLING_N_HORIZ_OUT_PIX];
#endif
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				res[k][l] =  -FLT_MAX;
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] = 0;
#endif
			}
		}

// load tile
		for( int b_j = lcl_id1; b_j < MLO_POOLING_LCL_DATA_HEIGHT; b_j += MLO_POOLING_GROUP_SZ1)
		{
			int bot_y_act = bot_y + b_j  - MLO_POOLING_PAD1;

			bool invisibleY = (bot_y_act < 0) || (bot_y_act >= MLO_POOLING_BOT_HEIGHT);

			int bot_y_off = bot_y_act * MLO_POOLING_BOT_STRIDE;

			int lcl_off_v = b_j * MLO_POOLING_LCL_DATA_WIDTH;

			for(int b_i = lcl_id0; b_i < MLO_POOLING_LCL_DATA_WIDTH; b_i += MLO_POOLING_GROUP_SZ0)
			{

				int bot_x_act = bot_x + b_i - MLO_POOLING_PAD0;

				bool invisibleX = (bot_x_act < 0) || (bot_x_act >= MLO_POOLING_BOT_WIDTH);
				bool invisible = invisibleX || invisibleY;

				int bot_idx = invisible ? 0 : (bot_off + bot_y_off + bot_x_act);
				_FLOAT bot_val = bot[bot_idx];

				bot_val = (invisible) ?

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				-FLT_MAX
#elif MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				0
#endif
				 : bot_val;
				bot_data[lcl_off_v + b_i] = bot_val;

			}
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		int lcl_y = lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX * MLO_POOLING_STRIDE1;
		int lcl_x = lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX * MLO_POOLING_STRIDE0;
		int lcl_off = lcl_y * MLO_POOLING_LCL_DATA_WIDTH + lcl_x;

		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			int y_dst = y+ lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX + k;
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
			int hstart = y_dst * MLO_POOLING_STRIDE1 - MLO_POOLING_PAD1;
			int hend = min(hstart + MLO_POOLING_KERNEL_SZ1, MLO_POOLING_BOT_HEIGHT + MLO_POOLING_PAD1);
#endif
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{

				int	x_dst = x+ lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX + l;
#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				int wstart = x_dst * MLO_POOLING_STRIDE0 - MLO_POOLING_PAD0;
				int wend = min(wstart + MLO_POOLING_KERNEL_SZ0, MLO_POOLING_BOT_WIDTH + MLO_POOLING_PAD0);
				int pool_size = (hend - hstart) * (wend - wstart);
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
				mask_private[k][l] = 0xFF;
#endif

				for( int j = 0; j < MLO_POOLING_KERNEL_SZ1; j++)
				{
					for(int i = 0; i < MLO_POOLING_KERNEL_SZ0; i++)
					{

						_FLOAT bot_val =  bot_data[lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i)];
#if 0
						if (y_dst == 0 && x_dst == 6)
						{

							printf("k: %d %f %f\n",
								lcl_off + (k * MLO_POOLING_STRIDE1+j)*MLO_POOLING_LCL_DATA_WIDTH + (l * MLO_POOLING_STRIDE0+i),
								res[k][l],
								bot_val
								);
						}
#endif
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
						if (bot_val > res[k][l])
						{
							res[k][l] = bot_val;
							mask_private[k][l] = i + MLO_POOLING_KERNEL_SZ0 * j;
						}
#else
						res[k][l] = MLO_POOLING_OP(res[k][l],bot_val);
#endif
					}
				}

#if MLO_POOLING_OP_ID == MLO_POOLING_OP_AVE
				res[k][l] *= (_FLOAT)1.f/ (_FLOAT)pool_size;
#endif
			}
		}

		int top_y = (y + lcl_id1 * MLO_POOLING_N_VERT_OUT_PIX);
		int top_x = (x + lcl_id0 * MLO_POOLING_N_HORIZ_OUT_PIX);
		int top_off = b * MLO_POOLING_TOP_BATCH_STRIDE + o * MLO_POOLING_TOP_CHANNEL_STRIDE + top_y * MLO_POOLING_TOP_STRIDE + top_x;
		for( int k = 0; k < MLO_POOLING_N_VERT_OUT_PIX; k++)
		{
			for(int l = 0; l < MLO_POOLING_N_HORIZ_OUT_PIX; l++)
			{
				if (top_y + k < MLO_POOLING_TOP_HEIGHT && top_x + l < MLO_POOLING_TOP_WIDTH)
				{
					top[top_off + k * MLO_POOLING_TOP_STRIDE +l] = res[k][l];
#if defined(MLO_POOLING_DO_BACKWARD) && MLO_POOLING_OP_ID == MLO_POOLING_OP_MAX
					mask[top_off + k * MLO_POOLING_TOP_STRIDE + l] = mask_private[k][l];
#endif
				}
			}
		}

}


sixtracklib/opencl: remove unnecessary printf statements from default tracking kernel

#ifndef SIXTRACKLIB_OPENCL_INTERNAL_TRACK_PARTICLES_KERNEL_CL__
#define SIXTRACKLIB_OPENCL_INTERNAL_TRACK_PARTICLES_KERNEL_CL__

#if !defined( SIXTRL_NO_INCLUDES )
    #include "sixtracklib/opencl/internal/default_compile_options.h"

    #include "sixtracklib/common/definitions.h"
    #include "sixtracklib/common/buffer/managed_buffer_minimal.h"
    #include "sixtracklib/common/buffer/managed_buffer_remap.h"
    #include "sixtracklib/common/internal/particles_defines.h"
    #include "sixtracklib/common/particles.h"
    #include "sixtracklib/common/track.h"
#endif /* !defined( SIXTRL_NO_INCLUDES ) */

#pragma OPENCL_EXTENSION cl_khr_int32_extended_atomics

__kernel void NS(Remap_particles_beam_elements_buffers_opencl)(
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char* SIXTRL_RESTRICT particles_buf,
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char* SIXTRL_RESTRICT beam_elements_buf,
    SIXTRL_BUFFER_DATAPTR_DEC int* SIXTRL_RESTRICT ptr_success_flag )
{
    typedef NS(buffer_size_t) buf_size_t;

    size_t const global_id                     = get_global_id( 0 );
    size_t const global_size                   = get_global_size( 0 );

    size_t const gid_to_remap_particles_buffer     = ( size_t )0u;
    size_t const gid_to_remap_beam_elements_buffer = ( global_size > 1u )
        ? ( gid_to_remap_particles_buffer + 1u )
        : ( gid_to_remap_particles_buffer );

    if( global_id <= gid_to_remap_beam_elements_buffer )
    {
        buf_size_t const slot_size = ( buf_size_t )8u;
        int success_flag = ( int )0u;

        if( global_id == gid_to_remap_particles_buffer )
        {
            if( ( particles_buf != SIXTRL_NULLPTR ) &&
                ( beam_elements_buf != particles_buf ) )
            {
                if( 0 != NS(ManagedBuffer_remap)( particles_buf, slot_size ) )
                {
                    success_flag |= -2;
                }
            }
            else
            {
                success_flag |= -1;
            }
        }

        if( global_id == gid_to_remap_beam_elements_buffer )
        {
            if( ( beam_elements_buf != SIXTRL_NULLPTR ) &&
                ( particles_buf != beam_elements_buf ) )
            {
                if( 0 != NS(ManagedBuffer_remap)(
                        beam_elements_buf, slot_size ) )
                {
                    success_flag |= -4;
                }
            }
            else
            {
                success_flag |= -1;
            }
        }

        if( ( success_flag != 0 ) && ( ptr_success_flag != SIXTRL_NULLPTR ) )
        {

            atomic_or( ptr_success_flag, success_flag );
        }
    }

    return;
}

__kernel void NS(Track_particles_beam_elements_opencl)(
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char* SIXTRL_RESTRICT particles_buf,
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char* SIXTRL_RESTRICT beam_elements_buf,
    SIXTRL_UINT64_T const num_turns,
    SIXTRL_BUFFER_DATAPTR_DEC int* SIXTRL_RESTRICT ptr_success_flag )
{
    typedef NS(buffer_size_t) buf_size_t;
    typedef SIXTRL_BUFFER_OBJ_ARGPTR_DEC  NS(Object)*        obj_iter_t;
    typedef SIXTRL_BUFFER_OBJ_ARGPTR_DEC  NS(Object) const*  obj_const_iter_t;
    typedef SIXTRL_BUFFER_DATAPTR_DEC     NS(Particles)*     ptr_particles_t;

    int success_flag = ( int )0u;
    buf_size_t const slot_size = ( buf_size_t )8u;

    if( ( !NS(ManagedBuffer_needs_remapping( particles_buf,     slot_size ) ) ) &&
        ( !NS(ManagedBuffer_needs_remapping( beam_elements_buf, slot_size ) ) ) )
    {
        size_t const stride             = get_local_size( 0 );
        size_t global_particle_id       = get_global_id( 0 );
        size_t object_begin_particle_id = ( size_t )0u;

        obj_iter_t part_block_it  = NS(ManagedBuffer_get_objects_index_begin)(
                particles_buf, slot_size );

        obj_iter_t part_block_end =
            NS(ManagedBuffer_get_objects_index_end)( particles_buf, slot_size );

        obj_const_iter_t be_begin =
            NS(ManagedBuffer_get_const_objects_index_begin)(
                beam_elements_buf, slot_size );

        obj_const_iter_t be_end   =
            NS(ManagedBuffer_get_const_objects_index_end)(
                beam_elements_buf, slot_size );

        for( ; part_block_it != part_block_end ; ++part_block_it )
        {
            ptr_particles_t particles = ( ptr_particles_t )(
                uintptr_t )NS(Object_get_begin_addr)( part_block_it );

            size_t const object_end_particle_id = object_begin_particle_id +
                NS(Particles_get_num_of_particles)( particles );

            SIXTRL_ASSERT( NS(Object_get_type_id)( part_block_it ) ==
                           NS(OBJECT_TYPE_PARTICLE) );

            if( ( global_particle_id <  object_end_particle_id   ) &&
                ( global_particle_id >= object_begin_particle_id ) )
            {
                size_t const particle_id =
                    global_particle_id - object_begin_particle_id;

                SIXTRL_UINT64_T turn = ( SIXTRL_UINT64_T )0u;

                SIXTRL_ASSERT( particle_id <
                    NS(Particles_get_num_of_particles)( particles ) );

                for( ; turn < num_turns ; ++turn )
                {
                    success_flag |= NS(Track_particle_beam_element_objs)(
                        particles, particle_id, be_begin, be_end );
                }
            }

            object_begin_particle_id = object_end_particle_id;
        }
    }
    else
    {
        if( NS(ManagedBuffer_needs_remapping( particles_buf, slot_size ) ) )
        {
            success_flag |= -2;
        }

        if( NS(ManagedBuffer_needs_remapping( beam_elements_buf, slot_size ) ) )
        {
            success_flag |= -4;
        }
    }

    if( ( success_flag != 0 ) && ( ptr_success_flag != SIXTRL_NULLPTR ) )
    {

        atomic_or( ptr_success_flag, success_flag );
    }

    return;
}

#endif /* SIXTRACKLIB_OPENCL_INTERNAL_TRACK_PARTICLES_KERNEL_CL__ */

/* end: sixtracklib/opencl/private/track_particles_kernel.cl */

#ifndef SIXTRACKLIB_OPENCL_INTERNAL_TRACK_PARTICLES_KERNEL_CL__
#define SIXTRACKLIB_OPENCL_INTERNAL_TRACK_PARTICLES_KERNEL_CL__

#if !defined( SIXTRL_NO_INCLUDES )
    #include "sixtracklib/opencl/internal/default_compile_options.h"

    #include "sixtracklib/common/definitions.h"
    #include "sixtracklib/common/buffer/managed_buffer_minimal.h"
    #include "sixtracklib/common/buffer/managed_buffer_remap.h"
    #include "sixtracklib/common/internal/particles_defines.h"
    #include "sixtracklib/common/particles.h"
    #include "sixtracklib/common/track.h"
#endif /* !defined( SIXTRL_NO_INCLUDES ) */

#pragma OPENCL_EXTENSION cl_khr_int32_extended_atomics

__kernel void NS(Remap_particles_beam_elements_buffers_opencl)(
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char* SIXTRL_RESTRICT particles_buf,
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char* SIXTRL_RESTRICT beam_elements_buf,
    SIXTRL_BUFFER_DATAPTR_DEC int* SIXTRL_RESTRICT ptr_success_flag )
{
    typedef NS(buffer_size_t) buf_size_t;

    size_t const global_id                     = get_global_id( 0 );
    size_t const global_size                   = get_global_size( 0 );

    size_t const gid_to_remap_particles_buffer     = ( size_t )0u;
    size_t const gid_to_remap_beam_elements_buffer = ( global_size > 1u )
        ? ( gid_to_remap_particles_buffer + 1u )
        : ( gid_to_remap_particles_buffer );

    if( global_id <= gid_to_remap_beam_elements_buffer )
    {
        buf_size_t const slot_size = ( buf_size_t )8u;
        int success_flag = ( int )0u;

        if( global_id == gid_to_remap_particles_buffer )
        {
            if( ( particles_buf != SIXTRL_NULLPTR ) &&
                ( beam_elements_buf != particles_buf ) )
            {
                if( 0 != NS(ManagedBuffer_remap)( particles_buf, slot_size ) )
                {
                    success_flag |= -2;
                }
            }
            else
            {
                success_flag |= -1;
            }
        }

        if( global_id == gid_to_remap_beam_elements_buffer )
        {
            if( ( beam_elements_buf != SIXTRL_NULLPTR ) &&
                ( particles_buf != beam_elements_buf ) )
            {
                if( 0 != NS(ManagedBuffer_remap)(
                        beam_elements_buf, slot_size ) )
                {
                    success_flag |= -4;
                }
            }
            else
            {
                success_flag |= -1;
            }
        }

        if( ( success_flag != 0 ) && ( ptr_success_flag != SIXTRL_NULLPTR ) )
        {

            atomic_or( ptr_success_flag, success_flag );
        }
    }

    return;
}

__kernel void NS(Track_particles_beam_elements_opencl)(
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char* SIXTRL_RESTRICT particles_buf,
    SIXTRL_BUFFER_DATAPTR_DEC unsigned char* SIXTRL_RESTRICT beam_elements_buf,
    SIXTRL_UINT64_T const num_turns,
    SIXTRL_BUFFER_DATAPTR_DEC int* SIXTRL_RESTRICT ptr_success_flag )
{
    typedef NS(buffer_size_t) buf_size_t;
    typedef SIXTRL_BUFFER_OBJ_ARGPTR_DEC  NS(Object)*        obj_iter_t;
    typedef SIXTRL_BUFFER_OBJ_ARGPTR_DEC  NS(Object) const*  obj_const_iter_t;
    typedef SIXTRL_BUFFER_DATAPTR_DEC     NS(Particles)*     ptr_particles_t;

    int success_flag = ( int )0u;
    buf_size_t const slot_size = ( buf_size_t )8u;

    if( ( !NS(ManagedBuffer_needs_remapping( particles_buf,     slot_size ) ) ) &&
        ( !NS(ManagedBuffer_needs_remapping( beam_elements_buf, slot_size ) ) ) )
    {
        size_t const stride             = get_local_size( 0 );
        size_t global_particle_id       = get_global_id( 0 );
        size_t object_begin_particle_id = ( size_t )0u;

        if( global_particle_id == 0 )
        {
            printf( "HERE\r\n" );
        }

        obj_iter_t part_block_it  = NS(ManagedBuffer_get_objects_index_begin)(
                particles_buf, slot_size );

        obj_iter_t part_block_end =
            NS(ManagedBuffer_get_objects_index_end)( particles_buf, slot_size );

        obj_const_iter_t be_begin =
            NS(ManagedBuffer_get_const_objects_index_begin)(
                beam_elements_buf, slot_size );

        obj_const_iter_t be_end   =
            NS(ManagedBuffer_get_const_objects_index_end)(
                beam_elements_buf, slot_size );

        for( ; part_block_it != part_block_end ; ++part_block_it )
        {
            ptr_particles_t particles = ( ptr_particles_t )(
                uintptr_t )NS(Object_get_begin_addr)( part_block_it );

            size_t const object_end_particle_id = object_begin_particle_id +
                NS(Particles_get_num_of_particles)( particles );

            SIXTRL_ASSERT( NS(Object_get_type_id)( part_block_it ) ==
                           NS(OBJECT_TYPE_PARTICLE) );

            if( ( global_particle_id <  object_end_particle_id   ) &&
                ( global_particle_id >= object_begin_particle_id ) )
            {
                size_t const particle_id =
                    global_particle_id - object_begin_particle_id;

                SIXTRL_UINT64_T turn = ( SIXTRL_UINT64_T )0u;

                SIXTRL_ASSERT( particle_id <
                    NS(Particles_get_num_of_particles)( particles ) );

                for( ; turn < num_turns ; ++turn )
                {
                    if( global_particle_id == 0 )
                    {
                        printf( "before charge_ratio %15.7f\r\n",
                                NS(Particles_get_charge_ratio_value)( particles, 0 ) );
                    }

                    success_flag |= NS(Track_particle_beam_element_objs)(
                        particles, particle_id, be_begin, be_end );

                    if( global_particle_id == 0 )
                    {
                        printf( "after  charge_ratio %15.7f\r\n",
                                NS(Particles_get_charge_ratio_value)( particles, 0 ) );
                    }
                }
            }

            object_begin_particle_id = object_end_particle_id;
        }
    }
    else
    {
        if( NS(ManagedBuffer_needs_remapping( particles_buf, slot_size ) ) )
        {
            success_flag |= -2;
        }

        if( NS(ManagedBuffer_needs_remapping( beam_elements_buf, slot_size ) ) )
        {
            success_flag |= -4;
        }
    }

    if( ( success_flag != 0 ) && ( ptr_success_flag != SIXTRL_NULLPTR ) )
    {

        atomic_or( ptr_success_flag, success_flag );
    }

    return;
}

#endif /* SIXTRACKLIB_OPENCL_INTERNAL_TRACK_PARTICLES_KERNEL_CL__ */

/* end: sixtracklib/opencl/private/track_particles_kernel.cl */


Fix area lighting

Fix area lighting

/**********************************************************************
Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
********************************************************************/

#include <../App/CL/common.cl>
#include <../App/CL/ray.cl>
#include <../App/CL/isect.cl>
#include <../App/CL/utils.cl>
#include <../App/CL/payload.cl>
#include <../App/CL/texture.cl>
#include <../App/CL/sampling.cl>
#include <../App/CL/normalmap.cl>
#include <../App/CL/bxdf.cl>
#include <../App/CL/light.cl>
#include <../App/CL/camera.cl>
#include <../App/CL/scene.cl>
#include <../App/CL/material.cl>
#include <../App/CL/volumetrics.cl>
#include <../App/CL/path.cl>

// This kernel only handles scattered paths.
// It applies direct illumination and generates
// path continuation if multiscattering is enabled.
__kernel void ShadeVolume(
    // Ray batch
    __global ray const* rays,
    // Intersection data
    __global Intersection const* isects,
    // Hit indices
    __global int const* hitindices,
    // Pixel indices
    __global int const* pixelindices,
    // Number of rays
    __global int const* numhits,
    // Vertices
    __global float3 const* vertices,
    // Normals
    __global float3 const* normals,
    // UVs
    __global float2 const* uvs,
    // Indices
    __global int const* indices,
    // Shapes
    __global Shape const* shapes,
    // Material IDs
    __global int const* materialids,
    // Materials
    __global Material const* materials,
    // Textures
    TEXTURE_ARG_LIST,
    // Environment texture index
    int envmapidx,
    // Envmap multiplier
    float envmapmul,
    // Emissives
    __global Light const* lights,
    // Number of emissive objects
    int num_lights,
    // RNG seed
    uint rngseed,
    // Sampler state
    __global uint* random,
    // Sobol matrices
    __global uint const* sobolmat,
    // Current bounce
    int bounce,
    // Current frame
    int frame,
    // Volume data
    __global Volume const* volumes,
    // Shadow rays
    __global ray* shadowrays,
    // Light samples
    __global float3* lightsamples,
    // Path throughput
    __global Path* paths,
    // Indirect rays (next path segment)
    __global ray* indirectrays,
    // Radiance
    __global float3* output
)
{
    int globalid = get_global_id(0);

    Scene scene =
    {
        vertices,
        normals,
        uvs,
        indices,
        shapes,
        materialids,
        materials,
        lights,
        envmapidx,
        envmapmul,
        num_lights
    };

    if (globalid < *numhits)
    {
        // Fetch index
        int hitidx = hitindices[globalid];
        int pixelidx = pixelindices[globalid];
        Intersection isect = isects[hitidx];

        __global Path* path = paths + pixelidx;

        // Only apply to scattered paths
        if (!Path_IsScattered(path))
        {
            return;
        }

        // Fetch incoming ray
        float3 o = rays[hitidx].o.xyz;
        float3 wi = rays[hitidx].d.xyz;

        Sampler sampler;
#if SAMPLER == SOBOL
        uint scramble = random[pixelidx] * 0x1fe3434f;
        Sampler_Init(&sampler, frame, SAMPLE_DIM_SURFACE_OFFSET + bounce * SAMPLE_DIMS_PER_BOUNCE + SAMPLE_DIM_VOLUME_EVALUATE_OFFSET, scramble);
#elif SAMPLER == RANDOM
        uint scramble = pixelidx * rngseed;
        Sampler_Init(&sampler, scramble);
#elif SAMPLER == CMJ
        uint rnd = random[pixelidx];
        uint scramble = rnd * 0x1fe3434f * ((frame + 13 * rnd) / (CMJ_DIM * CMJ_DIM));
        Sampler_Init(&sampler, frame % (CMJ_DIM * CMJ_DIM), SAMPLE_DIM_SURFACE_OFFSET + bounce * SAMPLE_DIMS_PER_BOUNCE + SAMPLE_DIM_VOLUME_EVALUATE_OFFSET, scramble);
#endif

        // Here we know that volidx != -1 since this is a precondition
        // for scattering event
        int volidx = Path_GetVolumeIdx(path);

        // Sample light source
        float pdf = 0.f;
        float selection_pdf = 0.f;
        float3 wo;

        int light_idx = Scene_SampleLight(&scene, Sampler_Sample1D(&sampler, SAMPLER_ARGS), &selection_pdf);

        // Here we need fake differential geometry for light sampling procedure
        DifferentialGeometry dg;
        // put scattering position in there (it is along the current ray at isect.distance
        // since EvaluateVolume has put it there
        dg.p = o + wi * Intersection_GetDistance(isects + hitidx);
        // Get light sample intencity
        float3 le = Light_Sample(light_idx, &scene, &dg, TEXTURE_ARGS, Sampler_Sample2D(&sampler, SAMPLER_ARGS), &wo, &pdf);

        // Generate shadow ray
        float shadow_ray_length = 0.999f * length(wo);
        Ray_Init(shadowrays + globalid, dg.p, normalize(wo), shadow_ray_length, 0.f, 0xFFFFFFFF);

        // Evaluate volume transmittion along the shadow ray (it is incorrect if the light source is outside of the
        // current volume, but in this case it will be discarded anyway since the intersection at the outer bound
        // of a current volume), so the result is fully correct.
        float3 tr = Volume_Transmittance(&volumes[volidx], &shadowrays[globalid], shadow_ray_length);

        // Volume emission is applied only if the light source is in the current volume(this is incorrect since the light source might be
        // outside of a volume and we have to compute fraction of ray in this case, but need to figure out how)
        float3 r = Volume_Emission(&volumes[volidx], &shadowrays[globalid], shadow_ray_length);

        // This is the estimate coming from a light source
        // TODO: remove hardcoded phase func and sigma
        r += tr * le * volumes[volidx].sigma_s * PhaseFunction_Uniform(wi, normalize(wo)) / pdf / selection_pdf;

        // Only if we have some radiance compute the visibility ray
        if (NON_BLACK(tr) && NON_BLACK(r) && pdf > 0.f)
        {
            // Put lightsample result
            lightsamples[globalid] = REASONABLE_RADIANCE(r * Path_GetThroughput(path));
        }
        else
        {
            // Nothing to compute
            lightsamples[globalid] = 0.f;
            // Otherwise make it incative to save intersector cycles (hopefully)
            Ray_SetInactive(shadowrays + globalid);
        }

#ifdef MULTISCATTER
        // This is highly brute-force
        // TODO: investigate importance sampling techniques here
        wo = Sample_MapToSphere(Sampler_Sample2D(&sampler, SAMPLER_ARGS));
        pdf = 1.f / (4.f * PI);

        // Generate new path segment
        Ray_Init(indirectrays + globalid, dg.p, normalize(wo), CRAZY_HIGH_DISTANCE, 0.f, 0xFFFFFFFF);

        // Update path throughput multiplying by phase function.
        Path_MulThroughput(path, volumes[volidx].sigma_s * PhaseFunction_Uniform(wi, normalize(wo)) / pdf);
#else
        // Single-scattering mode only,
        // kill the path and compact away on next iteration
        Path_Kill(path);
        Ray_SetInactive(indirectrays + globalid);
#endif
    }
}

// Handle ray-surface interaction possibly generating path continuation.
// This is only applied to non-scattered paths.
__kernel void ShadeSurface(
    // Ray batch
    __global ray const* rays,
    // Intersection data
    __global Intersection const* isects,
    // Hit indices
    __global int const* hitindices,
    // Pixel indices
    __global int const* pixelindices,
    // Number of rays
    __global int const* numhits,
    // Vertices
    __global float3 const* vertices,
    // Normals
    __global float3 const* normals,
    // UVs
    __global float2 const* uvs,
    // Indices
    __global int const* indices,
    // Shapes
    __global Shape const* shapes,
    // Material IDs
    __global int const* materialids,
    // Materials
    __global Material const* materials,
    // Textures
    TEXTURE_ARG_LIST,
    // Environment texture index
    int envmapidx,
    // Envmap multiplier
    float envmapmul,
    // Emissives
    __global Light const* lights,
    // Number of emissive objects
    int num_lights,
    // RNG seed
    uint rngseed,
    // Sampler states
    __global uint* random,
    // Sobol matrices
    __global uint const* sobolmat,
    // Current bounce
    int bounce,
    // Frame
    int frame,
    // Volume data
    __global Volume const* volumes,
    // Shadow rays
    __global ray* shadowrays,
    // Light samples
    __global float3* lightsamples,
    // Path throughput
    __global Path* paths,
    // Indirect rays
    __global ray* indirectrays,
    // Radiance
    __global float3* output
)
{
    int globalid = get_global_id(0);

    Scene scene =
    {
        vertices,
        normals,
        uvs,
        indices,
        shapes,
        materialids,
        materials,
        lights,
        envmapidx,
        envmapmul,
        num_lights
    };

    // Only applied to active rays after compaction
    if (globalid < *numhits)
    {
        // Fetch index
        int hitidx = hitindices[globalid];
        int pixelidx = pixelindices[globalid];
        Intersection isect = isects[hitidx];

        __global Path* path = paths + pixelidx;

        // Early exit for scattered paths
        if (Path_IsScattered(path))
        {
            return;
        }

        // Fetch incoming ray direction
        float3 wi = -normalize(rays[hitidx].d.xyz);

        Sampler sampler;
#if SAMPLER == SOBOL
        uint scramble = random[pixelidx] * 0x1fe3434f;
        Sampler_Init(&sampler, frame, SAMPLE_DIM_SURFACE_OFFSET + bounce * SAMPLE_DIMS_PER_BOUNCE, scramble);
#elif SAMPLER == RANDOM
        uint scramble = pixelidx * rngseed;
        Sampler_Init(&sampler, scramble);
#elif SAMPLER == CMJ
        uint rnd = random[pixelidx];
        uint scramble = rnd * 0x1fe3434f * ((frame + 331 * rnd) / (CMJ_DIM * CMJ_DIM));
        Sampler_Init(&sampler, frame % (CMJ_DIM * CMJ_DIM), SAMPLE_DIM_SURFACE_OFFSET + bounce * SAMPLE_DIMS_PER_BOUNCE, scramble);
#endif

        // Fill surface data
        DifferentialGeometry diffgeo;
        DifferentialGeometry_Fill(&scene, &isect, &diffgeo);

        // Check if we are hitting from the inside

        float backfacing = dot(diffgeo.ng, wi) < 0.f;
        int twosided = diffgeo.mat.twosided;
        if (twosided && backfacing)
        {
            // Reverse normal and tangents in this case
            // but not for BTDFs, since BTDFs rely
            // on normal direction in order to arrange
            // indices of refraction
            diffgeo.n = -diffgeo.n;
            diffgeo.dpdu = -diffgeo.dpdu;
            diffgeo.dpdv = -diffgeo.dpdv;
        }

        float ndotwi = dot(diffgeo.n, wi);

        // Select BxDF
        Material_Select(&scene, wi, &sampler, TEXTURE_ARGS, SAMPLER_ARGS, &diffgeo);

        // Terminate if emissive
        if (Bxdf_IsEmissive(&diffgeo))
        {
            if (!backfacing)
            {
                float weight = 1.f;

                if (bounce > 0 && !Path_IsSpecular(path))
                {
                    float2 extra = Ray_GetExtra(&rays[hitidx]);
                    float ld = isect.uvwt.w;
                    float denom = extra.y * diffgeo.area;
                    // TODO: num_lights should be num_emissies instead, presence of analytical lights breaks this code
                    float bxdflightpdf = denom > 0.f ? (ld * ld / denom / num_lights) : 0.f;
                    weight = BalanceHeuristic(1, extra.x, 1, bxdflightpdf);
                }

                {
                    // In this case we hit after an application of MIS process at previous step.
                    // That means BRDF weight has been already applied.
                    output[pixelidx] += Path_GetThroughput(path) * Emissive_GetLe(&diffgeo, TEXTURE_ARGS) * weight;
                }
            }

            Path_Kill(path);
            Ray_SetInactive(shadowrays + globalid);
            Ray_SetInactive(indirectrays + globalid);

            lightsamples[globalid] = 0.f;
            return;
        }

        float s = Bxdf_IsBtdf(&diffgeo) ? (-sign(ndotwi)) : 1.f;
        if (!twosided && backfacing && !Bxdf_IsBtdf(&diffgeo))
        {
             //Reverse normal and tangents in this case
             //but not for BTDFs, since BTDFs rely
             //on normal direction in order to arrange
             //indices of refraction
            diffgeo.n = -diffgeo.n;
            diffgeo.dpdu = -diffgeo.dpdu;
            diffgeo.dpdv = -diffgeo.dpdv;
        }

        // TODO: this is test code, need to
        // maintain proper volume stack here
        //if (Bxdf_IsBtdf(&diffgeo))
        //{
        //    // If we entering set the volume
        //    path->volume = !backfacing ? 0 : -1;
        //}

        // Check if we need to apply normal map
        //ApplyNormalMap(&diffgeo, TEXTURE_ARGS);
        DifferentialGeometry_ApplyBumpMap(&diffgeo, TEXTURE_ARGS);
        DifferentialGeometry_CalculateTangentTransforms(&diffgeo);

        float lightpdf = 0.f;
        float bxdflightpdf = 0.f;
        float bxdfpdf = 0.f;
        float lightbxdfpdf = 0.f;
        float selection_pdf = 0.f;
        float3 radiance = 0.f;
        float3 lightwo;
        float3 bxdfwo;
        float3 wo;
        float bxdfweight = 1.f;
        float lightweight = 1.f;

        int light_idx = Scene_SampleLight(&scene, Sampler_Sample1D(&sampler, SAMPLER_ARGS), &selection_pdf);

        float3 throughput = Path_GetThroughput(path);

        // Sample bxdf
        float3 bxdf = Bxdf_Sample(&diffgeo, wi, TEXTURE_ARGS, Sampler_Sample2D(&sampler, SAMPLER_ARGS), &bxdfwo, &bxdfpdf);

        // If we have light to sample we can hopefully do mis
        if (light_idx > -1)
        {
            // Sample light
            float3 le = Light_Sample(light_idx, &scene, &diffgeo, TEXTURE_ARGS, Sampler_Sample2D(&sampler, SAMPLER_ARGS), &lightwo, &lightpdf);
            lightbxdfpdf = Bxdf_GetPdf(&diffgeo, wi, normalize(lightwo), TEXTURE_ARGS);
            lightweight = Light_IsSingular(&scene.lights[light_idx]) ? 1.f : BalanceHeuristic(1, lightpdf, 1, lightbxdfpdf);

            // Apply MIS to account for both
            if (NON_BLACK(le) && lightpdf > 0.0f && !Bxdf_IsSingular(&diffgeo))
            {
                wo = lightwo;
                float ndotwo = fabs(dot(diffgeo.n, normalize(wo)));
                radiance = le * Bxdf_Evaluate(&diffgeo, wi, normalize(wo), TEXTURE_ARGS) * throughput * ndotwo * lightweight / lightpdf / selection_pdf;
            }
        }

        // If we have some light here generate a shadow ray
        if (NON_BLACK(radiance))
        {
            // Generate shadow ray
            float shadow_ray_length = 0.999f * (1.f - CRAZY_LOW_DISTANCE) * length(wo);
            float3 shadow_ray_dir = normalize(wo);
            float3 shadow_ray_o = diffgeo.p + CRAZY_LOW_DISTANCE * s * diffgeo.n;
            int shadow_ray_mask = Bxdf_IsSingular(&diffgeo) ? 0xFFFFFFFF : 0x0000FFFF;

            Ray_Init(shadowrays + globalid, shadow_ray_o, shadow_ray_dir, shadow_ray_length, 0.f, shadow_ray_mask);

            // Apply the volume to shadow ray if needed
            int volidx = Path_GetVolumeIdx(path);
            if (volidx != -1)
            {
                radiance *= Volume_Transmittance(&volumes[volidx], &shadowrays[globalid], shadow_ray_length);
                radiance += Volume_Emission(&volumes[volidx], &shadowrays[globalid], shadow_ray_length) * throughput;
            }

            // And write the light sample
            lightsamples[globalid] = REASONABLE_RADIANCE(radiance);
        }
        else
        {
            // Otherwise save some intersector cycles
            Ray_SetInactive(shadowrays + globalid);
            lightsamples[globalid] = 0;
        }

        // Apply Russian roulette
        float q = max(min(0.5f,
            // Luminance
            0.2126f * throughput.x + 0.7152f * throughput.y + 0.0722f * throughput.z), 0.01f);
        // Only if it is 3+ bounce
        bool rr_apply = bounce > 3;
        bool rr_stop = Sampler_Sample1D(&sampler, SAMPLER_ARGS) > q && rr_apply;

        if (rr_apply)
        {
            Path_MulThroughput(path, 1.f / q);
        }

        if (Bxdf_IsSingular(&diffgeo))
        {
            Path_SetSpecularFlag(path);
        }

        bxdfwo = normalize(bxdfwo);
        float3 t = bxdf * fabs(dot(diffgeo.n, bxdfwo));

        // Only continue if we have non-zero throughput & pdf
        if (NON_BLACK(t) && bxdfpdf > 0.f && !rr_stop)
        {
            // Update the throughput
            Path_MulThroughput(path, t / bxdfpdf);

            // Generate ray
            float3 indirect_ray_dir = bxdfwo;
            float3 indirect_ray_o = diffgeo.p + CRAZY_LOW_DISTANCE * s * diffgeo.n;

            Ray_Init(indirectrays + globalid, indirect_ray_o, indirect_ray_dir, CRAZY_HIGH_DISTANCE, 0.f, 0xFFFFFFFF);
            Ray_SetExtra(indirectrays + globalid, make_float2(bxdfpdf, fabs(dot(diffgeo.n, bxdfwo))));
        }
        else
        {
            // Otherwise kill the path
            Path_Kill(path);
            Ray_SetInactive(indirectrays + globalid);
        }
    }
}

///< Illuminate missing rays
__kernel void ShadeMiss(
    // Ray batch
    __global ray const* rays,
    // Intersection data
    __global Intersection const* isects,
    // Pixel indices
    __global int const* pixelindices,
    // Number of rays
    int numrays,
    // Textures
    TEXTURE_ARG_LIST,
    // Environment texture index
    int envmapidx,
    //
    __global Path const* paths,
    __global Volume const* volumes,
    // Output values
    __global float4* output
)
{
    int globalid = get_global_id(0);

    if (globalid < numrays)
    {
        int pixelidx = pixelindices[globalid];

        // In case of a miss
        if (isects[globalid].shapeid < 0)
        {
            // Multiply by throughput
            int volidx = paths[pixelidx].volume;

            if (volidx == -1)
                output[pixelidx].xyz += Texture_SampleEnvMap(rays[globalid].d.xyz, TEXTURE_ARGS_IDX(envmapidx));
            else
            {
                output[pixelidx].xyz += Texture_SampleEnvMap(rays[globalid].d.xyz, TEXTURE_ARGS_IDX(envmapidx)) *
                    Volume_Transmittance(&volumes[volidx], &rays[globalid], rays[globalid].o.w);

                output[pixelidx].xyz += Volume_Emission(&volumes[volidx], &rays[globalid], rays[globalid].o.w);
            }
        }

        if (isnan(output[pixelidx].x) || isnan(output[pixelidx].y) || isnan(output[pixelidx].z))
        {
            output[pixelidx] = make_float4(100.f, 0.f, 0.f, 1.f);
        }

        output[pixelidx].w += 1.f;
    }
}

///< Handle light samples and visibility info and add contribution to final buffer
__kernel void GatherLightSamples(
    // Pixel indices
    __global int const* pixelindices,
    // Number of rays
    __global int* numrays,
    // Shadow rays hits
    __global int const* shadowhits,
    // Light samples
    __global float3 const* lightsamples,
    // throughput
    __global Path const* paths,
    // Radiance sample buffer
    __global float4* output
)
{
    int globalid = get_global_id(0);

    if (globalid < *numrays)
    {
        // Get pixel id for this sample set
        int pixelidx = pixelindices[globalid];

        // Prepare accumulator variable
        float3 radiance = make_float3(0.f, 0.f, 0.f);

        // Start collecting samples
        {
            // If shadow ray didn't hit anything and reached skydome
            if (shadowhits[globalid] == -1)
            {
                // Add its contribution to radiance accumulator
                radiance += lightsamples[globalid];
            }
        }

        // Divide by number of light samples (samples already have built-in throughput)
        output[pixelidx].xyz += radiance;
    }
}

///< Handle light samples and visibility info and add contribution to final buffer
__kernel void GatherOcclusion(
    // Pixel indices
    __global int const* pixelindices,
    // Number of rays
    __global int const* numrays,
    // Shadow rays hits
    __global int const* shadowhits,
    // Light samples
    __global float3 const* lightsamples,
    // throughput
    __global float3 const* throughput,
    // Radiance sample buffer
    __global float4* output
)
{
    int globalid = get_global_id(0);

    if (globalid < *numrays)
    {
        // Get pixel id for this sample set
        int pixelidx = pixelindices[globalid];

        float visibility = (shadowhits[globalid] == 1) ? 0.f : 1.f;

        output[pixelidx].xyz += visibility;
        output[pixelidx].w += 1.f;
    }
}

///< Restore pixel indices after compaction
__kernel void RestorePixelIndices(
    // Compacted indices
    __global int const* compacted_indices,
    // Number of compacted indices
    __global int* numitems,
    // Previous pixel indices
    __global int const* previndices,
    // New pixel indices
    __global int* newindices
)
{
    int globalid = get_global_id(0);

    // Handle only working subset
    if (globalid < *numitems)
    {
        newindices[globalid] = previndices[compacted_indices[globalid]];
    }
}

///< Restore pixel indices after compaction
__kernel void FilterPathStream(
    // Intersections
    __global Intersection const* isects,
    // Number of compacted indices
    __global int const* numitems,
    // Pixel indices
    __global int const* pixelindices,
    // Paths
    __global Path* paths,
    // Predicate
    __global int* predicate
)
{
    int globalid = get_global_id(0);

    // Handle only working subset
    if (globalid < *numitems)
    {
        int pixelidx = pixelindices[globalid];

        __global Path* path = paths + pixelidx;

        if (Path_IsAlive(path))
        {
            bool kill = (length(Path_GetThroughput(path)) < CRAZY_LOW_THROUGHPUT);

            if (!kill)
            {
                predicate[globalid] = isects[globalid].shapeid >= 0 ? 1 : 0;
            }
            else
            {
                Path_Kill(path);
                predicate[globalid] = 0;
            }
        }
        else
        {
            predicate[globalid] = 0;
        }
    }
}

///< Illuminate missing rays
__kernel void ShadeBackground(
    // Ray batch
    __global ray const* rays,
    // Intersection data
    __global Intersection const* isects,
    // Pixel indices
    __global int const* pixelindices,
    // Number of rays
    __global int const* numrays,
    // Textures
    TEXTURE_ARG_LIST,
    // Environment texture index
    int envmapidx,
    float envmapmul,
    //
    int num_lights,
    __global Path const* paths,
    __global Volume const* volumes,
    // Output values
    __global float4* output
)
{
    int globalid = get_global_id(0);

    if (globalid < *numrays)
    {
        int pixelidx = pixelindices[globalid];

        __global Path const* path = paths + pixelidx;

        // In case of a miss
        if (isects[globalid].shapeid < 0 && Path_IsAlive(path))
        {
            float3 t = Path_GetThroughput(path);
            output[pixelidx].xyz += REASONABLE_RADIANCE(envmapmul * Texture_SampleEnvMap(rays[globalid].d.xyz, TEXTURE_ARGS_IDX(envmapidx)) * t);
        }
    }
}

// Copy data to interop texture if supported
__kernel void AccumulateData(
    __global float4 const* srcdata,
    int numelems,
    __global float4* dstdata
)
{
    int gid = get_global_id(0);

    if (gid < numelems)
    {
        float4 v = srcdata[gid];
        dstdata[gid] += v;
    }
}

// Copy data to interop texture if supported
__kernel void ApplyGammaAndCopyData(
    __global float4 const* data,
    int imgwidth,
    int imgheight,
    float gamma,
    write_only image2d_t img
)
{
    int gid = get_global_id(0);

    int gidx = gid % imgwidth;
    int gidy = gid / imgwidth;

    if (gidx < imgwidth && gidy < imgheight)
    {
        float4 v = data[gid];
        float4 val = clamp(native_powr(v / v.w, 1.f / gamma), 0.f, 1.f);
        write_imagef(img, make_int2(gidx, gidy), val);
    }
}

/**********************************************************************
Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
********************************************************************/

#include <../App/CL/common.cl>
#include <../App/CL/ray.cl>
#include <../App/CL/isect.cl>
#include <../App/CL/utils.cl>
#include <../App/CL/payload.cl>
#include <../App/CL/texture.cl>
#include <../App/CL/sampling.cl>
#include <../App/CL/normalmap.cl>
#include <../App/CL/bxdf.cl>
#include <../App/CL/light.cl>
#include <../App/CL/camera.cl>
#include <../App/CL/scene.cl>
#include <../App/CL/material.cl>
#include <../App/CL/volumetrics.cl>
#include <../App/CL/path.cl>

// This kernel only handles scattered paths.
// It applies direct illumination and generates
// path continuation if multiscattering is enabled.
__kernel void ShadeVolume(
    // Ray batch
    __global ray const* rays,
    // Intersection data
    __global Intersection const* isects,
    // Hit indices
    __global int const* hitindices,
    // Pixel indices
    __global int const* pixelindices,
    // Number of rays
    __global int const* numhits,
    // Vertices
    __global float3 const* vertices,
    // Normals
    __global float3 const* normals,
    // UVs
    __global float2 const* uvs,
    // Indices
    __global int const* indices,
    // Shapes
    __global Shape const* shapes,
    // Material IDs
    __global int const* materialids,
    // Materials
    __global Material const* materials,
    // Textures
    TEXTURE_ARG_LIST,
    // Environment texture index
    int envmapidx,
    // Envmap multiplier
    float envmapmul,
    // Emissives
    __global Light const* lights,
    // Number of emissive objects
    int num_lights,
    // RNG seed
    uint rngseed,
    // Sampler state
    __global uint* random,
    // Sobol matrices
    __global uint const* sobolmat,
    // Current bounce
    int bounce,
    // Current frame
    int frame,
    // Volume data
    __global Volume const* volumes,
    // Shadow rays
    __global ray* shadowrays,
    // Light samples
    __global float3* lightsamples,
    // Path throughput
    __global Path* paths,
    // Indirect rays (next path segment)
    __global ray* indirectrays,
    // Radiance
    __global float3* output
)
{
    int globalid = get_global_id(0);

    Scene scene =
    {
        vertices,
        normals,
        uvs,
        indices,
        shapes,
        materialids,
        materials,
        lights,
        envmapidx,
        envmapmul,
        num_lights
    };

    if (globalid < *numhits)
    {
        // Fetch index
        int hitidx = hitindices[globalid];
        int pixelidx = pixelindices[globalid];
        Intersection isect = isects[hitidx];

        __global Path* path = paths + pixelidx;

        // Only apply to scattered paths
        if (!Path_IsScattered(path))
        {
            return;
        }

        // Fetch incoming ray
        float3 o = rays[hitidx].o.xyz;
        float3 wi = rays[hitidx].d.xyz;

        Sampler sampler;
#if SAMPLER == SOBOL
        uint scramble = random[pixelidx] * 0x1fe3434f;
        Sampler_Init(&sampler, frame, SAMPLE_DIM_SURFACE_OFFSET + bounce * SAMPLE_DIMS_PER_BOUNCE + SAMPLE_DIM_VOLUME_EVALUATE_OFFSET, scramble);
#elif SAMPLER == RANDOM
        uint scramble = pixelidx * rngseed;
        Sampler_Init(&sampler, scramble);
#elif SAMPLER == CMJ
        uint rnd = random[pixelidx];
        uint scramble = rnd * 0x1fe3434f * ((frame + 13 * rnd) / (CMJ_DIM * CMJ_DIM));
        Sampler_Init(&sampler, frame % (CMJ_DIM * CMJ_DIM), SAMPLE_DIM_SURFACE_OFFSET + bounce * SAMPLE_DIMS_PER_BOUNCE + SAMPLE_DIM_VOLUME_EVALUATE_OFFSET, scramble);
#endif

        // Here we know that volidx != -1 since this is a precondition
        // for scattering event
        int volidx = Path_GetVolumeIdx(path);

        // Sample light source
        float pdf = 0.f;
        float selection_pdf = 0.f;
        float3 wo;

        int light_idx = Scene_SampleLight(&scene, Sampler_Sample1D(&sampler, SAMPLER_ARGS), &selection_pdf);

        // Here we need fake differential geometry for light sampling procedure
        DifferentialGeometry dg;
        // put scattering position in there (it is along the current ray at isect.distance
        // since EvaluateVolume has put it there
        dg.p = o + wi * Intersection_GetDistance(isects + hitidx);
        // Get light sample intencity
        float3 le = Light_Sample(light_idx, &scene, &dg, TEXTURE_ARGS, Sampler_Sample2D(&sampler, SAMPLER_ARGS), &wo, &pdf);

        // Generate shadow ray
        float shadow_ray_length = 0.999f * length(wo);
        Ray_Init(shadowrays + globalid, dg.p, normalize(wo), shadow_ray_length, 0.f, 0xFFFFFFFF);

        // Evaluate volume transmittion along the shadow ray (it is incorrect if the light source is outside of the
        // current volume, but in this case it will be discarded anyway since the intersection at the outer bound
        // of a current volume), so the result is fully correct.
        float3 tr = Volume_Transmittance(&volumes[volidx], &shadowrays[globalid], shadow_ray_length);

        // Volume emission is applied only if the light source is in the current volume(this is incorrect since the light source might be
        // outside of a volume and we have to compute fraction of ray in this case, but need to figure out how)
        float3 r = Volume_Emission(&volumes[volidx], &shadowrays[globalid], shadow_ray_length);

        // This is the estimate coming from a light source
        // TODO: remove hardcoded phase func and sigma
        r += tr * le * volumes[volidx].sigma_s * PhaseFunction_Uniform(wi, normalize(wo)) / pdf / selection_pdf;

        // Only if we have some radiance compute the visibility ray
        if (NON_BLACK(tr) && NON_BLACK(r) && pdf > 0.f)
        {
            // Put lightsample result
            lightsamples[globalid] = REASONABLE_RADIANCE(r * Path_GetThroughput(path));
        }
        else
        {
            // Nothing to compute
            lightsamples[globalid] = 0.f;
            // Otherwise make it incative to save intersector cycles (hopefully)
            Ray_SetInactive(shadowrays + globalid);
        }

#ifdef MULTISCATTER
        // This is highly brute-force
        // TODO: investigate importance sampling techniques here
        wo = Sample_MapToSphere(Sampler_Sample2D(&sampler, SAMPLER_ARGS));
        pdf = 1.f / (4.f * PI);

        // Generate new path segment
        Ray_Init(indirectrays + globalid, dg.p, normalize(wo), CRAZY_HIGH_DISTANCE, 0.f, 0xFFFFFFFF);

        // Update path throughput multiplying by phase function.
        Path_MulThroughput(path, volumes[volidx].sigma_s * PhaseFunction_Uniform(wi, normalize(wo)) / pdf);
#else
        // Single-scattering mode only,
        // kill the path and compact away on next iteration
        Path_Kill(path);
        Ray_SetInactive(indirectrays + globalid);
#endif
    }
}

// Handle ray-surface interaction possibly generating path continuation.
// This is only applied to non-scattered paths.
__kernel void ShadeSurface(
    // Ray batch
    __global ray const* rays,
    // Intersection data
    __global Intersection const* isects,
    // Hit indices
    __global int const* hitindices,
    // Pixel indices
    __global int const* pixelindices,
    // Number of rays
    __global int const* numhits,
    // Vertices
    __global float3 const* vertices,
    // Normals
    __global float3 const* normals,
    // UVs
    __global float2 const* uvs,
    // Indices
    __global int const* indices,
    // Shapes
    __global Shape const* shapes,
    // Material IDs
    __global int const* materialids,
    // Materials
    __global Material const* materials,
    // Textures
    TEXTURE_ARG_LIST,
    // Environment texture index
    int envmapidx,
    // Envmap multiplier
    float envmapmul,
    // Emissives
    __global Light const* lights,
    // Number of emissive objects
    int num_lights,
    // RNG seed
    uint rngseed,
    // Sampler states
    __global uint* random,
    // Sobol matrices
    __global uint const* sobolmat,
    // Current bounce
    int bounce,
    // Frame
    int frame,
    // Volume data
    __global Volume const* volumes,
    // Shadow rays
    __global ray* shadowrays,
    // Light samples
    __global float3* lightsamples,
    // Path throughput
    __global Path* paths,
    // Indirect rays
    __global ray* indirectrays,
    // Radiance
    __global float3* output
)
{
    int globalid = get_global_id(0);

    Scene scene =
    {
        vertices,
        normals,
        uvs,
        indices,
        shapes,
        materialids,
        materials,
        lights,
        envmapidx,
        envmapmul,
        num_lights
    };

    // Only applied to active rays after compaction
    if (globalid < *numhits)
    {
        // Fetch index
        int hitidx = hitindices[globalid];
        int pixelidx = pixelindices[globalid];
        Intersection isect = isects[hitidx];

        __global Path* path = paths + pixelidx;

        // Early exit for scattered paths
        if (Path_IsScattered(path))
        {
            return;
        }

        // Fetch incoming ray direction
        float3 wi = -normalize(rays[hitidx].d.xyz);

        Sampler sampler;
#if SAMPLER == SOBOL
        uint scramble = random[pixelidx] * 0x1fe3434f;
        Sampler_Init(&sampler, frame, SAMPLE_DIM_SURFACE_OFFSET + bounce * SAMPLE_DIMS_PER_BOUNCE, scramble);
#elif SAMPLER == RANDOM
        uint scramble = pixelidx * rngseed;
        Sampler_Init(&sampler, scramble);
#elif SAMPLER == CMJ
        uint rnd = random[pixelidx];
        uint scramble = rnd * 0x1fe3434f * ((frame + 331 * rnd) / (CMJ_DIM * CMJ_DIM));
        Sampler_Init(&sampler, frame % (CMJ_DIM * CMJ_DIM), SAMPLE_DIM_SURFACE_OFFSET + bounce * SAMPLE_DIMS_PER_BOUNCE, scramble);
#endif

        // Fill surface data
        DifferentialGeometry diffgeo;
        DifferentialGeometry_Fill(&scene, &isect, &diffgeo);

        // Check if we are hitting from the inside

        float backfacing = dot(diffgeo.ng, wi) < 0.f;
        int twosided = diffgeo.mat.twosided;
        if (twosided && backfacing)
        {
            // Reverse normal and tangents in this case
            // but not for BTDFs, since BTDFs rely
            // on normal direction in order to arrange
            // indices of refraction
            diffgeo.n = -diffgeo.n;
            diffgeo.dpdu = -diffgeo.dpdu;
            diffgeo.dpdv = -diffgeo.dpdv;
        }

        float ndotwi = dot(diffgeo.n, wi);

        // Select BxDF
        Material_Select(&scene, wi, &sampler, TEXTURE_ARGS, SAMPLER_ARGS, &diffgeo);

        // Terminate if emissive
        if (Bxdf_IsEmissive(&diffgeo))
        {
            if (!backfacing)
            {
                float weight = 1.f;

                if (bounce > 0 && !Path_IsSpecular(path))
                {
                    float2 extra = Ray_GetExtra(&rays[hitidx]);
                    float ld = isect.uvwt.w;
                    float denom = extra.y * diffgeo.area;
                    // TODO: num_lights should be num_emissies instead, presence of analytical lights breaks this code
                    float bxdflightpdf = denom > 0.f ? (ld * ld / denom / num_lights) : 0.f;
                    weight = BalanceHeuristic(1, extra.x, 1, bxdflightpdf);
                }

                {
                    // In this case we hit after an application of MIS process at previous step.
                    // That means BRDF weight has been already applied.
                    output[pixelidx] += Path_GetThroughput(path) * Emissive_GetLe(&diffgeo, TEXTURE_ARGS) * weight;
                }
            }

            Path_Kill(path);
            Ray_SetInactive(shadowrays + globalid);
            Ray_SetInactive(indirectrays + globalid);

            lightsamples[globalid] = 0.f;
            return;
        }

        float s = Bxdf_IsBtdf(&diffgeo) ? (-sign(ndotwi)) : 1.f;
        if (!twosided && backfacing && !Bxdf_IsBtdf(&diffgeo))
        {
             //Reverse normal and tangents in this case
             //but not for BTDFs, since BTDFs rely
             //on normal direction in order to arrange
             //indices of refraction
            diffgeo.n = -diffgeo.n;
            diffgeo.dpdu = -diffgeo.dpdu;
            diffgeo.dpdv = -diffgeo.dpdv;
        }

        // TODO: this is test code, need to
        // maintain proper volume stack here
        //if (Bxdf_IsBtdf(&diffgeo))
        //{
        //    // If we entering set the volume
        //    path->volume = !backfacing ? 0 : -1;
        //}

        // Check if we need to apply normal map
        //ApplyNormalMap(&diffgeo, TEXTURE_ARGS);
        DifferentialGeometry_ApplyBumpMap(&diffgeo, TEXTURE_ARGS);
        DifferentialGeometry_CalculateTangentTransforms(&diffgeo);

        float lightpdf = 0.f;
        float bxdflightpdf = 0.f;
        float bxdfpdf = 0.f;
        float lightbxdfpdf = 0.f;
        float selection_pdf = 0.f;
        float3 radiance = 0.f;
        float3 lightwo;
        float3 bxdfwo;
        float3 wo;
        float bxdfweight = 1.f;
        float lightweight = 1.f;

        int light_idx = Scene_SampleLight(&scene, Sampler_Sample1D(&sampler, SAMPLER_ARGS), &selection_pdf);

        float3 throughput = Path_GetThroughput(path);

        // Sample bxdf
        float3 bxdf = Bxdf_Sample(&diffgeo, wi, TEXTURE_ARGS, Sampler_Sample2D(&sampler, SAMPLER_ARGS), &bxdfwo, &bxdfpdf);

        // If we have light to sample we can hopefully do mis
        if (light_idx > -1)
        {
            // Sample light
            float3 le = Light_Sample(light_idx, &scene, &diffgeo, TEXTURE_ARGS, Sampler_Sample2D(&sampler, SAMPLER_ARGS), &lightwo, &lightpdf);
            lightbxdfpdf = Bxdf_GetPdf(&diffgeo, wi, normalize(lightwo), TEXTURE_ARGS);
            lightweight = Light_IsSingular(&scene.lights[light_idx]) ? 1.f : BalanceHeuristic(1, lightpdf, 1, lightbxdfpdf);

            // Apply MIS to account for both
            if (NON_BLACK(le) && lightpdf > 0.0f && !Bxdf_IsSingular(&diffgeo))
            {
                wo = lightwo;
                float ndotwo = fabs(dot(diffgeo.n, normalize(wo)));
                radiance = le * Bxdf_Evaluate(&diffgeo, wi, normalize(wo), TEXTURE_ARGS) * throughput * ndotwo * lightweight / lightpdf / selection_pdf;
            }
        }

        // If we have some light here generate a shadow ray
        if (NON_BLACK(radiance))
        {
            // Generate shadow ray
            float shadow_ray_length = (1.f - 2.f * CRAZY_LOW_DISTANCE) * length(wo);
            float3 shadow_ray_dir = normalize(wo);
            float3 shadow_ray_o = diffgeo.p + CRAZY_LOW_DISTANCE * s * diffgeo.n;
            int shadow_ray_mask = Bxdf_IsSingular(&diffgeo) ? 0xFFFFFFFF : 0x0000FFFF;

            Ray_Init(shadowrays + globalid, shadow_ray_o, shadow_ray_dir, shadow_ray_length, 0.f, shadow_ray_mask);

            // Apply the volume to shadow ray if needed
            int volidx = Path_GetVolumeIdx(path);
            if (volidx != -1)
            {
                radiance *= Volume_Transmittance(&volumes[volidx], &shadowrays[globalid], shadow_ray_length);
                radiance += Volume_Emission(&volumes[volidx], &shadowrays[globalid], shadow_ray_length) * throughput;
            }

            // And write the light sample
            lightsamples[globalid] = REASONABLE_RADIANCE(radiance);
        }
        else
        {
            // Otherwise save some intersector cycles
            Ray_SetInactive(shadowrays + globalid);
            lightsamples[globalid] = 0;
        }

        // Apply Russian roulette
        float q = max(min(0.5f,
            // Luminance
            0.2126f * throughput.x + 0.7152f * throughput.y + 0.0722f * throughput.z), 0.01f);
        // Only if it is 3+ bounce
        bool rr_apply = bounce > 3;
        bool rr_stop = Sampler_Sample1D(&sampler, SAMPLER_ARGS) > q && rr_apply;

        if (rr_apply)
        {
            Path_MulThroughput(path, 1.f / q);
        }

        if (Bxdf_IsSingular(&diffgeo))
        {
            Path_SetSpecularFlag(path);
        }

        bxdfwo = normalize(bxdfwo);
        float3 t = bxdf * fabs(dot(diffgeo.n, bxdfwo));

        // Only continue if we have non-zero throughput & pdf
        if (NON_BLACK(t) && bxdfpdf > 0.f && !rr_stop)
        {
            // Update the throughput
            Path_MulThroughput(path, t / bxdfpdf);

            // Generate ray
            float3 indirect_ray_dir = bxdfwo;
            float3 indirect_ray_o = diffgeo.p + CRAZY_LOW_DISTANCE * s * diffgeo.n;

            Ray_Init(indirectrays + globalid, indirect_ray_o, indirect_ray_dir, CRAZY_HIGH_DISTANCE, 0.f, 0xFFFFFFFF);
            Ray_SetExtra(indirectrays + globalid, make_float2(bxdfpdf, fabs(dot(diffgeo.n, bxdfwo))));
        }
        else
        {
            // Otherwise kill the path
            Path_Kill(path);
            Ray_SetInactive(indirectrays + globalid);
        }
    }
}

///< Illuminate missing rays
__kernel void ShadeMiss(
    // Ray batch
    __global ray const* rays,
    // Intersection data
    __global Intersection const* isects,
    // Pixel indices
    __global int const* pixelindices,
    // Number of rays
    int numrays,
    // Textures
    TEXTURE_ARG_LIST,
    // Environment texture index
    int envmapidx,
    //
    __global Path const* paths,
    __global Volume const* volumes,
    // Output values
    __global float4* output
)
{
    int globalid = get_global_id(0);

    if (globalid < numrays)
    {
        int pixelidx = pixelindices[globalid];

        // In case of a miss
        if (isects[globalid].shapeid < 0)
        {
            // Multiply by throughput
            int volidx = paths[pixelidx].volume;

            if (volidx == -1)
                output[pixelidx].xyz += Texture_SampleEnvMap(rays[globalid].d.xyz, TEXTURE_ARGS_IDX(envmapidx));
            else
            {
                output[pixelidx].xyz += Texture_SampleEnvMap(rays[globalid].d.xyz, TEXTURE_ARGS_IDX(envmapidx)) *
                    Volume_Transmittance(&volumes[volidx], &rays[globalid], rays[globalid].o.w);

                output[pixelidx].xyz += Volume_Emission(&volumes[volidx], &rays[globalid], rays[globalid].o.w);
            }
        }

        if (isnan(output[pixelidx].x) || isnan(output[pixelidx].y) || isnan(output[pixelidx].z))
        {
            output[pixelidx] = make_float4(100.f, 0.f, 0.f, 1.f);
        }

        output[pixelidx].w += 1.f;
    }
}

///< Handle light samples and visibility info and add contribution to final buffer
__kernel void GatherLightSamples(
    // Pixel indices
    __global int const* pixelindices,
    // Number of rays
    __global int* numrays,
    // Shadow rays hits
    __global int const* shadowhits,
    // Light samples
    __global float3 const* lightsamples,
    // throughput
    __global Path const* paths,
    // Radiance sample buffer
    __global float4* output
)
{
    int globalid = get_global_id(0);

    if (globalid < *numrays)
    {
        // Get pixel id for this sample set
        int pixelidx = pixelindices[globalid];

        // Prepare accumulator variable
        float3 radiance = make_float3(0.f, 0.f, 0.f);

        // Start collecting samples
        {
            // If shadow ray didn't hit anything and reached skydome
            if (shadowhits[globalid] == -1)
            {
                // Add its contribution to radiance accumulator
                radiance += lightsamples[globalid];
            }
        }

        // Divide by number of light samples (samples already have built-in throughput)
        output[pixelidx].xyz += radiance;
    }
}

///< Handle light samples and visibility info and add contribution to final buffer
__kernel void GatherOcclusion(
    // Pixel indices
    __global int const* pixelindices,
    // Number of rays
    __global int const* numrays,
    // Shadow rays hits
    __global int const* shadowhits,
    // Light samples
    __global float3 const* lightsamples,
    // throughput
    __global float3 const* throughput,
    // Radiance sample buffer
    __global float4* output
)
{
    int globalid = get_global_id(0);

    if (globalid < *numrays)
    {
        // Get pixel id for this sample set
        int pixelidx = pixelindices[globalid];

        float visibility = (shadowhits[globalid] == 1) ? 0.f : 1.f;

        output[pixelidx].xyz += visibility;
        output[pixelidx].w += 1.f;
    }
}

///< Restore pixel indices after compaction
__kernel void RestorePixelIndices(
    // Compacted indices
    __global int const* compacted_indices,
    // Number of compacted indices
    __global int* numitems,
    // Previous pixel indices
    __global int const* previndices,
    // New pixel indices
    __global int* newindices
)
{
    int globalid = get_global_id(0);

    // Handle only working subset
    if (globalid < *numitems)
    {
        newindices[globalid] = previndices[compacted_indices[globalid]];
    }
}

///< Restore pixel indices after compaction
__kernel void FilterPathStream(
    // Intersections
    __global Intersection const* isects,
    // Number of compacted indices
    __global int const* numitems,
    // Pixel indices
    __global int const* pixelindices,
    // Paths
    __global Path* paths,
    // Predicate
    __global int* predicate
)
{
    int globalid = get_global_id(0);

    // Handle only working subset
    if (globalid < *numitems)
    {
        int pixelidx = pixelindices[globalid];

        __global Path* path = paths + pixelidx;

        if (Path_IsAlive(path))
        {
            bool kill = (length(Path_GetThroughput(path)) < CRAZY_LOW_THROUGHPUT);

            if (!kill)
            {
                predicate[globalid] = isects[globalid].shapeid >= 0 ? 1 : 0;
            }
            else
            {
                Path_Kill(path);
                predicate[globalid] = 0;
            }
        }
        else
        {
            predicate[globalid] = 0;
        }
    }
}

///< Illuminate missing rays
__kernel void ShadeBackground(
    // Ray batch
    __global ray const* rays,
    // Intersection data
    __global Intersection const* isects,
    // Pixel indices
    __global int const* pixelindices,
    // Number of rays
    __global int const* numrays,
    // Textures
    TEXTURE_ARG_LIST,
    // Environment texture index
    int envmapidx,
    float envmapmul,
    //
    int num_lights,
    __global Path const* paths,
    __global Volume const* volumes,
    // Output values
    __global float4* output
)
{
    int globalid = get_global_id(0);

    if (globalid < *numrays)
    {
        int pixelidx = pixelindices[globalid];

        __global Path const* path = paths + pixelidx;

        // In case of a miss
        if (isects[globalid].shapeid < 0 && Path_IsAlive(path))
        {
            float3 t = Path_GetThroughput(path);
            output[pixelidx].xyz += REASONABLE_RADIANCE(envmapmul * Texture_SampleEnvMap(rays[globalid].d.xyz, TEXTURE_ARGS_IDX(envmapidx)) * t);
        }
    }
}

// Copy data to interop texture if supported
__kernel void AccumulateData(
    __global float4 const* srcdata,
    int numelems,
    __global float4* dstdata
)
{
    int gid = get_global_id(0);

    if (gid < numelems)
    {
        float4 v = srcdata[gid];
        dstdata[gid] += v;
    }
}

// Copy data to interop texture if supported
__kernel void ApplyGammaAndCopyData(
    __global float4 const* data,
    int imgwidth,
    int imgheight,
    float gamma,
    write_only image2d_t img
)
{
    int gid = get_global_id(0);

    int gidx = gid % imgwidth;
    int gidy = gid / imgwidth;

    if (gidx < imgwidth && gidy < imgheight)
    {
        float4 v = data[gid];
        float4 val = clamp(native_powr(v / v.w, 1.f / gamma), 0.f, 1.f);
        write_imagef(img, make_int2(gidx, gidy), val);
    }
}