Changing around how CSR-Stream reduction is performed. Removing a lot of multiplications that snuck their way into the code. 32-bit integer multiply is slow (up to 16x slower than addition on AMD GPUs), so replaced with full-speed addition or full-speed 24-bit multiply when required. Results in major performance gains on Fiji GPUs.

R"(
#ifdef DOUBLE
  #define FPTYPE double

  #ifdef cl_khr_fp64
      #pragma OPENCL EXTENSION cl_khr_fp64 : enable
  #elif defined(cl_amd_fp64)
      #pragma OPENCL EXTENSION cl_amd_fp64 : enable
  #else
      #error "Double precision floating point not supported by OpenCL implementation."
  #endif
#else
  #define FPTYPE float
#endif

#pragma OPENCL EXTENSION cl_khr_int64_base_atomics: enable
#pragma OPENCL EXTENSION cl_khr_int64_extended_atomics: enable

FPTYPE atomic_add_float_extended( global FPTYPE * restrict const ptr,
                                  const FPTYPE temp,
                                  FPTYPE * restrict const old_sum )
{
#ifdef DOUBLE
	unsigned long newVal;
	unsigned long prevVal;
	do
	{
		prevVal = as_ulong(*ptr);
		newVal = as_ulong(temp + *ptr);
	} while (atom_cmpxchg((global unsigned long *)ptr, prevVal, newVal) != prevVal);
    if (old_sum != 0)
        *old_sum = as_double(prevVal);
    return as_double(newVal);
#else
	unsigned int newVal;
	unsigned int prevVal;
	do
	{
		prevVal = as_uint(*ptr);
		newVal = as_uint(temp + *ptr);
	} while (atomic_cmpxchg((global unsigned int *)ptr, prevVal, newVal) != prevVal);
    if (old_sum != 0)
        *old_sum = as_float(prevVal);
    return as_float(newVal);
#endif
}

void atomic_add_float( global void * const ptr, const FPTYPE temp )
{
    atomic_add_float_extended(ptr, temp, 0);
}

// Knuth's Two-Sum algorithm, which allows us to add together two floating
// point numbers and exactly tranform the answer into a sum and a
// rounding error.
// Inputs: x and y, the two inputs to be aded together.
// In/Out: *sumk_err, which is incremented (by reference) -- holds the
//         error value as a result of the 2sum calculation.
// Returns: The non-corrected sum of inputs x and y.
FPTYPE two_sum( FPTYPE x,
                FPTYPE y,
                FPTYPE * restrict const sumk_err )
{
    const FPTYPE sumk_s = x + y;
#ifdef EXTENDED_PRECISION
    // We use this 2Sum algorithm to perform a compensated summation,
    // which can reduce the cummulative rounding errors in our SpMV summation.
    // Our compensated sumation is based on the SumK algorithm (with K==2) from
    // Ogita, Rump, and Oishi, "Accurate Sum and Dot Product" in
    // SIAM J. on Scientific Computing 26(6) pp 1955-1988, Jun. 2005.

    // 2Sum can be done in 6 FLOPs without a branch. However, calculating
    // double precision is slower than single precision on every existing GPU.
    // As such, replacing 2Sum with Fast2Sum when using DPFP results in better
    // performance (~5% higher total). This is true even though we must ensure
    // that |a| > |b|. Branch divergence is better than the DPFP slowdown.
    // Thus, for DPFP, our compensated summation algorithm is actually described
    // by both Pichat and Neumaier in "Correction d'une somme en arithmetique
    // a virgule flottante" (J. Numerische Mathematik 19(5) pp. 400-406, 1972)
    // and "Rundungsfehleranalyse einiger Verfahren zur Summation endlicher
    // Summen (ZAMM Z. Angewandte Mathematik und Mechanik 54(1) pp. 39-51,
    // 1974), respectively.
    if (fabs(x) < fabs(y))
    {
        const FPTYPE swap = x;
        x = y;
        y = swap;
    }
    (*sumk_err) += (y - (sumk_s - x));
    // Original 6 FLOP 2Sum algorithm.
    //const FPTYPE bp = sumk_s - x;
    //(*sumk_err) += ((x - (sumk_s - bp)) + (y - bp));
#endif
    return sumk_s;
}

// Performs (x_vals * x_vec) + y using an FMA.
// Ideally, we would perform an error-free transformation here and return the
// appropriate error. However, the EFT of an FMA is very expensive. As such,
// if we are in EXTENDED_PRECISION mode, this function devolves into two_sum
// with x_vals and x_vec inputs multiplied separately from the compensated add.
FPTYPE two_fma( const FPTYPE x_vals,
        const FPTYPE x_vec,
        FPTYPE y,
        FPTYPE * restrict const sumk_err )
{
#ifdef EXTENDED_PRECISION
    FPTYPE x = x_vals * x_vec;
    const FPTYPE sumk_s = x + y;
    if (fabs(x) < fabs(y))
    {
        const FPTYPE swap = x;
        x = y;
        y = swap;
    }
    (*sumk_err) += (y - (sumk_s - x));
    // 2Sum in the FMA case. Poor performance on low-DPFP GPUs.
    //const FPTYPE bp = fma(-x_vals, x_vec, sumk_s);
    //(*sumk_err) += (fma(x_vals, x_vec, -(sumk_s - bp)) + (y - bp));
    return sumk_s;
#else
    return fma(x_vals, x_vec, y);
#endif
}

FPTYPE atomic_two_sum_float( global FPTYPE * restrict const x_ptr,
                            FPTYPE y,
                            FPTYPE * restrict const sumk_err )
{
    // Have to wait until the return from the atomic op to know what X was.
    FPTYPE sumk_s = 0.;
#ifdef EXTENDED_PRECISION
    FPTYPE x;
    sumk_s = atomic_add_float_extended(x_ptr, y, &x);
    if (fabs(x) < fabs(y))
    {
        const FPTYPE swap = x;
        x = y;
        y = swap;
    }
    (*sumk_err) += (y - (sumk_s - x));
#else
    atomic_add_float(x_ptr, y);
#endif
    return sumk_s;
}

// A method of doing the final reduction in CSR-Vector without having to copy
// and paste it a bunch of times.
// The EXTENDED_PRECISION section is done as part of a compensated summation
// meant to reduce cummulative rounding errors. This can become a problem on
// GPUs because the reduction order is different than what would be used on a
// CPU. It is based on the PSumK algorithm (with K==2) from Yamanaka, Ogita,
// Rump, and Oishi, "A Parallel Algorithm of Accurate Dot Product," in
// J. Parallel Computing, 34(6-8), pp. 392-410, Jul. 2008.
// A version of this method is also used in csrmv_general.
// Inputs:  cur_sum: the input from which our sum starts
//          err: the current running cascade error for this final summation
//          partial: the local memory which holds the values to sum
//                  (we eventually use it to pass down temp. err vals as well)
//          lid: local ID of the work item calling this function.
//          thread_lane: The lane within this thread's row.
//          max_size: This parallel summation method operates in multiple rounds
//                  to do a parallel reduction. This is the length of each row.
//          reduc_size: As you reduce data down, this tells you how far away
//                 you will grab a value and add to your own local sum value.
FPTYPE sum2_reduce( FPTYPE cur_sum,
        FPTYPE * restrict const err,
        __local FPTYPE * restrict const partial,
        const unsigned int lid,
        const unsigned int thread_lane,
        const unsigned int max_size,
        const unsigned int reduc_size )
{
    if ( max_size > reduc_size )
    {
#ifdef EXTENDED_PRECISION
        const unsigned int partial_dest = lid + reduc_size;
        if (thread_lane < reduc_size)
            cur_sum  = two_sum(cur_sum, partial[partial_dest], err);
        // We reuse the LDS entries to move the error values down into lower
        // threads. This saves LDS space, allowing higher occupancy, but requires
        // more barriers, which can reduce performance.
        barrier(CLK_LOCAL_MEM_FENCE);
        // Have all of those upper threads pass their temporary errors
        // into a location that the lower threads can read.
        partial[lid] = (thread_lane >= reduc_size) ? *err : partial[lid];
        barrier(CLK_LOCAL_MEM_FENCE);
        if (thread_lane < reduc_size) // Add those errors in.
        {
            *err += partial[partial_dest];
            partial[lid] = cur_sum;
        }
#else
        cur_sum += partial[lid + reduc_size];
        partial[lid] = cur_sum;
#endif
    }
    return cur_sum;
}

__kernel void
csrmv_adaptive(__global const FPTYPE * restrict const vals,
                       __global const unsigned int * restrict const cols,
                       __global const unsigned int * restrict const rowPtrs,
                       __global const FPTYPE * restrict const vec,
                       __global FPTYPE * restrict const out,
                       __global unsigned long * restrict const rowBlocks,
                       __global const FPTYPE * restrict const pAlpha,
                       __global const FPTYPE * restrict const pBeta)
{
   __local FPTYPE partialSums[BLOCKSIZE];
   const unsigned int gid = get_group_id(0);
   const unsigned int lid = get_local_id(0);
   const FPTYPE alpha = *pAlpha;
   const FPTYPE beta = *pBeta;

   // The row blocks buffer holds a packed set of information used to inform each
   // workgroup about how to do its work:
   //
   // |6666 5555 5555 5544 4444 4444 3333 3333|3322 2222|2222 1111 1111 1100 0000 0000|
   // |3210 9876 5432 1098 7654 3210 9876 5432|1098 7654|3210 9876 5432 1098 7654 3210|
   // |------------Row Information------------|--------^|---WG ID within a long row---|
   // |                                       |    flag/|or # reduce threads for short|
   //
   // The upper 32 bits of each rowBlock entry tell the workgroup the ID of the first
   // row it will be working on. When one workgroup calculates multiple rows, this
   // rowBlock entry and the next one tell it the range of rows to work on.
   // The lower 24 bits are used whenever multiple workgroups calculate a single long
   // row. This tells each workgroup its ID within that row, so it knows which
   // part of the row to operate on.
   // Alternately, on "short" row blocks, the lower bits are used to communicate
   // the number of threads that should be used for the reduction. Pre-calculating
   // this on the CPU-side results in a noticable performance uplift on many matrices.
   // Bit 24 is a flag bit used so that the multiple WGs calculating a long row can
   // know when the first workgroup for that row has finished initializing the output
   // value. While this bit is the same as the first workgroup's flag bit, this
   // workgroup will spin-loop.
   unsigned int row = ((rowBlocks[gid] >> (64-ROWBITS)) & ((1UL << ROWBITS) - 1UL));
   const unsigned int stop_row = ((rowBlocks[gid + 1] >> (64-ROWBITS)) & ((1UL << ROWBITS) - 1UL));
   const unsigned int num_rows = stop_row - row;

   // Get the "workgroup within this long row" ID out of the bottom bits of the row block.
   const unsigned int wg = rowBlocks[gid] & ((1 << WGBITS) - 1);

   // Any workgroup only calculates, at most, BLOCK_MULTIPLIER*BLOCKSIZE items in a row.
   // If there are more items in this row, we assign more workgroups.
   unsigned int vecStart = mad24(wg, (unsigned int)(BLOCK_MULTIPLIER*BLOCKSIZE), rowPtrs[row]);
   unsigned int vecEnd = (rowPtrs[row + 1] > vecStart + BLOCK_MULTIPLIER*BLOCKSIZE) ? vecStart + BLOCK_MULTIPLIER*BLOCKSIZE : rowPtrs[row + 1];

   FPTYPE temp_sum = 0.;
   FPTYPE sumk_e = 0.;
   FPTYPE new_error = 0.;

   // If the next row block starts more than 2 rows away, then we choose CSR-Stream.
   // If this is zero (long rows) or one (final workgroup in a long row, or a single
   // row in a row block), we want to use the CSR-Vector algorithm(s).
   // We have found, through experimentation, that CSR-Vector is generally faster
   // when working on 2 rows, due to its simplicity and better reduction method.
   if (num_rows > ROWS_FOR_VECTOR)
   {
       // CSR-Stream case. See Sections III.A and III.B in the SC'14 paper:
       // "Efficient Sparse Matrix-Vector Multiplication on GPUs using the CSR Storage Format"
       // for a detailed description of CSR-Stream.
       // In a nutshell, the idea is to use all of the threads to stream the matrix
       // values into the local memory in a fast, coalesced manner. After that, the
       // per-row reductions are done out of the local memory, which is designed
       // to handle non-coalsced accesses.

       // The best method for reducing the local memory values depends on the number
       // of rows. The SC'14 paper discusses a "CSR-Scalar" style reduction where
       // each thread reduces its own row. This yields good performance if there
       // are many (relatively short) rows. However, if they are few (relatively
       // long) rows, it's actually better to perform a tree-style reduction where
       // multiple threads team up to reduce the same row.

       // The calculation below tells you how many threads this workgroup can allocate
       // to each row, assuming that every row gets the same number of threads.
       // We want the closest lower (or equal) power-of-2 to this number --
       // that is how many threads can work in each row's reduction using our algorithm.
       // For instance, with workgroup size 256, 2 rows = 128 threads, 3 rows = 64
       // threads, 4 rows = 64 threads, 5 rows = 32 threads, etc.
       //int numThreadsForRed = get_local_size(0) >> ((CHAR_BIT*sizeof(unsigned int))-clz(num_rows-1));
       const unsigned int numThreadsForRed = wg; // Same calculation as above, done on host.

       // Stream all of this row block's matrix values into local memory.
       // Perform the matvec in parallel with this work.
       const unsigned int col = rowPtrs[row] + lid;
      if (gid != (get_num_groups(0) - 1))
      {
          for(int i = 0; i < BLOCKSIZE; i += WGSIZE)
              partialSums[lid + i] = alpha * vals[col + i] * vec[cols[col + i]];
      }
      else
      {
          // This is required so that we stay in bounds for vals[] and cols[].
          // Otherwise, if the matrix's endpoints don't line up with BLOCKSIZE,
          // we will buffer overflow. On today's dGPUs, this doesn't cause problems.
          // The values are within a dGPU's page, which is zeroed out on allocation.
          // However, this may change in the future (e.g. with shared virtual memory.)
          // This causes a minor performance loss because this is the last workgroup
          // to be launched, and this loop can't be unrolled.
          const unsigned int max_to_load = rowPtrs[stop_row] - rowPtrs[row];
          for(int i = 0; i < max_to_load; i += WGSIZE)
              partialSums[lid + i] = alpha * vals[col + i] * vec[cols[col + i]];
      }
      barrier(CLK_LOCAL_MEM_FENCE);

      if(numThreadsForRed > 1)
      {
          // In this case, we want to have the workgroup perform a tree-style reduction
          // of each row. {numThreadsForRed} adjacent threads team up to linearly reduce
          // a row into {numThreadsForRed} locations in local memory.
          // After that, the entire workgroup does a parallel reduction, and each
          // row ends up with an individual answer.

          // {numThreadsForRed} adjacent threads all work on the same row, so their
          // start and end values are the same.
          // numThreadsForRed guaranteed to be a power of two, so the clz code below
          // avoids an integer divide. ~2% perf gain in EXTRA_PRECISION.
          //size_t st = lid/numThreadsForRed;
          const unsigned int local_row = row + (lid >> (31 - clz(numThreadsForRed)));
          const unsigned int local_first_val = rowPtrs[local_row] - rowPtrs[row];
          const unsigned int local_last_val = rowPtrs[local_row + 1] - rowPtrs[row];
          const unsigned int threadInBlock = lid & (numThreadsForRed - 1);

          // Not all row blocks are full -- they may have an odd number of rows. As such,
          // we need to ensure that adjacent-groups only work on real data for this rowBlock.
          if(local_row < stop_row)
          {
              // This is dangerous -- will infinite loop if your last value is within
              // numThreadsForRed of MAX_UINT. Noticable performance gain to avoid a
              // long induction variable here, though.
              for(unsigned int local_cur_val = local_first_val + threadInBlock;
                      local_cur_val < local_last_val;
                      local_cur_val += numThreadsForRed)
                  temp_sum = two_sum(partialSums[local_cur_val], temp_sum, &sumk_e);
          }
          barrier(CLK_LOCAL_MEM_FENCE);

          temp_sum = two_sum(temp_sum, sumk_e, &new_error);
          partialSums[lid] = temp_sum;

          // Step one of this two-stage reduction is done. Now each row has {numThreadsForRed}
          // values sitting in the local memory. This means that, roughly, the beginning of
          // LDS is full up to {workgroup size} entries.
          // Now we perform a parallel reduction that sums together the answers for each
          // row in parallel, leaving us an answer in 'temp_sum' for each row.
          for (int i = (WGSIZE >> 1); i > 0; i >>= 1)
          {
              barrier( CLK_LOCAL_MEM_FENCE );
              temp_sum = sum2_reduce(temp_sum, &new_error, partialSums, lid, threadInBlock, numThreadsForRed, i);
          }

          if (threadInBlock == 0 && local_row < stop_row)
          {
              // All of our write-outs check to see if the output vector should first be zeroed.
              // If so, just do a write rather than a read-write. Measured to be a slight (~5%)
              // performance improvement.
              if (beta != 0.)
                  temp_sum = two_fma(beta, out[local_row], temp_sum, &new_error);
              out[local_row] = temp_sum + new_error;
          }
      }
      else
      {
          // In this case, we want to have each thread perform the reduction for a single row.
          // Essentially, this looks like performing CSR-Scalar, except it is computed out of local memory.
          // However, this reduction is also much faster than CSR-Scalar, because local memory
          // is designed for scatter-gather operations.
          // We need a while loop because there may be more rows than threads in the WG.
          unsigned int local_row = row + lid;
          while(local_row < stop_row)
          {
              int local_first_val = (rowPtrs[local_row] - rowPtrs[row]);
              int local_last_val = rowPtrs[local_row + 1] - rowPtrs[row];
              temp_sum = 0.;
              sumk_e = 0.;
              for (int local_cur_val = local_first_val; local_cur_val < local_last_val; local_cur_val++)
                  temp_sum = two_sum(partialSums[local_cur_val], temp_sum, &sumk_e);

              // After you've done the reduction into the temp_sum register,
              // put that into the output for each row.
              if (beta != 0.)
                  temp_sum = two_fma(beta, out[local_row], temp_sum, &sumk_e);
              out[local_row] = temp_sum + sumk_e;
              local_row += WGSIZE;
          }
      }
   }
   else if (num_rows >= 1 && !wg) // CSR-Vector case.
   {
       // ^^ The above check says that if this workgroup is supposed to work on <= ROWS_VECTOR
       // number of rows then we should do the CSR-Vector algorithm. If we want this row to be
       // done with CSR-LongRows, then all of its workgroups (except the last one) will have the
       // same stop_row and row. The final workgroup in a LongRow will have stop_row and row
       // different, but the internal "wg" number will be non-zero.

       // If this workgroup is operating on multiple rows (because CSR-Stream is poor for small
       // numbers of rows), then it needs to iterate until it reaches the stop_row.
       // We don't check <= stop_row because of the potential for unsigned overflow.
       while (row < stop_row)
       {
           // Any workgroup only calculates, at most, BLOCKSIZE items in this row.
           // If there are more items in this row, we use CSR-LongRows.
           temp_sum = 0.;
           sumk_e = 0.;
           new_error = 0.;
           vecStart = rowPtrs[row];
           vecEnd = rowPtrs[row+1];

           // Load in a bunch of partial results into your register space, rather than LDS (no contention)
           // Then dump the partially reduced answers into the LDS for inter-work-item reduction.
           // Using a long induction variable to make sure unsigned int overflow doesn't break things.
           for (long j = vecStart + lid; j < vecEnd; j+=WGSIZE)
           {
               const unsigned int col = cols[(unsigned int)j];
               temp_sum = two_fma(alpha*vals[(unsigned int)j], vec[col], temp_sum, &sumk_e);
           }

           temp_sum = two_sum(temp_sum, sumk_e, &new_error);
           partialSums[lid] = temp_sum;

           // Reduce partial sums
           for (int i = (WGSIZE >> 1); i > 0; i >>= 1)
           {
               barrier( CLK_LOCAL_MEM_FENCE);
               temp_sum = sum2_reduce(temp_sum, &new_error, partialSums, lid, lid, WGSIZE, i);
           }

           if (lid == 0UL)
           {
               if (beta != 0.)
                   temp_sum = two_fma(beta, out[row], temp_sum, &new_error);
               out[row] = temp_sum + new_error;
           }
           row++;
       }
   }
   else
   {
       // In CSR-LongRows, we have more than one workgroup calculating this row.
       // The output values for those types of rows are stored using atomic_add, because
       // more than one parallel workgroup's value makes up the final answer.
       // Unfortunately, this makes it difficult to do y=Ax, rather than y=Ax+y, because
       // the values still left in y will be added in using the atomic_add.
       //
       // Our solution is to have the first workgroup in one of these long-rows cases
       // properly initaizlie the output vector. All the other workgroups working on this
       // row will spin-loop until that workgroup finishes its work.

       // First, figure out which workgroup you are in the row. Bottom 24 bits.
       // You can use that to find the global ID for the first workgroup calculating
       // this long row.
       const unsigned int first_wg_in_row = gid - (rowBlocks[gid] & ((1UL << WGBITS) - 1UL));
       const unsigned int compare_value = rowBlocks[gid] & (1UL << WGBITS);

       // Bit 24 in the first workgroup is the flag that everyone waits on.
       if(gid == first_wg_in_row && lid == 0UL)
       {
           // The first workgroup handles the output initialization.
           if (beta != 0.)
               out[row] *= beta;
           else
               out[row] = 0.;
#ifdef EXTENDED_PRECISION
           rowBlocks[get_num_groups(0) + gid + 1] = 0UL;
#endif
           atom_xor(&rowBlocks[first_wg_in_row], (1UL << WGBITS)); // Release other workgroups.
       }
       // For every other workgroup, bit 24 holds the value they wait on.
       // If your bit 24 == first_wg's bit 24, you spin loop.
       // The first workgroup will eventually flip this bit, and you can move forward.
       barrier(CLK_GLOBAL_MEM_FENCE);
       while(gid != first_wg_in_row &&
               lid == 0UL &&
               ((atom_max(&rowBlocks[first_wg_in_row], 0UL) & (1UL << WGBITS)) == compare_value));
       barrier(CLK_GLOBAL_MEM_FENCE);

       // After you've passed the barrier, update your local flag to make sure that
       // the next time through, you know what to wait on.
       if (gid != first_wg_in_row && lid == 0UL)
           rowBlocks[gid] ^= (1UL << WGBITS);

       // All but the final workgroup in a long-row collaboration have the same start_row
       // and stop_row. They only run for one iteration.
       // Load in a bunch of partial results into your register space, rather than LDS (no contention)
       // Then dump the partially reduced answers into the LDS for inter-work-item reduction.
       const unsigned int col = vecStart + lid;
       if (row == stop_row) // inner thread, we can hardcode/unroll this loop
       {
           // Don't put BLOCK_MULTIPLIER*BLOCKSIZE as the stop point, because
           // some GPU compilers will *aggressively* unroll this loop.
           // That increases register pressure and reduces occupancy.
           for (int j = 0; j < (int)(vecEnd - col); j += WGSIZE)
           {
               temp_sum = two_fma(alpha*vals[col + j], vec[cols[col + j]], temp_sum, &sumk_e);
#if 2*WGSIZE <= BLOCK_MULTIPLIER*BLOCKSIZE
               // If you can, unroll this loop once. It somewhat helps performance.
               j += WGSIZE;
               temp_sum = two_fma(alpha*vals[col + j], vec[cols[col + j]], temp_sum, &sumk_e);
#endif
           }
       }
       else
       {
           for(int j = 0; j < (int)(vecEnd - col); j += WGSIZE)
               temp_sum = two_fma(alpha*vals[col + j], vec[cols[col + j]], temp_sum, &sumk_e);
       }

       temp_sum = two_sum(temp_sum, sumk_e, &new_error);
       partialSums[lid] = temp_sum;

       // Reduce partial sums
       for (int i = (WGSIZE >> 1); i > 0; i >>= 1)
       {
           barrier( CLK_LOCAL_MEM_FENCE);
           temp_sum = sum2_reduce(temp_sum, &new_error, partialSums, lid, lid, WGSIZE, i);
       }

       if (lid == 0UL)
       {
           atomic_two_sum_float(&out[row], temp_sum, &new_error);

#ifdef EXTENDED_PRECISION
           unsigned int error_loc = get_num_groups(0) + first_wg_in_row + 1;
           // The last half of the rowBlocks buffer is used to hold errors.
           atomic_add_float(&(rowBlocks[error_loc]), new_error);
           // Coordinate across all of the workgroups in this coop in order to have
           // the last workgroup fix up the error values.
           // If this workgroup's row is different than the next workgroup's row
           // then this is the last workgroup -- it's this workgroup's job to add
           // the error values into the final sum.
           if (row != stop_row)
           {
               // Go forward once your ID is the same as the low order bits of the
               // coop's first workgroup. That value will be used to store the number
               // of threads that have completed so far. Once all the previous threads
               // are done, it's time to send out the errors!
               while((atom_max(&rowBlocks[first_wg_in_row], 0UL) & ((1UL << WGBITS) - 1)) != wg);

#ifdef DOUBLE
               new_error = as_double(rowBlocks[error_loc]);
#else
               new_error = as_float((int)rowBlocks[error_loc]);
#endif
               // Don't need to work atomically here, because this is the only workgroup
               // left working on this row.
               out[row] += new_error;
               rowBlocks[error_loc] = 0UL;

               // Reset the rowBlocks low order bits for next time.
               rowBlocks[first_wg_in_row] = rowBlocks[gid] - wg;
           }
           else
           {
               // Otherwise, increment the low order bits of the first thread in this
               // coop. We're using this to tell how many workgroups in a coop are done.
               // Do this with an atomic, since other threads may be doing this too.
               const unsigned long no_warn = atom_inc(&rowBlocks[first_wg_in_row]);
           }
#endif
       }
   }
}
)"

R"(
#ifdef DOUBLE
  #define FPTYPE double

  #ifdef cl_khr_fp64
      #pragma OPENCL EXTENSION cl_khr_fp64 : enable
  #elif defined(cl_amd_fp64)
      #pragma OPENCL EXTENSION cl_amd_fp64 : enable
  #else
      #error "Double precision floating point not supported by OpenCL implementation."
  #endif
#else
  #define FPTYPE float
#endif

#pragma OPENCL EXTENSION cl_khr_int64_base_atomics: enable
#pragma OPENCL EXTENSION cl_khr_int64_extended_atomics: enable

FPTYPE atomic_add_float_extended( global FPTYPE * restrict const ptr,
                                  const FPTYPE temp,
                                  FPTYPE * restrict const old_sum )
{
#ifdef DOUBLE
	unsigned long newVal;
	unsigned long prevVal;
	do
	{
		prevVal = as_ulong(*ptr);
		newVal = as_ulong(temp + *ptr);
	} while (atom_cmpxchg((global unsigned long *)ptr, prevVal, newVal) != prevVal);
    if (old_sum != 0)
        *old_sum = as_double(prevVal);
    return as_double(newVal);
#else
	unsigned int newVal;
	unsigned int prevVal;
	do
	{
		prevVal = as_uint(*ptr);
		newVal = as_uint(temp + *ptr);
	} while (atomic_cmpxchg((global unsigned int *)ptr, prevVal, newVal) != prevVal);
    if (old_sum != 0)
        *old_sum = as_float(prevVal);
    return as_float(newVal);
#endif
}

void atomic_add_float( global void * const ptr, const FPTYPE temp )
{
    atomic_add_float_extended(ptr, temp, 0);
}

// Knuth's Two-Sum algorithm, which allows us to add together two floating
// point numbers and exactly tranform the answer into a sum and a
// rounding error.
// Inputs: x and y, the two inputs to be aded together.
// In/Out: *sumk_err, which is incremented (by reference) -- holds the
//         error value as a result of the 2sum calculation.
// Returns: The non-corrected sum of inputs x and y.
FPTYPE two_sum( FPTYPE x,
                FPTYPE y,
                FPTYPE * restrict const sumk_err )
{
    const FPTYPE sumk_s = x + y;
#ifdef EXTENDED_PRECISION
    // We use this 2Sum algorithm to perform a compensated summation,
    // which can reduce the cummulative rounding errors in our SpMV summation.
    // Our compensated sumation is based on the SumK algorithm (with K==2) from
    // Ogita, Rump, and Oishi, "Accurate Sum and Dot Product" in
    // SIAM J. on Scientific Computing 26(6) pp 1955-1988, Jun. 2005.

    // 2Sum can be done in 6 FLOPs without a branch. However, calculating
    // double precision is slower than single precision on every existing GPU.
    // As such, replacing 2Sum with Fast2Sum when using DPFP results in better
    // performance (~5% higher total). This is true even though we must ensure
    // that |a| > |b|. Branch divergence is better than the DPFP slowdown.
    // Thus, for DPFP, our compensated summation algorithm is actually described
    // by both Pichat and Neumaier in "Correction d'une somme en arithmetique
    // a virgule flottante" (J. Numerische Mathematik 19(5) pp. 400-406, 1972)
    // and "Rundungsfehleranalyse einiger Verfahren zur Summation endlicher
    // Summen (ZAMM Z. Angewandte Mathematik und Mechanik 54(1) pp. 39-51,
    // 1974), respectively.
    if (fabs(x) < fabs(y))
    {
        const FPTYPE swap = x;
        x = y;
        y = swap;
    }
    (*sumk_err) += (y - (sumk_s - x));
    // Original 6 FLOP 2Sum algorithm.
    //const FPTYPE bp = sumk_s - x;
    //(*sumk_err) += ((x - (sumk_s - bp)) + (y - bp));
#endif
    return sumk_s;
}

// Performs (x_vals * x_vec) + y using an FMA.
// Ideally, we would perform an error-free transformation here and return the
// appropriate error. However, the EFT of an FMA is very expensive. As such,
// if we are in EXTENDED_PRECISION mode, this function devolves into two_sum
// with x_vals and x_vec inputs multiplied separately from the compensated add.
FPTYPE two_fma( const FPTYPE x_vals,
        const FPTYPE x_vec,
        FPTYPE y,
        FPTYPE * restrict const sumk_err )
{
#ifdef EXTENDED_PRECISION
    FPTYPE x = x_vals * x_vec;
    const FPTYPE sumk_s = x + y;
    if (fabs(x) < fabs(y))
    {
        const FPTYPE swap = x;
        x = y;
        y = swap;
    }
    (*sumk_err) += (y - (sumk_s - x));
    // 2Sum in the FMA case. Poor performance on low-DPFP GPUs.
    //const FPTYPE bp = fma(-x_vals, x_vec, sumk_s);
    //(*sumk_err) += (fma(x_vals, x_vec, -(sumk_s - bp)) + (y - bp));
    return sumk_s;
#else
    return fma(x_vals, x_vec, y);
#endif
}

FPTYPE atomic_two_sum_float( global FPTYPE * restrict const x_ptr,
                            FPTYPE y,
                            FPTYPE * restrict const sumk_err )
{
    // Have to wait until the return from the atomic op to know what X was.
    FPTYPE sumk_s = 0.;
#ifdef EXTENDED_PRECISION
    FPTYPE x;
    sumk_s = atomic_add_float_extended(x_ptr, y, &x);
    if (fabs(x) < fabs(y))
    {
        const FPTYPE swap = x;
        x = y;
        y = swap;
    }
    (*sumk_err) += (y - (sumk_s - x));
#else
    atomic_add_float(x_ptr, y);
#endif
    return sumk_s;
}

// A method of doing the final reduction in CSR-Vector without having to copy
// and paste it a bunch of times.
// The EXTENDED_PRECISION section is done as part of a compensated summation
// meant to reduce cummulative rounding errors. This can become a problem on
// GPUs because the reduction order is different than what would be used on a
// CPU. It is based on the PSumK algorithm (with K==2) from Yamanaka, Ogita,
// Rump, and Oishi, "A Parallel Algorithm of Accurate Dot Product," in
// J. Parallel Computing, 34(6-8), pp. 392-410, Jul. 2008.
// A version of this method is also used in csrmv_general.
// Inputs:  cur_sum: the input from which our sum starts
//          err: the current running cascade error for this final summation
//          partial: the local memory which holds the values to sum
//                  (we eventually use it to pass down temp. err vals as well)
//          lid: local ID of the work item calling this function.
//          thread_lane: The lane within this thread's row.
//          max_size: This parallel summation method operates in multiple rounds
//                  to do a parallel reduction. This is the length of each row.
//          reduc_size: As you reduce data down, this tells you how far away
//                 you will grab a value and add to your own local sum value.
FPTYPE sum2_reduce( FPTYPE cur_sum,
        FPTYPE * restrict const err,
        __local FPTYPE * restrict const partial,
        const unsigned int lid,
        const unsigned int thread_lane,
        const unsigned int max_size,
        const unsigned int reduc_size )
{
    if ( max_size > reduc_size )
    {
#ifdef EXTENDED_PRECISION
        const unsigned int partial_dest = lid + reduc_size;
        if (thread_lane < reduc_size)
            cur_sum  = two_sum(cur_sum, partial[partial_dest], err);
        // We reuse the LDS entries to move the error values down into lower
        // threads. This saves LDS space, allowing higher occupancy, but requires
        // more barriers, which can reduce performance.
        barrier(CLK_LOCAL_MEM_FENCE);
        // Have all of those upper threads pass their temporary errors
        // into a location that the lower threads can read.
        partial[lid] = (thread_lane >= reduc_size) ? *err : partial[lid];
        barrier(CLK_LOCAL_MEM_FENCE);
        if (thread_lane < reduc_size) // Add those errors in.
        {
            *err += partial[partial_dest];
            partial[lid] = cur_sum;
        }
#else
        cur_sum += partial[lid + reduc_size];
        partial[lid] = cur_sum;
#endif
    }
    return cur_sum;
}

__kernel void
csrmv_adaptive(__global const FPTYPE * restrict const vals,
                       __global const unsigned int * restrict const cols,
                       __global const unsigned int * restrict const rowPtrs,
                       __global const FPTYPE * restrict const vec,
                       __global FPTYPE * restrict const out,
                       __global unsigned long * restrict const rowBlocks,
                       __global const FPTYPE * restrict const pAlpha,
                       __global const FPTYPE * restrict const pBeta)
{
   __local FPTYPE partialSums[BLOCKSIZE];
   const unsigned int gid = get_group_id(0);
   const unsigned int lid = get_local_id(0);
   const FPTYPE alpha = *pAlpha;
   const FPTYPE beta = *pBeta;

   // The row blocks buffer holds a packed set of information used to inform each
   // workgroup about how to do its work:
   //
   // |6666 5555 5555 5544 4444 4444 3333 3333|3322 2222|2222 1111 1111 1100 0000 0000|
   // |3210 9876 5432 1098 7654 3210 9876 5432|1098 7654|3210 9876 5432 1098 7654 3210|
   // |------------Row Information------------|--------^|---WG ID within a long row---|
   // |                                       |    flag/|or # reduce threads for short|
   //
   // The upper 32 bits of each rowBlock entry tell the workgroup the ID of the first
   // row it will be working on. When one workgroup calculates multiple rows, this
   // rowBlock entry and the next one tell it the range of rows to work on.
   // The lower 24 bits are used whenever multiple workgroups calculate a single long
   // row. This tells each workgroup its ID within that row, so it knows which
   // part of the row to operate on.
   // Alternately, on "short" row blocks, the lower bits are used to communicate
   // the number of threads that should be used for the reduction. Pre-calculating
   // this on the CPU-side results in a noticable performance uplift on many matrices.
   // Bit 24 is a flag bit used so that the multiple WGs calculating a long row can
   // know when the first workgroup for that row has finished initializing the output
   // value. While this bit is the same as the first workgroup's flag bit, this
   // workgroup will spin-loop.
   unsigned int row = ((rowBlocks[gid] >> (64-ROWBITS)) & ((1UL << ROWBITS) - 1UL));
   const unsigned int stop_row = ((rowBlocks[gid + 1] >> (64-ROWBITS)) & ((1UL << ROWBITS) - 1UL));
   const unsigned int num_rows = stop_row - row;

   // Get the "workgroup within this long row" ID out of the bottom bits of the row block.
   const unsigned int wg = rowBlocks[gid] & ((1 << WGBITS) - 1);

   // Any workgroup only calculates, at most, BLOCK_MULTIPLIER*BLOCKSIZE items in a row.
   // If there are more items in this row, we assign more workgroups.
   unsigned int vecStart = rowPtrs[row] + (wg * BLOCK_MULTIPLIER*BLOCKSIZE);
   unsigned int vecEnd = (rowPtrs[row + 1] > vecStart + BLOCK_MULTIPLIER*BLOCKSIZE) ? vecStart + BLOCK_MULTIPLIER*BLOCKSIZE : rowPtrs[row + 1];

   FPTYPE temp_sum = 0.;
   FPTYPE sumk_e = 0.;
   FPTYPE new_error = 0.;

   // If the next row block starts more than 2 rows away, then we choose CSR-Stream.
   // If this is zero (long rows) or one (final workgroup in a long row, or a single
   // row in a row block), we want to use the CSR-Vector algorithm(s).
   // We have found, through experimentation, that CSR-Vector is generally faster
   // when working on 2 rows, due to its simplicity and better reduction method.
   if (num_rows > ROWS_FOR_VECTOR)
   {
       // CSR-Stream case. See Sections III.A and III.B in the SC'14 paper:
       // "Efficient Sparse Matrix-Vector Multiplication on GPUs using the CSR Storage Format"
       // for a detailed description of CSR-Stream.
       // In a nutshell, the idea is to use all of the threads to stream the matrix
       // values into the local memory in a fast, coalesced manner. After that, the
       // per-row reductions are done out of the local memory, which is designed
       // to handle non-coalsced accesses.

       // The best method for reducing the local memory values depends on the number
       // of rows. The SC'14 paper discusses a "CSR-Scalar" style reduction where
       // each thread reduces its own row. This yields good performance if there
       // are many (relatively short) rows. However, if they are few (relatively
       // long) rows, it's actually better to perform a tree-style reduction where
       // multiple threads team up to reduce the same row.

       // The calculation below tells you how many threads this workgroup can allocate
       // to each row, assuming that every row gets the same number of threads.
       // We want the closest lower (or equal) power-of-2 to this number --
       // that is how many threads can work in each row's reduction using our algorithm.
       // For instance, with workgroup size 256, 2 rows = 128 threads, 3 rows = 64
       // threads, 4 rows = 64 threads, 5 rows = 32 threads, etc.
       //int numThreadsForRed = get_local_size(0) >> ((CHAR_BIT*sizeof(unsigned int))-clz(num_rows-1));
       const unsigned int numThreadsForRed = wg; // Same calculation as above, done on host.

       // Stream all of this row block's matrix values into local memory.
       // Perform the matvec in parallel with this work.
       const unsigned int col = rowPtrs[row] + lid;
      if (gid != (get_num_groups(0) - 1))
      {
          for(int i = 0; i < BLOCKSIZE; i += WGSIZE)
              partialSums[lid + i] = alpha * vals[col + i] * vec[cols[col + i]];
      }
      else
      {
          // This is required so that we stay in bounds for vals[] and cols[].
          // Otherwise, if the matrix's endpoints don't line up with BLOCKSIZE,
          // we will buffer overflow. On today's dGPUs, this doesn't cause problems.
          // The values are within a dGPU's page, which is zeroed out on allocation.
          // However, this may change in the future (e.g. with shared virtual memory.)
          // This causes a minor performance loss because this is the last workgroup
          // to be launched, and this loop can't be unrolled.
          const unsigned int max_to_load = rowPtrs[stop_row] - rowPtrs[row];
          for(int i = 0; i < max_to_load; i += WGSIZE)
              partialSums[lid + i] = alpha * vals[col + i] * vec[cols[col + i]];
      }
      barrier(CLK_LOCAL_MEM_FENCE);

      if(numThreadsForRed > 1)
      {
          // In this case, we want to have the workgroup perform a tree-style reduction
          // of each row. {numThreadsForRed} adjacent threads team up to linearly reduce
          // a row into {numThreadsForRed} locations in local memory.
          // After that, the entire workgroup does a parallel reduction, and each
          // row ends up with an individual answer.

          // {numThreadsForRed} adjacent threads all work on the same row, so their
          // start and end values are the same.
          // numThreadsForRed guaranteed to be a power of two, so the clz code below
          // avoids an integer divide. ~2% perf gain in EXTRA_PRECISION.
          //size_t st = lid/numThreadsForRed;
          const unsigned int local_row = row + (lid >> (31 - clz(numThreadsForRed)));
          const unsigned int local_first_val = rowPtrs[local_row] - rowPtrs[row];
          const unsigned int local_last_val = rowPtrs[local_row + 1] - rowPtrs[row];
          const unsigned int threadInBlock = lid & (numThreadsForRed - 1);

          // Not all row blocks are full -- they may have an odd number of rows. As such,
          // we need to ensure that adjacent-groups only work on real data for this rowBlock.
          if(local_row < stop_row)
          {
              // only works when numThreadsForRed is a power of 2
              for(int i = 0; i < workForEachThread; i++)
                  temp_sum = two_sum(temp_sum, partialSums[local_first_val + i*numThreadsForRed + threadInBlock], &sumk_e);

              // The last few values (the remainder of this row) also need to be aded in.
              int local_cur_val = local_first_val + numThreadsForRed*workForEachThread;
              if(threadInBlock < local_last_val - local_cur_val)
                  temp_sum = two_sum(temp_sum, partialSums[local_cur_val + threadInBlock], &sumk_e);
          }
          barrier(CLK_LOCAL_MEM_FENCE);

          temp_sum = two_sum(temp_sum, sumk_e, &new_error);
          partialSums[lid] = temp_sum;

          // Step one of this two-stage reduction is done. Now each row has {numThreadsForRed}
          // values sitting in the local memory. This means that, roughly, the beginning of
          // LDS is full up to {workgroup size} entries.
          // Now we perform a parallel reduction that sums together the answers for each
          // row in parallel, leaving us an answer in 'temp_sum' for each row.
          for (int i = (WGSIZE >> 1); i > 0; i >>= 1)
          {
              barrier( CLK_LOCAL_MEM_FENCE );
              temp_sum = sum2_reduce(temp_sum, &new_error, partialSums, lid, threadInBlock, numThreadsForRed, i);
          }

          if (threadInBlock == 0 && local_row < stop_row)
          {
              // All of our write-outs check to see if the output vector should first be zeroed.
              // If so, just do a write rather than a read-write. Measured to be a slight (~5%)
              // performance improvement.
              if (beta != 0.)
                  temp_sum = two_fma(beta, out[local_row], temp_sum, &new_error);
              out[local_row] = temp_sum + new_error;
          }
      }
      else
      {
          // In this case, we want to have each thread perform the reduction for a single row.
          // Essentially, this looks like performing CSR-Scalar, except it is computed out of local memory.
          // However, this reduction is also much faster than CSR-Scalar, because local memory
          // is designed for scatter-gather operations.
          // We need a while loop because there may be more rows than threads in the WG.
          unsigned int local_row = row + lid;
          while(local_row < stop_row)
          {
              int local_first_val = (rowPtrs[local_row] - rowPtrs[row]);
              int local_last_val = rowPtrs[local_row + 1] - rowPtrs[row];
              temp_sum = 0.;
              sumk_e = 0.;
              for (int local_cur_val = local_first_val; local_cur_val < local_last_val; local_cur_val++)
                  temp_sum = two_sum(partialSums[local_cur_val], temp_sum, &sumk_e);

              // After you've done the reduction into the temp_sum register,
              // put that into the output for each row.
              if (beta != 0.)
                  temp_sum = two_fma(beta, out[local_row], temp_sum, &sumk_e);
              out[local_row] = temp_sum + sumk_e;
              local_row += WGSIZE;
          }
      }
   }
   else if (num_rows >= 1 && !wg) // CSR-Vector case.
   {
       // ^^ The above check says that if this workgroup is supposed to work on <= ROWS_VECTOR
       // number of rows then we should do the CSR-Vector algorithm. If we want this row to be
       // done with CSR-LongRows, then all of its workgroups (except the last one) will have the
       // same stop_row and row. The final workgroup in a LongRow will have stop_row and row
       // different, but the internal "wg" number will be non-zero.

       // If this workgroup is operating on multiple rows (because CSR-Stream is poor for small
       // numbers of rows), then it needs to iterate until it reaches the stop_row.
       // We don't check <= stop_row because of the potential for unsigned overflow.
       while (row < stop_row)
       {
           // Any workgroup only calculates, at most, BLOCKSIZE items in this row.
           // If there are more items in this row, we use CSR-LongRows.
           temp_sum = 0.;
           sumk_e = 0.;
           new_error = 0.;
           vecStart = rowPtrs[row];
           vecEnd = rowPtrs[row+1];

           // Load in a bunch of partial results into your register space, rather than LDS (no contention)
           // Then dump the partially reduced answers into the LDS for inter-work-item reduction.
           // Using a long induction variable to make sure unsigned int overflow doesn't break things.
           for (long j = vecStart + lid; j < vecEnd; j+=WGSIZE)
           {
               const unsigned int col = cols[(unsigned int)j];
               temp_sum = two_fma(alpha*vals[(unsigned int)j], vec[col], temp_sum, &sumk_e);
           }

           temp_sum = two_sum(temp_sum, sumk_e, &new_error);
           partialSums[lid] = temp_sum;

           // Reduce partial sums
           for (int i = (WGSIZE >> 1); i > 0; i >>= 1)
           {
               barrier( CLK_LOCAL_MEM_FENCE);
               temp_sum = sum2_reduce(temp_sum, &new_error, partialSums, lid, lid, WGSIZE, i);
           }

           if (lid == 0UL)
           {
               if (beta != 0.)
                   temp_sum = two_fma(beta, out[row], temp_sum, &new_error);
               out[row] = temp_sum + new_error;
           }
           row++;
       }
   }
   else
   {
       // In CSR-LongRows, we have more than one workgroup calculating this row.
       // The output values for those types of rows are stored using atomic_add, because
       // more than one parallel workgroup's value makes up the final answer.
       // Unfortunately, this makes it difficult to do y=Ax, rather than y=Ax+y, because
       // the values still left in y will be added in using the atomic_add.
       //
       // Our solution is to have the first workgroup in one of these long-rows cases
       // properly initaizlie the output vector. All the other workgroups working on this
       // row will spin-loop until that workgroup finishes its work.

       // First, figure out which workgroup you are in the row. Bottom 24 bits.
       // You can use that to find the global ID for the first workgroup calculating
       // this long row.
       const unsigned int first_wg_in_row = gid - (rowBlocks[gid] & ((1UL << WGBITS) - 1UL));
       const unsigned int compare_value = rowBlocks[gid] & (1UL << WGBITS);

       // Bit 24 in the first workgroup is the flag that everyone waits on.
       if(gid == first_wg_in_row && lid == 0UL)
       {
           // The first workgroup handles the output initialization.
           if (beta != 0.)
               out[row] *= beta;
           else
               out[row] = 0.;
#ifdef EXTENDED_PRECISION
           rowBlocks[get_num_groups(0) + gid + 1] = 0UL;
#endif
           atom_xor(&rowBlocks[first_wg_in_row], (1UL << WGBITS)); // Release other workgroups.
       }
       // For every other workgroup, bit 24 holds the value they wait on.
       // If your bit 24 == first_wg's bit 24, you spin loop.
       // The first workgroup will eventually flip this bit, and you can move forward.
       barrier(CLK_GLOBAL_MEM_FENCE);
       while(gid != first_wg_in_row &&
               lid == 0UL &&
               ((atom_max(&rowBlocks[first_wg_in_row], 0UL) & (1UL << WGBITS)) == compare_value));
       barrier(CLK_GLOBAL_MEM_FENCE);

       // After you've passed the barrier, update your local flag to make sure that
       // the next time through, you know what to wait on.
       if (gid != first_wg_in_row && lid == 0UL)
           rowBlocks[gid] ^= (1UL << WGBITS);

       // All but the final workgroup in a long-row collaboration have the same start_row
       // and stop_row. They only run for one iteration.
       // Load in a bunch of partial results into your register space, rather than LDS (no contention)
       // Then dump the partially reduced answers into the LDS for inter-work-item reduction.
       const unsigned int col = vecStart + lid;
       if (row == stop_row) // inner thread, we can hardcode/unroll this loop
       {
           // Don't put BLOCK_MULTIPLIER*BLOCKSIZE as the stop point, because
           // some GPU compilers will *aggressively* unroll this loop.
           // That increases register pressure and reduces occupancy.
           for (int j = 0; j < (int)(vecEnd - col); j += WGSIZE)
           {
               temp_sum = two_fma(alpha*vals[col + j], vec[cols[col + j]], temp_sum, &sumk_e);
#if 2*WGSIZE <= BLOCK_MULTIPLIER*BLOCKSIZE
               // If you can, unroll this loop once. It somewhat helps performance.
               j += WGSIZE;
               temp_sum = two_fma(alpha*vals[col + j], vec[cols[col + j]], temp_sum, &sumk_e);
#endif
           }
       }
       else
       {
           for(int j = 0; j < (int)(vecEnd - col); j += WGSIZE)
               temp_sum = two_fma(alpha*vals[col + j], vec[cols[col + j]], temp_sum, &sumk_e);
       }

       temp_sum = two_sum(temp_sum, sumk_e, &new_error);
       partialSums[lid] = temp_sum;

       // Reduce partial sums
       for (int i = (WGSIZE >> 1); i > 0; i >>= 1)
       {
           barrier( CLK_LOCAL_MEM_FENCE);
           temp_sum = sum2_reduce(temp_sum, &new_error, partialSums, lid, lid, WGSIZE, i);
       }

       if (lid == 0UL)
       {
           atomic_two_sum_float(&out[row], temp_sum, &new_error);

#ifdef EXTENDED_PRECISION
           unsigned int error_loc = get_num_groups(0) + first_wg_in_row + 1;
           // The last half of the rowBlocks buffer is used to hold errors.
           atomic_add_float(&(rowBlocks[error_loc]), new_error);
           // Coordinate across all of the workgroups in this coop in order to have
           // the last workgroup fix up the error values.
           // If this workgroup's row is different than the next workgroup's row
           // then this is the last workgroup -- it's this workgroup's job to add
           // the error values into the final sum.
           if (row != stop_row)
           {
               // Go forward once your ID is the same as the low order bits of the
               // coop's first workgroup. That value will be used to store the number
               // of threads that have completed so far. Once all the previous threads
               // are done, it's time to send out the errors!
               while((atom_max(&rowBlocks[first_wg_in_row], 0UL) & ((1UL << WGBITS) - 1)) != wg);

#ifdef DOUBLE
               new_error = as_double(rowBlocks[error_loc]);
#else
               new_error = as_float((int)rowBlocks[error_loc]);
#endif
               // Don't need to work atomically here, because this is the only workgroup
               // left working on this row.
               out[row] += new_error;
               rowBlocks[error_loc] = 0UL;

               // Reset the rowBlocks low order bits for next time.
               rowBlocks[first_wg_in_row] = rowBlocks[gid] - wg;
           }
           else
           {
               // Otherwise, increment the low order bits of the first thread in this
               // coop. We're using this to tell how many workgroups in a coop are done.
               // Do this with an atomic, since other threads may be doing this too.
               const unsigned long no_warn = atom_inc(&rowBlocks[first_wg_in_row]);
           }
#endif
       }
   }
}
)"


hacking on argmax.cl

hacking on argmax.cl

struct class_Eigen__array {
    int f0[1];
};
struct Eigen__Tuple {
    int f0;
    float f1;
};
struct Eigen__DSizes {
    struct class_Eigen__array f0;
};
struct Eigen__TensorEvaluator_0 {
    global int* f0;
    struct Eigen__DSizes f1;
    global struct Eigen__GpuDevice* f2;
    global struct class_Eigen__TensorMap* f3;
};
struct Eigen__GpuDevice {
    global struct class_Eigen__StreamInterface* f0;
    int f1;
    char f2[4];
};
struct class_Eigen__TensorMap {
    global int* f0;
    struct Eigen__DSizes f1;
    char f2[4];
};
struct class_Eigen__StreamInterface {
};
struct Eigen__TensorEvaluator_4 {
    global float* f0;
    struct Eigen__DSizes f1;
    global struct Eigen__GpuDevice* f2;
    global struct class_Eigen__TensorMap_5* f3;
};
struct class_Eigen__TensorMap_5 {
    global float* f0;
    struct Eigen__DSizes f1;
    char f2[4];
};
struct class_Eigen__array_nopointers {
    int f0[1];
};
struct Eigen__DSizes_nopointers {
    struct class_Eigen__array_nopointers f0;
};
struct Eigen__TensorEvaluator_0_nopointers {
    struct Eigen__DSizes_nopointers f0;
};
struct Eigen__TensorEvaluator_3 {
    struct Eigen__TensorEvaluator_4 f0;
};
struct class_Eigen__array_10 {
    char f0[1];
};
struct Eigen__Sizes {
    char f0;
};
struct Eigen__TensorEvaluator_4_nopointers {
    struct Eigen__DSizes_nopointers f0;
};
struct Eigen__TensorEvaluator_3_nopointers {
    struct Eigen__TensorEvaluator_4_nopointers f0;
};
struct class_Eigen__array_11 {
    int f0;
};
struct Eigen__internal__ArgMaxTupleReducer {
    char f0;
};
struct class_Eigen__array_10_nopointers {
    char f0[1];
};
struct Eigen__TensorEvaluator_9 {
    struct class_Eigen__array_10 f0;
    struct Eigen__Sizes f1;
    struct class_Eigen__array_11 f2;
    struct class_Eigen__array f3;
    struct class_Eigen__array f4;
    struct class_Eigen__array f5;
    struct Eigen__TensorEvaluator_3 f6;
    struct Eigen__internal__ArgMaxTupleReducer f7;
    global struct Eigen__Tuple* f8;
    global struct Eigen__GpuDevice* f9;
};
struct Eigen__Sizes_nopointers {
    char f0;
};
struct Eigen__TensorEvaluator_2 {
    struct Eigen__TensorEvaluator_3 f0;
    struct Eigen__TensorEvaluator_9 f1;
    int f2;
    struct class_Eigen__array f3;
    int f4;
    int f5;
};
struct class_Eigen__array_11_nopointers {
    int f0;
};
struct Eigen__TensorEvaluator {
    struct Eigen__TensorEvaluator_0 f0;
    struct Eigen__TensorEvaluator_2 f1;
};
struct Eigen__internal__ArgMaxTupleReducer_nopointers {
    char f0;
};
struct Eigen__TensorEvaluator_9_nopointers {
    struct class_Eigen__array_10_nopointers f0;
    struct Eigen__Sizes_nopointers f1;
    struct class_Eigen__array_11_nopointers f2;
    struct class_Eigen__array_nopointers f3;
    struct class_Eigen__array_nopointers f4;
    struct class_Eigen__array_nopointers f5;
    struct Eigen__TensorEvaluator_3_nopointers f6;
    struct Eigen__internal__ArgMaxTupleReducer_nopointers f7;
};
struct Eigen__TensorEvaluator_2_nopointers {
    struct Eigen__TensorEvaluator_3_nopointers f0;
    struct Eigen__TensorEvaluator_9_nopointers f1;
    int f2;
    struct class_Eigen__array_nopointers f3;
    int f4;
    int f5;
};
struct Eigen__TensorEvaluator_nopointers {
    struct Eigen__TensorEvaluator_0_nopointers f0;
    struct Eigen__TensorEvaluator_2_nopointers f1;
};

kernel void _ZN5Eigen8internal15EigenMetaKernelINS_15TensorEvaluatorIKNS_14TensorAssignOpINS_9TensorMapINS_6TensorIiLi1ELi1EiEELi16ENS_11MakePointerEEEKNS_20TensorTupleReducerOpINS0_18ArgMaxTupleReducerINS_5TupleIifEEEEKNS_5arrayIiLm1EEEKNS4_INS5_IfLi1ELi1EiEELi16ES7_EEEEEENS_9GpuDeviceEEEiEEvT_T0_(global struct Eigen__TensorEvaluator_nopointers* eval_nopointers, global int* eval_ptr0, uint eval_ptr0_offset, global float* eval_ptr1, uint eval_ptr1_offset, global float* eval_ptr2, uint eval_ptr2_offset, int size, local int *scratch);

kernel void _ZN5Eigen8internal15EigenMetaKernelINS_15TensorEvaluatorIKNS_14TensorAssignOpINS_9TensorMapINS_6TensorIiLi1ELi1EiEELi16ENS_11MakePointerEEEKNS_20TensorTupleReducerOpINS0_18ArgMaxTupleReducerINS_5TupleIifEEEEKNS_5arrayIiLm1EEEKNS4_INS5_IfLi1ELi1EiEELi16ES7_EEEEEENS_9GpuDeviceEEEiEEvT_T0_(global struct Eigen__TensorEvaluator_nopointers* eval_nopointers, global int* eval_ptr0, uint eval_ptr0_offset, global float* eval_ptr1, uint eval_ptr1_offset, global float* eval_ptr2, uint eval_ptr2_offset, int size, local int *scratch) {
    eval_ptr2 += eval_ptr2_offset;
    eval_ptr1 += eval_ptr1_offset;
    eval_ptr0 += eval_ptr0_offset;
struct Eigen__TensorEvaluator eval[1];
eval[0].f0.f1.f0.f0[0] = eval_nopointers[0].f0.f0.f0.f0[0];
eval[0].f1.f0.f0.f1.f0.f0[0] = eval_nopointers[0].f1.f0.f0.f0.f0.f0[0];
eval[0].f1.f1.f0.f0[0] = eval_nopointers[0].f1.f1.f0.f0[0];
eval[0].f1.f1.f1.f0 = eval_nopointers[0].f1.f1.f1.f0;
eval[0].f1.f1.f2.f0 = eval_nopointers[0].f1.f1.f2.f0;
eval[0].f1.f1.f3.f0[0] = eval_nopointers[0].f1.f1.f3.f0[0];

//eval_ptr0[0] = eval_nopointers[0].f1.f1.f3.f0[0] + 1;
//eval_ptr0[0] = 125.0f;
//return;

eval[0].f1.f1.f4.f0[0] = eval_nopointers[0].f1.f1.f4.f0[0];
eval[0].f1.f1.f5.f0[0] = eval_nopointers[0].f1.f1.f5.f0[0];
eval[0].f1.f1.f6.f0.f1.f0.f0[0] = eval_nopointers[0].f1.f1.f6.f0.f0.f0.f0[0];
eval[0].f1.f1.f7.f0 = eval_nopointers[0].f1.f1.f7.f0;
eval[0].f1.f2 = eval_nopointers[0].f1.f2;
eval[0].f1.f3.f0[0] = eval_nopointers[0].f1.f3.f0[0];
eval[0].f1.f4 = eval_nopointers[0].f1.f4;
eval[0].f1.f5 = eval_nopointers[0].f1.f5;
eval[0].f0.f0 = eval_ptr0;
eval[0].f1.f0.f0.f0 = eval_ptr1;
eval[0].f1.f1.f6.f0.f0 = eval_ptr2;

    bool v103;
    bool v111;
    bool v70;
    bool v86;
    bool v94;
    float v102;
    float v105;
    float v110;
    float v113;
    float v60;
    float max_so_far;
    float v69;
    float new_max_so_far;
    float v80;
    float v85;
    float v88;
    float v93;
    float v96;
    global float* input_data;
    global int* v126;
    global int* output_data;
    global struct Eigen__Tuple* v36;
    int v107;
    int v112;
    int v114;
    int v117;
    int v124;
    int v128;
    int group_id;
    int local_size;
    int local_id;
    int global_id;
    int num_groups;
    int inner_size;
    int v43;
    int inner_size_mod4;
    int global_pos;
    int v55;
    int v56;
    int segment_offset;
    int v58;
    int v61;
    int idx;
    int best_pos_so_far;
    int count;
    int candidate_pos;
    int new_best_pos_so_far;
    int new_idx;
    int new_count;
    int v77;
    int v78;
    int v79;
    int v81;
    int v82;
    int v90;
    int v99;
    int* v34;
    long v118;

    /* int group_id = call <unk> */;
    group_id = get_group_id(0);
    /* int local_size = call <unk> */;
    local_size = get_local_size(0);
    /* int v22 = mul local_size group_id */;
    /* int local_id = call <unk> */;
    local_id = get_local_id(0);
    /* int global_id = add v22 local_id */;
    global_id = (local_size * group_id) + local_id;
    /* int num_groups = call <unk> */;
    num_groups = get_num_groups(0);
    /* int v26 = mul num_groups local_size */;
    /* bool v27 = icmp global_id size */;
    /* if(v27) */
    if (global_id >= size) {
        goto exit;
    }

    /* struct Eigen__Tuple** v28 = getelementptr eval <unk> <unk> <unk> <unk> */;
    /* int* v30 = getelementptr eval v29 <unk> <unk> <unk> <unk> v29 */;
    /* float** v31 = getelementptr eval v29 <unk> <unk> <unk> <unk> <unk> */;
    /* int* v32 = getelementptr eval v29 <unk> <unk> */;
    /* int** v33 = getelementptr eval v29 <unk> <unk> */;
    /* int* v34 = getelementptr eval v29 <unk> <unk> */;
    v34 = &eval[0].f1.f4;
    /* long* v35 = bitcast v34 */;
    /* struct Eigen__Tuple* v36 = load v28 */;
    v36 = eval[0].f1.f1.f8;
    /* bool v37 = icmp v36 <unk> */;
    /* int inner_size = load v30 */;
    inner_size = eval[0].f1.f1.f3.f0[0];
    /* bool v40 = icmp inner_size <unk> */;
    /* float* input_data = load v31 */;
    input_data = eval_ptr2;
    /* int v43 = load v32 */;
    v43 = eval[0].f1.f2;
    /* bool v44 = icmp v43 0 */;
    /* int* output_data = load v33 */;
    output_data = eval_ptr0;
    /* int v46 = add inner_size <unk> */;
    /* int inner_size_mod4 = and inner_size <unk> */;
    inner_size_mod4 = inner_size & 3;
    /* bool v50 = icmp inner_size_mod4 0 */;
    /* bool v51 = icmp v46 v49 */;
    /* int global_pos = phi global_id */
    global_pos = global_id;

block3:;
    /* if(v37) */
    if (v36 == 0) {
        goto block5;
    }

    /* long v53 = sext global_pos */;
    /* int* v54 = getelementptr v36 v53 0 */;
    /* int v55 = load v54 */;
//eval_ptr0[0] = global_pos;
//eval_ptr0[0] = 125.0f;
//return;
    v56 = v36[global_pos].f0;
    /* int v56 = phi v55 */
    // v56 = v55;
    goto block15;
block5:;
    /* int segment_offset = mul inner_size global_pos */;
    segment_offset = inner_size * global_pos;
    /* if(v40) */
    if (inner_size <= 0) {
        /* int v56 = phi 0 */
        v56 = 0;
        goto block15;
    }

    /* if(v50) */
    if (inner_size_mod4 == 0) {
        /* int v58 = phi 0 */
        v58 = 0;
        /* float v60 = phi -INFINITY */
        v60 = -INFINITY;
        /* int v61 = phi 0 */
        v61 = 0;
        goto block10;
    }

    /* int idx = phi 0 */
    idx = 0;
    /* float max_so_far = phi -INFINITY */
    max_so_far = -INFINITY;
    /* int best_pos_so_far = phi 0 */
    best_pos_so_far = 0;
    /* int count = phi inner_size_mod4 */
    count = inner_size_mod4;

block8:;
    /* int candidate_pos = add idx segment_offset */;
    candidate_pos = idx + segment_offset;
    /* long v67 = sext candidate_pos */;
    /* float* v68 = getelementptr input_data v67 */;
    /* float v69 = load v68 */;
    // v69 = input_data[candidate_pos];
    /* bool v70 = fcmp v69 max_so_far */;
    // v70 = v69 <= max_so_far;
    /* int new_best_pos_so_far = select v70 best_pos_so_far candidate_pos */;
    new_best_pos_so_far = input_data[candidate_pos] <= max_so_far ? best_pos_so_far : candidate_pos;
    /* float new_max_so_far = select v70 max_so_far v69 */;
    new_max_so_far = input_data[candidate_pos] <= max_so_far ? max_so_far : input_data[candidate_pos];
    /* int new_idx = add idx <unk> */;
    new_idx = idx + 1;
    /* int new_count = add count v47 */;
    new_count = count - 1;
    /* bool v76 = icmp new_count 0 */;
    /* if(v76) */
    if(new_count != 0) {
        /* int idx = phi new_idx */
        idx = new_idx;
        /* float max_so_far = phi new_max_so_far */
        max_so_far = new_max_so_far;
        /* int best_pos_so_far = phi new_best_pos_so_far */
        best_pos_so_far = new_best_pos_so_far;
        /* int count = phi new_count */
        count = new_count;
        goto block8;
    }

    /* int v77 = phi new_best_pos_so_far */
    v77 = new_best_pos_so_far;
    /* int v58 = phi new_idx */
    v58 = new_idx;
    /* float v60 = phi new_max_so_far */
    v60 = new_max_so_far;
    /* int v61 = phi new_best_pos_so_far */
    v61 = new_best_pos_so_far;

block10:;
    /* if(v51) */
    if (inner_size < 4) {
        /* int v78 = phi v77 */
        v78 = v77;
        goto block14;
    }

    /* int v79 = phi v58 */
    v79 = v58;
    /* float v80 = phi v60 */
    v80 = v60;
    /* int v81 = phi v61 */
    v81 = v61;

block12:;
    /* int v82 = add v79 segment_offset */;
    v82 = v79 + segment_offset;
    /* long v83 = sext v82 */;
    /* float* v84 = getelementptr input_data v83 */;
    /* float v85 = load v84 */;
    v85 = (&(input_data[v82]))[0];
    /* bool v86 = fcmp v85 v80 */;
    v86 = v85 <= v80;
    /* int v87 = select v86 v81 v82 */;
    /* float v88 = select v86 v80 v85 */;
    v88 = v86 ? v80 : v85;
    /* int v89 = add v79 v74 */;
    /* int v90 = add v89 segment_offset */;
    v90 = (v79 + 1) + segment_offset;
    /* long v91 = sext v90 */;
    /* float* v92 = getelementptr input_data v91 */;
    /* float v93 = load v92 */;
    v93 = (&(input_data[v90]))[0];
    /* bool v94 = fcmp v93 v88 */;
    v94 = v93 <= v88;
    /* int v95 = select v94 v87 v90 */;
    /* float v96 = select v94 v88 v93 */;
    v96 = v94 ? v88 : v93;
    /* int v97 = add v79 <unk> */;
    /* int v99 = add v97 segment_offset */;
    v99 = (v79 + 2) + segment_offset;
    /* long v100 = sext v99 */;
    /* float* v101 = getelementptr input_data v100 */;
    /* float v102 = load v101 */;
    v102 = (&(input_data[v99]))[0];
    /* bool v103 = fcmp v102 v96 */;
    v103 = v102 <= v96;
    /* int v104 = select v103 v95 v99 */;
    /* float v105 = select v103 v96 v102 */;
    v105 = v103 ? v96 : v102;
    /* int v106 = add v79 v49 */;
    /* int v107 = add v106 segment_offset */;
    v107 = (v79 + 3) + segment_offset;
    /* long v108 = sext v107 */;
    /* float* v109 = getelementptr input_data v108 */;
    /* float v110 = load v109 */;
    v110 = (&(input_data[v107]))[0];
    /* bool v111 = fcmp v110 v105 */;
    v111 = v110 <= v105;
    /* int v112 = select v111 v104 v107 */;
    v112 = v111 ? (v103 ? (v94 ? (v86 ? v81 : v82) : v90) : v99) : v107;
    /* float v113 = select v111 v105 v110 */;
    v113 = v111 ? v105 : v110;
    /* int v114 = add v79 <unk> */;
    v114 = v79 + 4;
    /* bool v116 = icmp v114 inner_size */;
    /* if(v116) */
    if(v114 != inner_size) {
        /* int v79 = phi v114 */
        v79 = v114;
        /* float v80 = phi v113 */
        v80 = v113;
        /* int v81 = phi v112 */
        v81 = v112;
        goto block12;
    }

    /* int v78 = phi v112 */
    v78 = v112;

block14:;
    /* int v56 = phi v78 */
    v56 = v78;

block15:;
    /* if(v44) */
    if (v43 < 0) {
        /* int v117 = phi v56 */
        output_data[0] =12344;
        return;
        v117 = v56;
        goto block17;
    }

    /* long v118 = load v35 */;
    v118 = ((long*)v34)[0];
    /* int v119 = trunc v118 */;
    /* int v120 = srem v56 v119 */;
    /* long v121 = lshr v118 <unk> */;
    /* int v123 = trunc v121 */;
    /* int v124 = sdiv v120 v123 */;
    v124 = (v56 % (int)v118) / (int)(v118 >> 32);
    /* int v117 = phi v124 */
    v117 = v124;

block17:;
    /* long v125 = sext global_pos */;
    /* int* v126 = getelementptr output_data v125 */;
    //v126 = &output_data[global_pos];
    /* void v127 = store v117 v126 */;
    output_data[global_pos] = v117;
    /* int v128 = add global_pos v26 */;
    global_pos += (num_groups * local_size);
    /* bool v129 = icmp v128 size */;
    /* if(v129) */
    if (global_pos < size) {
        /* int global_pos = phi v128 */
        goto block3;
    }
v18:;
exit:;
    return;
}

struct class_Eigen__array {
    int f0[1];
};
struct Eigen__Tuple {
    int f0;
    float f1;
};
struct Eigen__DSizes {
    struct class_Eigen__array f0;
};
struct Eigen__TensorEvaluator_0 {
    global int* f0;
    struct Eigen__DSizes f1;
    global struct Eigen__GpuDevice* f2;
    global struct class_Eigen__TensorMap* f3;
};
struct Eigen__GpuDevice {
    global struct class_Eigen__StreamInterface* f0;
    int f1;
    char f2[4];
};
struct class_Eigen__TensorMap {
    global int* f0;
    struct Eigen__DSizes f1;
    char f2[4];
};
struct class_Eigen__StreamInterface {
};
struct Eigen__TensorEvaluator_4 {
    global float* f0;
    struct Eigen__DSizes f1;
    global struct Eigen__GpuDevice* f2;
    global struct class_Eigen__TensorMap_5* f3;
};
struct class_Eigen__TensorMap_5 {
    global float* f0;
    struct Eigen__DSizes f1;
    char f2[4];
};
struct class_Eigen__array_nopointers {
    int f0[1];
};
struct Eigen__DSizes_nopointers {
    struct class_Eigen__array_nopointers f0;
};
struct Eigen__TensorEvaluator_0_nopointers {
    struct Eigen__DSizes_nopointers f0;
};
struct Eigen__TensorEvaluator_3 {
    struct Eigen__TensorEvaluator_4 f0;
};
struct class_Eigen__array_10 {
    char f0[1];
};
struct Eigen__Sizes {
    char f0;
};
struct Eigen__TensorEvaluator_4_nopointers {
    struct Eigen__DSizes_nopointers f0;
};
struct Eigen__TensorEvaluator_3_nopointers {
    struct Eigen__TensorEvaluator_4_nopointers f0;
};
struct class_Eigen__array_11 {
    int f0;
};
struct Eigen__internal__ArgMaxTupleReducer {
    char f0;
};
struct class_Eigen__array_10_nopointers {
    char f0[1];
};
struct Eigen__TensorEvaluator_9 {
    struct class_Eigen__array_10 f0;
    struct Eigen__Sizes f1;
    struct class_Eigen__array_11 f2;
    struct class_Eigen__array f3;
    struct class_Eigen__array f4;
    struct class_Eigen__array f5;
    struct Eigen__TensorEvaluator_3 f6;
    struct Eigen__internal__ArgMaxTupleReducer f7;
    global struct Eigen__Tuple* f8;
    global struct Eigen__GpuDevice* f9;
};
struct Eigen__Sizes_nopointers {
    char f0;
};
struct Eigen__TensorEvaluator_2 {
    struct Eigen__TensorEvaluator_3 f0;
    struct Eigen__TensorEvaluator_9 f1;
    int f2;
    struct class_Eigen__array f3;
    int f4;
    int f5;
};
struct class_Eigen__array_11_nopointers {
    int f0;
};
struct Eigen__TensorEvaluator {
    struct Eigen__TensorEvaluator_0 f0;
    struct Eigen__TensorEvaluator_2 f1;
};
struct Eigen__internal__ArgMaxTupleReducer_nopointers {
    char f0;
};
struct Eigen__TensorEvaluator_9_nopointers {
    struct class_Eigen__array_10_nopointers f0;
    struct Eigen__Sizes_nopointers f1;
    struct class_Eigen__array_11_nopointers f2;
    struct class_Eigen__array_nopointers f3;
    struct class_Eigen__array_nopointers f4;
    struct class_Eigen__array_nopointers f5;
    struct Eigen__TensorEvaluator_3_nopointers f6;
    struct Eigen__internal__ArgMaxTupleReducer_nopointers f7;
};
struct Eigen__TensorEvaluator_2_nopointers {
    struct Eigen__TensorEvaluator_3_nopointers f0;
    struct Eigen__TensorEvaluator_9_nopointers f1;
    int f2;
    struct class_Eigen__array_nopointers f3;
    int f4;
    int f5;
};
struct Eigen__TensorEvaluator_nopointers {
    struct Eigen__TensorEvaluator_0_nopointers f0;
    struct Eigen__TensorEvaluator_2_nopointers f1;
};

kernel void _ZN5Eigen8internal15EigenMetaKernelINS_15TensorEvaluatorIKNS_14TensorAssignOpINS_9TensorMapINS_6TensorIiLi1ELi1EiEELi16ENS_11MakePointerEEEKNS_20TensorTupleReducerOpINS0_18ArgMaxTupleReducerINS_5TupleIifEEEEKNS_5arrayIiLm1EEEKNS4_INS5_IfLi1ELi1EiEELi16ES7_EEEEEENS_9GpuDeviceEEEiEEvT_T0_(global struct Eigen__TensorEvaluator_nopointers* eval_nopointers, global int* eval_ptr0, uint eval_ptr0_offset, global float* eval_ptr1, uint eval_ptr1_offset, global float* eval_ptr2, uint eval_ptr2_offset, int size, local int *scratch);

kernel void _ZN5Eigen8internal15EigenMetaKernelINS_15TensorEvaluatorIKNS_14TensorAssignOpINS_9TensorMapINS_6TensorIiLi1ELi1EiEELi16ENS_11MakePointerEEEKNS_20TensorTupleReducerOpINS0_18ArgMaxTupleReducerINS_5TupleIifEEEEKNS_5arrayIiLm1EEEKNS4_INS5_IfLi1ELi1EiEELi16ES7_EEEEEENS_9GpuDeviceEEEiEEvT_T0_(global struct Eigen__TensorEvaluator_nopointers* eval_nopointers, global int* eval_ptr0, uint eval_ptr0_offset, global float* eval_ptr1, uint eval_ptr1_offset, global float* eval_ptr2, uint eval_ptr2_offset, int size, local int *scratch) {
    eval_ptr2 += eval_ptr2_offset;
    eval_ptr1 += eval_ptr1_offset;
    eval_ptr0 += eval_ptr0_offset;
struct Eigen__TensorEvaluator eval[1];
eval[0].f0.f1.f0.f0[0] = eval_nopointers[0].f0.f0.f0.f0[0];
eval[0].f1.f0.f0.f1.f0.f0[0] = eval_nopointers[0].f1.f0.f0.f0.f0.f0[0];
eval[0].f1.f1.f0.f0[0] = eval_nopointers[0].f1.f1.f0.f0[0];
eval[0].f1.f1.f1.f0 = eval_nopointers[0].f1.f1.f1.f0;
eval[0].f1.f1.f2.f0 = eval_nopointers[0].f1.f1.f2.f0;
eval[0].f1.f1.f3.f0[0] = eval_nopointers[0].f1.f1.f3.f0[0];
eval[0].f1.f1.f4.f0[0] = eval_nopointers[0].f1.f1.f4.f0[0];
eval[0].f1.f1.f5.f0[0] = eval_nopointers[0].f1.f1.f5.f0[0];
eval[0].f1.f1.f6.f0.f1.f0.f0[0] = eval_nopointers[0].f1.f1.f6.f0.f0.f0.f0[0];
eval[0].f1.f1.f7.f0 = eval_nopointers[0].f1.f1.f7.f0;
eval[0].f1.f2 = eval_nopointers[0].f1.f2;
eval[0].f1.f3.f0[0] = eval_nopointers[0].f1.f3.f0[0];
eval[0].f1.f4 = eval_nopointers[0].f1.f4;
eval[0].f1.f5 = eval_nopointers[0].f1.f5;
eval[0].f0.f0 = eval_ptr0;
eval[0].f1.f0.f0.f0 = eval_ptr1;
eval[0].f1.f1.f6.f0.f0 = eval_ptr2;

    bool v103;
    bool v111;
    bool v70;
    bool v86;
    bool v94;
    float v102;
    float v105;
    float v110;
    float v113;
    float v60;
    float max_so_far;
    float v69;
    float new_max_so_far;
    float v80;
    float v85;
    float v88;
    float v93;
    float v96;
    global float* input_data;
    global int* v126;
    global int* output_data;
    global struct Eigen__Tuple* v36;
    int v107;
    int v112;
    int v114;
    int v117;
    int v124;
    int v128;
    int group_id;
    int local_size;
    int local_id;
    int global_id;
    int num_groups;
    int inner_size;
    int v43;
    int inner_size_mod4;
    int global_pos;
    int v55;
    int v56;
    int segment_offset;
    int v58;
    int v61;
    int idx;
    int best_pos_so_far;
    int count;
    int candidate_pos;
    int new_best_pos_so_far;
    int new_idx;
    int new_count;
    int v77;
    int v78;
    int v79;
    int v81;
    int v82;
    int v90;
    int v99;
    int* v34;
    long v118;

    /* int group_id = call <unk> */;
    group_id = get_group_id(0);
    /* int local_size = call <unk> */;
    local_size = get_local_size(0);
    /* int v22 = mul local_size group_id */;
    /* int local_id = call <unk> */;
    local_id = get_local_id(0);
    /* int global_id = add v22 local_id */;
    global_id = (local_size * group_id) + local_id;
    /* int num_groups = call <unk> */;
    num_groups = get_num_groups(0);
    /* int v26 = mul num_groups local_size */;
    /* bool v27 = icmp global_id size */;
    /* if(v27) */
    if (global_id >= size) {
        goto exit;
    }

    /* struct Eigen__Tuple** v28 = getelementptr eval <unk> <unk> <unk> <unk> */;
    /* int* v30 = getelementptr eval v29 <unk> <unk> <unk> <unk> v29 */;
    /* float** v31 = getelementptr eval v29 <unk> <unk> <unk> <unk> <unk> */;
    /* int* v32 = getelementptr eval v29 <unk> <unk> */;
    /* int** v33 = getelementptr eval v29 <unk> <unk> */;
    /* int* v34 = getelementptr eval v29 <unk> <unk> */;
    v34 = eval[0].f1.f4;
    /* long* v35 = bitcast v34 */;
    /* struct Eigen__Tuple* v36 = load v28 */;
    v36 = eval[0].f1.f1.f8;
    /* bool v37 = icmp v36 <unk> */;
    /* int inner_size = load v30 */;
    inner_size = eval[0].f1.f1.f3.f0;
    /* bool v40 = icmp inner_size <unk> */;
    /* float* input_data = load v31 */;
    input_data = eval_ptr2;
    /* int v43 = load v32 */;
    v43 = eval[0].f1.f2;
    /* bool v44 = icmp v43 0 */;
    /* int* output_data = load v33 */;
    output_data = eval_ptr0;
    /* int v46 = add inner_size <unk> */;
    /* int inner_size_mod4 = and inner_size <unk> */;
    inner_size_mod4 = inner_size & 3;
    /* bool v50 = icmp inner_size_mod4 0 */;
    /* bool v51 = icmp v46 v49 */;
    /* int global_pos = phi global_id */
    global_pos = global_id;

block3:;
    /* if(v37) */
    if (v36 == 0) {
        goto block5;
    }

    /* long v53 = sext global_pos */;
    /* int* v54 = getelementptr v36 v53 0 */;
    /* int v55 = load v54 */;
    v56 = v36[global_pos].f0;
    /* int v56 = phi v55 */
    // v56 = v55;
    goto block15;
block5:;
    /* int segment_offset = mul inner_size global_pos */;
    segment_offset = inner_size * global_pos;
    /* if(v40) */
    if (inner_size <= 0) {
        /* int v56 = phi 0 */
        v56 = 0;
        goto block15;
    }

    /* if(v50) */
    if (inner_size_mod4 == 0) {
        /* int v58 = phi 0 */
        v58 = 0;
        /* float v60 = phi -INFINITY */
        v60 = -INFINITY;
        /* int v61 = phi 0 */
        v61 = 0;
        goto block10;
    }

    /* int idx = phi 0 */
    idx = 0;
    /* float max_so_far = phi -INFINITY */
    max_so_far = -INFINITY;
    /* int best_pos_so_far = phi 0 */
    best_pos_so_far = 0;
    /* int count = phi inner_size_mod4 */
    count = inner_size_mod4;

block8:;
    /* int candidate_pos = add idx segment_offset */;
    candidate_pos = idx + segment_offset;
    /* long v67 = sext candidate_pos */;
    /* float* v68 = getelementptr input_data v67 */;
    /* float v69 = load v68 */;
    // v69 = input_data[candidate_pos];
    /* bool v70 = fcmp v69 max_so_far */;
    // v70 = v69 <= max_so_far;
    /* int new_best_pos_so_far = select v70 best_pos_so_far candidate_pos */;
    new_best_pos_so_far = input_data[candidate_pos] <= max_so_far ? best_pos_so_far : candidate_pos;
    /* float new_max_so_far = select v70 max_so_far v69 */;
    new_max_so_far = input_data[candidate_pos] <= max_so_far ? max_so_far : input_data[candidate_pos];
    /* int new_idx = add idx <unk> */;
    new_idx = idx + 1;
    /* int new_count = add count v47 */;
    new_count = count - 1;
    /* bool v76 = icmp new_count 0 */;
    /* if(v76) */
    if(new_count != 0) {
        /* int idx = phi new_idx */
        idx = new_idx;
        /* float max_so_far = phi new_max_so_far */
        max_so_far = new_max_so_far;
        /* int best_pos_so_far = phi new_best_pos_so_far */
        best_pos_so_far = new_best_pos_so_far;
        /* int count = phi new_count */
        count = new_count;
        goto block8;
    }

    /* int v77 = phi new_best_pos_so_far */
    v77 = new_best_pos_so_far;
    /* int v58 = phi new_idx */
    v58 = new_idx;
    /* float v60 = phi new_max_so_far */
    v60 = new_max_so_far;
    /* int v61 = phi new_best_pos_so_far */
    v61 = new_best_pos_so_far;

block10:;
    /* if(v51) */
    if (inner_size < 4) {
        /* int v78 = phi v77 */
        v78 = v77;
        goto block14;
    }

    /* int v79 = phi v58 */
    v79 = v58;
    /* float v80 = phi v60 */
    v80 = v60;
    /* int v81 = phi v61 */
    v81 = v61;

block12:;
    /* int v82 = add v79 segment_offset */;
    v82 = v79 + segment_offset;
    /* long v83 = sext v82 */;
    /* float* v84 = getelementptr input_data v83 */;
    /* float v85 = load v84 */;
    v85 = (&(input_data[v82]))[0];
    /* bool v86 = fcmp v85 v80 */;
    v86 = v85 <= v80;
    /* int v87 = select v86 v81 v82 */;
    /* float v88 = select v86 v80 v85 */;
    v88 = v86 ? v80 : v85;
    /* int v89 = add v79 v74 */;
    /* int v90 = add v89 segment_offset */;
    v90 = (v79 + 1) + segment_offset;
    /* long v91 = sext v90 */;
    /* float* v92 = getelementptr input_data v91 */;
    /* float v93 = load v92 */;
    v93 = (&(input_data[v90]))[0];
    /* bool v94 = fcmp v93 v88 */;
    v94 = v93 <= v88;
    /* int v95 = select v94 v87 v90 */;
    /* float v96 = select v94 v88 v93 */;
    v96 = v94 ? v88 : v93;
    /* int v97 = add v79 <unk> */;
    /* int v99 = add v97 segment_offset */;
    v99 = (v79 + 2) + segment_offset;
    /* long v100 = sext v99 */;
    /* float* v101 = getelementptr input_data v100 */;
    /* float v102 = load v101 */;
    v102 = (&(input_data[v99]))[0];
    /* bool v103 = fcmp v102 v96 */;
    v103 = v102 <= v96;
    /* int v104 = select v103 v95 v99 */;
    /* float v105 = select v103 v96 v102 */;
    v105 = v103 ? v96 : v102;
    /* int v106 = add v79 v49 */;
    /* int v107 = add v106 segment_offset */;
    v107 = (v79 + 3) + segment_offset;
    /* long v108 = sext v107 */;
    /* float* v109 = getelementptr input_data v108 */;
    /* float v110 = load v109 */;
    v110 = (&(input_data[v107]))[0];
    /* bool v111 = fcmp v110 v105 */;
    v111 = v110 <= v105;
    /* int v112 = select v111 v104 v107 */;
    v112 = v111 ? (v103 ? (v94 ? (v86 ? v81 : v82) : v90) : v99) : v107;
    /* float v113 = select v111 v105 v110 */;
    v113 = v111 ? v105 : v110;
    /* int v114 = add v79 <unk> */;
    v114 = v79 + 4;
    /* bool v116 = icmp v114 inner_size */;
    /* if(v116) */
    if(v114 != inner_size) {
        /* int v79 = phi v114 */
        v79 = v114;
        /* float v80 = phi v113 */
        v80 = v113;
        /* int v81 = phi v112 */
        v81 = v112;
        goto block12;
    }

    /* int v78 = phi v112 */
    v78 = v112;

block14:;
    /* int v56 = phi v78 */
    v56 = v78;

block15:;
    /* if(v44) */
    if (v43 < 0) {
        /* int v117 = phi v56 */
        v117 = v56;
        goto block17;
    }

    /* long v118 = load v35 */;
    v118 = ((long*)v34)[0];
    /* int v119 = trunc v118 */;
    /* int v120 = srem v56 v119 */;
    /* long v121 = lshr v118 <unk> */;
    /* int v123 = trunc v121 */;
    /* int v124 = sdiv v120 v123 */;
    v124 = (v56 % (int)v118) / (int)(v118 >> 32);
    /* int v117 = phi v124 */
    v117 = v124;

block17:;
    /* long v125 = sext global_pos */;
    /* int* v126 = getelementptr output_data v125 */;
    //v126 = &output_data[global_pos];
    /* void v127 = store v117 v126 */;
    output_data[global_pos] = v117;
    /* int v128 = add global_pos v26 */;
    global_pos += (num_groups * local_size);
    /* bool v129 = icmp v128 size */;
    /* if(v129) */
    if (global_pos < size) {
        /* int global_pos = phi v128 */
        goto block3;
    }
v18:;
exit:;
    return;
}