A minor fix.

A minor fix.

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Shengen Yan,yanshengen@gmail.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

///////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////Macro for border type////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////////////

#if defined (DOUBLE_SUPPORT) && defined (cl_khr_fp64)
#pragma OPENCL EXTENSION cl_khr_fp64:enable
#define FPTYPE double
#else
#define FPTYPE float
#endif

#ifdef BORDER_CONSTANT
#elif defined BORDER_REPLICATE
#define EXTRAPOLATE(x, maxV) \
    { \
        x = max(min(x, maxV - 1), 0); \
    }
#elif defined BORDER_WRAP
#define EXTRAPOLATE(x, maxV) \
    { \
        if (x < 0) \
            x -= ((x - maxV + 1) / maxV) * maxV; \
        if (x >= maxV) \
            x %= maxV; \
    }
#elif defined(BORDER_REFLECT) || defined(BORDER_REFLECT101)
#define EXTRAPOLATE_(x, maxV, delta) \
    { \
        if (maxV == 1) \
            x = 0; \
        else \
            do \
            { \
                if ( x < 0 ) \
                    x = -x - 1 + delta; \
                else \
                    x = maxV - 1 - (x - maxV) - delta; \
            } \
            while (x >= maxV || x < 0); \
    }
#ifdef BORDER_REFLECT
#define EXTRAPOLATE(x, maxV) EXTRAPOLATE_(x, maxV, 0)
#else
#define EXTRAPOLATE(x, maxV) EXTRAPOLATE_(x, maxV, 1)
#endif
#else
#error No extrapolation method
#endif

#define THREADS 256

///////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////calcHarris////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////

__kernel void calcHarris(__global const float *Dx, __global const float *Dy, __global float *dst,
                         int dx_offset, int dx_whole_rows, int dx_whole_cols, int dx_step,
                         int dy_offset, int dy_whole_rows, int dy_whole_cols, int dy_step,
                         int dst_offset, int dst_rows, int dst_cols, int dst_step, float k)
{
    int col = get_local_id(0);
    int gX = get_group_id(0);
    int gY = get_group_id(1);
    int gly = get_global_id(1);

    int dx_x_off = (dx_offset % dx_step) >> 2;
    int dx_y_off = dx_offset / dx_step;
    int dy_x_off = (dy_offset % dy_step) >> 2;
    int dy_y_off = dy_offset / dy_step;
    int dst_x_off = (dst_offset % dst_step) >> 2;
    int dst_y_off = dst_offset / dst_step;

    int dx_startX = gX * (THREADS-ksX+1) - anX + dx_x_off;
    int dx_startY = (gY << 1) - anY + dx_y_off;
    int dy_startX = gX * (THREADS-ksX+1) - anX + dy_x_off;
    int dy_startY = (gY << 1) - anY + dy_y_off;
    int dst_startX = gX * (THREADS-ksX+1) + dst_x_off;
    int dst_startY = (gY << 1) + dst_y_off;

    float dx_data[ksY+1],dy_data[ksY+1], data[3][ksY+1];
    __local FPTYPE temp[6][THREADS];

#ifdef BORDER_CONSTANT
    for (int i=0; i < ksY+1; i++)
    {
        bool dx_con = dx_startX+col >= 0 && dx_startX+col < dx_whole_cols && dx_startY+i >= 0 && dx_startY+i < dx_whole_rows;
        int indexDx = (dx_startY+i)*(dx_step>>2)+(dx_startX+col);
        float dx_s = dx_con ? Dx[indexDx] : 0.0f;
        dx_data[i] = dx_s;

        bool dy_con = dy_startX+col >= 0 && dy_startX+col < dy_whole_cols && dy_startY+i >= 0 && dy_startY+i < dy_whole_rows;
        int indexDy = (dy_startY+i)*(dy_step>>2)+(dy_startX+col);
        float dy_s = dy_con ? Dy[indexDy] : 0.0f;
        dy_data[i] = dy_s;

        data[0][i] = dx_data[i] * dx_data[i];
        data[1][i] = dx_data[i] * dy_data[i];
        data[2][i] = dy_data[i] * dy_data[i];
    }
#else
    int clamped_col = min(2*dst_cols, col);
    for (int i=0; i < ksY+1; i++)
    {
        int dx_selected_row = dx_startY+i, dx_selected_col = dx_startX+clamped_col;
        EXTRAPOLATE(dx_selected_row, dx_whole_rows)
        EXTRAPOLATE(dx_selected_col, dx_whole_cols)
        dx_data[i] = Dx[dx_selected_row * (dx_step>>2) + dx_selected_col];

        int dy_selected_row = dy_startY+i, dy_selected_col = dy_startX+clamped_col;
        EXTRAPOLATE(dy_selected_row, dy_whole_rows)
        EXTRAPOLATE(dy_selected_col, dy_whole_cols)
        dy_data[i] = Dy[dy_selected_row * (dy_step>>2) + dy_selected_col];

        data[0][i] = dx_data[i] * dx_data[i];
        data[1][i] = dx_data[i] * dy_data[i];
        data[2][i] = dy_data[i] * dy_data[i];
    }
#endif
    FPTYPE sum0 = 0.0f, sum1 = 0.0f, sum2 = 0.0f;
    for (int i=1; i < ksY; i++)
    {
        sum0 += data[0][i];
        sum1 += data[1][i];
        sum2 += data[2][i];
    }

    FPTYPE sum01 = sum0 + data[0][0];
    FPTYPE sum02 = sum0 + data[0][ksY];
    temp[0][col] = sum01;
    temp[1][col] = sum02;
    FPTYPE sum11 = sum1 + data[1][0];
    FPTYPE sum12 = sum1 + data[1][ksY];
    temp[2][col] = sum11;
    temp[3][col] = sum12;
    FPTYPE sum21 = sum2 + data[2][0];
    FPTYPE sum22 = sum2 + data[2][ksY];
    temp[4][col] = sum21;
    temp[5][col] = sum22;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (col < (THREADS- (ksX - 1)))
    {
        col += anX;
        int posX = dst_startX - dst_x_off + col - anX;
        int posY = (gly << 1);
        int till = (ksX + 1)%2;
        float tmp_sum[6] = { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
        for (int k=0; k<6; k++)
        {
            FPTYPE temp_sum = 0;
            for (int i=-anX; i<=anX - till; i++)
            {
                temp_sum += temp[k][col+i];
            }
            tmp_sum[k] = temp_sum;
        }

        if (posX < dst_cols && (posY) < dst_rows)
        {
            dst[(dst_startY+0) * (dst_step>>2)+ dst_startX + col - anX] =
                    tmp_sum[0] * tmp_sum[4] - tmp_sum[2] * tmp_sum[2] - k * (tmp_sum[0] + tmp_sum[4]) * (tmp_sum[0] + tmp_sum[4]);
        }
        if (posX < dst_cols && (posY + 1) < dst_rows)
        {
            dst[(dst_startY+1) * (dst_step>>2)+ dst_startX + col - anX] =
                    tmp_sum[1] * tmp_sum[5] - tmp_sum[3] * tmp_sum[3] - k * (tmp_sum[1] + tmp_sum[5]) * (tmp_sum[1] + tmp_sum[5]);
        }
    }
}

/*M///////////////////////////////////////////////////////////////////////////////////////
//
//  IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
//
//  By downloading, copying, installing or using the software you agree to this license.
//  If you do not agree to this license, do not download, install,
//  copy or use the software.
//
//
//                           License Agreement
//                For Open Source Computer Vision Library
//
// Copyright (C) 2010-2012, Institute Of Software Chinese Academy Of Science, all rights reserved.
// Copyright (C) 2010-2012, Advanced Micro Devices, Inc., all rights reserved.
// Third party copyrights are property of their respective owners.
//
// @Authors
//    Shengen Yan,yanshengen@gmail.com
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
//   * Redistribution's of source code must retain the above copyright notice,
//     this list of conditions and the following disclaimer.
//
//   * Redistribution's in binary form must reproduce the above copyright notice,
//     this list of conditions and the following disclaimer in the documentation
//     and/or other materials provided with the distribution.
//
//   * The name of the copyright holders may not be used to endorse or promote products
//     derived from this software without specific prior written permission.
//
// This software is provided by the copyright holders and contributors as is and
// any express or implied warranties, including, but not limited to, the implied
// warranties of merchantability and fitness for a particular purpose are disclaimed.
// In no event shall the Intel Corporation or contributors be liable for any direct,
// indirect, incidental, special, exemplary, or consequential damages
// (including, but not limited to, procurement of substitute goods or services;
// loss of use, data, or profits; or business interruption) however caused
// and on any theory of liability, whether in contract, strict liability,
// or tort (including negligence or otherwise) arising in any way out of
// the use of this software, even if advised of the possibility of such damage.
//
//M*/

///////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////Macro for border type////////////////////////////////////////////
/////////////////////////////////////////////////////////////////////////////////////////////////

#if defined (DOUBLE_SUPPORT) && defined (cl_khr_fp64)
#pragma OPENCL EXTENSION cl_khr_fp64:enable
#define FPTYPE double
#else
#define FPTYPE float
#endif

#ifdef BORDER_CONSTANT
#elif defined BORDER_REPLICATE
#define EXTRAPOLATE(x, maxV) \
    { \
        x = max(min(x, maxV - 1), 0); \
    }
#elif defined BORDER_WRAP
#define EXTRAPOLATE(x, maxV) \
    { \
        if (x < 0) \
            x -= ((x - maxV + 1) / maxV) * maxV; \
        if (x >= maxV) \
            x %= maxV; \
    }
#elif defined(BORDER_REFLECT) || defined(BORDER_REFLECT101)
#define EXTRAPOLATE_(x, maxV, delta) \
    { \
        if (maxV == 1) \
            x = 0; \
        else \
            do \
            { \
                if ( x < 0 ) \
                    x = -x - 1 + delta; \
                else \
                    x = maxV - 1 - (x - maxV) - delta; \
            } \
            while (x >= maxV || x < 0); \
    }
#ifdef BORDER_REFLECT
#define EXTRAPOLATE(x, maxV) EXTRAPOLATE_(x, maxV, 0)
#else
#define EXTRAPOLATE(x, maxV) EXTRAPOLATE_(x, maxV, 1)
#endif
#else
#error No extrapolation method
#endif

#define THREADS 256

///////////////////////////////////////////////////////////////////////////////////////////////////
/////////////////////////////////////calcHarris////////////////////////////////////////////////////
///////////////////////////////////////////////////////////////////////////////////////////////////

__kernel void calcHarris(__global const float *Dx, __global const float *Dy, __global float *dst,
                         int dx_offset, int dx_whole_rows, int dx_whole_cols, int dx_step,
                         int dy_offset, int dy_whole_rows, int dy_whole_cols, int dy_step,
                         int dst_offset, int dst_rows, int dst_cols, int dst_step, float k)
{
    int col = get_local_id(0);
    int gX = get_group_id(0);
    int gY = get_group_id(1);
    int gly = get_global_id(1);

    int dx_x_off = (dx_offset % dx_step) >> 2;
    int dx_y_off = dx_offset / dx_step;
    int dy_x_off = (dy_offset % dy_step) >> 2;
    int dy_y_off = dy_offset / dy_step;
    int dst_x_off = (dst_offset % dst_step) >> 2;
    int dst_y_off = dst_offset / dst_step;

    int dx_startX = gX * (THREADS-ksX+1) - anX + dx_x_off;
    int dx_startY = (gY << 1) - anY + dx_y_off;
    int dy_startX = gX * (THREADS-ksX+1) - anX + dy_x_off;
    int dy_startY = (gY << 1) - anY + dy_y_off;
    int dst_startX = gX * (THREADS-ksX+1) + dst_x_off;
    int dst_startY = (gY << 1) + dst_y_off;

    float dx_data[ksY+1],dy_data[ksY+1], data[3][ksY+1];
    __local FPTYPE temp[6][THREADS];

#ifdef BORDER_CONSTANT
    for (int i=0; i < ksY+1; i++)
    {
        bool dx_con = dx_startX+col >= 0 && dx_startX+col < dx_whole_cols && dx_startY+i >= 0 && dx_startY+i < dx_whole_rows;
        int indexDx = (dx_startY+i)*(dx_step>>2)+(dx_startX+col);
        float dx_s = dx_con ? Dx[indexDx] : 0.0f;
        dx_data[i] = dx_s;

        bool dy_con = dy_startX+col >= 0 && dy_startX+col < dy_whole_cols && dy_startY+i >= 0 && dy_startY+i < dy_whole_rows;
        int indexDy = (dy_startY+i)*(dy_step>>2)+(dy_startX+col);
        float dy_s = dy_con ? Dy[indexDy] : 0.0f;
        dy_data[i] = dy_s;

        data[0][i] = dx_data[i] * dx_data[i];
        data[1][i] = dx_data[i] * dy_data[i];
        data[2][i] = dy_data[i] * dy_data[i];
    }
#else
    int clamped_col = min(2*dst_cols, col);
    for (int i=0; i < ksY+1; i++)
    {
        int dx_selected_row = dx_startY+i, dx_selected_col = dx_startX+clamped_col;
        EXTRAPOLATE(dx_selected_row, dx_whole_rows)
        EXTRAPOLATE(dx_selected_col, dx_whole_cols)
        dx_data[i] = Dx[dx_selected_row * (dx_step>>2) + dx_selected_col];

        int dy_selected_row = dy_startY+i, dy_selected_col = dy_startX+clamped_col;
        EXTRAPOLATE(dy_selected_row, dy_whole_rows)
        EXTRAPOLATE(dy_selected_col, dy_whole_cols)
        dy_data[i] = Dy[dy_selected_row * (dy_step>>2) + dy_selected_col];

        data[0][i] = dx_data[i] * dx_data[i];
        data[1][i] = dx_data[i] * dy_data[i];
        data[2][i] = dy_data[i] * dy_data[i];
    }
#endif
    FPTYPE sum0 = 0.0f, sum1 = 0.0f, sum2 = 0.0f;
    for (int i=1; i < ksY; i++)
    {
        sum0 += data[0][i];
        sum1 += data[1][i];
        sum2 += data[2][i];
    }

    FPTYPE sum01 = sum0 + data[0][0];
    FPTYPE sum02 = sum0 + data[0][ksY];
    temp[0][col] = sum01;
    temp[1][col] = sum02;
    FPTYPE sum11 = sum1 + data[1][0];
    FPTYPE sum12 = sum1 + data[1][ksY];
    temp[2][col] = sum11;
    temp[3][col] = sum12;
    FPTYPE sum21 = sum2 + data[2][0];
    FPTYPE sum22 = sum2 + data[2][ksY];
    temp[4][col] = sum21;
    temp[5][col] = sum22;
    barrier(CLK_LOCAL_MEM_FENCE);

    if (col < (THREADS- (ksX - 1)))
    {
        col += anX;
        int posX = dst_startX - dst_x_off + col - anX;
        int posY = (gly << 1);
        int till = (ksX + 1)%2;
        float tmp_sum[6] = { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f };
        for (int k=0; k<6; k++)
        {
            FPTYPE temp_sum = 0;
            for (int i=-anX; i<=anX - till; i++)
            {
                temp_sum += temp[k][col+i];
                tmp_sum[k] = temp_sum;
            }
        }

        if (posX < dst_cols && (posY) < dst_rows)
        {
            dst[(dst_startY+0) * (dst_step>>2)+ dst_startX + col - anX] =
                    tmp_sum[0] * tmp_sum[4] - tmp_sum[2] * tmp_sum[2] - k * (tmp_sum[0] + tmp_sum[4]) * (tmp_sum[0] + tmp_sum[4]);
        }
        if (posX < dst_cols && (posY + 1) < dst_rows)
        {
            dst[(dst_startY+1) * (dst_step>>2)+ dst_startX + col - anX] =
                    tmp_sum[1] * tmp_sum[5] - tmp_sum[3] * tmp_sum[3] - k * (tmp_sum[1] + tmp_sum[5]) * (tmp_sum[1] + tmp_sum[5]);
        }
    }
}


Added #ifndef __CL_PLATFORM_H usage.

Added #ifndef __CL_PLATFORM_H usage.

git-svn-id: 602a26e73ae9f2a14e2bb95676f2e137fcc39919@3067 7c0444ca-5550-0410-9008-faecdbe78eb5

/* rng/taus.c
 *
 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 2007 James Theiler, Brian Gough
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

/*
 * The following adapted from rng/taus.c by Craig Rasmussen, 12/27/2010
 * for OpenCL.  Also converted (hopefully) to use 32 bit integers.
 *
 * It uses a uint4 to store state and a result:
 *    state.s0 => random number
 *    state.s1 => state s1 value
 *    state.s2 => state s2 value
 *    state.s3 => state s3 value
 */

#define cl_random_state(s) (cl_taus_get(s))

#ifndef __CL_PLATFORM_H
   typedef struct uint4_ {
      int s0, s1, s2, s3;
   } uint4 __attribute__((aligned(16)));
#endif

static inline uint4
cl_taus_get(uint4 state)
{
   uint4 result;

#define TAUSWORTHE(s,a,b,c,d) (((s &c) <<d)) ^ ((((s <<a))^s) >>b)

  result.s1 = TAUSWORTHE (state.s1, 13, 19, 4294967294UL, 12);
  result.s2 = TAUSWORTHE (state.s2,  2, 25, 4294967288UL, 4);
  result.s3 = TAUSWORTHE (state.s3,  3, 11, 4294967280UL, 17);

  result.s0 = (state.s1 ^ state.s2 ^ state.s3);

  return result;
}

static inline float cl_random_prob(uint4 state)
{
   return (float) ((double) state.s0 / (double) 4294967296.0);
}

/* rng/taus.c
 *
 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 2007 James Theiler, Brian Gough
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

/*
 * The following adapted from rng/taus.c by Craig Rasmussen, 12/27/2010
 * for OpenCL.  Also converted (hopefully) to use 32 bit integers.
 *
 * It uses a uint4 to store state and a result:
 *    state.s0 => random number
 *    state.s1 => state s1 value
 *    state.s2 => state s2 value
 *    state.s3 => state s3 value
 */

#define cl_random_state(s) (cl_taus_get(s))

#ifndef PF_USE_OPENCL
   typedef struct uint4_ {
      int s0, s1, s2, s3;
   } uint4;
#endif

static inline uint4
cl_taus_get(uint4 state)
{
   uint4 result;

#define TAUSWORTHE(s,a,b,c,d) (((s &c) <<d)) ^ ((((s <<a))^s) >>b)

  result.s1 = TAUSWORTHE (state.s1, 13, 19, 4294967294UL, 12);
  result.s2 = TAUSWORTHE (state.s2,  2, 25, 4294967288UL, 4);
  result.s3 = TAUSWORTHE (state.s3,  3, 11, 4294967280UL, 17);

  result.s0 = (state.s1 ^ state.s2 ^ state.s3);

  return result;
}

static inline float cl_random_prob(uint4 state)
{
   return (float) ((double) state.s0 / (double) 4294967296.0);
}


Tile tensor product in m direction.

Tile tensor product in m direction.

#pragma OPENCL EXTENSION cl_khr_fp64 : enable

__kernel
void matvec_single(__global const double *A, __global const double *x,
                   __global double *b, int n) {

  int gid = get_global_id(0);
  const int stride = get_global_size(0);

  double sum;
  while(gid<n) {

    sum = 0.0;
    for(int i=0; i<n; ++i)
      sum += A[gid*n+i]*x[i];
    b[gid] = sum;

    gid += stride;

  }

}

__kernel
void matvec_tile(__global const double *A, __global const double *x,
                 __global double *b, int n, int tile_size) {

  // Assume n%tile_size==0

  // gid is i_outer
  int gid = get_global_id(0)*tile_size;
  const int stride = get_global_size(0)*tile_size;
  const int T = tile_size;

  double sum;
  while(gid<n) {

    for(int j_outer=0; j_outer<n/T; ++j_outer) {
      for(int i_inner=0; i_inner<T; ++i_inner) {

          sum = 0.0;
          for(int j_inner=0; j_inner<T; ++j_inner)
            sum += A[(gid+i_inner)*n+j_outer*T+j_inner]*x[j_outer*T+j_inner];

          b[gid+i_inner] += sum;

      }
    }

    gid += stride;

  }

}

// A is nxn, M is nxk, B is nxk
__kernel
void matmat_single(__global const double *A, __global const double *M,
                   __global double *B, int n, int k) {

  int gid = get_global_id(0);
  const int stride = get_global_size(0);

  double sum;
  while(gid<n) {

    for(int j=0; j<k; ++j) {

      sum = 0.0;
      for(int i=0; i<n; ++i)
        sum += A[gid*n+i]*M[k*i+j];

      B[gid*k+j] = sum;

    }

    gid += stride;

  }

}

// A is nxn, M is nxk, B is nxk
__kernel
void matmat_tile(__global const double *A, __global const double *M,
                 __global double *B, int n, int k, int tile_size) {

  // Assume n%tile_size==0

  // gid is i_outer
  int gid = get_global_id(0)*tile_size;
  const int stride = get_global_size(0)*tile_size;
  const int T = tile_size;

  double sum;
  while(gid<n) {

    for(int j_outer=0; j_outer<n/T; ++j_outer) {
      for(int k_outer=0; k_outer<k/T; ++k_outer) {
        for(int i_inner=0; i_inner<T; ++i_inner) {
          for(int k_inner=0; k_inner<T; ++k_inner) {

            sum = 0.0;
            for(int j_inner=0; j_inner<T; ++j_inner)
              sum += A[(gid+i_inner)*n+j_outer*T+j_inner]*M[(j_outer*T+j_inner)*k+k_outer*T+k_inner];

            B[(gid+i_inner)*k+k_outer*T+k_inner] += sum;

          }
        }

      }
    }

    gid += stride;

  }

}

// A is nxn, M is nxk, B is nxk
__kernel
void tensor_IIA_single(__global const double *A, __global const double *M,
                       __global double *B, int n) {

  int gid = get_global_id(0);
  const int stride = get_global_size(0);

  double sum;
  while(gid<n) {

    for(int k=0; k<n; ++k) {
      for(int j=0; j<n; ++j) {

        sum = 0.0;
        for(int i=0; i<n; ++i)
          sum += A[n*gid+i]*M[k*n*n+n*j+i];

        B[k*n*n+n*j+gid] = sum;

      }

    }

    gid += stride;

  }

}

// A is nxn, M is nxk, B is nxk
__kernel
void tensor_IAI_single(__global const double *A, __global const double *M,
                       __global double *B, int n) {

  int gid = get_global_id(0);
  const int stride = get_global_size(0);

  double sum;
  while(gid<n) {

    for(int k=0; k<n; ++k) {
      for(int j=0; j<n; ++j) {

        sum = 0.0;
        for(int i=0; i<n; ++i)
          sum += A[n*gid+i]*M[k*n*n+n*i+j];

        B[k*n*n+n*gid+j] = sum;

      }

    }

    gid += stride;

  }

}

// A is nxn, M is nxk, B is nxk
__kernel
void tensor_AII_single(__global const double *A, __global const double *M,
                       __global double *B, int n) {

  int gid = get_global_id(0);
  const int stride = get_global_size(0);

  double sum;
  while(gid<n) {

    for(int k=0; k<n; ++k) {
      for(int j=0; j<n; ++j) {

        sum = 0.0;
        for(int i=0; i<n; ++i)
          sum += A[n*gid+i]*M[i*n*n+n*k+j];

        B[gid*n*n+n*k+j] = sum;

      }

    }

    gid += stride;

  }

}

__kernel
void tensor_tile(__global const double *A, __global const double *M,
                 __global double *B, int n, int tile_size, int imat) {

  // Assume n%tile_size==0

  // gid is i_outer*T
  int gid = get_global_id(0)*tile_size;
  const int stride = get_global_size(0)*tile_size;
  const int T = tile_size;

  int ni, nj, nk, nm;
  if(imat==0) {
    ni = 1;
    nj = 1;
    nk = n*n;
    nm = n;
  }
  else if(imat==1) {
    ni = n;
    nj = n;
    nk = 1;
    nm = n*n;
  }
  else if(imat==2) {
    ni = n*n;
    nj = n*n;
    nk = 1;
    nm = n;
  }

  double sum;
  while(gid<n) {

    for(int m_outer=0; m_outer<n/T; ++m_outer) {
      for(int j_outer=0; j_outer<n/T; ++j_outer) {
        for(int k_outer=0; k_outer<n/T; ++k_outer) {

          for(int m_inner=0; m_inner<T; ++m_inner) {
            for(int i_inner=0; i_inner<T; ++i_inner) {
              for(int k_inner=0; k_inner<T; ++k_inner) {

                sum = 0.0;
                for(int j_inner=0; j_inner<T; ++j_inner)
                  sum += A[(gid+i_inner)*n+j_outer*T+j_inner]*M[(m_outer*T+m_inner)*nm+
                                                                (j_outer*T+j_inner)*nj+
                                                                (k_outer*T+k_inner)*nk];

                B[(m_outer*T+m_inner)*nm+(gid+i_inner)*ni+(k_outer*T+k_inner)*nk] += sum;

              }
            }

          }
        }
      }
    }

    gid += stride;

  }

}

// __kernel
// void tensor_IIA_tile(__global const double *A, __global const double *M,
//                      __global double *B, int n, int tile_size) {

//   // Assume n%tile_size==0

//   // gid is i_outer*T
//   int gid = get_global_id(0)*tile_size;
//   const int stride = get_global_size(0)*tile_size;
//   const int T = tile_size;

//   double sum;
//   while(gid<n) {

//     for(int m=0; m<n; ++m) {

//       for(int j_outer=0; j_outer<n/T; ++j_outer) {
//         for(int k_outer=0; k_outer<n/T; ++k_outer) {
//           for(int i_inner=0; i_inner<T; ++i_inner) {
//             for(int k_inner=0; k_inner<T; ++k_inner) {

//               sum = 0.0;
//               for(int j_inner=0; j_inner<T; ++j_inner)
//                 sum += A[(gid+i_inner)*n+j_outer*T+j_inner]*M[m*n+
//                                                               (j_outer*T+j_inner)+
//                                                               (k_outer*T+k_inner)*n*n];

//               B[m*n+(gid+i_inner)+(k_outer*T+k_inner)*n*n] += sum;

//             }
//           }

//         }
//       }

//     }

//     gid += stride;

//   }

// }

// __kernel
// void tensor_IAI_tile(__global const double *A, __global const double *M,
//                      __global double *B, int n, int tile_size) {

//   // Assume n%tile_size==0

//   // gid is i_outer
//   int gid = get_global_id(0)*tile_size;
//   const int stride = get_global_size(0)*tile_size;
//   const int T = tile_size;

//   double sum;
//   while(gid<n) {

//     for(int m=0; m<n; ++m) {

//       for(int j_outer=0; j_outer<n/T; ++j_outer) {
//         for(int k_outer=0; k_outer<n/T; ++k_outer) {
//           for(int i_inner=0; i_inner<T; ++i_inner) {
//             for(int k_inner=0; k_inner<T; ++k_inner) {

//               sum = 0.0;
//               for(int j_inner=0; j_inner<T; ++j_inner)
//                 sum += A[(gid+i_inner)*n+j_outer*T+j_inner]*M[(j_outer*T+j_inner)*n+
//                                                               k_outer*T+k_inner+
//                                                               m*n*n];

//               B[(gid+i_inner)*n+k_outer*T+k_inner+m*n*n] += sum;

//             }
//           }

//         }
//       }

//     }

//     gid += stride;

//   }

// }

// __kernel
// void tensor_AII_tile(__global const double *A, __global const double *M,
//                      __global double *B, int n, int tile_size) {

//   // Assume n%tile_size==0

//   // gid is i_outer
//   int gid = get_global_id(0)*tile_size;
//   const int stride = get_global_size(0)*tile_size;
//   const int T = tile_size;

//   double sum;
//   while(gid<n) {

//     for(int m=0; m<n; ++m) {

//       for(int j_outer=0; j_outer<n/T; ++j_outer) {
//         for(int k_outer=0; k_outer<n/T; ++k_outer) {
//           for(int i_inner=0; i_inner<T; ++i_inner) {
//             for(int k_inner=0; k_inner<T; ++k_inner) {

//               sum = 0.0;
//               for(int j_inner=0; j_inner<T; ++j_inner)
//                 sum += A[(gid+i_inner)*n+j_outer*T+j_inner]*M[m*n+
//                                                               (j_outer*T+j_inner)*n*n+
//                                                               (k_outer*T+k_inner)];

//               B[m*n+(gid+i_inner)*n*n+(k_outer*T+k_inner)] += sum;

//             }
//           }

//         }
//       }

//     }

//     gid += stride;

//   }

// }

#pragma OPENCL EXTENSION cl_khr_fp64 : enable

__kernel
void matvec_single(__global const double *A, __global const double *x,
                   __global double *b, int n) {

  int gid = get_global_id(0);
  const int stride = get_global_size(0);

  double sum;
  while(gid<n) {

    sum = 0.0;
    for(int i=0; i<n; ++i)
      sum += A[gid*n+i]*x[i];
    b[gid] = sum;

    gid += stride;

  }

}

__kernel
void matvec_tile(__global const double *A, __global const double *x,
                 __global double *b, int n, int tile_size) {

  // Assume n%tile_size==0

  // gid is i_outer
  int gid = get_global_id(0)*tile_size;
  const int stride = get_global_size(0)*tile_size;
  const int T = tile_size;

  double sum;
  while(gid<n) {

    for(int j_outer=0; j_outer<n/T; ++j_outer) {
      for(int i_inner=0; i_inner<T; ++i_inner) {

          sum = 0.0;
          for(int j_inner=0; j_inner<T; ++j_inner)
            sum += A[(gid+i_inner)*n+j_outer*T+j_inner]*x[j_outer*T+j_inner];

          b[gid+i_inner] += sum;

      }
    }

    gid += stride;

  }

}

// A is nxn, M is nxk, B is nxk
__kernel
void matmat_single(__global const double *A, __global const double *M,
                   __global double *B, int n, int k) {

  int gid = get_global_id(0);
  const int stride = get_global_size(0);

  double sum;
  while(gid<n) {

    for(int j=0; j<k; ++j) {

      sum = 0.0;
      for(int i=0; i<n; ++i)
        sum += A[gid*n+i]*M[k*i+j];

      B[gid*k+j] = sum;

    }

    gid += stride;

  }

}

// A is nxn, M is nxk, B is nxk
__kernel
void matmat_tile(__global const double *A, __global const double *M,
                 __global double *B, int n, int k, int tile_size) {

  // Assume n%tile_size==0

  // gid is i_outer
  int gid = get_global_id(0)*tile_size;
  const int stride = get_global_size(0)*tile_size;
  const int T = tile_size;

  double sum;
  while(gid<n) {

    for(int j_outer=0; j_outer<n/T; ++j_outer) {
      for(int k_outer=0; k_outer<k/T; ++k_outer) {
        for(int i_inner=0; i_inner<T; ++i_inner) {
          for(int k_inner=0; k_inner<T; ++k_inner) {

            sum = 0.0;
            for(int j_inner=0; j_inner<T; ++j_inner)
              sum += A[(gid+i_inner)*n+j_outer*T+j_inner]*M[(j_outer*T+j_inner)*k+k_outer*T+k_inner];

            B[(gid+i_inner)*k+k_outer*T+k_inner] += sum;

          }
        }

      }
    }

    gid += stride;

  }

}

// A is nxn, M is nxk, B is nxk
__kernel
void tensor_IIA_single(__global const double *A, __global const double *M,
                       __global double *B, int n) {

  int gid = get_global_id(0);
  const int stride = get_global_size(0);

  double sum;
  while(gid<n) {

    for(int k=0; k<n; ++k) {
      for(int j=0; j<n; ++j) {

        sum = 0.0;
        for(int i=0; i<n; ++i)
          sum += A[n*gid+i]*M[k*n*n+n*j+i];

        B[k*n*n+n*j+gid] = sum;

      }

    }

    gid += stride;

  }

}

// A is nxn, M is nxk, B is nxk
__kernel
void tensor_IAI_single(__global const double *A, __global const double *M,
                       __global double *B, int n) {

  int gid = get_global_id(0);
  const int stride = get_global_size(0);

  double sum;
  while(gid<n) {

    for(int k=0; k<n; ++k) {
      for(int j=0; j<n; ++j) {

        sum = 0.0;
        for(int i=0; i<n; ++i)
          sum += A[n*gid+i]*M[k*n*n+n*i+j];

        B[k*n*n+n*gid+j] = sum;

      }

    }

    gid += stride;

  }

}

// A is nxn, M is nxk, B is nxk
__kernel
void tensor_AII_single(__global const double *A, __global const double *M,
                       __global double *B, int n) {

  int gid = get_global_id(0);
  const int stride = get_global_size(0);

  double sum;
  while(gid<n) {

    for(int k=0; k<n; ++k) {
      for(int j=0; j<n; ++j) {

        sum = 0.0;
        for(int i=0; i<n; ++i)
          sum += A[n*gid+i]*M[i*n*n+n*k+j];

        B[gid*n*n+n*k+j] = sum;

      }

    }

    gid += stride;

  }

}

__kernel
void tensor_tile(__global const double *A, __global const double *M,
                 __global double *B, int n, int tile_size, int imat) {

  // Assume n%tile_size==0

  // gid is i_outer*T
  int gid = get_global_id(0)*tile_size;
  const int stride = get_global_size(0)*tile_size;
  const int T = tile_size;

  int ni, nj, nk, nm;
  if(imat==0) {
    ni = 1;
    nj = 1;
    nk = n*n;
    nm = n;
  }
  else if(imat==1) {
    ni = n;
    nj = n;
    nk = 1;
    nm = n*n;
  }
  else if(imat==2) {
    ni = n*n;
    nj = n*n;
    nk = 1;
    nm = n;
  }

  double sum;
  while(gid<n) {

    for(int m=0; m<n; ++m) {

      for(int j_outer=0; j_outer<n/T; ++j_outer) {
        for(int k_outer=0; k_outer<n/T; ++k_outer) {
          for(int i_inner=0; i_inner<T; ++i_inner) {
            for(int k_inner=0; k_inner<T; ++k_inner) {

              sum = 0.0;
              for(int j_inner=0; j_inner<T; ++j_inner)
                sum += A[(gid+i_inner)*n+j_outer*T+j_inner]*M[m*nm+
                                                              (j_outer*T+j_inner)*nj+
                                                              (k_outer*T+k_inner)*nk];

              B[m*nm+(gid+i_inner)*ni+(k_outer*T+k_inner)*nk] += sum;

            }
          }

        }
      }

    }

    gid += stride;

  }

}

// __kernel
// void tensor_IIA_tile(__global const double *A, __global const double *M,
//                      __global double *B, int n, int tile_size) {

//   // Assume n%tile_size==0

//   // gid is i_outer*T
//   int gid = get_global_id(0)*tile_size;
//   const int stride = get_global_size(0)*tile_size;
//   const int T = tile_size;

//   double sum;
//   while(gid<n) {

//     for(int m=0; m<n; ++m) {

//       for(int j_outer=0; j_outer<n/T; ++j_outer) {
//         for(int k_outer=0; k_outer<n/T; ++k_outer) {
//           for(int i_inner=0; i_inner<T; ++i_inner) {
//             for(int k_inner=0; k_inner<T; ++k_inner) {

//               sum = 0.0;
//               for(int j_inner=0; j_inner<T; ++j_inner)
//                 sum += A[(gid+i_inner)*n+j_outer*T+j_inner]*M[m*n+
//                                                               (j_outer*T+j_inner)+
//                                                               (k_outer*T+k_inner)*n*n];

//               B[m*n+(gid+i_inner)+(k_outer*T+k_inner)*n*n] += sum;

//             }
//           }

//         }
//       }

//     }

//     gid += stride;

//   }

// }

// __kernel
// void tensor_IAI_tile(__global const double *A, __global const double *M,
//                      __global double *B, int n, int tile_size) {

//   // Assume n%tile_size==0

//   // gid is i_outer
//   int gid = get_global_id(0)*tile_size;
//   const int stride = get_global_size(0)*tile_size;
//   const int T = tile_size;

//   double sum;
//   while(gid<n) {

//     for(int m=0; m<n; ++m) {

//       for(int j_outer=0; j_outer<n/T; ++j_outer) {
//         for(int k_outer=0; k_outer<n/T; ++k_outer) {
//           for(int i_inner=0; i_inner<T; ++i_inner) {
//             for(int k_inner=0; k_inner<T; ++k_inner) {

//               sum = 0.0;
//               for(int j_inner=0; j_inner<T; ++j_inner)
//                 sum += A[(gid+i_inner)*n+j_outer*T+j_inner]*M[(j_outer*T+j_inner)*n+
//                                                               k_outer*T+k_inner+
//                                                               m*n*n];

//               B[(gid+i_inner)*n+k_outer*T+k_inner+m*n*n] += sum;

//             }
//           }

//         }
//       }

//     }

//     gid += stride;

//   }

// }

// __kernel
// void tensor_AII_tile(__global const double *A, __global const double *M,
//                      __global double *B, int n, int tile_size) {

//   // Assume n%tile_size==0

//   // gid is i_outer
//   int gid = get_global_id(0)*tile_size;
//   const int stride = get_global_size(0)*tile_size;
//   const int T = tile_size;

//   double sum;
//   while(gid<n) {

//     for(int m=0; m<n; ++m) {

//       for(int j_outer=0; j_outer<n/T; ++j_outer) {
//         for(int k_outer=0; k_outer<n/T; ++k_outer) {
//           for(int i_inner=0; i_inner<T; ++i_inner) {
//             for(int k_inner=0; k_inner<T; ++k_inner) {

//               sum = 0.0;
//               for(int j_inner=0; j_inner<T; ++j_inner)
//                 sum += A[(gid+i_inner)*n+j_outer*T+j_inner]*M[m*n+
//                                                               (j_outer*T+j_inner)*n*n+
//                                                               (k_outer*T+k_inner)];

//               B[m*n+(gid+i_inner)*n*n+(k_outer*T+k_inner)] += sum;

//             }
//           }

//         }
//       }

//     }

//     gid += stride;

//   }

// }


Improve ans decode kernel for NVIDIA cards

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable

// This doesn't seem to make much of a difference on AMD but very
// significant on NVIDIA....
#define USE_LOCAL_TABLE

// Not too sure about this -- bank conflicts on writes aren't particularly
// that bad from what I understand, but maybe this makes a difference, idk?
// #define AVOID_GLOBAL_BANK_CONFLICTS

#define ANS_TABLE_SIZE_LOG  11
#define ANS_TABLE_SIZE      (1 << ANS_TABLE_SIZE_LOG)
#define NUM_ENCODED_SYMBOLS 256
#define ANS_DECODER_K       (1 << 4)
#define ANS_DECODER_L       (ANS_DECODER_K * ANS_TABLE_SIZE)

typedef struct AnsTableEntry_Struct {
	ushort freq;
	ushort cum_freq;
	uchar  symbol;
} AnsTableEntry;

__kernel void ans_decode(const __constant AnsTableEntry *global_table,
						 const __global   uchar         *data,
						       __global   uchar         *out_stream)
{
#ifdef USE_LOCAL_TABLE
	// Load everything into local memory
	__local AnsTableEntry table[ANS_TABLE_SIZE];
	size_t entries_per_thread = (ANS_TABLE_SIZE + get_local_size(0) - 1) / get_local_size(0);
	size_t table_idx = get_local_id(0) * entries_per_thread;
	size_t table_limit = min(table_idx + entries_per_thread, (size_t)ANS_TABLE_SIZE);
	for (size_t i = table_idx; i < table_limit; ++i) {
		table[i].freq = global_table[i].freq;
		table[i].cum_freq = global_table[i].cum_freq;
		table[i].symbol = global_table[i].symbol;
	}
#endif

	__local uint normalization_mask;
    if (0 == get_local_id(0)) {
      normalization_mask = 0;
    }

	uint offset = ((const __global uint *)data)[get_group_id(0)];
	uint state = ((const __global uint *)(data + offset) - get_local_size(0))[get_local_id(0)];
    uint next_to_read = (offset - (get_local_size(0) * 4)) / 2;
    const __global ushort *stream_data = (const __global ushort *)data;

	barrier(CLK_LOCAL_MEM_FENCE);

#ifdef AVOID_GLOBAL_BANK_CONFLICTS
	uchar result[NUM_ENCODED_SYMBOLS];
#endif  // AVOID_GLOBAL_BANK_CONFLICTS

	for (int i = 0; i < NUM_ENCODED_SYMBOLS; ++i) {
		const uint symbol = state & (ANS_TABLE_SIZE - 1);
#ifdef USE_LOCAL_TABLE
		const __local AnsTableEntry *entry = table + symbol;
#else
		const __constant AnsTableEntry *entry = global_table + symbol;
#endif  // USE_LOCAL_TABLE
		state = (state >> ANS_TABLE_SIZE_LOG) * entry->freq
			- entry->cum_freq + symbol;

		// Set the bit for this invocation...
		const uint normalization_bit =
		  ((uint)(state < ANS_DECODER_L)) << get_local_id(0);
		atomic_or(&normalization_mask, normalization_bit);

		barrier(CLK_LOCAL_MEM_FENCE);

		// If we need to renormalize, then do so...
		const uint total_to_read = popcount(normalization_mask);
		if (normalization_bit != 0) {
		  const uint up_to_me_mask = normalization_bit - 1;
		  uint num_to_skip = total_to_read;
		  num_to_skip -= popcount(normalization_mask & up_to_me_mask) + 1;
		  state = (state << 16) | stream_data[next_to_read - num_to_skip - 1];
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		// Clear the bit in the normalization mask...
		atomic_and(&normalization_mask, ~normalization_bit);

		// Advance the read pointer by the number of shorts read
		next_to_read -= total_to_read;
#ifdef AVOID_GLOBAL_BANK_CONFLICTS
		// Write the result
		const int gidx = get_global_id(0) * NUM_ENCODED_SYMBOLS + (255 - i);
		out_stream[gidx] = entry->symbol;
	}
#else
		// Write the result
		const int gidx = get_global_id(0) * NUM_ENCODED_SYMBOLS + (255 - i);
		out_stream[gidx] = entry->symbol;
	}
#endif // AVOID_GLOBAL_BANK_CONFLICTS
}

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable

// This doesn't seem to make much of a difference....
#define USE_LOCAL_TABLE

#define ANS_TABLE_SIZE_LOG  11
#define ANS_TABLE_SIZE      (1 << ANS_TABLE_SIZE_LOG)
#define NUM_ENCODED_SYMBOLS 256
#define ANS_DECODER_K       (1 << 4)
#define ANS_DECODER_L       (ANS_DECODER_K * ANS_TABLE_SIZE)

typedef struct AnsTableEntry_Struct {
	ushort freq;
	ushort cum_freq;
	uchar  symbol;
} AnsTableEntry;

__kernel void ans_decode(const __constant AnsTableEntry *global_table,
						 const __global   uchar         *data,
						       __global   uchar         *out_stream)
{
#ifdef USE_LOCAL_TABLE
	// Load everything into local memory
	__local AnsTableEntry table[ANS_TABLE_SIZE];
	size_t entries_per_thread = (ANS_TABLE_SIZE + get_local_size(0) - 1) / get_local_size(0);
	size_t table_idx = get_local_id(0) * entries_per_thread;
	size_t table_limit = min(table_idx + entries_per_thread, (size_t)ANS_TABLE_SIZE);
	for (size_t i = table_idx; i < table_limit; ++i) {
		table[i].freq = global_table[i].freq;
		table[i].cum_freq = global_table[i].cum_freq;
		table[i].symbol = global_table[i].symbol;
	}
#endif

	__local uint normalization_mask;
	normalization_mask = 0;

	uint offset = ((const __global uint *)data)[get_group_id(0)];
	uint state = ((const __global uint *)(data + offset) - get_local_size(0))[get_local_id(0)];
    uint next_to_read = (offset - (get_local_size(0) * 4)) / 2;
    const __global ushort *stream_data = (const __global ushort *)data;

	barrier(CLK_LOCAL_MEM_FENCE);

	for (int i = 0; i < NUM_ENCODED_SYMBOLS; ++i) {
		const uint symbol = state & (ANS_TABLE_SIZE - 1);
#ifdef USE_LOCAL_TABLE
		const __local AnsTableEntry *entry = table + symbol;
#else
		const __constant AnsTableEntry *entry = global_table + symbol;
#endif
		state = (state >> ANS_TABLE_SIZE_LOG) * entry->freq
			- entry->cum_freq + symbol;

		// Set the bit for this invocation...
		const uint normalization_bit =
		  ((uint)(state < ANS_DECODER_L)) << get_local_id(0);
		atomic_or(&normalization_mask, normalization_bit);

		barrier(CLK_LOCAL_MEM_FENCE);

		// If we need to renormalize, then do so...
		const uint total_to_read = popcount(normalization_mask);
		if (normalization_bit != 0) {
		  const uint up_to_me_mask = normalization_bit - 1;
		  uint num_to_skip = total_to_read;
		  num_to_skip -= popcount(normalization_mask & up_to_me_mask) + 1;
		  state = (state << 16) | stream_data[next_to_read - num_to_skip - 1];
		}

		barrier(CLK_LOCAL_MEM_FENCE);

		// Clear the bit in the normalization mask...
		atomic_and(&normalization_mask, ~normalization_bit);

		// Advance the read pointer by the number of shorts read
		next_to_read -= total_to_read;

		// Write the result
		const int gidx = get_global_id(0) * NUM_ENCODED_SYMBOLS + (255 - i);
		out_stream[gidx] = entry->symbol;
	}
}


Added short comment about getting the next symbol.

#ifdef KERNEL_TEST_ENABLED
#include "../tests/kerneltypes.h"
#endif

#include "SharedMacros.h"
#include "SharedTypes.h"

// zero symbol, add 32-bit unsigned integer to it to return pointer as symbol
#define ZERO 0x6240000000000000ULL

/* Used to create, manipulate and access packet information. */
#define PKT_TYPE_SHIFT 0
#define PKT_DEST_SHIFT 2
#define PKT_ARG_SHIFT  10
#define PKT_SUB_SHIFT  14
#define PKT_TYPE_MASK  0x3      // 00000000000000000000000000000011
#define PKT_DEST_MASK  0x3FC    // 00000000000000000000001111111100
#define PKT_ARG_MASK   0x3C00   // 00000000000000000011110000000000
#define PKT_SUB_MASK   0xFFC000 // 00000000111111111100000000000000

/* Used to access symbol information. */
#define SYMBOL_KIND_SHIFT    63
#define SYMBOL_QUOTED_SHIFT  54
#define SYMBOL_SNId_SHIFT    24
#define SYMBOL_SUBTASK_SHIFT 32
#define SYMBOL_NARGS_SHIFT   32
#define SYMBOL_KIND_MASK     0xF000000000000000
#define SYMBOL_QUOTED_MASK   0x00C0000000000000
#define SYMBOL_OPCODE_MASK   0xFFFFFFFF
#define SYMBOL_SNId_MASK     0xFF000000
#define SYMBOL_SUBTASK_MASK  0xFFFF00000000
#define SYMBOL_NARGS_MASK    0xD00000000

/* Definition of symbol kinds. */
#define K_S 0
#define K_R 4
#define K_B 6

/* Used to access the information stored within a subtask record. */
#define NARGS_ABSENT_SHIFT   0
#define SUBTREC_STATUS_SHIFT 4
#define NARGS_ABSENT_MASK    0xF
#define SUBTREC_STATUS_MASK  0xF0

/* Packet Types. */
#define ERROR     0
#define REFERENCE 1
#define DATA      2

/* Subtask record status. */
#define NEW        0
#define PROCESSING 1
#define PENDING    2

/* Arg status. */
#define ABSENT     0
#define REQUESTING 1
#define PRESENT    2

/***********************************/
/******* Function Prototypes *******/
/***********************************/
void parse_pkt(packet p, __global uint2 *q, int n, __global bytecode *cStore, __global subt *subt, __global char *scratch);
void parse_subtask(uint code_addr,
                   bytecode symbol,
                   __global uint2 *q,
                   int n,
                   __global bytecode *cStore,
                   __global subt *subt,
                   __global char *scratch);
bytecode service_compute(__global subt* subt, __global char *scratch);

uint symbol_get_kind(ulong s);
bool symbol_is_quoted(ulong s);
uint symbol_get_opcode(ulong s);
uint symbol_get_SNId(ulong s);
uint symbol_get_subtask(ulong s);
uint symbol_get_nargs(ulong s);

bool cunit_q_is_empty(size_t gid, __global uint2 *q, int n);
bool cunit_q_is_full(size_t gid, __global uint2 *q, int n);
uint cunit_q_size(size_t gid, __global uint2 *q, int n);
void transferRQ(__global uint2 *rq,  __global uint2 *q, int n);

void subt_store_payload(uint payload, uint arg_pos, ushort i, __global subt *subt);
bool subt_is_ready(ushort i, __global subt *subt);
__global subt_rec *subt_get_rec(ushort i, __global subt *subt);
bool subt_push(ushort i, __global subt *subt);
bool subt_pop(ushort *result, __global subt *subt);
bool subt_is_full(__global subt *subt);
bool subt_is_empty(__global subt *subt);
void subt_cleanup(ushort i, __global subt *subt);
ushort subt_top(__global subt *subt);
void subt_set_top(__global subt *subt, ushort i);

uint subt_rec_get_service_id(__global subt_rec *r);
uint subt_rec_get_arg(__global subt_rec *r, uint arg_pos);
uint subt_rec_get_arg_status(__global subt_rec *r, uint arg_pos);
uint subt_rec_get_subt_status(__global subt_rec *r);
uint subt_rec_get_nargs_absent(__global subt_rec *r);
uint subt_rec_get_return_to(__global subt_rec *r);
uint subt_rec_get_return_as(__global subt_rec *r);
void subt_rec_set_service_id(__global subt_rec *r, uint service_id);
void subt_rec_set_arg(__global subt_rec *r, uint arg_pos, uint arg);
void subt_rec_set_arg_status(__global subt_rec *r, uint arg_pos, uint status);
void subt_rec_set_subt_status(__global subt_rec *r, uint status);
void subt_rec_set_nargs_absent(__global subt_rec *r, uint n);
void subt_rec_set_return_to(__global subt_rec *r, uint return_to);
void subt_rec_set_return_as(__global subt_rec *r, uint return_as);

uint pkt_get_type(packet p);
uint pkt_get_dest(packet p);
uint pkt_get_arg_pos(packet p);
uint pkt_get_sub(packet p);
uint pkt_get_payload(packet p);
void pkt_set_type(packet *p, uint type);
void pkt_set_dest(packet *p, uint dest);
void pkt_set_arg_pos(packet *p, uint arg);
void pkt_set_sub(packet *p, uint sub);
void pkt_set_payload(packet *p, uint payload);
packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload);

uint q_get_head_index(size_t id, size_t gid, __global packet *q, int n);
uint q_get_tail_index(size_t id, size_t gid, __global packet *q, int n);
void q_set_head_index(uint index, size_t id, size_t gid, __global packet *q, int n);
void q_set_tail_index(uint index, size_t id, size_t gid, __global packet *q, int n);
void q_set_last_op(uint type, size_t id, size_t gid,__global packet *q, int n);
bool q_last_op_is_read(size_t id, size_t gid, __global packet *q, int n);
bool q_last_op_is_write(size_t id, size_t gid, __global packet *q, int n);
bool q_is_empty(size_t id, size_t gid, __global packet *q, int n);
bool q_is_full(size_t id, size_t gid,__global packet *q, int n);
uint q_size(size_t id, size_t gid, __global packet *q, int n);
bool q_read(packet *result, size_t id, __global packet *q, int n);
bool q_write(packet value, size_t id, __global packet *q, int n);

/**************************/
/******* The Kernel *******/
/**************************/
__kernel void vm(__global packet *q,            /* Compute unit queues. */
                 __global packet *rq,           /* Transfer queues for READ state. */
                 int n,                         /* The number of compute units. */
                 __global int *state,           /* Are we in the READ or WRITE state? */
                 __global bytecode *cStore,     /* The code store. */
                 __global subt *subt,           /* The subtask table. */
                 __global char *scratch         /* Scratch memory for temporary results. */
                 ) {
  size_t gid = get_global_id(0);

  if (*state == WRITE) {
    transferRQ(rq, q, n);
  } else {
    for (int i = 0; i < n; i++) {
      packet p;
      while (q_read(&p, i, q, n)) {
        parse_pkt(p, rq, n, cStore, subt, scratch);
      }
    }
  }
}

void parse_pkt(packet p, __global uint2 *q, int n, __global bytecode *cStore, __global subt *subt, __global char *scratch) {
  uint type = pkt_get_type(p);
  uint destination = pkt_get_dest(p);
  uint arg_pos = pkt_get_arg_pos(p);
  uint subtask = pkt_get_sub(p);
  uint payload = pkt_get_payload(p);

  /* Get the bytecode symbol from the code store. */
  bytecode symbol = *(cStore + payload);

  switch (type) {
  case ERROR:
    break;

  case REFERENCE: {
    /* Get the subtask field which is identical to the CodeAddress. */
    uint code_addr = symbol_get_subtask(symbol);

    parse_subtask(code_addr, symbol, q, n, cStore, subt, scratch);

    if (subt_is_ready(subtask, subt)) {
      /* If subtask is ready, call the service core with the subtask table and scratch memory as arguments. */
      bytecode result = service_compute(subt, scratch);

      /* The service core returns a result (a symbol, 64-bit word), put into data packet and return to caller. */
      // How to put 64-word in 32-bit payload space?
      // How to determine caller?
      // packet p = pkt_create(DATA, ... );

      /* If the service core has returned, clean up -> recycle subtask record by pushing back onto stack. */
      subt_cleanup(subtask, subt);
    }
    break;
  }

  case DATA:
    subt_store_payload(symbol, arg_pos, subtask, subt);
    if (subt_is_ready(subtask, subt)) {
      // Perform computation => needs scratch; will return data, needs to create packet & put in queue
      bytecode result = service_compute(subt, scratch);
      subt_cleanup(subtask, subt);
    }
    break;
  }
}

void parse_subtask(uint code_addr,
		   bytecode symbol,
		   __global uint2 *q,
		   int n,
		   __global bytecode *cStore,
		   __global subt *subt,
		   __global char *scratch
		   ) {
  /* Get a subtask record from the stack */
  ushort av_index;
  if (!subt_pop(&av_index, subt)) return;
  __global subt_rec *rec = subt_get_rec(av_index, subt);

  // Set record return_as to code_addr? Is code_addr the parent subtask?

  /* Assume very first bytecode is of type K_S.
     Get the number of args and opcode and put in subtask table. */
  uint nargs = symbol_get_nargs(symbol);
  uint opcode = symbol_get_opcode(symbol);
  subt_rec_set_nargs_absent(rec, nargs);
  subt_rec_set_service_id(rec, opcode);

  for (int arg_pos = 0; arg_pos < nargs; arg_pos++) {
    // symbol = get next symbol.

    switch (symbol_get_kind(symbol)) {
    case K_R:
      if (!symbol_is_quoted(symbol)) {
        uint dest = symbol_get_SNId(symbol);
        packet p = pkt_create(REFERENCE, dest, arg_pos, av_index, symbol);
        q_write(p, dest, q, n);
      } else {
        subt_store_payload(symbol, arg_pos, av_index, subt);
      }
      break;

    case K_B:
      subt_store_payload(symbol, arg_pos, av_index, subt);
      break;
    }
  }
}

bytecode service_compute(__global subt* subt, __global char *scratch) {
  return 0;
}

uint symbol_get_kind(bytecode s) {
  return (s & SYMBOL_KIND_MASK) >> SYMBOL_KIND_SHIFT;
}

bool symbol_is_quoted(bytecode s) {
  return (s & SYMBOL_QUOTED_MASK) >> SYMBOL_QUOTED_SHIFT;
}

uint symbol_get_opcode(bytecode s) {
  return s & SYMBOL_OPCODE_MASK;
}

uint symbol_get_SNId(bytecode s) {
  return s & SYMBOL_SNId_MASK;
}

uint symbol_get_subtask(bytecode s) {
  return (s & SYMBOL_SUBTASK_MASK) >> SYMBOL_SUBTASK_SHIFT;
}

uint symbol_get_nargs(bytecode s) {
  return (s & SYMBOL_NARGS_MASK) >> SYMBOL_NARGS_SHIFT;
}

/**************************************/
/**** Compute Unit Queue Functions ****/
/**************************************/

/* Returns true if all queues owned by the specified compute unit are empty, false otherwise. */
bool cunit_q_is_empty(size_t gid, __global uint2 *q, int n) {
  for (int i = 0; i < n; i++) {
    if (!q_is_empty(gid, i, q, n)) {
      return false;
    }
  }

  return true;
}

/* Returns true if all queues owned by the specified compute unit are full, false otherwise. */
bool cunit_q_is_full(size_t gid, __global uint2 *q, int n) {
  for (int i = 0; i < n; i++) {
    if (!q_is_full(gid, i, q, n)) {
      return false;
    }
  }

  return true;
}

/* Return the total size of all compute unit owned queues. */
uint cunit_q_size(size_t gid, __global uint2 *q, int n) {
  uint size = 0;
  for (int i = 0; i < n; i++) {
    size += q_size(gid, i, q, n);
  }

  return size;
}

/* Copy all compute unit owned queue values from the readQueue into the real queues. */
void transferRQ(__global uint2 *rq, __global uint2 *q, int n) {
  uint2 packet;
  for (int i = 0; i < n; i++) {
    while (q_read(&packet, i, rq, n)) {
      q_write(packet, i, q, n);
    }
  }
}

/*********************************/
/**** Subtask Table Functions ****/
/*********************************/

void subt_store_payload(uint payload, uint arg_pos, ushort i, __global subt *subt) {
  __global subt_rec *rec = subt_get_rec(i, subt);
  uint nargs_absent = subt_rec_get_nargs_absent(rec) - 1;
  subt_rec_set_arg(rec, arg_pos, payload);
  subt_rec_set_arg_status(rec, arg_pos, PRESENT);
  subt_rec_set_nargs_absent(rec, nargs_absent);
}

bool subt_is_ready(ushort i, __global subt *subt) {
  __global subt_rec *rec = subt_get_rec(i, subt);
  return subt_rec_get_nargs_absent(rec) == 0;
}

__global subt_rec *subt_get_rec(ushort i, __global subt *subt) {
  return &(subt->recs[i]);
}

bool subt_push(ushort i, __global subt *subt) {
  if (subt_is_empty(subt)) {
    return false;
  }

  ushort top = subt_top(subt);
  subt->av_recs[top - 1] = i;
  subt_set_top(subt, top - 1);
  return true;
}

bool subt_pop(ushort *av_index, __global subt *subt) {
  if (subt_is_full(subt)) {
    return false;
  }

  ushort top = subt_top(subt);
  *av_index = subt->av_recs[top];
  subt_set_top(subt, top + 1);
  return true;
}

void subt_cleanup(ushort i, __global subt *subt) {
  subt_push(i, subt);
}

bool subt_is_full(__global subt *subt) {
  return subt_top(subt) == SUBT_SIZE + 1;
}

bool subt_is_empty(__global subt *subt) {
  return subt_top(subt) == 1;
}

ushort subt_top(__global subt *subt) {
  return subt->av_recs[0];
}

void subt_set_top(__global subt *subt, ushort i) {
  subt->av_recs[0] = i;
}

/**********************************/
/**** Subtask Record Functions ****/
/**********************************/
uint subt_rec_get_service_id(__global subt_rec *r) {
  return r->service_id;
}

uint subt_rec_get_arg(__global subt_rec *r, uint arg_pos) {
  return r->args[arg_pos];
}

uint subt_rec_get_arg_status(__global subt_rec *r, uint arg_pos) {
  return r->arg_status[arg_pos];
}

uint subt_rec_get_subt_status(__global subt_rec *r) {
  return (r->subt_status & SUBTREC_STATUS_MASK) >> SUBTREC_STATUS_SHIFT;
}

uint subt_rec_get_nargs_absent(__global subt_rec *r) {
  return (r->subt_status & NARGS_ABSENT_MASK);
}

uint subt_rec_get_return_to(__global subt_rec *r) {
  return r->return_to;
}

uint subt_rec_get_return_as(__global subt_rec *r) {
  return r->return_as;
}

void subt_rec_set_service_id(__global subt_rec *r, uint service_id) {
  r->service_id = service_id;
}

void subt_rec_set_arg(__global subt_rec *r, uint arg_pos, uint arg) {
  r->args[arg_pos] = arg;
}

void subt_rec_set_arg_status(__global subt_rec *r, uint arg_pos, uint status) {
  r->arg_status[arg_pos] = status;
}

void subt_rec_set_subt_status(__global subt_rec *r, uint status) {
  r->subt_status = (r->subt_status & ~SUBTREC_STATUS_MASK)
    | ((status << SUBTREC_STATUS_SHIFT) & SUBTREC_STATUS_MASK);
}

void subt_rec_set_nargs_absent(__global subt_rec *r, uint n) {
  r->subt_status = (r->subt_status & ~NARGS_ABSENT_MASK)
    | ((n << NARGS_ABSENT_SHIFT) & NARGS_ABSENT_MASK);
}

void subt_rec_set_return_to(__global subt_rec *r, uint return_to) {
  r->return_to = return_to;
}

void subt_rec_set_return_as(__global subt_rec *r, uint return_as) {
  r->return_as = return_as;
}

/**************************/
/**** Packet Functions ****/
/**************************/

/* Return the packet type. */
uint pkt_get_type(packet p) {
  return (p.x & PKT_TYPE_MASK) >> PKT_TYPE_SHIFT;
}

/* Return the packet destination address. */
uint pkt_get_dest(packet p) {
  return (p.x & PKT_DEST_MASK) >> PKT_DEST_SHIFT;
}

/* Return the packet argument position. */
uint pkt_get_arg_pos(packet p) {
  return (p.x & PKT_ARG_MASK) >> PKT_ARG_SHIFT;
}

/* Return the packet subtask. */
uint pkt_get_sub(packet p) {
  return (p.x & PKT_SUB_MASK) >> PKT_SUB_SHIFT;
}

/* Return the packet payload. */
uint pkt_get_payload(packet p) {
  return p.y;
}

/* Set the packet type. */
void pkt_set_type(packet *p, uint type) {
  (*p).x = ((*p).x & ~PKT_TYPE_MASK) | ((type << PKT_TYPE_SHIFT) & PKT_TYPE_MASK);
}

/* Set the packet destination address. */
void pkt_set_dest(packet *p, uint dest) {
  (*p).x = ((*p).x & ~PKT_DEST_MASK) | ((dest << PKT_DEST_SHIFT) & PKT_DEST_MASK);
}

/* Set the packet argument position. */
void pkt_set_arg_pos(packet *p, uint arg) {
  (*p).x = ((*p).x & ~PKT_ARG_MASK) | ((arg << PKT_ARG_SHIFT) & PKT_ARG_MASK);
}

/* Set the packet subtask. */
void pkt_set_sub(packet *p, uint sub) {
  (*p).x = ((*p).x & ~PKT_SUB_MASK) | ((sub << PKT_SUB_SHIFT) & PKT_SUB_MASK);
}

/* Set the packet payload. */
void pkt_set_payload(packet *p, uint payload) {
  (*p).y = payload;
}

/* Return a newly created packet. */
packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload) {
  packet p;
  pkt_set_type(&p, type);
  pkt_set_dest(&p, dest);
  pkt_set_arg_pos(&p, arg);
  pkt_set_sub(&p, sub);
  pkt_set_payload(&p, payload);
  return p;
}

/*************************/
/**** Queue Functions ****/
/*************************/

/* Returns the array index of the head element of the queue. */
uint q_get_head_index(size_t id, size_t gid, __global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  return indices.x;
}

/* Returns the array index of the tail element of the queue. */
uint q_get_tail_index(size_t id, size_t gid,__global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  return indices.y;
}

/* Set the array index of the head element of the queue. */
void q_set_head_index(uint index, size_t id, size_t gid, __global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  indices.x = index;
  q[id * n + gid].x = as_uint(indices);
}

/* Set the array index of the tail element of the queue. */
void q_set_tail_index(uint index, size_t id, size_t gid,__global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  indices.y = index;
  q[id * n + gid].x = as_uint(indices);
}

/* Set the type of the operation last performed on the queue. */
void q_set_last_op(uint type, size_t id, size_t gid, __global uint2 *q, int n) {
  q[id * n + gid].y = type;
}

/* Returns true if the last operation performed on the queue is a read, false otherwise. */
bool q_last_op_is_read(size_t id, size_t gid,__global uint2 *q, int n) {
  return q[id * n + gid].y == READ;
}

/* Returns true if the last operation performed on the queue is a write, false otherwise. */
bool q_last_op_is_write(size_t id, size_t gid,__global uint2 *q, int n) {
  return q[id * n + gid].y == WRITE;
}

/* Returns true if the queue is empty, false otherwise. */
bool q_is_empty(size_t id, size_t gid,  __global uint2 *q, int n) {
  return (q_get_head_index(id, gid, q, n) == q_get_tail_index(id, gid, q, n))
    && q_last_op_is_read(id, gid, q, n);
}

/* Returns true if the queue is full, false otherwise. */
bool q_is_full(size_t id, size_t gid, __global uint2 *q, int n) {
  return (q_get_head_index(id, gid, q, n) == q_get_tail_index(id, gid, q, n))
    && q_last_op_is_write(id, gid, q, n);
}

/* Return the size of the queue. */
uint q_size(size_t id, size_t gid, __global uint2 *q, int n) {
  if (q_is_full(id, gid, q, n)) return QUEUE_SIZE;
  if (q_is_empty(id, gid, q, n)) return 0;
  uint head = q_get_head_index(id, gid, q, n);
  uint tail = q_get_tail_index(id, gid, q, n);
  return (tail > head) ? (tail - head) : QUEUE_SIZE - head;
}

/* Read the value located at the head index of the queue into 'result'.
 * Returns true if succcessful (queue is not empty), false otherwise. */
bool q_read(uint2 *result, size_t id, __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  if (q_is_empty(gid, id, q, n)) {
    return false;
  }

  int index = q_get_head_index(gid, id, q, n);
  *result = q[(n*n) + (gid * n * QUEUE_SIZE) + (id * QUEUE_SIZE) + index];
  q_set_head_index((index + 1) % QUEUE_SIZE, gid, id, q, n);
  q_set_last_op(READ, gid, id, q, n);
  return true;
}

/* Write a value into the tail index of the queue.
 * Returns true if successful (queue is not full), false otherwise. */
bool q_write(uint2 value, size_t id, __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  if (q_is_full(id, gid, q, n)) {
    return false;
  }

  int index = q_get_tail_index(id, gid, q, n);
  q[(n*n) + (id * n * QUEUE_SIZE) + (gid * QUEUE_SIZE) + index] = value;
  q_set_tail_index((index + 1) % QUEUE_SIZE, id, gid, q, n);
  q_set_last_op(WRITE, id, gid, q, n);
  return true;
}

#ifdef KERNEL_TEST_ENABLED
#include "../tests/kerneltypes.h"
#endif

#include "SharedMacros.h"
#include "SharedTypes.h"

// zero symbol, add 32-bit unsigned integer to it to return pointer as symbol
#define ZERO 0x6240000000000000ULL

/* Used to create, manipulate and access packet information. */
#define PKT_TYPE_SHIFT 0
#define PKT_DEST_SHIFT 2
#define PKT_ARG_SHIFT  10
#define PKT_SUB_SHIFT  14
#define PKT_TYPE_MASK  0x3      // 00000000000000000000000000000011
#define PKT_DEST_MASK  0x3FC    // 00000000000000000000001111111100
#define PKT_ARG_MASK   0x3C00   // 00000000000000000011110000000000
#define PKT_SUB_MASK   0xFFC000 // 00000000111111111100000000000000

/* Used to access symbol information. */
#define SYMBOL_KIND_SHIFT    63
#define SYMBOL_QUOTED_SHIFT  54
#define SYMBOL_SNId_SHIFT    24
#define SYMBOL_SUBTASK_SHIFT 32
#define SYMBOL_NARGS_SHIFT   32
#define SYMBOL_KIND_MASK     0xF000000000000000
#define SYMBOL_QUOTED_MASK   0x00C0000000000000
#define SYMBOL_OPCODE_MASK   0xFFFFFFFF
#define SYMBOL_SNId_MASK     0xFF000000
#define SYMBOL_SUBTASK_MASK  0xFFFF00000000
#define SYMBOL_NARGS_MASK    0xD00000000

/* Definition of symbol kinds. */
#define K_S 0
#define K_R 4
#define K_B 6

/* Used to access the information stored within a subtask record. */
#define NARGS_ABSENT_SHIFT   0
#define SUBTREC_STATUS_SHIFT 4
#define NARGS_ABSENT_MASK    0xF
#define SUBTREC_STATUS_MASK  0xF0

/* Packet Types. */
#define ERROR     0
#define REFERENCE 1
#define DATA      2

/* Subtask record status. */
#define NEW        0
#define PROCESSING 1
#define PENDING    2

/* Arg status. */
#define ABSENT     0
#define REQUESTING 1
#define PRESENT    2

/***********************************/
/******* Function Prototypes *******/
/***********************************/
void parse_pkt(packet p, __global uint2 *q, int n, __global bytecode *cStore, __global subt *subt, __global char *scratch);
void parse_subtask(uint code_addr,
                   bytecode symbol,
                   __global uint2 *q,
                   int n,
                   __global bytecode *cStore,
                   __global subt *subt,
                   __global char *scratch);
bytecode service_compute(__global subt* subt, __global char *scratch);

uint symbol_get_kind(ulong s);
bool symbol_is_quoted(ulong s);
uint symbol_get_opcode(ulong s);
uint symbol_get_SNId(ulong s);
uint symbol_get_subtask(ulong s);
uint symbol_get_nargs(ulong s);

bool cunit_q_is_empty(size_t gid, __global uint2 *q, int n);
bool cunit_q_is_full(size_t gid, __global uint2 *q, int n);
uint cunit_q_size(size_t gid, __global uint2 *q, int n);
void transferRQ(__global uint2 *rq,  __global uint2 *q, int n);

void subt_store_payload(uint payload, uint arg_pos, ushort i, __global subt *subt);
bool subt_is_ready(ushort i, __global subt *subt);
__global subt_rec *subt_get_rec(ushort i, __global subt *subt);
bool subt_push(ushort i, __global subt *subt);
bool subt_pop(ushort *result, __global subt *subt);
bool subt_is_full(__global subt *subt);
bool subt_is_empty(__global subt *subt);
void subt_cleanup(ushort i, __global subt *subt);
ushort subt_top(__global subt *subt);
void subt_set_top(__global subt *subt, ushort i);

uint subt_rec_get_service_id(__global subt_rec *r);
uint subt_rec_get_arg(__global subt_rec *r, uint arg_pos);
uint subt_rec_get_arg_status(__global subt_rec *r, uint arg_pos);
uint subt_rec_get_subt_status(__global subt_rec *r);
uint subt_rec_get_nargs_absent(__global subt_rec *r);
uint subt_rec_get_return_to(__global subt_rec *r);
uint subt_rec_get_return_as(__global subt_rec *r);
void subt_rec_set_service_id(__global subt_rec *r, uint service_id);
void subt_rec_set_arg(__global subt_rec *r, uint arg_pos, uint arg);
void subt_rec_set_arg_status(__global subt_rec *r, uint arg_pos, uint status);
void subt_rec_set_subt_status(__global subt_rec *r, uint status);
void subt_rec_set_nargs_absent(__global subt_rec *r, uint n);
void subt_rec_set_return_to(__global subt_rec *r, uint return_to);
void subt_rec_set_return_as(__global subt_rec *r, uint return_as);

uint pkt_get_type(packet p);
uint pkt_get_dest(packet p);
uint pkt_get_arg_pos(packet p);
uint pkt_get_sub(packet p);
uint pkt_get_payload(packet p);
void pkt_set_type(packet *p, uint type);
void pkt_set_dest(packet *p, uint dest);
void pkt_set_arg_pos(packet *p, uint arg);
void pkt_set_sub(packet *p, uint sub);
void pkt_set_payload(packet *p, uint payload);
packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload);

uint q_get_head_index(size_t id, size_t gid, __global packet *q, int n);
uint q_get_tail_index(size_t id, size_t gid, __global packet *q, int n);
void q_set_head_index(uint index, size_t id, size_t gid, __global packet *q, int n);
void q_set_tail_index(uint index, size_t id, size_t gid, __global packet *q, int n);
void q_set_last_op(uint type, size_t id, size_t gid,__global packet *q, int n);
bool q_last_op_is_read(size_t id, size_t gid, __global packet *q, int n);
bool q_last_op_is_write(size_t id, size_t gid, __global packet *q, int n);
bool q_is_empty(size_t id, size_t gid, __global packet *q, int n);
bool q_is_full(size_t id, size_t gid,__global packet *q, int n);
uint q_size(size_t id, size_t gid, __global packet *q, int n);
bool q_read(packet *result, size_t id, __global packet *q, int n);
bool q_write(packet value, size_t id, __global packet *q, int n);

/**************************/
/******* The Kernel *******/
/**************************/
__kernel void vm(__global packet *q,            /* Compute unit queues. */
                 __global packet *rq,           /* Transfer queues for READ state. */
                 int n,                         /* The number of compute units. */
                 __global int *state,           /* Are we in the READ or WRITE state? */
                 __global bytecode *cStore,     /* The code store. */
                 __global subt *subt,           /* The subtask table. */
                 __global char *scratch         /* Scratch memory for temporary results. */
                 ) {
  size_t gid = get_global_id(0);

  if (*state == WRITE) {
    transferRQ(rq, q, n);
  } else {
    for (int i = 0; i < n; i++) {
      packet p;
      while (q_read(&p, i, q, n)) {
        parse_pkt(p, rq, n, cStore, subt, scratch);
      }
    }
  }
}

void parse_pkt(packet p, __global uint2 *q, int n, __global bytecode *cStore, __global subt *subt, __global char *scratch) {
  uint type = pkt_get_type(p);
  uint destination = pkt_get_dest(p);
  uint arg_pos = pkt_get_arg_pos(p);
  uint subtask = pkt_get_sub(p);
  uint payload = pkt_get_payload(p);

  /* Get the bytecode symbol from the code store. */
  bytecode symbol = *(cStore + payload);

  switch (type) {
  case ERROR:
    break;

  case REFERENCE: {
    /* Get the subtask field which is identical to the CodeAddress. */
    uint code_addr = symbol_get_subtask(symbol);

    parse_subtask(code_addr, symbol, q, n, cStore, subt, scratch);

    if (subt_is_ready(subtask, subt)) {
      /* If subtask is ready, call the service core with the subtask table and scratch memory as arguments. */
      bytecode result = service_compute(subt, scratch);

      /* The service core returns a result (a symbol, 64-bit word), put into data packet and return to caller. */
      // How to put 64-word in 32-bit payload space?
      // How to determine caller?
      // packet p = pkt_create(DATA, ... );

      /* If the service core has returned, clean up -> recycle subtask record by pushing back onto stack. */
      subt_cleanup(subtask, subt);
    }
    break;
  }

  case DATA:
    subt_store_payload(symbol, arg_pos, subtask, subt);
    if (subt_is_ready(subtask, subt)) {
      // Perform computation => needs scratch; will return data, needs to create packet & put in queue
      bytecode result = service_compute(subt, scratch);
      subt_cleanup(subtask, subt);
    }
    break;
  }
}

void parse_subtask(uint code_addr,
		   bytecode symbol,
		   __global uint2 *q,
		   int n,
		   __global bytecode *cStore,
		   __global subt *subt,
		   __global char *scratch
		   ) {
  /* Get a subtask record from the stack */
  ushort av_index;
  if (!subt_pop(&av_index, subt)) return;
  __global subt_rec *rec = subt_get_rec(av_index, subt);

  // Set record return_as to code_addr? Is code_addr the parent subtask?

  /* Assume very first bytecode is of type K_S.
     Get the number of args and opcode and put in subtask table. */
  uint nargs = symbol_get_nargs(symbol);
  uint opcode = symbol_get_opcode(symbol);
  subt_rec_set_nargs_absent(rec, nargs);
  subt_rec_set_service_id(rec, opcode);

  for (int arg_pos = 0; arg_pos < nargs; arg_pos++) {
    switch (symbol_get_kind(symbol)) {
    case K_R:
      if (!symbol_is_quoted(symbol)) {
        uint dest = symbol_get_SNId(symbol);
        packet p = pkt_create(REFERENCE, dest, arg_pos, av_index, symbol);
        q_write(p, dest, q, n);
      } else {
        subt_store_payload(symbol, arg_pos, av_index, subt);
      }
      break;

    case K_B:
      subt_store_payload(symbol, arg_pos, av_index, subt);
      break;
    }
  }
}

bytecode service_compute(__global subt* subt, __global char *scratch) {
  return 0;
}

uint symbol_get_kind(bytecode s) {
  return (s & SYMBOL_KIND_MASK) >> SYMBOL_KIND_SHIFT;
}

bool symbol_is_quoted(bytecode s) {
  return (s & SYMBOL_QUOTED_MASK) >> SYMBOL_QUOTED_SHIFT;
}

uint symbol_get_opcode(bytecode s) {
  return s & SYMBOL_OPCODE_MASK;
}

uint symbol_get_SNId(bytecode s) {
  return s & SYMBOL_SNId_MASK;
}

uint symbol_get_subtask(bytecode s) {
  return (s & SYMBOL_SUBTASK_MASK) >> SYMBOL_SUBTASK_SHIFT;
}

uint symbol_get_nargs(bytecode s) {
  return (s & SYMBOL_NARGS_MASK) >> SYMBOL_NARGS_SHIFT;
}

/**************************************/
/**** Compute Unit Queue Functions ****/
/**************************************/

/* Returns true if all queues owned by the specified compute unit are empty, false otherwise. */
bool cunit_q_is_empty(size_t gid, __global uint2 *q, int n) {
  for (int i = 0; i < n; i++) {
    if (!q_is_empty(gid, i, q, n)) {
      return false;
    }
  }

  return true;
}

/* Returns true if all queues owned by the specified compute unit are full, false otherwise. */
bool cunit_q_is_full(size_t gid, __global uint2 *q, int n) {
  for (int i = 0; i < n; i++) {
    if (!q_is_full(gid, i, q, n)) {
      return false;
    }
  }

  return true;
}

/* Return the total size of all compute unit owned queues. */
uint cunit_q_size(size_t gid, __global uint2 *q, int n) {
  uint size = 0;
  for (int i = 0; i < n; i++) {
    size += q_size(gid, i, q, n);
  }

  return size;
}

/* Copy all compute unit owned queue values from the readQueue into the real queues. */
void transferRQ(__global uint2 *rq, __global uint2 *q, int n) {
  uint2 packet;
  for (int i = 0; i < n; i++) {
    while (q_read(&packet, i, rq, n)) {
      q_write(packet, i, q, n);
    }
  }
}

/*********************************/
/**** Subtask Table Functions ****/
/*********************************/

void subt_store_payload(uint payload, uint arg_pos, ushort i, __global subt *subt) {
  __global subt_rec *rec = subt_get_rec(i, subt);
  uint nargs_absent = subt_rec_get_nargs_absent(rec) - 1;
  subt_rec_set_arg(rec, arg_pos, payload);
  subt_rec_set_arg_status(rec, arg_pos, PRESENT);
  subt_rec_set_nargs_absent(rec, nargs_absent);
}

bool subt_is_ready(ushort i, __global subt *subt) {
  __global subt_rec *rec = subt_get_rec(i, subt);
  return subt_rec_get_nargs_absent(rec) == 0;
}

__global subt_rec *subt_get_rec(ushort i, __global subt *subt) {
  return &(subt->recs[i]);
}

bool subt_push(ushort i, __global subt *subt) {
  if (subt_is_empty(subt)) {
    return false;
  }

  ushort top = subt_top(subt);
  subt->av_recs[top - 1] = i;
  subt_set_top(subt, top - 1);
  return true;
}

bool subt_pop(ushort *av_index, __global subt *subt) {
  if (subt_is_full(subt)) {
    return false;
  }

  ushort top = subt_top(subt);
  *av_index = subt->av_recs[top];
  subt_set_top(subt, top + 1);
  return true;
}

void subt_cleanup(ushort i, __global subt *subt) {
  subt_push(i, subt);
}

bool subt_is_full(__global subt *subt) {
  return subt_top(subt) == SUBT_SIZE + 1;
}

bool subt_is_empty(__global subt *subt) {
  return subt_top(subt) == 1;
}

ushort subt_top(__global subt *subt) {
  return subt->av_recs[0];
}

void subt_set_top(__global subt *subt, ushort i) {
  subt->av_recs[0] = i;
}

/**********************************/
/**** Subtask Record Functions ****/
/**********************************/
uint subt_rec_get_service_id(__global subt_rec *r) {
  return r->service_id;
}

uint subt_rec_get_arg(__global subt_rec *r, uint arg_pos) {
  return r->args[arg_pos];
}

uint subt_rec_get_arg_status(__global subt_rec *r, uint arg_pos) {
  return r->arg_status[arg_pos];
}

uint subt_rec_get_subt_status(__global subt_rec *r) {
  return (r->subt_status & SUBTREC_STATUS_MASK) >> SUBTREC_STATUS_SHIFT;
}

uint subt_rec_get_nargs_absent(__global subt_rec *r) {
  return (r->subt_status & NARGS_ABSENT_MASK);
}

uint subt_rec_get_return_to(__global subt_rec *r) {
  return r->return_to;
}

uint subt_rec_get_return_as(__global subt_rec *r) {
  return r->return_as;
}

void subt_rec_set_service_id(__global subt_rec *r, uint service_id) {
  r->service_id = service_id;
}

void subt_rec_set_arg(__global subt_rec *r, uint arg_pos, uint arg) {
  r->args[arg_pos] = arg;
}

void subt_rec_set_arg_status(__global subt_rec *r, uint arg_pos, uint status) {
  r->arg_status[arg_pos] = status;
}

void subt_rec_set_subt_status(__global subt_rec *r, uint status) {
  r->subt_status = (r->subt_status & ~SUBTREC_STATUS_MASK)
    | ((status << SUBTREC_STATUS_SHIFT) & SUBTREC_STATUS_MASK);
}

void subt_rec_set_nargs_absent(__global subt_rec *r, uint n) {
  r->subt_status = (r->subt_status & ~NARGS_ABSENT_MASK)
    | ((n << NARGS_ABSENT_SHIFT) & NARGS_ABSENT_MASK);
}

void subt_rec_set_return_to(__global subt_rec *r, uint return_to) {
  r->return_to = return_to;
}

void subt_rec_set_return_as(__global subt_rec *r, uint return_as) {
  r->return_as = return_as;
}

/**************************/
/**** Packet Functions ****/
/**************************/

/* Return the packet type. */
uint pkt_get_type(packet p) {
  return (p.x & PKT_TYPE_MASK) >> PKT_TYPE_SHIFT;
}

/* Return the packet destination address. */
uint pkt_get_dest(packet p) {
  return (p.x & PKT_DEST_MASK) >> PKT_DEST_SHIFT;
}

/* Return the packet argument position. */
uint pkt_get_arg_pos(packet p) {
  return (p.x & PKT_ARG_MASK) >> PKT_ARG_SHIFT;
}

/* Return the packet subtask. */
uint pkt_get_sub(packet p) {
  return (p.x & PKT_SUB_MASK) >> PKT_SUB_SHIFT;
}

/* Return the packet payload. */
uint pkt_get_payload(packet p) {
  return p.y;
}

/* Set the packet type. */
void pkt_set_type(packet *p, uint type) {
  (*p).x = ((*p).x & ~PKT_TYPE_MASK) | ((type << PKT_TYPE_SHIFT) & PKT_TYPE_MASK);
}

/* Set the packet destination address. */
void pkt_set_dest(packet *p, uint dest) {
  (*p).x = ((*p).x & ~PKT_DEST_MASK) | ((dest << PKT_DEST_SHIFT) & PKT_DEST_MASK);
}

/* Set the packet argument position. */
void pkt_set_arg_pos(packet *p, uint arg) {
  (*p).x = ((*p).x & ~PKT_ARG_MASK) | ((arg << PKT_ARG_SHIFT) & PKT_ARG_MASK);
}

/* Set the packet subtask. */
void pkt_set_sub(packet *p, uint sub) {
  (*p).x = ((*p).x & ~PKT_SUB_MASK) | ((sub << PKT_SUB_SHIFT) & PKT_SUB_MASK);
}

/* Set the packet payload. */
void pkt_set_payload(packet *p, uint payload) {
  (*p).y = payload;
}

/* Return a newly created packet. */
packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload) {
  packet p;
  pkt_set_type(&p, type);
  pkt_set_dest(&p, dest);
  pkt_set_arg_pos(&p, arg);
  pkt_set_sub(&p, sub);
  pkt_set_payload(&p, payload);
  return p;
}

/*************************/
/**** Queue Functions ****/
/*************************/

/* Returns the array index of the head element of the queue. */
uint q_get_head_index(size_t id, size_t gid, __global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  return indices.x;
}

/* Returns the array index of the tail element of the queue. */
uint q_get_tail_index(size_t id, size_t gid,__global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  return indices.y;
}

/* Set the array index of the head element of the queue. */
void q_set_head_index(uint index, size_t id, size_t gid, __global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  indices.x = index;
  q[id * n + gid].x = as_uint(indices);
}

/* Set the array index of the tail element of the queue. */
void q_set_tail_index(uint index, size_t id, size_t gid,__global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  indices.y = index;
  q[id * n + gid].x = as_uint(indices);
}

/* Set the type of the operation last performed on the queue. */
void q_set_last_op(uint type, size_t id, size_t gid, __global uint2 *q, int n) {
  q[id * n + gid].y = type;
}

/* Returns true if the last operation performed on the queue is a read, false otherwise. */
bool q_last_op_is_read(size_t id, size_t gid,__global uint2 *q, int n) {
  return q[id * n + gid].y == READ;
}

/* Returns true if the last operation performed on the queue is a write, false otherwise. */
bool q_last_op_is_write(size_t id, size_t gid,__global uint2 *q, int n) {
  return q[id * n + gid].y == WRITE;
}

/* Returns true if the queue is empty, false otherwise. */
bool q_is_empty(size_t id, size_t gid,  __global uint2 *q, int n) {
  return (q_get_head_index(id, gid, q, n) == q_get_tail_index(id, gid, q, n))
    && q_last_op_is_read(id, gid, q, n);
}

/* Returns true if the queue is full, false otherwise. */
bool q_is_full(size_t id, size_t gid, __global uint2 *q, int n) {
  return (q_get_head_index(id, gid, q, n) == q_get_tail_index(id, gid, q, n))
    && q_last_op_is_write(id, gid, q, n);
}

/* Return the size of the queue. */
uint q_size(size_t id, size_t gid, __global uint2 *q, int n) {
  if (q_is_full(id, gid, q, n)) return QUEUE_SIZE;
  if (q_is_empty(id, gid, q, n)) return 0;
  uint head = q_get_head_index(id, gid, q, n);
  uint tail = q_get_tail_index(id, gid, q, n);
  return (tail > head) ? (tail - head) : QUEUE_SIZE - head;
}

/* Read the value located at the head index of the queue into 'result'.
 * Returns true if succcessful (queue is not empty), false otherwise. */
bool q_read(uint2 *result, size_t id, __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  if (q_is_empty(gid, id, q, n)) {
    return false;
  }

  int index = q_get_head_index(gid, id, q, n);
  *result = q[(n*n) + (gid * n * QUEUE_SIZE) + (id * QUEUE_SIZE) + index];
  q_set_head_index((index + 1) % QUEUE_SIZE, gid, id, q, n);
  q_set_last_op(READ, gid, id, q, n);
  return true;
}

/* Write a value into the tail index of the queue.
 * Returns true if successful (queue is not full), false otherwise. */
bool q_write(uint2 value, size_t id, __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  if (q_is_full(id, gid, q, n)) {
    return false;
  }

  int index = q_get_tail_index(id, gid, q, n);
  q[(n*n) + (id * n * QUEUE_SIZE) + (gid * QUEUE_SIZE) + index] = value;
  q_set_tail_index((index + 1) % QUEUE_SIZE, id, gid, q, n);
  q_set_last_op(WRITE, id, gid, q, n);
  return true;
}


Fixed a bug in the packet get functions.

/* Used to create, manipulate and access packet information. */
#define PKT_TYPE_SHIFT 0
#define PKT_DEST_SHIFT 2
#define PKT_ARG_SHIFT  10
#define PKT_SUB_SHIFT  14
#define PKT_TYPE_MASK 0x3      // 00000000000000000000000000000011
#define PKT_DEST_MASK 0x3FC    // 00000000000000000000001111111100
#define PKT_ARG_MASK  0x3C00   // 00000000000000000011110000000000
#define PKT_SUB_MASK  0xFFC000 // 00000000111111111100000000000000

/* Packet Types */
#define ERROR     0
#define REFERENCE 1
#define DATA      2
#define REQUEST   3

typedef uint2 packet;

uint pkt_get_type(packet p);
uint pkt_get_dest(packet p);
uint pkt_get_arg(packet p);
uint pkt_get_sub(packet p);
uint pkt_get_payload(packet p);
void pkt_set_type(packet *p, uint type);
void pkt_set_dest(packet *p, uint dest);
void pkt_set_arg(packet *p, uint arg);
void pkt_set_sub(packet *p, uint sub);
void pkt_set_payload(packet *p, uint payload);
packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload);

void transferRQ(__global uint2 *q, __global uint3 *qDetails, __global uint2 *rq, __global uint3 *rqDetails, int n);
uint q_get_head_index(size_t id, size_t gid, __global uint3 *qDetails, int n);
uint q_get_tail_index(size_t id, size_t gid, __global uint3 *qDetails, int n);
void q_set_head_index(uint index, size_t id, size_t gid, __global uint3 *qDetails, int n);
void q_set_tail_index(uint index, size_t id, size_t gid, __global uint3 *qDetails, int n);
void q_set_last_op(uint type, size_t id, size_t gid,__global uint3 *qDetails, int n);
bool q_last_op_is_read(size_t id, size_t gid, __global uint3 *qDetails, int n);
bool q_last_op_is_write(size_t id, size_t gid, __global uint3 *qDetails, int n);
bool q_is_empty(size_t id, size_t gid, __global uint3 *qDetails, int n);
bool q_is_full(size_t id, size_t gid,__global uint3 *qDetails, int n);
bool q_read(uint2 *result, size_t id, __global uint2 *q, __global uint3 *qDetails, int n);
bool q_write(uint2 value, size_t id, __global uint2 *q, __global uint3 *qDetails, int n);

/**************************/
/******* The Kernel *******/
/**************************/
__kernel void qtest(__global uint2 *q, __global uint3 *qDetails,
		    __global uint2 *rq, __global uint3 *rqDetails,
		    int n,
		    __global int *state)
{
  size_t gid = get_global_id(0);

  packet p = pkt_create(ERROR, 7, 0, 0, 0);
  if (*state == WRITE) {
    transferRQ(rq, rqDetails, q, qDetails, n);
  } else {
    q_write(p, gid, q, qDetails, n);
  }

  *state = COMPLETE;
}

/**************************/
/**** Packet Functions ****/
/**************************/

uint pkt_get_type(packet p) {
  return (p.x & PKT_TYPE_MASK) >> PKT_TYPE_SHIFT;
}

uint pkt_get_dest(packet p) {
  return (p.x & PKT_DEST_MASK) >> PKT_DEST_SHIFT;
}

uint pkt_get_arg(packet p) {
  return (p.x & PKT_ARG_MASK) >> PKT_ARG_SHIFT;
}

uint pkt_get_sub(packet p) {
  return (p.x & PKT_SUB_MASK) >> PKT_SUB_SHIFT;
}

uint pkt_get_payload(packet p) {
  return p.y;
}

void pkt_set_type(packet *p, uint type) {
  (*p).x = ((*p).x & ~PKT_TYPE_MASK) | ((type << PKT_TYPE_SHIFT) & PKT_TYPE_MASK);
}

void pkt_set_dest(packet *p, uint dest) {
  (*p).x = ((*p).x & ~PKT_DEST_MASK) | ((dest << PKT_DEST_SHIFT) & PKT_DEST_MASK);
}

void pkt_set_arg(packet *p, uint arg) {
  (*p).x = ((*p).x & ~PKT_ARG_MASK) | ((arg << PKT_ARG_SHIFT) & PKT_ARG_MASK);
}

void pkt_set_sub(packet *p, uint sub) {
  (*p).x = ((*p).x & ~PKT_SUB_MASK) | ((sub << PKT_SUB_SHIFT) & PKT_SUB_MASK);
}

void pkt_set_payload(packet *p, uint payload) {
  (*p).y = payload;
}

packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload) {
  packet p = 0;
  pkt_set_type(&p, type);
  pkt_set_dest(&p, dest);
  pkt_set_arg(&p, arg);
  pkt_set_sub(&p, sub);
  pkt_set_payload(&p, payload);
  return p;
}

/*************************/
/**** Queue Functions ****/
/*************************/

/* Returns the array index of the head element of the queue specifided by 'id' */
uint q_get_head_index(size_t id, size_t gid, __global uint3 *qDetails, int n) {
  return qDetails[id * n + gid].x;
}

/* Returns the array index of the tail element of the queue specified by 'id'. */
uint q_get_tail_index(size_t id, size_t gid,__global uint3 *qDetails, int n) {
  return qDetails[id * n + gid].y;
}

/* Set the array index of the head element of the queue specified by 'id'. */
void q_set_head_index(uint index, size_t id, size_t gid, __global uint3 *qDetails, int n) {
  qDetails[id * n + gid].x = index;
}

/* Set the array index of the tail element of the queue specified by 'id'. */
void q_set_tail_index(uint index, size_t id, size_t gid,__global uint3 *qDetails, int n) {
  qDetails[id * n + gid].y = index;
}

/* Set the type of the operation last performed on the queue specified by 'id'. */
void q_set_last_op(uint type, size_t id, size_t gid, __global uint3 *qDetails, int n) {
  qDetails[id * n + gid].z = type;
}

/* Returns true if the last operation performed on the queue specified by 'id' is a read, false otherwise. */
bool q_last_op_is_read(size_t id, size_t gid,__global uint3 *qDetails, int n) {
  return qDetails[id * n + gid].z == READ;
}

/* Returns true if the last operation performed on the queue specified by 'id' is a write, false otherwise. */
bool q_last_op_is_write(size_t id, size_t gid,__global uint3 *qDetails, int n) {
  return qDetails[id * n + gid].z != READ;
}

/* Returns true if the queue is empty, false otherwise. */
bool q_is_empty(size_t id, size_t gid,  __global uint3 *qDetails, int n) {
  return (q_get_head_index(id, gid, qDetails, n) == q_get_tail_index(id, gid, qDetails, n))
    && q_last_op_is_read(id, gid, qDetails, n);
}

/* Returns true if the queue is full, false otherwise. */
bool q_is_full(size_t id, size_t gid, __global uint3 *qDetails, int n) {
  return (q_get_head_index(id, gid, qDetails, n) == q_get_tail_index(id, gid, qDetails, n))
    && q_last_op_is_write(id, gid, qDetails, n);
}

/* Read the value located at the head index into 'result' from the queue specified by 'id'.
 * Returns true if succcessful (queue is not empty), false otherwise. */
bool q_read(uint2 *result, size_t id, __global uint2 *q, __global uint3 *qDetails, int n) {
  size_t gid = get_global_id(0);
  if (q_is_empty(id, gid, qDetails, n)) {
    return false;
  }

  int index = q_get_head_index(id, gid, qDetails, n);
  *result = q[(gid * n * QUEUE_SIZE) + (id * QUEUE_SIZE) + index];
  q_set_head_index((index + 1) % QUEUE_SIZE, id, gid, qDetails, n);
  q_set_last_op(READ, id, gid, qDetails, n);
  return true;
}

/* Write a value into the tail index of the queue specified by 'id'.
 * Returns true if successful (queue is not full), false otherwise. */
bool q_write(uint2 value, size_t id, __global uint2 *q, __global uint3 *qDetails, int n) {
  size_t gid = get_global_id(0);
  if (q_is_full(id, gid, qDetails, n)) {
    return false;
  }

  int index = q_get_tail_index(id, gid, qDetails, n);
  q[(gid * n * QUEUE_SIZE) + (id * QUEUE_SIZE) + index] = value;
  q_set_tail_index((index + 1) % QUEUE_SIZE, id, gid, qDetails, n);
  q_set_last_op(WRITE, id, gid, qDetails, n);
  return true;
}

void transferRQ(__global uint2 *rq, __global uint3 *rqDetails, __global uint2 *q, __global uint3 *qDetails, int n) {
  size_t gid = get_global_id(0);
  uint2 packet;
  for (int i = 0; i < n; i++) {
    while (!q_is_empty(i, gid, rqDetails, n)) {
      q_read(&packet, i, rq, rqDetails, n);
      q_write(packet, i, q, qDetails, n);
    }
  }
}

/* Used to create, manipulate and access packet information. */
#define PKT_TYPE_BIT_POS 0
#define PKT_DEST_BIT_POS 2
#define PKT_ARG_BIT_POS  10
#define PKT_SUB_BIT_POS  14
#define PKT_TYPE_MASK 0x3      // 00000000000000000000000000000011
#define PKT_DEST_MASK 0x3FC    // 00000000000000000000001111111100
#define PKT_ARG_MASK  0x3C00   // 00000000000000000011110000000000
#define PKT_SUB_MASK  0xFFC000 // 00000000111111111100000000000000

/* Packet Types */
#define DEFAULT   0
#define REFERENCE 1

typedef uint2 packet;

uint pkt_get_type(packet p);
uint pkt_get_dest(packet p);
uint pkt_get_arg(packet p);
uint pkt_get_sub(packet p);
uint pkt_get_payload(packet p);
void pkt_set_type(packet *p, uint type);
void pkt_set_dest(packet *p, uint dest);
void pkt_set_arg(packet *p, uint arg);
void pkt_set_sub(packet *p, uint sub);
void pkt_set_payload(packet *p, uint payload);
packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload);

void transferRQ(__global uint2 *q, __global uint3 *qDetails, __global uint2 *rq, __global uint3 *rqDetails, int n);
uint q_get_head_index(size_t id, size_t gid, __global uint3 *qDetails, int n);
uint q_get_tail_index(size_t id, size_t gid, __global uint3 *qDetails, int n);
void q_set_head_index(uint index, size_t id, size_t gid, __global uint3 *qDetails, int n);
void q_set_tail_index(uint index, size_t id, size_t gid, __global uint3 *qDetails, int n);
void q_set_last_op(uint type, size_t id, size_t gid,__global uint3 *qDetails, int n);
bool q_last_op_is_read(size_t id, size_t gid, __global uint3 *qDetails, int n);
bool q_last_op_is_write(size_t id, size_t gid, __global uint3 *qDetails, int n);
bool q_is_empty(size_t id, size_t gid, __global uint3 *qDetails, int n);
bool q_is_full(size_t id, size_t gid,__global uint3 *qDetails, int n);
bool q_read(uint2 *result, size_t id, __global uint2 *q, __global uint3 *qDetails, int n);
bool q_write(uint2 value, size_t id, __global uint2 *q, __global uint3 *qDetails, int n);

/**************************/
/******* The Kernel *******/
/**************************/
__kernel void qtest(__global uint2 *q, __global uint3 *qDetails,
		    __global uint2 *rq, __global uint3 *rqDetails,
		    int n,
		    __global int *state)
{
  size_t gid = get_global_id(0);

  packet p = pkt_create(DEFAULT, 7, 0, 0, 0);
  if (*state == WRITE) {
    transferRQ(rq, rqDetails, q, qDetails, n);
  } else {
    q_write(p, gid, q, qDetails, n);
  }

  *state = COMPLETE;
}

/**************************/
/**** Packet Functions ****/
/**************************/

uint pkt_get_type(packet p) {
  return p.x & PKT_TYPE_MASK;
}

uint pkt_get_dest(packet p) {
  return p.x & PKT_DEST_MASK;
}

uint pkt_get_arg(packet p) {
  return p.x & PKT_ARG_MASK;
}

uint pkt_get_sub(packet p) {
  return p.x & PKT_SUB_MASK;
}

uint pkt_get_payload(packet p) {
  return p.y;
}

void pkt_set_type(packet *p, uint type) {
  (*p).x = ((*p).x & ~PKT_TYPE_MASK) | ((type << PKT_TYPE_BIT_POS) & PKT_TYPE_MASK);
}

void pkt_set_dest(packet *p, uint dest) {
  (*p).x = ((*p).x & ~PKT_DEST_MASK) | ((dest << PKT_DEST_BIT_POS) & PKT_DEST_MASK);
}

void pkt_set_arg(packet *p, uint arg) {
  (*p).x = ((*p).x & ~PKT_ARG_MASK) | ((arg << PKT_ARG_BIT_POS) & PKT_ARG_MASK);
}

void pkt_set_sub(packet *p, uint sub) {
  (*p).x = ((*p).x & ~PKT_SUB_MASK) | ((sub << PKT_SUB_BIT_POS) & PKT_SUB_MASK);
}

void pkt_set_payload(packet *p, uint payload) {
  (*p).y = payload;
}

packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload) {
  packet p = 0;
  pkt_set_type(&p, type);
  pkt_set_dest(&p, dest);
  pkt_set_arg(&p, arg);
  pkt_set_sub(&p, sub);
  pkt_set_payload(&p, payload);
  return p;
}

/*************************/
/**** Queue Functions ****/
/*************************/

/* Returns the array index of the head element of the queue specifided by 'id' */
uint q_get_head_index(size_t id, size_t gid, __global uint3 *qDetails, int n) {
  return qDetails[id * n + gid].x;
}

/* Returns the array index of the tail element of the queue specified by 'id'. */
uint q_get_tail_index(size_t id, size_t gid,__global uint3 *qDetails, int n) {
  return qDetails[id * n + gid].y;
}

/* Set the array index of the head element of the queue specified by 'id'. */
void q_set_head_index(uint index, size_t id, size_t gid, __global uint3 *qDetails, int n) {
  qDetails[id * n + gid].x = index;
}

/* Set the array index of the tail element of the queue specified by 'id'. */
void q_set_tail_index(uint index, size_t id, size_t gid,__global uint3 *qDetails, int n) {
  qDetails[id * n + gid].y = index;
}

/* Set the type of the operation last performed on the queue specified by 'id'. */
void q_set_last_op(uint type, size_t id, size_t gid, __global uint3 *qDetails, int n) {
  qDetails[id * n + gid].z = type;
}

/* Returns true if the last operation performed on the queue specified by 'id' is a read, false otherwise. */
bool q_last_op_is_read(size_t id, size_t gid,__global uint3 *qDetails, int n) {
  return qDetails[id * n + gid].z == READ;
}

/* Returns true if the last operation performed on the queue specified by 'id' is a write, false otherwise. */
bool q_last_op_is_write(size_t id, size_t gid,__global uint3 *qDetails, int n) {
  return qDetails[id * n + gid].z != READ;
}

/* Returns true if the queue is empty, false otherwise. */
bool q_is_empty(size_t id, size_t gid,  __global uint3 *qDetails, int n) {
  return (q_get_head_index(id, gid, qDetails, n) == q_get_tail_index(id, gid, qDetails, n))
    && q_last_op_is_read(id, gid, qDetails, n);
}

/* Returns true if the queue is full, false otherwise. */
bool q_is_full(size_t id, size_t gid, __global uint3 *qDetails, int n) {
  return (q_get_head_index(id, gid, qDetails, n) == q_get_tail_index(id, gid, qDetails, n))
    && q_last_op_is_write(id, gid, qDetails, n);
}

/* Read the value located at the head index into 'result' from the queue specified by 'id'.
 * Returns true if succcessful (queue is not empty), false otherwise. */
bool q_read(uint2 *result, size_t id, __global uint2 *q, __global uint3 *qDetails, int n) {
  size_t gid = get_global_id(0);
  if (q_is_empty(id, gid, qDetails, n)) {
    return false;
  }

  int index = q_get_head_index(id, gid, qDetails, n);
  *result = q[(gid * n * QUEUE_SIZE) + (id * QUEUE_SIZE) + index];
  q_set_head_index((index + 1) % QUEUE_SIZE, id, gid, qDetails, n);
  q_set_last_op(READ, id, gid, qDetails, n);
  return true;
}

/* Write a value into the tail index of the queue specified by 'id'.
 * Returns true if successful (queue is not full), false otherwise. */
bool q_write(uint2 value, size_t id, __global uint2 *q, __global uint3 *qDetails, int n) {
  size_t gid = get_global_id(0);
  if (q_is_full(id, gid, qDetails, n)) {
    return false;
  }

  int index = q_get_tail_index(id, gid, qDetails, n);
  q[(gid * n * QUEUE_SIZE) + (id * QUEUE_SIZE) + index] = value;
  q_set_tail_index((index + 1) % QUEUE_SIZE, id, gid, qDetails, n);
  q_set_last_op(WRITE, id, gid, qDetails, n);
  return true;
}

void transferRQ(__global uint2 *rq, __global uint3 *rqDetails, __global uint2 *q, __global uint3 *qDetails, int n) {
  size_t gid = get_global_id(0);
  uint2 packet;
  for (int i = 0; i < n; i++) {
    while (!q_is_empty(i, gid, rqDetails, n)) {
      q_read(&packet, i, rq, rqDetails, n);
      q_write(packet, i, q, qDetails, n);
    }
  }
}