Implemented the subt_get_pending function.

#ifdef KERNEL_TEST_ENABLED
#include "../tests/kerneltypes.h"
#endif

/* Used to create, manipulate and access packet information. */
#define PKT_TYPE_SHIFT 0
#define PKT_DEST_SHIFT 2
#define PKT_ARG_SHIFT  10
#define PKT_SUB_SHIFT  14
#define PKT_TYPE_MASK  0x3      // 00000000000000000000000000000011
#define PKT_DEST_MASK  0x3FC    // 00000000000000000000001111111100
#define PKT_ARG_MASK   0x3C00   // 00000000000000000011110000000000
#define PKT_SUB_MASK   0xFFC000 // 00000000111111111100000000000000

/* Used to access the information stored within a subtask record. */
#define NARGS_ABSENT_SHIFT   0
#define SUBTREC_STATUS_SHIFT 4
#define NARGS_ABSENT_MASK    0xF
#define SUBTREC_STATUS_MASK  0xF0

/* Packet Types. */
#define ERROR     0
#define REFERENCE 1
#define REQUEST   2
#define DATA      3

/* Subtask record status. */
#define NEW        0
#define PROCESSING 1
#define PENDING    2

/* Arg status. */
#define ABSENT     0
#define REQUESTING 1
#define PRESENT    2

/* Arg mode. */
#define BY_VAL 0
#define BY_REF 1

/* A packet is 2x32bit integers. */
typedef uint2 packet;

/* Bytecode is 64bit. */
typedef ulong bytecode;

/* A subtask table record. */
typedef struct subt_rec {
  uint service_id;            // [32bits] Opcode
  uint args[QUEUE_SIZE];      // [32bits] Pointers to data or constants.
  uchar arg_mode[QUEUE_SIZE]; // [4bits] Arg status, [4bits] Arg mode
  uchar subt_status;          // [4bits]  Subtask status, [4bits] number of args absent.
  uchar return_to;            // [8bits]
  ushort return_as;           // [16bits] Subtask address + argument position.
} subt_rec;

/***********************************/
/******* Function Prototypes *******/
/***********************************/
bool cunit_q_is_empty(size_t gid, __global uint2 *q, int n);
bool cunit_q_is_full(size_t gid, __global uint2 *q, int n);
uint cunit_q_size(size_t gid, __global uint2 *q, int n);
void transferRQ(__global uint2 *rq,  __global uint2 *q, int n);

bool subt_add_rec(packet p, subt_rec *subt, size_t *avSubtRecs);

bool avSubtRecs_push(size_t i, size_t *avSubtRecs);
bool avSubtRecs_pop(size_t *result, size_t *avSubtRecs);
bool avSubtRecs_is_empty(size_t *avSubtRecs);
bool avSubtRecs_is_full(size_t *avSubtRecs);
size_t avSubtRecs_top(size_t *avSubtRecs);
void avSubtRecs_set_top(size_t top, size_t *avSubtRecs);

uint subt_rec_get_service_id(subt_rec r);
uint subt_rec_get_arg(subt_rec r, uint arg_pos);
uint subt_rec_get_arg_mode(subt_rec r, uint arg_pos);
uint subt_rec_get_subt_status(subt_rec r);
uint subt_rec_get_nargs_absent(subt_rec r);
uint subt_rec_get_return_to(subt_rec r);
uint subt_rec_get_return_as(subt_rec r);
void subt_rec_set_service_id(subt_rec *r, uint service_id);
void subt_rec_set_arg(subt_rec *r, uint arg_pos, uint arg);
void subt_rec_set_arg_mode(subt_rec *r, uint arg_pos, uint mode);
void subt_rec_set_subt_status(subt_rec *r, uint status);
void subt_rec_set_nargs_absent(subt_rec *r, uint n);
void subt_rec_set_return_to(subt_rec *r, uint return_to);
void subt_rec_set_return_as(subt_rec *r, uint return_as);

uint pkt_get_type(packet p);
uint pkt_get_dest(packet p);
uint pkt_get_arg_pos(packet p);
uint pkt_get_sub(packet p);
uint pkt_get_payload(packet p);
void pkt_set_type(packet *p, uint type);
void pkt_set_dest(packet *p, uint dest);
void pkt_set_arg_pos(packet *p, uint arg);
void pkt_set_sub(packet *p, uint sub);
void pkt_set_payload(packet *p, uint payload);
packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload);

uint q_get_head_index(size_t id, size_t gid, __global uint2 *q, int n);
uint q_get_tail_index(size_t id, size_t gid, __global uint2 *q, int n);
void q_set_head_index(uint index, size_t id, size_t gid, __global uint2 *q, int n);
void q_set_tail_index(uint index, size_t id, size_t gid, __global uint2 *q, int n);
void q_set_last_op(uint type, size_t id, size_t gid,__global uint2 *q, int n);
bool q_last_op_is_read(size_t id, size_t gid, __global uint2 *q, int n);
bool q_last_op_is_write(size_t id, size_t gid, __global uint2 *q, int n);
bool q_is_empty(size_t id, size_t gid, __global uint2 *q, int n);
bool q_is_full(size_t id, size_t gid,__global uint2 *q, int n);
uint q_size(size_t id, size_t gid, __global uint2 *q, int n);
bool q_read(uint2 *result, size_t id, __global uint2 *q, int n);
bool q_write(uint2 value, size_t id, __global uint2 *q, int n);

/**************************/
/******* The Kernel *******/
/**************************/
__kernel void vm(__global packet *q,            /* Compute unit queues. */
		 __global packet *rq,           /* Transfer queues for READ state. */
		 int n,                        /* The number of compute units. */
		 __global int *state,          /* Are we in the READ or WRITE state? */
		 __global bytecode *cStore,    /* The code store. */
		 __global subt_rec *subt,      /* The subtask table. */
		 __global size_t *avSubtRecs   /* The available subtask table records. */
		 ) {
  size_t gid = get_global_id(0);

  if (*state == WRITE) {
    transferRQ(rq, q, n);

    switch (gid) {
    case 0:
      break;
    case 1:
      break;
    case 2:
      break;
    case 3:
      if (!cunit_q_is_empty(0, q, n)) {
	*state = COMPLETE;
      }
      break;
    }
  } else {
    switch (gid) {
    case 0:
      if (cunit_q_is_empty(1, q, n)) {
	packet p = pkt_create(REFERENCE, 0, 1, 0, 0);
	q_write(p, 1, rq, n);
	q_write(p, 2, rq, n);
	q_write(p, 3, rq, n);
      }
      break;

    case 1:
      if (!cunit_q_is_empty(gid, q, n)) {
	packet p = pkt_create(REFERENCE, 0, 1, 0, 0);
	q_write(p, 2, rq, n);
      }
      break;
    case 2:
      if (cunit_q_size(gid, q, n) == 2) {
	packet p = pkt_create(REFERENCE, 0, 1, 0, 0);
	q_write(p, 0, rq, n);
      }
      break;
    case 3:
      break;
    }
  }
}

/**************************************/
/**** Compute Unit Queue Functions ****/
/**************************************/

/* Returns true if all queues owned by the specified compute unit are empty, false otherwise. */
bool cunit_q_is_empty(size_t gid, __global uint2 *q, int n) {
  for (int i = 0; i < n; i++) {
    if (!q_is_empty(gid, i, q, n)) {
      return false;
    }
  }

  return true;
}

/* Returns true if all queues owned by the specified compute unit are full, false otherwise. */
bool cunit_q_is_full(size_t gid, __global uint2 *q, int n) {
  for (int i = 0; i < n; i++) {
    if (!q_is_full(gid, i, q, n)) {
      return false;
    }
  }

  return true;
}

/* Return the total size of all compute unit owned queues. */
uint cunit_q_size(size_t gid, __global uint2 *q, int n) {
  uint size = 0;
  for (int i = 0; i < n; i++) {
    size += q_size(gid, i, q, n);
  }

  return size;
}

/* Copy all compute unit owned queue values from the readQueue into the real queues. */
void transferRQ(__global uint2 *rq, __global uint2 *q, int n) {
  uint2 packet;
  for (int i = 0; i < n; i++) {
    while (q_read(&packet, i, rq, n)) {
      q_write(packet, i, q, n);
    }
  }
}

/*********************************/
/**** Subtask Table Functions ****/
/*********************************/

/* Insert a record into the subtask table.
 * Returns true if successful (table is not full), false otherwise. */
bool subt_add_rec(packet p, subt_rec *subt, size_t *avSubtRecs) {
  size_t i;
  if (!avSubtRecs_pop(&i, avSubtRecs)) {
    return false;
  }

  subt_rec r;
  subt_rec_set_subt_status(&r, NEW);
  subt_rec_set_arg(&r, pkt_get_arg_pos(p), pkt_get_payload(p)); // If packet has DATA type.

  subt[i] = r;
  return true;
}

/* Get the first pending subtask record from the subtask table. */
subt_rec *subt_get_pending(subt_rec *subt) {
  for (int i = 0; i < SUBT_SIZE; i++) {
    if (subt_rec_get_subt_status(subt[i]) == PENDING) {
      return subt + i;
    }
  }

  return NULL;
}

/*********************************************************/
/**** Available Subtask Table Records Stack Functions ****/
/*********************************************************/

bool avSubtRecs_push(size_t i, size_t *avSubtRecs) {
  if (avSubtRecs_is_full(avSubtRecs)) {
    return false;
  }

  size_t top = avSubtRecs_top(avSubtRecs) - 1;
  avSubtRecs[top] = i;
  avSubtRecs_set_top(top, avSubtRecs);
  return true;
}

bool avSubtRecs_pop(size_t *result, size_t *avSubtRecs) {
  if (avSubtRecs_is_empty(avSubtRecs)) {
    return false;
  }

  size_t top = avSubtRecs_top(avSubtRecs);
  *result = avSubtRecs[top];
  avSubtRecs_set_top(top + 1, avSubtRecs);
  return true;
}

bool avSubtRecs_is_empty(size_t *avSubtRecs) {
  return avSubtRecs_top(avSubtRecs) == SUBT_SIZE - 1;
}

bool avSubtRecs_is_full(size_t *avSubtRecs) {
  return avSubtRecs_top(avSubtRecs) == 0;
}

size_t avSubtRecs_top(size_t *avSubtRecs) {
  return avSubtRecs[0];
}

void avSubtRecs_set_top(size_t top, size_t *avSubtRecs) {
  avSubtRecs[0] = top;
}

/**********************************/
/**** Subtask Record Functions ****/
/**********************************/
uint subt_rec_get_service_id(subt_rec r) {
  return r.service_id;
}

uint subt_rec_get_arg(subt_rec r, uint arg_pos) {
  return r.args[arg_pos];
}

uint subt_rec_get_arg_mode(subt_rec r, uint arg_pos) {
  return r.arg_mode[arg_pos];
}

uint subt_rec_get_subt_status(subt_rec r) {
  return (r.subt_status & SUBTREC_STATUS_MASK) >> SUBTREC_STATUS_SHIFT;
}

uint subt_rec_get_nargs_absent(subt_rec r) {
  return (r.subt_status & NARGS_ABSENT_MASK);
}

uint subt_rec_get_return_to(subt_rec r) {
  return r.return_to;
}

uint subt_rec_get_return_as(subt_rec r) {
  return r.return_as;
}

void subt_rec_set_service_id(subt_rec *r, uint service_id) {
  r->service_id = service_id;
}

void subt_rec_set_arg(subt_rec *r, uint arg_pos, uint arg) {
  r->args[arg_pos] = arg;
}

void subt_rec_set_arg_mode(subt_rec *r, uint arg_pos, uint mode) {
  r->arg_mode[arg_pos] = mode;
}

void subt_rec_set_subt_status(subt_rec *r, uint status) {
  r->subt_status = (r->subt_status & ~SUBTREC_STATUS_MASK)
    | ((status << SUBTREC_STATUS_SHIFT) & SUBTREC_STATUS_MASK);
}

void subt_rec_set_nargs_absent(subt_rec *r, uint n) {
  r->subt_status = (r->subt_status & ~NARGS_ABSENT_MASK)
    | ((n << NARGS_ABSENT_SHIFT) & NARGS_ABSENT_MASK);
}

void subt_rec_set_return_to(subt_rec *r, uint return_to) {
  r->return_to = return_to;
}

void subt_rec_set_return_as(subt_rec *r, uint return_as) {
  r->return_as = return_as;
}

/**************************/
/**** Packet Functions ****/
/**************************/

/* Return the packet type. */
uint pkt_get_type(packet p) {
  return (p.x & PKT_TYPE_MASK) >> PKT_TYPE_SHIFT;
}

/* Return the packet destination address. */
uint pkt_get_dest(packet p) {
  return (p.x & PKT_DEST_MASK) >> PKT_DEST_SHIFT;
}

/* Return the packet argument position. */
uint pkt_get_arg_pos(packet p) {
  return (p.x & PKT_ARG_MASK) >> PKT_ARG_SHIFT;
}

/* Return the packet subtask. */
uint pkt_get_sub(packet p) {
  return (p.x & PKT_SUB_MASK) >> PKT_SUB_SHIFT;
}

/* Return the packet payload. */
uint pkt_get_payload(packet p) {
  return p.y;
}

/* Set the packet type. */
void pkt_set_type(packet *p, uint type) {
  (*p).x = ((*p).x & ~PKT_TYPE_MASK) | ((type << PKT_TYPE_SHIFT) & PKT_TYPE_MASK);
}

/* Set the packet destination address. */
void pkt_set_dest(packet *p, uint dest) {
  (*p).x = ((*p).x & ~PKT_DEST_MASK) | ((dest << PKT_DEST_SHIFT) & PKT_DEST_MASK);
}

/* Set the packet argument position. */
void pkt_set_arg_pos(packet *p, uint arg) {
  (*p).x = ((*p).x & ~PKT_ARG_MASK) | ((arg << PKT_ARG_SHIFT) & PKT_ARG_MASK);
}

/* Set the packet subtask. */
void pkt_set_sub(packet *p, uint sub) {
  (*p).x = ((*p).x & ~PKT_SUB_MASK) | ((sub << PKT_SUB_SHIFT) & PKT_SUB_MASK);
}

/* Set the packet payload. */
void pkt_set_payload(packet *p, uint payload) {
  (*p).y = payload;
}

/* Return a newly created packet. */
packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload) {
  packet p;
  pkt_set_type(&p, type);
  pkt_set_dest(&p, dest);
  pkt_set_arg_pos(&p, arg);
  pkt_set_sub(&p, sub);
  pkt_set_payload(&p, payload);
  return p;
}

/*************************/
/**** Queue Functions ****/
/*************************/

/* Returns the array index of the head element of the queue. */
uint q_get_head_index(size_t id, size_t gid, __global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  return indices.x;
}

/* Returns the array index of the tail element of the queue. */
uint q_get_tail_index(size_t id, size_t gid,__global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  return indices.y;
}

/* Set the array index of the head element of the queue. */
void q_set_head_index(uint index, size_t id, size_t gid, __global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  indices.x = index;
  q[id * n + gid].x = as_uint(indices);
}

/* Set the array index of the tail element of the queue. */
void q_set_tail_index(uint index, size_t id, size_t gid,__global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  indices.y = index;
  q[id * n + gid].x = as_uint(indices);
}

/* Set the type of the operation last performed on the queue. */
void q_set_last_op(uint type, size_t id, size_t gid, __global uint2 *q, int n) {
  q[id * n + gid].y = type;
}

/* Returns true if the last operation performed on the queue is a read, false otherwise. */
bool q_last_op_is_read(size_t id, size_t gid,__global uint2 *q, int n) {
  return q[id * n + gid].y == READ;
}

/* Returns true if the last operation performed on the queue is a write, false otherwise. */
bool q_last_op_is_write(size_t id, size_t gid,__global uint2 *q, int n) {
  return q[id * n + gid].y == WRITE;
}

/* Returns true if the queue is empty, false otherwise. */
bool q_is_empty(size_t id, size_t gid,  __global uint2 *q, int n) {
  return (q_get_head_index(id, gid, q, n) == q_get_tail_index(id, gid, q, n))
    && q_last_op_is_read(id, gid, q, n);
}

/* Returns true if the queue is full, false otherwise. */
bool q_is_full(size_t id, size_t gid, __global uint2 *q, int n) {
  return (q_get_head_index(id, gid, q, n) == q_get_tail_index(id, gid, q, n))
    && q_last_op_is_write(id, gid, q, n);
}

/* Return the size of the queue. */
uint q_size(size_t id, size_t gid, __global uint2 *q, int n) {
  if (q_is_full(id, gid, q, n)) return QUEUE_SIZE;
  if (q_is_empty(id, gid, q, n)) return 0;
  uint head = q_get_head_index(id, gid, q, n);
  uint tail = q_get_tail_index(id, gid, q, n);
  return (tail > head) ? (tail - head) : QUEUE_SIZE - head;
}

/* Read the value located at the head index of the queue into 'result'.
 * Returns true if succcessful (queue is not empty), false otherwise. */
bool q_read(uint2 *result, size_t id, __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  if (q_is_empty(id, gid, q, n)) {
    return false;
  }

  int index = q_get_head_index(id, gid, q, n);
  *result = q[(n*n) + (id * n * QUEUE_SIZE) + (gid * QUEUE_SIZE) + index];
  q_set_head_index((index + 1) % QUEUE_SIZE, id, gid, q, n);
  q_set_last_op(READ, id, gid, q, n);
  return true;
}

/* Write a value into the tail index of the queue.
 * Returns true if successful (queue is not full), false otherwise. */
bool q_write(uint2 value, size_t id, __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  if (q_is_full(id, gid, q, n)) {
    return false;
  }

  int index = q_get_tail_index(id, gid, q, n);
  q[(n*n) + (id * n * QUEUE_SIZE) + (gid * QUEUE_SIZE) + index] = value;
  q_set_tail_index((index + 1) % QUEUE_SIZE, id, gid, q, n);
  q_set_last_op(WRITE, id, gid, q, n);
  return true;
}

#ifdef KERNEL_TEST_ENABLED
#include "../tests/kerneltypes.h"
#endif

/* Used to create, manipulate and access packet information. */
#define PKT_TYPE_SHIFT 0
#define PKT_DEST_SHIFT 2
#define PKT_ARG_SHIFT  10
#define PKT_SUB_SHIFT  14
#define PKT_TYPE_MASK  0x3      // 00000000000000000000000000000011
#define PKT_DEST_MASK  0x3FC    // 00000000000000000000001111111100
#define PKT_ARG_MASK   0x3C00   // 00000000000000000011110000000000
#define PKT_SUB_MASK   0xFFC000 // 00000000111111111100000000000000

/* Used to access the information stored within a subtask record. */
#define NARGS_ABSENT_SHIFT   0
#define SUBTREC_STATUS_SHIFT 4
#define NARGS_ABSENT_MASK    0xF
#define SUBTREC_STATUS_MASK  0xF0

/* Packet Types. */
#define ERROR     0
#define REFERENCE 1
#define REQUEST   2
#define DATA      3

/* Subtask record status. */
#define NEW        0
#define PROCESSING 1
#define PENDING    2

/* Arg status. */
#define ABSENT     0
#define REQUESTING 1
#define PRESENT    2

/* Arg mode. */
#define BY_VAL 0
#define BY_REF 1

/* A packet is 2x32bit integers. */
typedef uint2 packet;

/* Bytecode is 64bit. */
typedef ulong bytecode;

/* A subtask table record. */
typedef struct subt_rec {
  uint service_id;            // [32bits] Opcode
  uint args[QUEUE_SIZE];      // [32bits] Pointers to data or constants.
  uchar arg_mode[QUEUE_SIZE]; // [4bits] Arg status, [4bits] Arg mode
  uchar subt_status;          // [4bits]  Subtask status, [4bits] number of args absent.
  uchar return_to;            // [8bits]
  ushort return_as;           // [16bits] Subtask address + argument position.
} subt_rec;

/***********************************/
/******* Function Prototypes *******/
/***********************************/
bool cunit_q_is_empty(size_t gid, __global uint2 *q, int n);
bool cunit_q_is_full(size_t gid, __global uint2 *q, int n);
uint cunit_q_size(size_t gid, __global uint2 *q, int n);
void transferRQ(__global uint2 *rq,  __global uint2 *q, int n);

bool subt_add_rec(packet p, subt_rec *subt, size_t *avSubtRecs);

bool avSubtRecs_push(size_t i, size_t *avSubtRecs);
bool avSubtRecs_pop(size_t *result, size_t *avSubtRecs);
bool avSubtRecs_is_empty(size_t *avSubtRecs);
bool avSubtRecs_is_full(size_t *avSubtRecs);
size_t avSubtRecs_top(size_t *avSubtRecs);
void avSubtRecs_set_top(size_t top, size_t *avSubtRecs);

uint subt_rec_get_service_id(subt_rec r);
uint subt_rec_get_arg(subt_rec r, uint arg_pos);
uint subt_rec_get_arg_mode(subt_rec r, uint arg_pos);
uint subt_rec_get_subt_status(subt_rec r);
uint subt_rec_get_nargs_absent(subt_rec r);
uint subt_rec_get_return_to(subt_rec r);
uint subt_rec_get_return_as(subt_rec r);
void subt_rec_set_service_id(subt_rec *r, uint service_id);
void subt_rec_set_arg(subt_rec *r, uint arg_pos, uint arg);
void subt_rec_set_arg_mode(subt_rec *r, uint arg_pos, uint mode);
void subt_rec_set_subt_status(subt_rec *r, uint status);
void subt_rec_set_nargs_absent(subt_rec *r, uint n);
void subt_rec_set_return_to(subt_rec *r, uint return_to);
void subt_rec_set_return_as(subt_rec *r, uint return_as);

uint pkt_get_type(packet p);
uint pkt_get_dest(packet p);
uint pkt_get_arg_pos(packet p);
uint pkt_get_sub(packet p);
uint pkt_get_payload(packet p);
void pkt_set_type(packet *p, uint type);
void pkt_set_dest(packet *p, uint dest);
void pkt_set_arg_pos(packet *p, uint arg);
void pkt_set_sub(packet *p, uint sub);
void pkt_set_payload(packet *p, uint payload);
packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload);

uint q_get_head_index(size_t id, size_t gid, __global uint2 *q, int n);
uint q_get_tail_index(size_t id, size_t gid, __global uint2 *q, int n);
void q_set_head_index(uint index, size_t id, size_t gid, __global uint2 *q, int n);
void q_set_tail_index(uint index, size_t id, size_t gid, __global uint2 *q, int n);
void q_set_last_op(uint type, size_t id, size_t gid,__global uint2 *q, int n);
bool q_last_op_is_read(size_t id, size_t gid, __global uint2 *q, int n);
bool q_last_op_is_write(size_t id, size_t gid, __global uint2 *q, int n);
bool q_is_empty(size_t id, size_t gid, __global uint2 *q, int n);
bool q_is_full(size_t id, size_t gid,__global uint2 *q, int n);
uint q_size(size_t id, size_t gid, __global uint2 *q, int n);
bool q_read(uint2 *result, size_t id, __global uint2 *q, int n);
bool q_write(uint2 value, size_t id, __global uint2 *q, int n);

/**************************/
/******* The Kernel *******/
/**************************/
__kernel void vm(__global packet *q,            /* Compute unit queues. */
		 __global packet *rq,           /* Transfer queues for READ state. */
		 int n,                        /* The number of compute units. */
		 __global int *state,          /* Are we in the READ or WRITE state? */
		 __global bytecode *cStore,    /* The code store. */
		 __global subt_rec *subt,      /* The subtask table. */
		 __global size_t *avSubtRecs   /* The available subtask table records. */
		 ) {
  size_t gid = get_global_id(0);

  if (*state == WRITE) {
    transferRQ(rq, q, n);

    switch (gid) {
    case 0:
      break;
    case 1:
      break;
    case 2:
      break;
    case 3:
      if (!cunit_q_is_empty(0, q, n)) {
	*state = COMPLETE;
      }
      break;
    }
  } else {
    switch (gid) {
    case 0:
      if (cunit_q_is_empty(1, q, n)) {
	packet p = pkt_create(REFERENCE, 0, 1, 0, 0);
	q_write(p, 1, rq, n);
	q_write(p, 2, rq, n);
	q_write(p, 3, rq, n);
      }
      break;

    case 1:
      if (!cunit_q_is_empty(gid, q, n)) {
	packet p = pkt_create(REFERENCE, 0, 1, 0, 0);
	q_write(p, 2, rq, n);
      }
      break;
    case 2:
      if (cunit_q_size(gid, q, n) == 2) {
	packet p = pkt_create(REFERENCE, 0, 1, 0, 0);
	q_write(p, 0, rq, n);
      }
      break;
    case 3:
      break;
    }
  }
}

/**************************************/
/**** Compute Unit Queue Functions ****/
/**************************************/

/* Returns true if all queues owned by the specified compute unit are empty, false otherwise. */
bool cunit_q_is_empty(size_t gid, __global uint2 *q, int n) {
  for (int i = 0; i < n; i++) {
    if (!q_is_empty(gid, i, q, n)) {
      return false;
    }
  }

  return true;
}

/* Returns true if all queues owned by the specified compute unit are full, false otherwise. */
bool cunit_q_is_full(size_t gid, __global uint2 *q, int n) {
  for (int i = 0; i < n; i++) {
    if (!q_is_full(gid, i, q, n)) {
      return false;
    }
  }

  return true;
}

/* Return the total size of all compute unit owned queues. */
uint cunit_q_size(size_t gid, __global uint2 *q, int n) {
  uint size = 0;
  for (int i = 0; i < n; i++) {
    size += q_size(gid, i, q, n);
  }

  return size;
}

/* Copy all compute unit owned queue values from the readQueue into the real queues. */
void transferRQ(__global uint2 *rq, __global uint2 *q, int n) {
  uint2 packet;
  for (int i = 0; i < n; i++) {
    while (q_read(&packet, i, rq, n)) {
      q_write(packet, i, q, n);
    }
  }
}

/*********************************/
/**** Subtask Table Functions ****/
/*********************************/

/* Insert a record into the subtask table.
 * Returns true if successful (table is not full), false otherwise. */
bool subt_add_rec(packet p, subt_rec *subt, size_t *avSubtRecs) {
  size_t i;
  if (!avSubtRecs_pop(&i, avSubtRecs)) {
    return false;
  }

  subt_rec r;
  subt_rec_set_subt_status(&r, NEW);
  subt_rec_set_arg(&r, pkt_get_arg_pos(p), pkt_get_payload(p)); // If packet has DATA type.

  subt[i] = r;
  return true;
}

/*********************************************************/
/**** Available Subtask Table Records Stack Functions ****/
/*********************************************************/

bool avSubtRecs_push(size_t i, size_t *avSubtRecs) {
  if (avSubtRecs_is_full(avSubtRecs)) {
    return false;
  }

  size_t top = avSubtRecs_top(avSubtRecs) - 1;
  avSubtRecs[top] = i;
  avSubtRecs_set_top(top, avSubtRecs);
  return true;
}

bool avSubtRecs_pop(size_t *result, size_t *avSubtRecs) {
  if (avSubtRecs_is_empty(avSubtRecs)) {
    return false;
  }

  size_t top = avSubtRecs_top(avSubtRecs);
  *result = avSubtRecs[top];
  avSubtRecs_set_top(top + 1, avSubtRecs);
  return true;
}

bool avSubtRecs_is_empty(size_t *avSubtRecs) {
  return avSubtRecs_top(avSubtRecs) == SUBT_SIZE - 1;
}

bool avSubtRecs_is_full(size_t *avSubtRecs) {
  return avSubtRecs_top(avSubtRecs) == 0;
}

size_t avSubtRecs_top(size_t *avSubtRecs) {
  return avSubtRecs[0];
}

void avSubtRecs_set_top(size_t top, size_t *avSubtRecs) {
  avSubtRecs[0] = top;
}

/**********************************/
/**** Subtask Record Functions ****/
/**********************************/
uint subt_rec_get_service_id(subt_rec r) {
  return r.service_id;
}

uint subt_rec_get_arg(subt_rec r, uint arg_pos) {
  return r.args[arg_pos];
}

uint subt_rec_get_arg_mode(subt_rec r, uint arg_pos) {
  return r.arg_mode[arg_pos];
}

uint subt_rec_get_subt_status(subt_rec r) {
  return (r.subt_status & SUBTREC_STATUS_MASK) >> SUBTREC_STATUS_SHIFT;
}

uint subt_rec_get_nargs_absent(subt_rec r) {
  return (r.subt_status & NARGS_ABSENT_MASK);
}

uint subt_rec_get_return_to(subt_rec r) {
  return r.return_to;
}

uint subt_rec_get_return_as(subt_rec r) {
  return r.return_as;
}

void subt_rec_set_service_id(subt_rec *r, uint service_id) {
  r->service_id = service_id;
}

void subt_rec_set_arg(subt_rec *r, uint arg_pos, uint arg) {
  r->args[arg_pos] = arg;
}

void subt_rec_set_arg_mode(subt_rec *r, uint arg_pos, uint mode) {
  r->arg_mode[arg_pos] = mode;
}

void subt_rec_set_subt_status(subt_rec *r, uint status) {
  r->subt_status = (r->subt_status & ~SUBTREC_STATUS_MASK)
    | ((status << SUBTREC_STATUS_SHIFT) & SUBTREC_STATUS_MASK);
}

void subt_rec_set_nargs_absent(subt_rec *r, uint n) {
  r->subt_status = (r->subt_status & ~NARGS_ABSENT_MASK)
    | ((n << NARGS_ABSENT_SHIFT) & NARGS_ABSENT_MASK);
}

void subt_rec_set_return_to(subt_rec *r, uint return_to) {
  r->return_to = return_to;
}

void subt_rec_set_return_as(subt_rec *r, uint return_as) {
  r->return_as = return_as;
}

/**************************/
/**** Packet Functions ****/
/**************************/

/* Return the packet type. */
uint pkt_get_type(packet p) {
  return (p.x & PKT_TYPE_MASK) >> PKT_TYPE_SHIFT;
}

/* Return the packet destination address. */
uint pkt_get_dest(packet p) {
  return (p.x & PKT_DEST_MASK) >> PKT_DEST_SHIFT;
}

/* Return the packet argument position. */
uint pkt_get_arg_pos(packet p) {
  return (p.x & PKT_ARG_MASK) >> PKT_ARG_SHIFT;
}

/* Return the packet subtask. */
uint pkt_get_sub(packet p) {
  return (p.x & PKT_SUB_MASK) >> PKT_SUB_SHIFT;
}

/* Return the packet payload. */
uint pkt_get_payload(packet p) {
  return p.y;
}

/* Set the packet type. */
void pkt_set_type(packet *p, uint type) {
  (*p).x = ((*p).x & ~PKT_TYPE_MASK) | ((type << PKT_TYPE_SHIFT) & PKT_TYPE_MASK);
}

/* Set the packet destination address. */
void pkt_set_dest(packet *p, uint dest) {
  (*p).x = ((*p).x & ~PKT_DEST_MASK) | ((dest << PKT_DEST_SHIFT) & PKT_DEST_MASK);
}

/* Set the packet argument position. */
void pkt_set_arg_pos(packet *p, uint arg) {
  (*p).x = ((*p).x & ~PKT_ARG_MASK) | ((arg << PKT_ARG_SHIFT) & PKT_ARG_MASK);
}

/* Set the packet subtask. */
void pkt_set_sub(packet *p, uint sub) {
  (*p).x = ((*p).x & ~PKT_SUB_MASK) | ((sub << PKT_SUB_SHIFT) & PKT_SUB_MASK);
}

/* Set the packet payload. */
void pkt_set_payload(packet *p, uint payload) {
  (*p).y = payload;
}

/* Return a newly created packet. */
packet pkt_create(uint type, uint dest, uint arg, uint sub, uint payload) {
  packet p;
  pkt_set_type(&p, type);
  pkt_set_dest(&p, dest);
  pkt_set_arg_pos(&p, arg);
  pkt_set_sub(&p, sub);
  pkt_set_payload(&p, payload);
  return p;
}

/*************************/
/**** Queue Functions ****/
/*************************/

/* Returns the array index of the head element of the queue. */
uint q_get_head_index(size_t id, size_t gid, __global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  return indices.x;
}

/* Returns the array index of the tail element of the queue. */
uint q_get_tail_index(size_t id, size_t gid,__global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  return indices.y;
}

/* Set the array index of the head element of the queue. */
void q_set_head_index(uint index, size_t id, size_t gid, __global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  indices.x = index;
  q[id * n + gid].x = as_uint(indices);
}

/* Set the array index of the tail element of the queue. */
void q_set_tail_index(uint index, size_t id, size_t gid,__global uint2 *q, int n) {
  ushort2 indices = as_ushort2(q[id * n + gid].x);
  indices.y = index;
  q[id * n + gid].x = as_uint(indices);
}

/* Set the type of the operation last performed on the queue. */
void q_set_last_op(uint type, size_t id, size_t gid, __global uint2 *q, int n) {
  q[id * n + gid].y = type;
}

/* Returns true if the last operation performed on the queue is a read, false otherwise. */
bool q_last_op_is_read(size_t id, size_t gid,__global uint2 *q, int n) {
  return q[id * n + gid].y == READ;
}

/* Returns true if the last operation performed on the queue is a write, false otherwise. */
bool q_last_op_is_write(size_t id, size_t gid,__global uint2 *q, int n) {
  return q[id * n + gid].y == WRITE;
}

/* Returns true if the queue is empty, false otherwise. */
bool q_is_empty(size_t id, size_t gid,  __global uint2 *q, int n) {
  return (q_get_head_index(id, gid, q, n) == q_get_tail_index(id, gid, q, n))
    && q_last_op_is_read(id, gid, q, n);
}

/* Returns true if the queue is full, false otherwise. */
bool q_is_full(size_t id, size_t gid, __global uint2 *q, int n) {
  return (q_get_head_index(id, gid, q, n) == q_get_tail_index(id, gid, q, n))
    && q_last_op_is_write(id, gid, q, n);
}

/* Return the size of the queue. */
uint q_size(size_t id, size_t gid, __global uint2 *q, int n) {
  if (q_is_full(id, gid, q, n)) return QUEUE_SIZE;
  if (q_is_empty(id, gid, q, n)) return 0;
  uint head = q_get_head_index(id, gid, q, n);
  uint tail = q_get_tail_index(id, gid, q, n);
  return (tail > head) ? (tail - head) : QUEUE_SIZE - head;
}

/* Read the value located at the head index of the queue into 'result'.
 * Returns true if succcessful (queue is not empty), false otherwise. */
bool q_read(uint2 *result, size_t id, __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  if (q_is_empty(id, gid, q, n)) {
    return false;
  }

  int index = q_get_head_index(id, gid, q, n);
  *result = q[(n*n) + (id * n * QUEUE_SIZE) + (gid * QUEUE_SIZE) + index];
  q_set_head_index((index + 1) % QUEUE_SIZE, id, gid, q, n);
  q_set_last_op(READ, id, gid, q, n);
  return true;
}

/* Write a value into the tail index of the queue.
 * Returns true if successful (queue is not full), false otherwise. */
bool q_write(uint2 value, size_t id, __global uint2 *q, int n) {
  size_t gid = get_global_id(0);
  if (q_is_full(id, gid, q, n)) {
    return false;
  }

  int index = q_get_tail_index(id, gid, q, n);
  q[(n*n) + (id * n * QUEUE_SIZE) + (gid * QUEUE_SIZE) + index] = value;
  q_set_tail_index((index + 1) % QUEUE_SIZE, id, gid, q, n);
  q_set_last_op(WRITE, id, gid, q, n);
  return true;
}


added probBase for non-spiking input, doesn't change default behavior and can be helpful for
providing uniform input

git-svn-id: 602a26e73ae9f2a14e2bb95676f2e137fcc39919@3930 7c0444ca-5550-0410-9008-faecdbe78eb5

#include "Retina_params.h"
#include "cl_random.hcl"

#ifndef PI
#  define PI 3.1415926535897932
#endif

#ifndef PV_USE_OPENCL

#  include <math.h>
#  define EXP  expf
#  define COS  cosf
#  define FMOD fmodf
#  define CL_KERNEL
#  define CL_MEM_GLOBAL
#  define CL_MEM_LOCAL

#else

#  define EXP exp
#  define COS  __cosf
#  define FMOD __fmodf
#  define CL_KERNEL     __kernel
#  define CL_MEM_GLOBAL __global
#  define CL_MEM_LOCAL  __local
#  include "conversions.hcl"

#endif /* PV_USE_OPENCL */

/*
 * Spiking method for Retina
 * Returns 1 if an event should occur, 0 otherwise. This is a stochastic model.
 *
 * REMARKS:
 *      - During ABS_REFACTORY_PERIOD a neuron does not spike
 *      - The neurons that correspond to stimuli (on Image pixels)
 *        spike with probability probStim.
 *      - The neurons that correspond to background image pixels
 *        spike with probability probBase.
 *      - After ABS_REFACTORY_PERIOD the spiking probability
 *        grows exponentially to probBase and probStim respectively.
 *      - The burst of the retina is periodic with period T set by
 *        T = 1000/burstFreq in miliseconds
 *      - When the time t is such that mT < t < mT + burstDuration, where m is
 *        an integer, the burstStatus is set to 1.
 *      - The burstStatus is also determined by the condition that
 *        beginStim < t < endStim. These parameters are set in the input
 *        params file params.stdp
 *      - sinAmp modulates the spiking probability only when burstDuration <= 0
 *        or burstFreq = 0
 *      - probSpike is set to probBase for all neurons.
 *      - for neurons exposed to Image on pixels, probSpike increases with probStim.
 *      - When the probability is negative, the neuron does not spike.
 *
 * NOTES:
 *      - time is measured in milliseconds.
 *
 */
static inline
int spike(float time, float dt,
          float prev, float stimFactor, uint4 * rnd_state, CL_MEM_GLOBAL Retina_params * params)
{
   float probSpike;
   float burstStatus = 1;
   float sinAmp = 1.0;

   // input parameters
   //
   float probBase  = params->probBase;
   float probStim  = params->probStim * stimFactor;

   // see if neuron is in a refactory period
   //
   if ((time - prev) < params->abs_refactory_period) {
      return 0;
   }
   else {
      float delta = time - prev - params->abs_refactory_period;
      float refact = 1.0f - EXP(-delta/params->refactory_period);
      refact = (refact < 0) ? 0 : refact;
      probBase *= refact;
      probStim *= refact;
   }

   if (params->burstDuration <= 0 || params->burstFreq == 0) {
      sinAmp = COS( 2*PI*time * params->burstFreq / 1000. );
   }
   else {
      burstStatus = FMOD(time, 1000. / params->burstFreq);
      burstStatus = burstStatus <= params->burstDuration;
   }

   burstStatus *= (int) ( (time >= params->beginStim) && (time < params->endStim) );
   probSpike = probBase;

   if ((int)burstStatus) {
      probSpike += probStim * sinAmp;  // negative prob is OK
   }

   *rnd_state = cl_random_get(*rnd_state);
   int spike_flag = (cl_random_prob(*rnd_state) < probSpike);
   return spike_flag;
}

//
// update the state of a retinal layer (spiking)
//
//    assume called with 1D kernel
//
CL_KERNEL
void Retina_spiking_update_state (
    const float time,
    const float dt,

    const int nx,
    const int ny,
    const int nf,
    const int nb,

    CL_MEM_GLOBAL Retina_params * params,
    CL_MEM_GLOBAL uint4 * rnd,
    CL_MEM_GLOBAL float * phiExc,
    CL_MEM_GLOBAL float * phiInh,
    CL_MEM_GLOBAL float * activity,
    CL_MEM_GLOBAL float * prevTime)
{
   int k;
#ifndef PV_USE_OPENCL
for (k = 0; k < nx*ny*nf; k++) {
#else
   k = get_global_id(0);
#endif

   int kex = kIndexExtended(k, nx, ny, nf, nb);

   //
   // kernel (nonheader part) begins here
   //

   // load local variables from global memory
   //
   uint4 l_rnd = rnd[k];
   float l_phiExc = phiExc[k];
   float l_phiInh = phiInh[k];
   float l_prev   = prevTime[kex];
   float l_activ;

   l_activ = (float) spike(time, dt, l_prev, (l_phiExc - l_phiInh), &l_rnd, params);
   l_prev  = (l_activ > 0.0f) ? time : l_prev;

   l_phiExc = 0.0f;
   l_phiInh = 0.0f;

   // store local variables back to global memory
   //
   rnd[k] = l_rnd;
   phiExc[k] = l_phiExc;
   phiInh[k] = l_phiInh;
   prevTime[kex] = l_prev;
   activity[kex] = l_activ;

#ifndef PV_USE_OPENCL
   }
#endif

}

//
// update the state of a retinal layer (non-spiking)
//
//    assume called with 1D kernel
//
CL_KERNEL
void Retina_nonspiking_update_state (
    const float time,
    const float dt,

    const int nx,
    const int ny,
    const int nf,
    const int nb,

    CL_MEM_GLOBAL Retina_params * params,
    CL_MEM_GLOBAL float * phiExc,
    CL_MEM_GLOBAL float * phiInh,
    CL_MEM_GLOBAL float * activity)
{
   int k;
#ifndef PV_USE_OPENCL
for (k = 0; k < nx*ny*nf; k++) {
#else
   k = get_global_id(0);
#endif

   int kex = kIndexExtended(k, nx, ny, nf, nb);

   //
   // kernel (nonheader part) begins here
   //

   // load local variables from global memory
   //
   float l_phiExc = phiExc[k];
   float l_phiInh = phiInh[k];
   float l_activ;

   // adding base prob should not change default behavior
   l_activ = params->probStim*(l_phiExc - l_phiInh) + params->probBase;

   l_phiExc = 0.0f;
   l_phiInh = 0.0f;

   // store local variables back to global memory
   //
   phiExc[k] = l_phiExc;
   phiInh[k] = l_phiInh;
   activity[kex] = l_activ;

#ifndef PV_USE_OPENCL
   }
#endif

}

#include "Retina_params.h"
#include "cl_random.hcl"

#ifndef PI
#  define PI 3.1415926535897932
#endif

#ifndef PV_USE_OPENCL

#  include <math.h>
#  define EXP  expf
#  define COS  cosf
#  define FMOD fmodf
#  define CL_KERNEL
#  define CL_MEM_GLOBAL
#  define CL_MEM_LOCAL

#else

#  define EXP exp
#  define COS  __cosf
#  define FMOD __fmodf
#  define CL_KERNEL     __kernel
#  define CL_MEM_GLOBAL __global
#  define CL_MEM_LOCAL  __local
#  include "conversions.hcl"

#endif /* PV_USE_OPENCL */

/*
 * Spiking method for Retina
 * Returns 1 if an event should occur, 0 otherwise. This is a stochastic model.
 *
 * REMARKS:
 *      - During ABS_REFACTORY_PERIOD a neuron does not spike
 *      - The neurons that correspond to stimuli (on Image pixels)
 *        spike with probability probStim.
 *      - The neurons that correspond to background image pixels
 *        spike with probability probBase.
 *      - After ABS_REFACTORY_PERIOD the spiking probability
 *        grows exponentially to probBase and probStim respectively.
 *      - The burst of the retina is periodic with period T set by
 *        T = 1000/burstFreq in miliseconds
 *      - When the time t is such that mT < t < mT + burstDuration, where m is
 *        an integer, the burstStatus is set to 1.
 *      - The burstStatus is also determined by the condition that
 *        beginStim < t < endStim. These parameters are set in the input
 *        params file params.stdp
 *      - sinAmp modulates the spiking probability only when burstDuration <= 0
 *        or burstFreq = 0
 *      - probSpike is set to probBase for all neurons.
 *      - for neurons exposed to Image on pixels, probSpike increases with probStim.
 *      - When the probability is negative, the neuron does not spike.
 *
 * NOTES:
 *      - time is measured in milliseconds.
 *
 */
static inline
int spike(float time, float dt,
          float prev, float stimFactor, uint4 * rnd_state, CL_MEM_GLOBAL Retina_params * params)
{
   float probSpike;
   float burstStatus = 1;
   float sinAmp = 1.0;

   // input parameters
   //
   float probBase  = params->probBase;
   float probStim  = params->probStim * stimFactor;

   // see if neuron is in a refactory period
   //
   if ((time - prev) < params->abs_refactory_period) {
      return 0;
   }
   else {
      float delta = time - prev - params->abs_refactory_period;
      float refact = 1.0f - EXP(-delta/params->refactory_period);
      refact = (refact < 0) ? 0 : refact;
      probBase *= refact;
      probStim *= refact;
   }

   if (params->burstDuration <= 0 || params->burstFreq == 0) {
      sinAmp = COS( 2*PI*time * params->burstFreq / 1000. );
   }
   else {
      burstStatus = FMOD(time, 1000. / params->burstFreq);
      burstStatus = burstStatus <= params->burstDuration;
   }

   burstStatus *= (int) ( (time >= params->beginStim) && (time < params->endStim) );
   probSpike = probBase;

   if ((int)burstStatus) {
      probSpike += probStim * sinAmp;  // negative prob is OK
   }

   *rnd_state = cl_random_get(*rnd_state);
   int spike_flag = (cl_random_prob(*rnd_state) < probSpike);
   return spike_flag;
}

//
// update the state of a retinal layer (spiking)
//
//    assume called with 1D kernel
//
CL_KERNEL
void Retina_spiking_update_state (
    const float time,
    const float dt,

    const int nx,
    const int ny,
    const int nf,
    const int nb,

    CL_MEM_GLOBAL Retina_params * params,
    CL_MEM_GLOBAL uint4 * rnd,
    CL_MEM_GLOBAL float * phiExc,
    CL_MEM_GLOBAL float * phiInh,
    CL_MEM_GLOBAL float * activity,
    CL_MEM_GLOBAL float * prevTime)
{
   int k;
#ifndef PV_USE_OPENCL
for (k = 0; k < nx*ny*nf; k++) {
#else
   k = get_global_id(0);
#endif

   int kex = kIndexExtended(k, nx, ny, nf, nb);

   //
   // kernel (nonheader part) begins here
   //

   // load local variables from global memory
   //
   uint4 l_rnd = rnd[k];
   float l_phiExc = phiExc[k];
   float l_phiInh = phiInh[k];
   float l_prev   = prevTime[kex];
   float l_activ;

   l_activ = (float) spike(time, dt, l_prev, (l_phiExc - l_phiInh), &l_rnd, params);
   l_prev  = (l_activ > 0.0f) ? time : l_prev;

   l_phiExc = 0.0f;
   l_phiInh = 0.0f;

   // store local variables back to global memory
   //
   rnd[k] = l_rnd;
   phiExc[k] = l_phiExc;
   phiInh[k] = l_phiInh;
   prevTime[kex] = l_prev;
   activity[kex] = l_activ;

#ifndef PV_USE_OPENCL
   }
#endif

}

//
// update the state of a retinal layer (non-spiking)
//
//    assume called with 1D kernel
//
CL_KERNEL
void Retina_nonspiking_update_state (
    const float time,
    const float dt,

    const int nx,
    const int ny,
    const int nf,
    const int nb,

    CL_MEM_GLOBAL Retina_params * params,
    CL_MEM_GLOBAL float * phiExc,
    CL_MEM_GLOBAL float * phiInh,
    CL_MEM_GLOBAL float * activity)
{
   int k;
#ifndef PV_USE_OPENCL
for (k = 0; k < nx*ny*nf; k++) {
#else
   k = get_global_id(0);
#endif

   int kex = kIndexExtended(k, nx, ny, nf, nb);

   //
   // kernel (nonheader part) begins here
   //

   // load local variables from global memory
   //
   float l_phiExc = phiExc[k];
   float l_phiInh = phiInh[k];
   float l_activ;

   l_activ = params->probStim*(l_phiExc - l_phiInh);

   l_phiExc = 0.0f;
   l_phiInh = 0.0f;

   // store local variables back to global memory
   //
   phiExc[k] = l_phiExc;
   phiInh[k] = l_phiInh;
   activity[kex] = l_activ;

#ifndef PV_USE_OPENCL
   }
#endif

}


Remove unused arguments

Remove unused arguments

// For optimizations regarding bank conflicts, look at:
// http://http.developer.nvidia.com/GPUGems3/gpugems3_ch39.html
// TODO: Get efficient scan to work with arbitrary n > 0
// scan consisting of upsweep, null_last and downsweep does not work!!!

kernel void lazy_scan(global uint* restrict in, global uint* restrict out,
                      uint len) {
  uint idx = get_global_id(0);
  if (idx == 0) {
    out[0] = 0;
    for (uint i = 1; i < len; ++i)
      out[i] = out[i - 1] + in[i - 1];
  }
}

/// Prefix scan in local memory, tmp size should be == n.
/// Requires n/2 threads. N must be a power of 2.
/// From: http://http.developer.nvidia.com/GPUGems3/gpugems3_ch39.html
kernel void prescan(global uint* restrict data, local uint* tmp, uint n) {
  // extern __shared__ float tmp[];  // allocated on invocation
  uint thread = get_local_id(0); // threadIdx.x;
  uint offset = 1;
  // A (2 lines) --> load input into shared memory
  tmp[2 * thread    ] = data[2 * thread    ];
  tmp[2 * thread + 1] = data[2 * thread + 1];
  // build sum in place up the tree
  for (uint d = n >> 1; d > 0; d >>= 1) {
    barrier(CLK_LOCAL_MEM_FENCE); //__syncthreads();
    if (thread < d) {
      // B (2 lines)
      int ai = offset * (2 * thread + 1) - 1;
      int bi = offset * (2 * thread + 2) - 1;
      tmp[bi] += tmp[ai];
    }
    offset *= 2;
  }
  // C (2 lines) --> clear the last element
  if (thread == 0)
    tmp[n - 1] = 0;
  // traverse down tree & build scan
  for (uint d = 1; d < n; d *= 2) {
    offset >>= 1;
    barrier(CLK_LOCAL_MEM_FENCE); //__syncthreads();
    if (thread < d) {
      // D (2 lines)
      int ai = offset * (2 * thread + 1) - 1;
      int bi = offset * (2 * thread + 2) - 1;
      uint t = tmp[ai];
      tmp[ai] = tmp[bi];
      tmp[bi] += t;
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  // E (2 line) --> write results to device memory
  data[2 * thread    ] = tmp[2 * thread    ];
  data[2 * thread + 1] = tmp[2 * thread + 1];
}

#define NUM_BANKS 16
#define LOG_NUM_BANKS 4
#define CONFLICT_FREE_OFFSET(n) \
    ((n) >> (NUM_BANKS + (n)) >> (2 * LOG_NUM_BANKS))

/// Prefix scan in local memory, tmp size should be == n.
/// Requires n/2 threads. N must be a power of 2.
/// Optimized for memory bank comflicts.
/// From: http://http.developer.nvidia.com/GPUGems3/gpugems3_ch39.html
kernel void prescan2(global uint* restrict data, local uint* tmp, uint n) {
  // extern __shared__ float tmp[];  // allocated on invocation
  uint thread = get_local_id(0); // threadIdx.x;
  uint offset = 1;
  // New A (6 lines) --> load input into shared memory
  uint ai = thread;
  uint bi = thread + (n / 2);
  uint bankOffsetA = CONFLICT_FREE_OFFSET(ai);
  uint bankOffsetB = CONFLICT_FREE_OFFSET(bi);
  tmp[ai + bankOffsetA] = data[ai];
  tmp[bi + bankOffsetB] = data[bi];
  // build sum in place up the tree
  for (uint d = n >> 1; d > 0; d >>= 1) {
    barrier(CLK_LOCAL_MEM_FENCE); //__syncthreads();
    if (thread < d) {
      // New B (4 lines)
      uint ai = offset * (2 * thread + 1) - 1;
      uint bi = offset * (2 * thread + 2) - 1;
      ai += CONFLICT_FREE_OFFSET(ai);
      bi += CONFLICT_FREE_OFFSET(bi);
      tmp[bi] += tmp[ai];
    }
    offset *= 2;
  }
  // New C (2 lines) --> clear the last element
  if (thread == 0)
    tmp[n - 1 + CONFLICT_FREE_OFFSET(n - 1)] = 0;
  // traverse down tree & build scan
  for (uint d = 1; d < n; d *= 2) {
    offset >>= 1;
    barrier(CLK_LOCAL_MEM_FENCE); //__syncthreads();
    if (thread < d) {
      // New D (4 lines)
      uint ai = offset * (2 * thread + 1) - 1;
      uint bi = offset * (2 * thread + 2) - 1;
      ai += CONFLICT_FREE_OFFSET(ai);
      bi += CONFLICT_FREE_OFFSET(bi);
      uint t = tmp[ai];
      tmp[ai] = tmp[bi];
      tmp[bi] += t;
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  // New E (2 lines) --> write results to device memory
  data[ai] = tmp[ai + bankOffsetA];
  data[bi] = tmp[bi + bankOffsetB];
}

/// Global exclusive scan, phase 1.
/// From: http://http.developer.nvidia.com/GPUGems3/gpugems3_ch39.html
kernel void es_phase_1(global uint* restrict data,
                       global uint* restrict increments,
                       local uint* tmp, uint len) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint threads_per_block = get_local_size(0);
  const uint elements_per_block = threads_per_block * 2;
  const uint global_offset = block * elements_per_block;
  const uint n = elements_per_block;

  uint offset = 1;
  // A (2 lines) --> load input into shared memory
  tmp[2 * thread    ] = (2 * thread     < len) ? data[global_offset + (2 * thread    )] : 0;
  tmp[2 * thread + 1] = (2 * thread + 1 < len) ? data[global_offset + (2 * thread + 1)] : 0;
  // build sum in place up the tree
  for (uint d = n >> 1; d > 0; d >>= 1) {
    barrier(CLK_LOCAL_MEM_FENCE); //__syncthreads();
    if (thread < d) {
      // B (2 lines)
      int ai = offset * (2 * thread + 1) - 1;
      int bi = offset * (2 * thread + 2) - 1;
      tmp[bi] += tmp[ai];
    }
    offset *= 2;
  }
  // C (2 lines) --> clear the last element
  if (thread == 0) {
    increments[block] = tmp[n - 1];
    tmp[n - 1] = 0;
  }
  // traverse down tree & build scan
  for (uint d = 1; d < n; d *= 2) {
    offset >>= 1;
    barrier(CLK_LOCAL_MEM_FENCE); //__syncthreads();
    if (thread < d) {
      // D (2 lines)
      int ai = offset * (2 * thread + 1) - 1;
      int bi = offset * (2 * thread + 2) - 1;
      uint t = tmp[ai];
      tmp[ai] = tmp[bi];
      tmp[bi] += t;
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  // E (2 line) --> write results to device memory
  if (2 * thread < len)
    data[global_offset + (2 * thread    )] = tmp[2 * thread    ];
  if (2 * thread + 1 < len)
    data[global_offset + (2 * thread + 1)] = tmp[2 * thread + 1];
}

/// Global exclusive scan, phase 2.
kernel void es_phase_2(global uint* restrict data,
                       global uint* restrict increments,
                       local uint* tmp, uint len) {
  (void) data; // just to keep the arguments similar
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint threads_per_block = get_local_size(0);
  const uint elements_per_block = threads_per_block * 2;
  const uint global_offset = block * elements_per_block;
  const uint n = elements_per_block;

  // block-wise scan, save "block sum" into increments
  uint offset = 1;
  uint ai = thread;
  uint bi = thread + (n / 2);
  uint bankOffsetA = CONFLICT_FREE_OFFSET(ai);
  uint bankOffsetB = CONFLICT_FREE_OFFSET(bi);
  uint ai_global = ai + global_offset;
  uint bi_global = bi + global_offset;
  tmp[ai + bankOffsetA] = (ai_global < len) ? increments[ai_global] : 0;
  tmp[bi + bankOffsetB] = (bi_global < len) ? increments[bi_global] : 0;
  for (uint d = n >> 1; d > 0; d >>= 1) {
    barrier(CLK_LOCAL_MEM_FENCE);
    if (thread < d) {
      uint ai = offset * (2 * thread + 1) - 1;
      uint bi = offset * (2 * thread + 2) - 1;
      ai += CONFLICT_FREE_OFFSET(ai);
      bi += CONFLICT_FREE_OFFSET(bi);
      tmp[bi] += tmp[ai];
    }
    offset *= 2;
  }
  if (thread == 0)
    tmp[n - 1 + CONFLICT_FREE_OFFSET(n - 1)] = 0;
  for (uint d = 1; d < n; d *= 2) {
    offset >>= 1;
    barrier(CLK_LOCAL_MEM_FENCE);
    if (thread < d) {
      int ai = offset * (2 * thread + 1) - 1;
      int bi = offset * (2 * thread + 2) - 1;
      ai += CONFLICT_FREE_OFFSET(ai);
      bi += CONFLICT_FREE_OFFSET(bi);
      uint t = tmp[ai];
      tmp[ai] = tmp[bi];
      tmp[bi] += t;
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  if (ai_global < len) increments[ai_global] = tmp[ai + bankOffsetA];
  if (bi_global < len) increments[bi_global] = tmp[bi + bankOffsetB];
}

kernel void es_phase_3(global uint* restrict data,
                       global uint* restrict increments,
                       uint len) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint threads_per_block = get_local_size(0);
  const uint elements_per_block = threads_per_block * 2;
  const uint global_offset = block * elements_per_block;

  // add the appropriate value to each block
  uint ai = 2 * thread;
  uint bi = 2 * thread + 1;
  uint ai_global = ai + global_offset;
  uint bi_global = bi + global_offset;
  uint increment = increments[block];
  if (ai_global < len) data[ai_global] += increment;
  if (bi_global < len) data[bi_global] += increment;
}

// For optimizations regarding bank conflicts, look at:
// http://http.developer.nvidia.com/GPUGems3/gpugems3_ch39.html
// TODO: Get efficient scan to work with arbitrary n > 0
// scan consisting of upsweep, null_last and downsweep does not work!!!

kernel void lazy_scan(global uint* restrict in, global uint* restrict out,
                      uint len) {
  uint idx = get_global_id(0);
  if (idx == 0) {
    out[0] = 0;
    for (uint i = 1; i < len; ++i)
      out[i] = out[i - 1] + in[i - 1];
  }
}

/// Prefix scan in local memory, tmp size should be == n.
/// Requires n/2 threads. N must be a power of 2.
/// From: http://http.developer.nvidia.com/GPUGems3/gpugems3_ch39.html
kernel void prescan(global uint* restrict data, local uint* tmp, uint n) {
  // extern __shared__ float tmp[];  // allocated on invocation
  uint thread = get_local_id(0); // threadIdx.x;
  uint offset = 1;
  // A (2 lines) --> load input into shared memory
  tmp[2 * thread    ] = data[2 * thread    ];
  tmp[2 * thread + 1] = data[2 * thread + 1];
  // build sum in place up the tree
  for (uint d = n >> 1; d > 0; d >>= 1) {
    barrier(CLK_LOCAL_MEM_FENCE); //__syncthreads();
    if (thread < d) {
      // B (2 lines)
      int ai = offset * (2 * thread + 1) - 1;
      int bi = offset * (2 * thread + 2) - 1;
      tmp[bi] += tmp[ai];
    }
    offset *= 2;
  }
  // C (2 lines) --> clear the last element
  if (thread == 0)
    tmp[n - 1] = 0;
  // traverse down tree & build scan
  for (uint d = 1; d < n; d *= 2) {
    offset >>= 1;
    barrier(CLK_LOCAL_MEM_FENCE); //__syncthreads();
    if (thread < d) {
      // D (2 lines)
      int ai = offset * (2 * thread + 1) - 1;
      int bi = offset * (2 * thread + 2) - 1;
      uint t = tmp[ai];
      tmp[ai] = tmp[bi];
      tmp[bi] += t;
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  // E (2 line) --> write results to device memory
  data[2 * thread    ] = tmp[2 * thread    ];
  data[2 * thread + 1] = tmp[2 * thread + 1];
}

#define NUM_BANKS 16
#define LOG_NUM_BANKS 4
#define CONFLICT_FREE_OFFSET(n) \
    ((n) >> (NUM_BANKS + (n)) >> (2 * LOG_NUM_BANKS))

/// Prefix scan in local memory, tmp size should be == n.
/// Requires n/2 threads. N must be a power of 2.
/// Optimized for memory bank comflicts.
/// From: http://http.developer.nvidia.com/GPUGems3/gpugems3_ch39.html
kernel void prescan2(global uint* restrict data, local uint* tmp, uint n) {
  // extern __shared__ float tmp[];  // allocated on invocation
  uint thread = get_local_id(0); // threadIdx.x;
  uint offset = 1;
  // New A (6 lines) --> load input into shared memory
  uint ai = thread;
  uint bi = thread + (n / 2);
  uint bankOffsetA = CONFLICT_FREE_OFFSET(ai);
  uint bankOffsetB = CONFLICT_FREE_OFFSET(bi);
  tmp[ai + bankOffsetA] = data[ai];
  tmp[bi + bankOffsetB] = data[bi];
  // build sum in place up the tree
  for (uint d = n >> 1; d > 0; d >>= 1) {
    barrier(CLK_LOCAL_MEM_FENCE); //__syncthreads();
    if (thread < d) {
      // New B (4 lines)
      uint ai = offset * (2 * thread + 1) - 1;
      uint bi = offset * (2 * thread + 2) - 1;
      ai += CONFLICT_FREE_OFFSET(ai);
      bi += CONFLICT_FREE_OFFSET(bi);
      tmp[bi] += tmp[ai];
    }
    offset *= 2;
  }
  // New C (2 lines) --> clear the last element
  if (thread == 0)
    tmp[n - 1 + CONFLICT_FREE_OFFSET(n - 1)] = 0;
  // traverse down tree & build scan
  for (uint d = 1; d < n; d *= 2) {
    offset >>= 1;
    barrier(CLK_LOCAL_MEM_FENCE); //__syncthreads();
    if (thread < d) {
      // New D (4 lines)
      uint ai = offset * (2 * thread + 1) - 1;
      uint bi = offset * (2 * thread + 2) - 1;
      ai += CONFLICT_FREE_OFFSET(ai);
      bi += CONFLICT_FREE_OFFSET(bi);
      uint t = tmp[ai];
      tmp[ai] = tmp[bi];
      tmp[bi] += t;
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  // New E (2 lines) --> write results to device memory
  data[ai] = tmp[ai + bankOffsetA];
  data[bi] = tmp[bi + bankOffsetB];
}

/// Global exclusive scan, phase 1.
/// From: http://http.developer.nvidia.com/GPUGems3/gpugems3_ch39.html
kernel void es_phase_1(global uint* restrict data,
                       global uint* restrict increments,
                       local uint* tmp, uint len) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint threads_per_block = get_local_size(0);
  const uint elements_per_block = threads_per_block * 2;
  const uint global_offset = block * elements_per_block;
  const uint n = elements_per_block;

  uint offset = 1;
  // A (2 lines) --> load input into shared memory
  tmp[2 * thread    ] = (2 * thread     < len) ? data[global_offset + (2 * thread    )] : 0;
  tmp[2 * thread + 1] = (2 * thread + 1 < len) ? data[global_offset + (2 * thread + 1)] : 0;
  // build sum in place up the tree
  for (uint d = n >> 1; d > 0; d >>= 1) {
    barrier(CLK_LOCAL_MEM_FENCE); //__syncthreads();
    if (thread < d) {
      // B (2 lines)
      int ai = offset * (2 * thread + 1) - 1;
      int bi = offset * (2 * thread + 2) - 1;
      tmp[bi] += tmp[ai];
    }
    offset *= 2;
  }
  // C (2 lines) --> clear the last element
  if (thread == 0) {
    increments[block] = tmp[n - 1];
    tmp[n - 1] = 0;
  }
  // traverse down tree & build scan
  for (uint d = 1; d < n; d *= 2) {
    offset >>= 1;
    barrier(CLK_LOCAL_MEM_FENCE); //__syncthreads();
    if (thread < d) {
      // D (2 lines)
      int ai = offset * (2 * thread + 1) - 1;
      int bi = offset * (2 * thread + 2) - 1;
      uint t = tmp[ai];
      tmp[ai] = tmp[bi];
      tmp[bi] += t;
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  // E (2 line) --> write results to device memory
  if (2 * thread < len)
    data[global_offset + (2 * thread    )] = tmp[2 * thread    ];
  if (2 * thread + 1 < len)
    data[global_offset + (2 * thread + 1)] = tmp[2 * thread + 1];
}

/// Global exclusive scan, phase 2.
kernel void es_phase_2(global uint* restrict data,
                       global uint* restrict increments,
                       local uint* tmp, uint len) {
  (void) data; // just to keep the arguments similar
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint threads_per_block = get_local_size(0);
  const uint elements_per_block = threads_per_block * 2;
  const uint global_offset = block * elements_per_block;
  const uint n = elements_per_block;

  // block-wise scan, save "block sum" into increments
  uint offset = 1;
  uint ai = thread;
  uint bi = thread + (n / 2);
  uint bankOffsetA = CONFLICT_FREE_OFFSET(ai);
  uint bankOffsetB = CONFLICT_FREE_OFFSET(bi);
  uint ai_global = ai + global_offset;
  uint bi_global = bi + global_offset;
  tmp[ai + bankOffsetA] = (ai_global < len) ? increments[ai_global] : 0;
  tmp[bi + bankOffsetB] = (bi_global < len) ? increments[bi_global] : 0;
  for (uint d = n >> 1; d > 0; d >>= 1) {
    barrier(CLK_LOCAL_MEM_FENCE);
    if (thread < d) {
      uint ai = offset * (2 * thread + 1) - 1;
      uint bi = offset * (2 * thread + 2) - 1;
      ai += CONFLICT_FREE_OFFSET(ai);
      bi += CONFLICT_FREE_OFFSET(bi);
      tmp[bi] += tmp[ai];
    }
    offset *= 2;
  }
  if (thread == 0)
    tmp[n - 1 + CONFLICT_FREE_OFFSET(n - 1)] = 0;
  for (uint d = 1; d < n; d *= 2) {
    offset >>= 1;
    barrier(CLK_LOCAL_MEM_FENCE);
    if (thread < d) {
      int ai = offset * (2 * thread + 1) - 1;
      int bi = offset * (2 * thread + 2) - 1;
      ai += CONFLICT_FREE_OFFSET(ai);
      bi += CONFLICT_FREE_OFFSET(bi);
      uint t = tmp[ai];
      tmp[ai] = tmp[bi];
      tmp[bi] += t;
    }
  }
  barrier(CLK_LOCAL_MEM_FENCE);
  if (ai_global < len) increments[ai_global] = tmp[ai + bankOffsetA];
  if (bi_global < len) increments[bi_global] = tmp[bi + bankOffsetB];
}

kernel void es_phase_3(global uint* restrict data,
                       global uint* restrict increments,
                       uint len) {
  const uint thread = get_local_id(0);
  const uint block = get_group_id(0);
  const uint threads_per_block = get_local_size(0);
  const uint elements_per_block = threads_per_block * 2;
  const uint global_offset = block * elements_per_block;
  const uint n = elements_per_block;
  const uint index = global_offset + thread;

  // add the appropriate value to each block
  uint ai = 2 * thread;
  uint bi = 2 * thread + 1;
  uint ai_global = ai + global_offset;
  uint bi_global = bi + global_offset;
  uint increment = increments[block];
  if (ai_global < len) data[ai_global] += increment;
  if (bi_global < len) data[bi_global] += increment;
}


Commented version of short_idct4x4llm_1 kernel.

Need to find a video to test with before actually using the new code.

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable

__constant int cospi8sqrt2minus1 = 20091;
__constant int sinpi8sqrt2      = 35468;
__constant int rounding = 0;

__kernel void vp8_short_idct4x4llm_kernel(
    __global short *input,
    __global short *output,
    int pitch
)
{
    int i;
    int a1, b1, c1, d1;

    __global short *ip = input;
    __global short *op = output;
    int temp1, temp2;
    int shortpitch = pitch >> 1;

    for (i = 0; i < 4; i++)
    {
        a1 = ip[0] + ip[8];
        b1 = ip[0] - ip[8];

        temp1 = (ip[4] * sinpi8sqrt2 + rounding) >> 16;
        temp2 = ip[12] + ((ip[12] * cospi8sqrt2minus1 + rounding) >> 16);
        c1 = temp1 - temp2;

        temp1 = ip[4] + ((ip[4] * cospi8sqrt2minus1 + rounding) >> 16);
        temp2 = (ip[12] * sinpi8sqrt2 + rounding) >> 16;
        d1 = temp1 + temp2;

        op[shortpitch*0] = a1 + d1;
        op[shortpitch*3] = a1 - d1;

        op[shortpitch*1] = b1 + c1;
        op[shortpitch*2] = b1 - c1;

        ip++;
        op++;
    }

    ip = output;
    op = output;

    for (i = 0; i < 4; i++)
    {
        a1 = ip[0] + ip[2];
        b1 = ip[0] - ip[2];

        temp1 = (ip[1] * sinpi8sqrt2 + rounding) >> 16;
        temp2 = ip[3] + ((ip[3] * cospi8sqrt2minus1 + rounding) >> 16);
        c1 = temp1 - temp2;

        temp1 = ip[1] + ((ip[1] * cospi8sqrt2minus1 + rounding) >> 16);
        temp2 = (ip[3] * sinpi8sqrt2 + rounding) >> 16;
        d1 = temp1 + temp2;

        op[0] = (a1 + d1 + 4) >> 3;
        op[3] = (a1 - d1 + 4) >> 3;

        op[1] = (b1 + c1 + 4) >> 3;
        op[2] = (b1 - c1 + 4) >> 3;

        ip += shortpitch;
        op += shortpitch;
    }

}

__kernel void vp8_short_idct4x4llm_1_kernel(
    __global short *input,
    __global short *output,
    int pitch
)
{
    int a1;
    int out_offset;
    int shortpitch = pitch >> 1;
    //short4 a;
    a1 = ((input[0] + 4) >> 3);
    //a = a1;

    int tid = get_global_id(0);
    if (tid < 4){
        out_offset = shortpitch * tid;
        //vstore4(a,0,&output[out_offset];
        output[out_offset] = a1;
        output[out_offset+1] = a1;
        output[out_offset+2] = a1;
        output[out_offset+3] = a1;
    }
}

__kernel void vp8_dc_only_idct_add_kernel(
    short input_dc,
    __global unsigned char *pred_ptr,
    __global unsigned char *dst_ptr,
    int pitch,
    int stride
)
{
    int a1 = ((input_dc + 4) >> 3);
    int r, c;
    int pred_offset,dst_offset;

    int tid = get_global_id(0);
    if (tid < 16){
        r = tid / 4;
        c = tid % 4;

        pred_offset = r * pitch;
        dst_offset = r * stride;
        int a = a1 + pred_ptr[pred_offset + c] ;

        if (a < 0)
            a = 0;
        else if (a > 255)
            a = 255;

        dst_ptr[dst_offset + c] = (unsigned char) a ;
    }
}

__kernel void vp8_short_inv_walsh4x4_kernel(
    __global short *input,
    __global short *output
)
{

#define VEC
#ifdef VEC
    //4-short vectors to calculate things in
    short4 a,b,c,d, a2v, b2v, c2v, d2v, a1t, b1t, c1t, d1t;
#else
    int i;
    int a1, b1, c1, d1;
    int a2, b2, c2, d2;
    __global short *ip = input;
    __global short *op = output;
#endif
    int tid = get_global_id(0);

#ifdef VEC
    if (tid == 0){
        //first pass loop in vector form
        a = vload4(0,input) + vload4(3,input);
        b = vload4(1,input) + vload4(2,input);
        c = vload4(1,input) - vload4(2,input);
        d = vload4(0,input) - vload4(3,input);
        vstore4(a + b, 0, output);
        vstore4(c + d, 1, output);
        vstore4(a - b, 2, output);
        vstore4(d - c, 3, output);

        //2nd pass
        a = (short4)(output[0], output[4], output[8], output[12]);
        b = (short4)(output[1], output[5], output[9], output[13]);
        c = (short4)(output[1], output[5], output[9], output[13]);
        d = (short4)(output[0], output[4], output[8], output[12]);
        a1t = (short4)(output[3], output[7], output[11], output[15]);
        b1t = (short4)(output[2], output[6], output[10], output[14]);
        c1t = (short4)(output[2], output[6], output[10], output[14]);
        d1t = (short4)(output[3], output[7], output[11], output[15]);

        a = a + a1t + 3;
        b = b + b1t;
        c = c - c1t;
        d = d - d1t + 3;

        a2v = (a + b) >> 3;
        b2v = (c + d) >> 3;
        c2v = (a - b) >> 3;
        d2v = (d - c) >> 3;

        output[0] = a2v.x;
        output[1] = b2v.x;
        output[2] = c2v.x;
        output[3] = d2v.x;
        output[4] = a2v.y;
        output[5] = b2v.y;
        output[6] = c2v.y;
        output[7] = d2v.y;
        output[8] = a2v.z;
        output[9] = b2v.z;
        output[10] = c2v.z;
        output[11] = d2v.z;
        output[12] = a2v.w;
        output[13] = b2v.w;
        output[14] = c2v.w;
        output[15] = d2v.w;
#else
        ip = output;
        op = output;

        for (i = 0; i < 4; i++)
        {
            a1 = ip[0] + ip[3];
            b1 = ip[1] + ip[2];
            c1 = ip[1] - ip[2];
            d1 = ip[0] - ip[3];

            a2 = a1 + b1;
            b2 = c1 + d1;
            c2 = a1 - b1;
            d2 = d1 - c1;

            op[0] = (a2 + 3) >> 3;
            op[1] = (b2 + 3) >> 3;
            op[2] = (c2 + 3) >> 3;
            op[3] = (d2 + 3) >> 3;

            ip += 4;
            op += 4;
        }
#endif
    }
}

__kernel void vp8_short_inv_walsh4x4_1_kernel(
    __global short *input,
    int input_offset,
    __global short *output,
    int output_offset
){
    int a1;
    int tid = get_global_id(0);
    short16 a;

    if (tid == 0)
    {
        a1 = ((input[input_offset] + 3) >> 3);
        a = (short)a1; //Set all elements of vector to a1
        vstore16(a, 0, output);
    }
}

#pragma OPENCL EXTENSION cl_khr_byte_addressable_store : enable

__constant int cospi8sqrt2minus1 = 20091;
__constant int sinpi8sqrt2      = 35468;
__constant int rounding = 0;

__kernel void vp8_short_idct4x4llm_kernel(
    __global short *input,
    __global short *output,
    int pitch
)
{
    int i;
    int a1, b1, c1, d1;

    __global short *ip = input;
    __global short *op = output;
    int temp1, temp2;
    int shortpitch = pitch >> 1;

    for (i = 0; i < 4; i++)
    {
        a1 = ip[0] + ip[8];
        b1 = ip[0] - ip[8];

        temp1 = (ip[4] * sinpi8sqrt2 + rounding) >> 16;
        temp2 = ip[12] + ((ip[12] * cospi8sqrt2minus1 + rounding) >> 16);
        c1 = temp1 - temp2;

        temp1 = ip[4] + ((ip[4] * cospi8sqrt2minus1 + rounding) >> 16);
        temp2 = (ip[12] * sinpi8sqrt2 + rounding) >> 16;
        d1 = temp1 + temp2;

        op[shortpitch*0] = a1 + d1;
        op[shortpitch*3] = a1 - d1;

        op[shortpitch*1] = b1 + c1;
        op[shortpitch*2] = b1 - c1;

        ip++;
        op++;
    }

    ip = output;
    op = output;

    for (i = 0; i < 4; i++)
    {
        a1 = ip[0] + ip[2];
        b1 = ip[0] - ip[2];

        temp1 = (ip[1] * sinpi8sqrt2 + rounding) >> 16;
        temp2 = ip[3] + ((ip[3] * cospi8sqrt2minus1 + rounding) >> 16);
        c1 = temp1 - temp2;

        temp1 = ip[1] + ((ip[1] * cospi8sqrt2minus1 + rounding) >> 16);
        temp2 = (ip[3] * sinpi8sqrt2 + rounding) >> 16;
        d1 = temp1 + temp2;

        op[0] = (a1 + d1 + 4) >> 3;
        op[3] = (a1 - d1 + 4) >> 3;

        op[1] = (b1 + c1 + 4) >> 3;
        op[2] = (b1 - c1 + 4) >> 3;

        ip += shortpitch;
        op += shortpitch;
    }

}

__kernel void vp8_short_idct4x4llm_1_kernel(
    __global short *input,
    __global short *output,
    int pitch
)
{
    int a1;
    int out_offset;
    int shortpitch = pitch >> 1;
    a1 = ((input[0] + 4) >> 3);

    int tid = get_global_id(0);
    if (tid < 4){
        out_offset = shortpitch * tid;
        output[out_offset] = a1;
        output[out_offset+1] = a1;
        output[out_offset+2] = a1;
        output[out_offset+3] = a1;
    }
}

__kernel void vp8_dc_only_idct_add_kernel(
    short input_dc,
    __global unsigned char *pred_ptr,
    __global unsigned char *dst_ptr,
    int pitch,
    int stride
)
{
    int a1 = ((input_dc + 4) >> 3);
    int r, c;
    int pred_offset,dst_offset;

    int tid = get_global_id(0);
    if (tid < 16){
        r = tid / 4;
        c = tid % 4;

        pred_offset = r * pitch;
        dst_offset = r * stride;
        int a = a1 + pred_ptr[pred_offset + c] ;

        if (a < 0)
            a = 0;
        else if (a > 255)
            a = 255;

        dst_ptr[dst_offset + c] = (unsigned char) a ;
    }
}

__kernel void vp8_short_inv_walsh4x4_kernel(
    __global short *input,
    __global short *output
)
{

#define VEC
#ifdef VEC
    //4-short vectors to calculate things in
    short4 a,b,c,d, a2v, b2v, c2v, d2v, a1t, b1t, c1t, d1t;
#else
    int i;
    int a1, b1, c1, d1;
    int a2, b2, c2, d2;
    __global short *ip = input;
    __global short *op = output;
#endif
    int tid = get_global_id(0);

#ifdef VEC
    if (tid == 0){
        //first pass loop in vector form
        a = vload4(0,input) + vload4(3,input);
        b = vload4(1,input) + vload4(2,input);
        c = vload4(1,input) - vload4(2,input);
        d = vload4(0,input) - vload4(3,input);
        vstore4(a + b, 0, output);
        vstore4(c + d, 1, output);
        vstore4(a - b, 2, output);
        vstore4(d - c, 3, output);

        //2nd pass
        a = (short4)(output[0], output[4], output[8], output[12]);
        b = (short4)(output[1], output[5], output[9], output[13]);
        c = (short4)(output[1], output[5], output[9], output[13]);
        d = (short4)(output[0], output[4], output[8], output[12]);
        a1t = (short4)(output[3], output[7], output[11], output[15]);
        b1t = (short4)(output[2], output[6], output[10], output[14]);
        c1t = (short4)(output[2], output[6], output[10], output[14]);
        d1t = (short4)(output[3], output[7], output[11], output[15]);

        a = a + a1t + 3;
        b = b + b1t;
        c = c - c1t;
        d = d - d1t + 3;

        a2v = (a + b) >> 3;
        b2v = (c + d) >> 3;
        c2v = (a - b) >> 3;
        d2v = (d - c) >> 3;

        output[0] = a2v.x;
        output[1] = b2v.x;
        output[2] = c2v.x;
        output[3] = d2v.x;
        output[4] = a2v.y;
        output[5] = b2v.y;
        output[6] = c2v.y;
        output[7] = d2v.y;
        output[8] = a2v.z;
        output[9] = b2v.z;
        output[10] = c2v.z;
        output[11] = d2v.z;
        output[12] = a2v.w;
        output[13] = b2v.w;
        output[14] = c2v.w;
        output[15] = d2v.w;
#else
        ip = output;
        op = output;

        for (i = 0; i < 4; i++)
        {
            a1 = ip[0] + ip[3];
            b1 = ip[1] + ip[2];
            c1 = ip[1] - ip[2];
            d1 = ip[0] - ip[3];

            a2 = a1 + b1;
            b2 = c1 + d1;
            c2 = a1 - b1;
            d2 = d1 - c1;

            op[0] = (a2 + 3) >> 3;
            op[1] = (b2 + 3) >> 3;
            op[2] = (c2 + 3) >> 3;
            op[3] = (d2 + 3) >> 3;

            ip += 4;
            op += 4;
        }
#endif
    }
}

__kernel void vp8_short_inv_walsh4x4_1_kernel(
    __global short *input,
    int input_offset,
    __global short *output,
    int output_offset
){
    int a1;
    int tid = get_global_id(0);
    short16 a;

    if (tid == 0)
    {
        a1 = ((input[input_offset] + 3) >> 3);
        a = (short)a1; //Set all elements of vector to a1
        vstore16(a, 0, output);
    }
}