vstore_half: fix for double arguments and rounding

Add properly rounding float2half and double2half implementations.
Pocl now passes conformance test on halfs.

/* OpenCL built-in library: vstore_half()

   Copyright (c) 2011 Universidad Rey Juan Carlos
   Copyright (c) 2017 Michal Babej / Tampere University of Technology

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

/* The following code is taken & adapted from half library @
 * http://half.sourceforge.net
 *
 * half - IEEE 754-based half-precision floating point library.
 *
 * Copyright (c) 2012-2017 Christian Rau <rauy@users.sourceforge.net>
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation
 * files (the "Software"), to deal in the Software without restriction,
 * including without limitation the rights to use, copy,
 * modify, merge, publish, distribute, sublicense, and/or sell copies of the
 * Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE
 * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 */

#define ROUND_TOWARD_INFINITY 1
#define ROUND_TOWARD_NEG_INFINITY 2
#define ROUND_TOWARD_ZERO 3
#define ROUND_TO_NEAREST 4

static ushort
_cl_float2half_round (float f, int round_mode)
{
  uint bits = as_uint (f);
  ushort hbits = (bits >> 16) & 0x8000;
  bits &= 0x7FFFFFFF;
  int exp = bits >> 23;
  if (exp == 255)
    {
      ushort temp = (((bits & 0x7FFFFF) != 0) ? 0x03FF : 0x0);
      return (hbits | 0x7C00 | temp);
    }
  if (exp > 142)
    {
      if (round_mode == ROUND_TOWARD_INFINITY)
        return hbits | 0x7C00 - (hbits >> 15);
      if (round_mode == ROUND_TOWARD_NEG_INFINITY)
        return hbits | 0x7BFF + (hbits >> 15);
      return hbits | 0x7BFF + (round_mode != ROUND_TOWARD_ZERO);
    }
  int g, s;
  if (exp > 112)
    {
      g = (bits >> 12) & 1;
      s = (bits & 0xFFF) != 0;
      hbits |= ((exp - 112) << 10) | ((bits >> 13) & 0x3FF);
    }
  else if (exp > 101)
    {
      int i = 125 - exp;
      bits = (bits & 0x7FFFFF) | 0x800000;
      g = (bits >> i) & 1;
      s = (bits & ((1L << i) - 1)) != 0;
      hbits |= bits >> (i + 1);
    }
  else
    {
      g = 0;
      s = bits != 0;
    }
  if (round_mode == ROUND_TO_NEAREST)
    hbits += g & (s | hbits);
  else if (round_mode == ROUND_TOWARD_INFINITY)
    hbits += ~(hbits >> 15) & (s | g);
  else if (round_mode == ROUND_TOWARD_NEG_INFINITY)
    hbits += (hbits >> 15) & (g | s);
  return hbits;
}

static ushort
_cl_float2half (float d)
{
  return _cl_float2half_round (d, ROUND_TO_NEAREST);
}

static ushort
_cl_float2half_rte (float d)
{
  return _cl_float2half_round (d, ROUND_TO_NEAREST);
}

static ushort
_cl_float2half_rtz (float d)
{
  return _cl_float2half_round (d, ROUND_TOWARD_ZERO);
}

static ushort
_cl_float2half_rtn (float d)
{
  return _cl_float2half_round (d, ROUND_TOWARD_NEG_INFINITY);
}

static ushort
_cl_float2half_rtp (float d)
{
  return _cl_float2half_round (d, ROUND_TOWARD_INFINITY);
}

#ifdef cl_khr_fp64

static ushort
_cl_double2half_round (double value, int round_mode)
{
  ulong bits = as_ulong (value);
  uint hi = (bits >> 32);
  uint lo = (bits & 0xFFFFFFFF);
  ushort hbits = (hi >> 16) & 0x8000;
  hi &= 0x7FFFFFFF;
  int exp = hi >> 20;
  if (exp == 2047)
    {
      ushort temp = ((bits & 0xFFFFFFFFFFFFF) != 0 ? 0x03FF : 0x0);
      return (hbits | 0x7C00 | temp);
    }
  if (exp > 1038)
    {
      if (round_mode == ROUND_TOWARD_INFINITY)
        return (hbits | 0x7C00 - (hbits >> 15));
      if (round_mode == ROUND_TOWARD_NEG_INFINITY)
        return (hbits | 0x7BFF + (hbits >> 15));
      return (hbits | 0x7BFF + (round_mode != ROUND_TOWARD_ZERO));
    }
  int g;
  int s = (lo != 0);
  if (exp > 1008)
    {
      g = (hi >> 9) & 1;
      s |= (hi & 0x1FF) != 0;
      hbits |= ((exp - 1008) << 10) | ((hi >> 10) & 0x3FF);
    }
  else if (exp > 997)
    {
      int i = 1018 - exp;
      hi = (hi & 0xFFFFF) | 0x100000;
      g = (hi >> i) & 1;
      s |= (hi & ((1L << i) - 1)) != 0;
      hbits |= hi >> (i + 1);
    }
  else
    {
      g = 0;
      s |= hi != 0;
    }
  if (round_mode == ROUND_TO_NEAREST)
    hbits += g & (s | hbits);
  else if (round_mode == ROUND_TOWARD_INFINITY)
    hbits += ~(hbits >> 15) & (s | g);
  else if (round_mode == ROUND_TOWARD_NEG_INFINITY)
    hbits += (hbits >> 15) & (g | s);
  return hbits;
}

static ushort
_cl_double2half (double d)
{
  return _cl_double2half_round (d, ROUND_TO_NEAREST);
}

static ushort
_cl_double2half_rte (double d)
{
  return _cl_double2half_round (d, ROUND_TO_NEAREST);
}

static ushort
_cl_double2half_rtz (double d)
{
  return _cl_double2half_round (d, ROUND_TOWARD_ZERO);
}

static ushort
_cl_double2half_rtn (double d)
{
  return _cl_double2half_round (d, ROUND_TOWARD_NEG_INFINITY);
}

static ushort
_cl_double2half_rtp (double d)
{
  return _cl_double2half_round (d, ROUND_TOWARD_INFINITY);
}

#endif

#ifdef __F16C__

ushort4 _cl_float2half4 (const float4 data);
ushort8 _cl_float2half8 (const float8 data);
ushort4 _cl_float2half4_rte (const float4 data);
ushort8 _cl_float2half8_rte (const float8 data);
ushort4 _cl_float2half4_rtn (const float4 data);
ushort8 _cl_float2half8_rtn (const float8 data);
ushort4 _cl_float2half4_rtp (const float4 data);
ushort8 _cl_float2half8_rtp (const float8 data);
ushort4 _cl_float2half4_rtz (const float4 data);
ushort8 _cl_float2half8_rtz (const float8 data);

#define IMPLEMENT_VSTORE_HALF(MOD, SUFFIX)                                    \
                                                                              \
  void _CL_OVERLOADABLE vstore_half##SUFFIX (float data, size_t offset,       \
                                             MOD half *p)                     \
  {                                                                           \
    ((MOD ushort *)p)[offset] = _cl_float2half##SUFFIX (data);                \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half2##SUFFIX (float2 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.lo, offset * 2, p);                             \
    vstore_half##SUFFIX (data.hi, offset * 2 + 1, p);                         \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half3##SUFFIX (float3 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.x, offset * 3, p);                              \
    vstore_half##SUFFIX (data.y, offset * 3 + 1, p);                          \
    vstore_half##SUFFIX (data.z, offset * 3 + 2, p);                          \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half4##SUFFIX (float4 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort4 *)p)[offset] = _cl_float2half4##SUFFIX (data);              \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half8##SUFFIX (float8 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort8 *)p)[offset] = _cl_float2half8##SUFFIX (data);              \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half16##SUFFIX (float16 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    ((MOD ushort8 *)p)[offset * 2] = _cl_float2half8##SUFFIX (data.lo);       \
    ((MOD ushort8 *)p)[offset * 2 + 1] = _cl_float2half8##SUFFIX (data.hi);   \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half##SUFFIX (float data, size_t offset,      \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort *)p)[offset] = _cl_float2half##SUFFIX (data);                \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half2##SUFFIX (float2 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half##SUFFIX (data.lo, offset * 2, p);                            \
    vstorea_half##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half3##SUFFIX (float3 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half2##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half2##SUFFIX ((float2) (data.z, 0.0f), offset * 2 + 1, p);       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half4##SUFFIX (float4 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    ((MOD ushort4 *)p)[offset] = _cl_float2half4##SUFFIX (data);              \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half8##SUFFIX (float8 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    ((MOD ushort8 *)p)[offset] = _cl_float2half8##SUFFIX (data);              \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half16##SUFFIX (float16 data, size_t offset,  \
                                                MOD half *p)                  \
  {                                                                           \
    ((MOD ushort8 *)p)[offset * 2] = _cl_float2half8##SUFFIX (data.lo);       \
    ((MOD ushort8 *)p)[offset * 2 + 1] = _cl_float2half8##SUFFIX (data.hi);   \
  }

// __F16C__
#else

#define IMPLEMENT_VSTORE_HALF(MOD, SUFFIX)                                    \
                                                                              \
  void _CL_OVERLOADABLE vstore_half##SUFFIX (float data, size_t offset,       \
                                             MOD half *p)                     \
  {                                                                           \
    ((MOD ushort *)p)[offset] = _cl_float2half##SUFFIX (data);                \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half2##SUFFIX (float2 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.lo, offset * 2, p);                             \
    vstore_half##SUFFIX (data.hi, offset * 2 + 1, p);                         \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half3##SUFFIX (float3 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.x, offset * 3, p);                              \
    vstore_half##SUFFIX (data.y, offset * 3 + 1, p);                          \
    vstore_half##SUFFIX (data.z, offset * 3 + 2, p);                          \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half4##SUFFIX (float4 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half2##SUFFIX (data.lo, offset * 2, p);                            \
    vstore_half2##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half8##SUFFIX (float8 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half4##SUFFIX (data.lo, offset * 2, p);                            \
    vstore_half4##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half16##SUFFIX (float16 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstore_half8##SUFFIX (data.lo, offset * 2, p);                            \
    vstore_half8##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half##SUFFIX (float data, size_t offset,      \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort *)p)[offset] = _cl_float2half##SUFFIX (data);                \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half2##SUFFIX (float2 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half##SUFFIX (data.lo, offset * 2, p);                            \
    vstorea_half##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half3##SUFFIX (float3 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half2##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half2##SUFFIX ((float2) (data.z, 0.0f), offset * 2 + 1, p);       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half4##SUFFIX (float4 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half2##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half2##SUFFIX (data.hi, offset * 2 + 1, p);                       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half8##SUFFIX (float8 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half4##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half4##SUFFIX (data.hi, offset * 2 + 1, p);                       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half16##SUFFIX (float16 data, size_t offset,  \
                                                MOD half *p)                  \
  {                                                                           \
    vstorea_half8##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half8##SUFFIX (data.hi, offset * 2 + 1, p);                       \
  }

#endif

IMPLEMENT_VSTORE_HALF (__global, )
IMPLEMENT_VSTORE_HALF (__global, _rte)
IMPLEMENT_VSTORE_HALF (__global, _rtz)
IMPLEMENT_VSTORE_HALF (__global, _rtp)
IMPLEMENT_VSTORE_HALF (__global, _rtn)
IMPLEMENT_VSTORE_HALF (__local, )
IMPLEMENT_VSTORE_HALF (__local, _rte)
IMPLEMENT_VSTORE_HALF (__local, _rtz)
IMPLEMENT_VSTORE_HALF (__local, _rtp)
IMPLEMENT_VSTORE_HALF (__local, _rtn)
IMPLEMENT_VSTORE_HALF (__private, )
IMPLEMENT_VSTORE_HALF (__private, _rte)
IMPLEMENT_VSTORE_HALF (__private, _rtz)
IMPLEMENT_VSTORE_HALF (__private, _rtp)
IMPLEMENT_VSTORE_HALF (__private, _rtn)

#ifdef cl_khr_fp64

///#ifdef __F16C__
#if 0

#define IMPLEMENT_VSTORE_HALF_DBL(MOD, SUFFIX)                                \
                                                                              \
  void _CL_OVERLOADABLE vstore_half##SUFFIX (double data, size_t offset,      \
                                             MOD half *p)                     \
  {                                                                           \
    ((MOD ushort *)p)[offset] = _cl_double2half##SUFFIX (data);               \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half2##SUFFIX (double2 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.lo, offset * 2, p);                             \
    vstore_half##SUFFIX (data.hi, offset * 2 + 1, p);                         \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half3##SUFFIX (double3 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.x, offset * 3, p);                              \
    vstore_half##SUFFIX (data.y, offset * 3 + 1, p);                          \
    vstore_half##SUFFIX (data.z, offset * 3 + 2, p);                          \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half4##SUFFIX (double4 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort4 *)p)[offset]                                                \
        = _cl_float2half4##SUFFIX (convert_float4##SUFFIX (data));            \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half8##SUFFIX (double8 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort8 *)p)[offset]                                                \
        = _cl_float2half8##SUFFIX (convert_float8##SUFFIX (data));            \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half16##SUFFIX (double16 data, size_t offset,  \
                                               MOD half *p)                   \
  {                                                                           \
    ((MOD ushort8 *)p)[offset * 2]                                            \
        = _cl_float2half8##SUFFIX (convert_float8##SUFFIX (data.lo));         \
    ((MOD ushort8 *)p)[offset * 2 + 1]                                        \
        = _cl_float2half8##SUFFIX (convert_float8##SUFFIX (data.hi));         \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half##SUFFIX (double data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort *)p)[offset] = _cl_double2half##SUFFIX (data);               \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half2##SUFFIX (double2 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half##SUFFIX (data.lo, offset * 2, p);                            \
    vstorea_half##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half3##SUFFIX (double3 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half2##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half2##SUFFIX ((float2) (data.z, 0.0f), offset * 2 + 1, p);       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half4##SUFFIX (double4 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    ((MOD ushort4 *)p)[offset]                                                \
        = _cl_float2half4##SUFFIX (convert_float4##SUFFIX (data));            \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half8##SUFFIX (double8 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    ((MOD ushort8 *)p)[offset]                                                \
        = _cl_float2half8##SUFFIX (convert_float8##SUFFIX (data));            \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half16##SUFFIX (double16 data, size_t offset, \
                                                MOD half *p)                  \
  {                                                                           \
    ((MOD ushort8 *)p)[offset * 2]                                            \
        = _cl_float2half8##SUFFIX (convert_float8##SUFFIX (data.lo));         \
    ((MOD ushort8 *)p)[offset * 2 + 1]                                        \
        = _cl_float2half8##SUFFIX (convert_float8##SUFFIX (data.hi));         \
  }

// __F16C__
#else

#define IMPLEMENT_VSTORE_HALF_DBL(MOD, SUFFIX)                                \
                                                                              \
  void _CL_OVERLOADABLE vstore_half##SUFFIX (double data, size_t offset,      \
                                             MOD half *p)                     \
  {                                                                           \
    ((MOD ushort *)p)[offset] = _cl_double2half##SUFFIX (data);               \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half2##SUFFIX (double2 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.lo, offset * 2, p);                             \
    vstore_half##SUFFIX (data.hi, offset * 2 + 1, p);                         \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half3##SUFFIX (double3 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.x, offset * 3, p);                              \
    vstore_half##SUFFIX (data.y, offset * 3 + 1, p);                          \
    vstore_half##SUFFIX (data.z, offset * 3 + 2, p);                          \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half4##SUFFIX (double4 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half2##SUFFIX (data.lo, offset * 2, p);                            \
    vstore_half2##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half8##SUFFIX (double8 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half4##SUFFIX (data.lo, offset * 2, p);                            \
    vstore_half4##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half16##SUFFIX (double16 data, size_t offset,  \
                                               MOD half *p)                   \
  {                                                                           \
    vstore_half8##SUFFIX (data.lo, offset * 2, p);                            \
    vstore_half8##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half##SUFFIX (double data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort *)p)[offset] = _cl_double2half##SUFFIX (data);               \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half2##SUFFIX (double2 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half##SUFFIX (data.lo, offset * 2, p);                            \
    vstorea_half##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half3##SUFFIX (double3 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half2##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half2##SUFFIX ((double2) (data.z, 0.0), offset * 2 + 1, p);       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half4##SUFFIX (double4 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half2##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half2##SUFFIX (data.hi, offset * 2 + 1, p);                       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half8##SUFFIX (double8 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half4##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half4##SUFFIX (data.hi, offset * 2 + 1, p);                       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half16##SUFFIX (double16 data, size_t offset, \
                                                MOD half *p)                  \
  {                                                                           \
    vstorea_half8##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half8##SUFFIX (data.hi, offset * 2 + 1, p);                       \
  }

#endif

IMPLEMENT_VSTORE_HALF_DBL (__global, )
IMPLEMENT_VSTORE_HALF_DBL (__global, _rte)
IMPLEMENT_VSTORE_HALF_DBL (__global, _rtz)
IMPLEMENT_VSTORE_HALF_DBL (__global, _rtp)
IMPLEMENT_VSTORE_HALF_DBL (__global, _rtn)
IMPLEMENT_VSTORE_HALF_DBL (__local, )
IMPLEMENT_VSTORE_HALF_DBL (__local, _rte)
IMPLEMENT_VSTORE_HALF_DBL (__local, _rtz)
IMPLEMENT_VSTORE_HALF_DBL (__local, _rtp)
IMPLEMENT_VSTORE_HALF_DBL (__local, _rtn)
IMPLEMENT_VSTORE_HALF_DBL (__private, )
IMPLEMENT_VSTORE_HALF_DBL (__private, _rte)
IMPLEMENT_VSTORE_HALF_DBL (__private, _rtz)
IMPLEMENT_VSTORE_HALF_DBL (__private, _rtp)
IMPLEMENT_VSTORE_HALF_DBL (__private, _rtn)

#endif

/* OpenCL built-in library: vstore_half()

   Copyright (c) 2011 Universidad Rey Juan Carlos
   Copyright (c) 2017 Michal Babej / Tampere University of Technology

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/

/* The following function is rewritten to OpenCL from a
 * FP16 header-only library using The MIT License (MIT)
 *
 * https://github.com/Maratyszcza/FP16
 */

ushort
_cl_float2half (float f)
{
  const float scale_to_inf = 0x1.0p+112f;
  const float scale_to_zero = 0x1.0p-110f;
  float base = (fabs(f) * scale_to_inf) * scale_to_zero;
  const uint w = as_uint(f);
  const uint shl1_w = w + w;
  const uint sign = w & 0x80000000;
  uint bias = shl1_w & 0xFF000000;
  if (bias < (uint)(0x71000000)) {
    bias = (uint) (0x71000000);
  }
  base = as_float((bias >> 1) + (uint)(0x07800000)) + base;
  const uint bits = as_uint(base);
  const uint exp_bits = (bits >> 13) & (uint)(0x00007C00);
  const uint mantissa_bits = bits & (uint)(0x00000FFF);
  const uint nonsign = exp_bits + mantissa_bits;
  return ((sign >> 16) | (shl1_w > (uint)(0xFF000000) ? (ushort)0x7E00 : convert_ushort(nonsign)));
}

#define _cl_float2half_rte _cl_float2half
#define _cl_float2half_rtn _cl_float2half
#define _cl_float2half_rtz _cl_float2half
#define _cl_float2half_rtp _cl_float2half

#ifdef __F16C__

ushort4 _cl_float2half4 (const float4 data);
ushort8 _cl_float2half8 (const float8 data);
ushort4 _cl_float2half4_rte (const float4 data);
ushort8 _cl_float2half8_rte (const float8 data);
ushort4 _cl_float2half4_rtn (const float4 data);
ushort8 _cl_float2half8_rtn (const float8 data);
ushort4 _cl_float2half4_rtp (const float4 data);
ushort8 _cl_float2half8_rtp (const float8 data);
ushort4 _cl_float2half4_rtz (const float4 data);
ushort8 _cl_float2half8_rtz (const float8 data);

#define IMPLEMENT_VSTORE_HALF(MOD, SUFFIX)                                    \
                                                                              \
  void _CL_OVERLOADABLE vstore_half##SUFFIX (float data, size_t offset,       \
                                             MOD half *p)                     \
  {                                                                           \
    ((MOD ushort *)p)[offset] = _cl_float2half (data);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half2##SUFFIX (float2 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.lo, offset * 2, p);                             \
    vstore_half##SUFFIX (data.hi, offset * 2 + 1, p);                         \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half3##SUFFIX (float3 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.x, offset * 3, p);                              \
    vstore_half##SUFFIX (data.y, offset * 3 + 1, p);                          \
    vstore_half##SUFFIX (data.z, offset * 3 + 2, p);                          \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half4##SUFFIX (float4 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort4 *)p)[offset] = _cl_float2half4##SUFFIX (data);              \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half8##SUFFIX (float8 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort8 *)p)[offset] = _cl_float2half8##SUFFIX (data);              \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half16##SUFFIX (float16 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    ((MOD ushort8 *)p)[offset * 2] = _cl_float2half8##SUFFIX (data.lo);       \
    ((MOD ushort8 *)p)[offset * 2 + 1] = _cl_float2half8##SUFFIX (data.hi);   \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half##SUFFIX (float data, size_t offset,      \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort *)p)[offset] = _cl_float2half (data);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half2##SUFFIX (float2 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half##SUFFIX (data.lo, offset * 2, p);                            \
    vstorea_half##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half3##SUFFIX (float3 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half2##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half2##SUFFIX ((float2) (data.z, 0.0f), offset * 2 + 1, p);       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half4##SUFFIX (float4 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    ((MOD ushort4 *)p)[offset] = _cl_float2half4##SUFFIX (data);              \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half8##SUFFIX (float8 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    ((MOD ushort8 *)p)[offset] = _cl_float2half8##SUFFIX (data);              \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half16##SUFFIX (float16 data, size_t offset,  \
                                                MOD half *p)                  \
  {                                                                           \
    ((MOD ushort8 *)p)[offset * 2] = _cl_float2half8##SUFFIX (data.lo);       \
    ((MOD ushort8 *)p)[offset * 2 + 1] = _cl_float2half8##SUFFIX (data.hi);   \
  }

// __F16C__
#else

#define IMPLEMENT_VSTORE_HALF(MOD, SUFFIX)                                    \
                                                                              \
  void _CL_OVERLOADABLE vstore_half##SUFFIX (float data, size_t offset,       \
                                             MOD half *p)                     \
  {                                                                           \
    if (isnan (data))                                                         \
      ((MOD ushort *)p)[offset] = (ushort)0xffff;                             \
    else                                                                      \
      ((MOD ushort *)p)[offset] = _cl_float2half##SUFFIX (data);              \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half2##SUFFIX (float2 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.lo, offset * 2, p);                             \
    vstore_half##SUFFIX (data.hi, offset * 2 + 1, p);                         \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half3##SUFFIX (float3 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.x, offset * 3, p);                              \
    vstore_half##SUFFIX (data.y, offset * 3 + 1, p);                          \
    vstore_half##SUFFIX (data.z, offset * 3 + 2, p);                          \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half4##SUFFIX (float4 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half2##SUFFIX (data.lo, offset * 2, p);                            \
    vstore_half2##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half8##SUFFIX (float8 data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half4##SUFFIX (data.lo, offset * 2, p);                            \
    vstore_half4##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half16##SUFFIX (float16 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstore_half8##SUFFIX (data.lo, offset * 2, p);                            \
    vstore_half8##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half##SUFFIX (float data, size_t offset,      \
                                              MOD half *p)                    \
  {                                                                           \
    if (isnan (data))                                                         \
      ((MOD ushort *)p)[offset] = (ushort)0xffff;                             \
    else                                                                      \
      ((MOD ushort *)p)[offset] = _cl_float2half##SUFFIX (data);              \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half2##SUFFIX (float2 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half##SUFFIX (data.lo, offset * 2, p);                            \
    vstorea_half##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half3##SUFFIX (float3 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half2##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half2##SUFFIX ((float2) (data.z, 0.0f), offset * 2 + 1, p);       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half4##SUFFIX (float4 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half2##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half2##SUFFIX (data.hi, offset * 2 + 1, p);                       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half8##SUFFIX (float8 data, size_t offset,    \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half4##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half4##SUFFIX (data.hi, offset * 2 + 1, p);                       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half16##SUFFIX (float16 data, size_t offset,  \
                                                MOD half *p)                  \
  {                                                                           \
    vstorea_half8##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half8##SUFFIX (data.hi, offset * 2 + 1, p);                       \
  }

#endif

IMPLEMENT_VSTORE_HALF (__global, )
IMPLEMENT_VSTORE_HALF (__global, _rte)
IMPLEMENT_VSTORE_HALF (__global, _rtz)
IMPLEMENT_VSTORE_HALF (__global, _rtp)
IMPLEMENT_VSTORE_HALF (__global, _rtn)
IMPLEMENT_VSTORE_HALF (__local, )
IMPLEMENT_VSTORE_HALF (__local, _rte)
IMPLEMENT_VSTORE_HALF (__local, _rtz)
IMPLEMENT_VSTORE_HALF (__local, _rtp)
IMPLEMENT_VSTORE_HALF (__local, _rtn)
IMPLEMENT_VSTORE_HALF (__private, )
IMPLEMENT_VSTORE_HALF (__private, _rte)
IMPLEMENT_VSTORE_HALF (__private, _rtz)
IMPLEMENT_VSTORE_HALF (__private, _rtp)
IMPLEMENT_VSTORE_HALF (__private, _rtn)

#ifdef cl_khr_fp64

#ifdef __F16C__

#define IMPLEMENT_VSTORE_HALF_DBL(MOD, SUFFIX)                                \
                                                                              \
  void _CL_OVERLOADABLE vstore_half##SUFFIX (double data, size_t offset,      \
                                             MOD half *p)                     \
  {                                                                           \
    ((MOD ushort *)p)[offset]                                                 \
        = _cl_float2half##SUFFIX (convert_float##SUFFIX (data));              \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half2##SUFFIX (double2 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.lo, offset * 2, p);                             \
    vstore_half##SUFFIX (data.hi, offset * 2 + 1, p);                         \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half3##SUFFIX (double3 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.x, offset * 3, p);                              \
    vstore_half##SUFFIX (data.y, offset * 3 + 1, p);                          \
    vstore_half##SUFFIX (data.z, offset * 3 + 2, p);                          \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half4##SUFFIX (double4 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort4 *)p)[offset]                                                \
        = _cl_float2half4##SUFFIX (convert_float4##SUFFIX (data));            \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half8##SUFFIX (double8 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort8 *)p)[offset]                                                \
        = _cl_float2half8##SUFFIX (convert_float8##SUFFIX (data));            \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half16##SUFFIX (double16 data, size_t offset,  \
                                               MOD half *p)                   \
  {                                                                           \
    ((MOD ushort8 *)p)[offset * 2]                                            \
        = _cl_float2half8##SUFFIX (convert_float8##SUFFIX (data.lo));         \
    ((MOD ushort8 *)p)[offset * 2 + 1]                                        \
        = _cl_float2half8##SUFFIX (convert_float8##SUFFIX (data.hi));         \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half##SUFFIX (double data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    ((MOD ushort *)p)[offset]                                                 \
        = _cl_float2half (convert_float##SUFFIX (data));                      \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half2##SUFFIX (double2 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half##SUFFIX (data.lo, offset * 2, p);                            \
    vstorea_half##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half3##SUFFIX (double3 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half2##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half2##SUFFIX ((float2) (data.z, 0.0f), offset * 2 + 1, p);       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half4##SUFFIX (double4 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    ((MOD ushort4 *)p)[offset]                                                \
        = _cl_float2half4##SUFFIX (convert_float4##SUFFIX (data));            \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half8##SUFFIX (double8 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    ((MOD ushort8 *)p)[offset]                                                \
        = _cl_float2half8##SUFFIX (convert_float8##SUFFIX (data));            \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half16##SUFFIX (double16 data, size_t offset, \
                                                MOD half *p)                  \
  {                                                                           \
    ((MOD ushort8 *)p)[offset * 2]                                            \
        = _cl_float2half8##SUFFIX (convert_float8##SUFFIX (data.lo));         \
    ((MOD ushort8 *)p)[offset * 2 + 1]                                        \
        = _cl_float2half8##SUFFIX (convert_float8##SUFFIX (data.hi));         \
  }

// __F16C__
#else

#define IMPLEMENT_VSTORE_HALF_DBL(MOD, SUFFIX)                                \
                                                                              \
  void _CL_OVERLOADABLE vstore_half##SUFFIX (double data, size_t offset,      \
                                             MOD half *p)                     \
  {                                                                           \
    if (isnan (data))                                                         \
      ((MOD ushort *)p)[offset] = (ushort)0xffff;                             \
    else                                                                      \
      ((MOD ushort *)p)[offset]                                               \
          = _cl_float2half (convert_float##SUFFIX (data));                    \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half2##SUFFIX (double2 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.lo, offset * 2, p);                             \
    vstore_half##SUFFIX (data.hi, offset * 2 + 1, p);                         \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half3##SUFFIX (double3 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half##SUFFIX (data.x, offset * 3, p);                              \
    vstore_half##SUFFIX (data.y, offset * 3 + 1, p);                          \
    vstore_half##SUFFIX (data.z, offset * 3 + 2, p);                          \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half4##SUFFIX (double4 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half2##SUFFIX (data.lo, offset * 2, p);                            \
    vstore_half2##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half8##SUFFIX (double8 data, size_t offset,    \
                                              MOD half *p)                    \
  {                                                                           \
    vstore_half4##SUFFIX (data.lo, offset * 2, p);                            \
    vstore_half4##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstore_half16##SUFFIX (double16 data, size_t offset,  \
                                               MOD half *p)                   \
  {                                                                           \
    vstore_half8##SUFFIX (data.lo, offset * 2, p);                            \
    vstore_half8##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half##SUFFIX (double data, size_t offset,     \
                                              MOD half *p)                    \
  {                                                                           \
    if (isnan (data))                                                         \
      ((MOD ushort *)p)[offset] = (ushort)0xffff;                             \
    else                                                                      \
      ((MOD ushort *)p)[offset]                                               \
          = _cl_float2half (convert_float##SUFFIX (data));                    \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half2##SUFFIX (double2 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half##SUFFIX (data.lo, offset * 2, p);                            \
    vstorea_half##SUFFIX (data.hi, offset * 2 + 1, p);                        \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half3##SUFFIX (double3 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half2##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half2##SUFFIX ((double2) (data.z, 0.0), offset * 2 + 1, p);       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half4##SUFFIX (double4 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half2##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half2##SUFFIX (data.hi, offset * 2 + 1, p);                       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half8##SUFFIX (double8 data, size_t offset,   \
                                               MOD half *p)                   \
  {                                                                           \
    vstorea_half4##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half4##SUFFIX (data.hi, offset * 2 + 1, p);                       \
  }                                                                           \
                                                                              \
  void _CL_OVERLOADABLE vstorea_half16##SUFFIX (double16 data, size_t offset, \
                                                MOD half *p)                  \
  {                                                                           \
    vstorea_half8##SUFFIX (data.lo, offset * 2, p);                           \
    vstorea_half8##SUFFIX (data.hi, offset * 2 + 1, p);                       \
  }

#endif

IMPLEMENT_VSTORE_HALF_DBL (__global, )
IMPLEMENT_VSTORE_HALF_DBL (__global, _rte)
IMPLEMENT_VSTORE_HALF_DBL (__global, _rtz)
IMPLEMENT_VSTORE_HALF_DBL (__global, _rtp)
IMPLEMENT_VSTORE_HALF_DBL (__global, _rtn)
IMPLEMENT_VSTORE_HALF_DBL (__local, )
IMPLEMENT_VSTORE_HALF_DBL (__local, _rte)
IMPLEMENT_VSTORE_HALF_DBL (__local, _rtz)
IMPLEMENT_VSTORE_HALF_DBL (__local, _rtp)
IMPLEMENT_VSTORE_HALF_DBL (__local, _rtn)
IMPLEMENT_VSTORE_HALF_DBL (__private, )
IMPLEMENT_VSTORE_HALF_DBL (__private, _rte)
IMPLEMENT_VSTORE_HALF_DBL (__private, _rtz)
IMPLEMENT_VSTORE_HALF_DBL (__private, _rtp)
IMPLEMENT_VSTORE_HALF_DBL (__private, _rtn)

#endif


RLC working on MSB CUP

RLC working on MSB CUP

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

/*

Bit Plane Coder
===============

A. Scan Pattern

Xx4 stripes

The stripes are scanned from top to bottom, and the columns within a
stripe are scanned from left to right.

B. Serial State Variables

Pixels are stored in Sign + Magnitude format

sigma[y][x]			equal to 1 if at least one non-zero bit has been coded, otherwise zero

sigma_prime[y][x]   equal to 1 if magnitude refinement coding (MRC) has been applied to this pixel, otherwise zero

eta[y][x]		    equal to 1 if zero coding (ZC) has been applied in SPP for this bit plane, otherwise zero

Note: eta gets reset to zero at the start of each bit plane, while sigma and sigma_prime do not.

C. Concepts

Preferred Neighbourhood: at least one of the eight neighbours has sigma[y][x] == 1

Sign Coding (SC)
Zero Coding (ZC)
Magnitude Refinement Coding (MRC)
Run Length Encoding (RLC) (applied only to first pixel in stripe column)
	- if we are at the first pixel in the column, and all four pixels in column have sigma == 0 and eta == 0,
      then apply RLC to current pixel
	- occurrence of first 1 bit in column determines how RLC is coded

C. Serial Algorithm

Three passes per bit plane : Significance Propagation Pass (SPP)
                             Magnitude Refinement Pass	 (MRP)
						     Clean Up Pass				 (CUP)

Only the CUP is performed on the most significant bit plane (MSB)

Algorithm:

i. most significant bit plane is calculated

ii.  SPP
     // not significant, and in preferred nbhd
     if in preferred nbhd and sigma[y][x] == 0 then
	     ZC
		 eta[y][x] = 1
		 if (encoded bit == 1) then
			SC
			sigma[y][x] = 1
		 end if
	end if

iii. MRP
     // significant, but ZC has not been performed in SPP
	`if (sigma[y][x] == 1 and eta[y][x] == 0 then
		MRC
		sigma_prime = 1
	end if

iv. CUP
    // not significant, and not in preferred nbhd
	if sigma[y][x] == 0 and eta[y][x] == 0 then
		if pixel meets RLC conditions then
		   RLC
		else
		   ZC
		endif
*/

#define CODEBLOCKY_QUARTER 8

#define BOUNDARY 1
#define BOUNDARY_X2 2

#define CODEBLOCKY_X4 128

//////////////////////////////////////////////////////
// State Buffer

#define STATE_BUFFER_SIZE 1156
#define STATE_BOTTOM_BOUNDARY_OFFSET 1124

#define STATE_BUFFER_STRIDE 34
#define STATE_BUFFER_STRIDE_X2 68
#define STATE_BUFFER_STRIDE_X3 102
#define STATE_BUFFER_STRIDE_X4 136

#define LEFT_TOP -35
#define TOP  -34
#define RIGHT_TOP -33

#define LEFT   -1
#define RIGHT   1

#define LEFT_BOTTOM  33
#define BOTTOM  34
#define RIGHT_BOTTOM  35

////////////////////////////////////////////////////
// State Variables

// bit positions (0 based indices)
#define INPUT_SIGN_BITPOS  15  // assumes 16 bit signed input

#define SIGMA_NEW_BITPOS   0x0
#define SIGMA_OLD_BITPOS   0x1
#define NBH_BITPOS         0x2
#define PIXEL_START_BITPOS 0xA
#define PIXEL_END_BITPOS   0x18
#define SIGN_BITPOS        0x19

// bit flags
#define SIGMA_NEW_F			 0x1		  //position 0
#define SIGMA_OLD_F			 0x10		  //position  1
#define NBH_F				 0x20		  //position  2
#define PIXEL_F				 0x7FFF0000   //positions 10-24
#define SIGN_F				 0x2000000    //position  25

#define NOT_SIGMA_NEW_F      0xFFFFFFFE   // ~SIGMA_NEW
#define SIGMA_OLD_AND_NEW_F  0x11

#define INPUT_TO_SIGN_SHIFT 10
#define SIGMA_OLD_TO_NBH_SHIFT 0x1

#define BIT(pix) (((pix)>>(bp))&1)
#define NBH(pix) ((pix) & NBH_F)
#define SIGMA_OLD(pix) ((pix) & SIGMA_OLD_F)
#define SIGMA_NEW(pix) ((pix) & SIGMA_NEW_F)
#define SIGMA_OLD_AND_NEW(pix) ((pix) & SIGMA_OLD_AND_NEW_F)
#define SIGN(pix) ((pix) & SIGN_F)

#define SET_SIGMA_NEW(pix) ( (pix) |= SIGMA_NEW_F )
#define CLEAR_SIGMA_NEW(pix) ( (pix) &= NOT_SIGMA_NEW_F )

/////////////////////////////////////
// Context Variables

/*
0-2		(CX,D) pairs counter
3		SPP flag
4		MRP flag
5		SPP flag
8		D4
9-13	CX4
14		D3
15-19	CX3
20		D2
21-25	CX2
26		D1
27-31	CX1

*/

/////////////////////////////////////////////////////

// TABLES
/*

// ZERO CODING /////////////////////////////////////

// LL and LH sub-bands
values					Context
sumH	sumV	sumD	CX
2		x		x		8
1 		1		x		7
1		0 		1		6
1		0		0		5
0		2		x		4
0		1		x		3
0		0 		2		2
0		0		1		1
0		0		0		0

//HH sub-band

values					Context
sum(H + V )		sumD	CX
2 				3		8
1				2		7
1				2		6
1				1		5
0				1		4
0				1		3
0				0		2
0				0		1
0				0		0

// HL sub-band

 values Context
sumH	sumV	sumD	CX
x		2		x		8
1		1		x		7
0		1 		1		6
0		1		0		5
2		0		x		4
1		0		x		3
0		0 		2		2
0		0		1		1
0		0		0		0

// SIGN CODING ///////////////////////

H	V	X^	 CX
1	1	0	 13
1	0	0	 12
1	-1	0	 11
0	1	0	 10
0	0	0	 9
0	-1	1	 10
-1	1	1	 11
-1	0	1	 12
-1 -1	1	 13

// Magnitude Refinement //////////////////

sigm_prime [y; x]	(H + V + D)	CX
1					x			16
0 					1			15
0					0			14

*/

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE  | CLK_FILTER_NEAREST;

void KERNEL run(read_only image2d_t channel) {

	// state buffer
	LOCAL uint state[STATE_BUFFER_SIZE];
	LOCAL uint cxd[STATE_BUFFER_SIZE];

	///////////////////////////////////////////////////////////////////////////////////
	//1. Calculate MSB

	LOCAL int msbScratch[CODEBLOCKX]; // between -1 and 31 - negative one value indicates that this code block is identically zero

	int maxSigBit;
	if (getLocalId(1) == 0) {

		uint maxVal = 0;
		state[getLocalId(0)] = 0;   //top boundary
		LOCAL uint* statePtr = state + (BOUNDARY + getLocalId(0));
		int2 posIn = (int2)(getGlobalId(0),  (getGlobalId(1) >> 3)*CODEBLOCKY);

		for (uint i = 0; i < CODEBLOCKY; ++i) {
			int pixel = read_imagei(channel, sampler, posIn).x;
			uint absPixel = abs(pixel);
			maxVal = max(maxVal, absPixel);
			pixel = (absPixel << PIXEL_START_BITPOS) | SIGN((pixel << INPUT_TO_SIGN_SHIFT));
			statePtr[0] = pixel;

			posIn.y++;
			statePtr += STATE_BUFFER_STRIDE;
		}
		state[ getLocalId(0) + STATE_BOTTOM_BOUNDARY_OFFSET] = 0;		//bottom boundary

		//initialize full left and right boundary columns
		if (getLocalId(0) == 0 || getLocalId(0) == CODEBLOCKX-1) {
			int delta = -1 + ((getLocalId(0)/(CODEBLOCKX-1)) << 1); // -1 or +1
			statePtr = state + BOUNDARY + getLocalId(0) + delta;
			for (uint i = 0; i < CODEBLOCKY+ BOUNDARY_X2; ++i) {
				 *statePtr = 0;
				 statePtr += STATE_BUFFER_STRIDE;
			}
		}

		// calculate column-msb
		maxSigBit = 31 - clz(maxVal);
		msbScratch[getLocalId(0)] =maxSigBit;
	}
	localMemoryFence();

	//calculate global msb
	if (!getLocalId(0) && !getLocalId(1) ) {
		int4 mx = (int4)(maxSigBit);

		LOCAL int* scratchPtr = msbScratch;
		for(uint i=0; i < CODEBLOCKX; i+=4) {
			mx = max(mx,(int4)(msbScratch[i],msbScratch[i+1],msbScratch[i+2],msbScratch[i+3]));
		}
		maxSigBit = mx.x;
		maxSigBit = max(maxSigBit, mx.y);
		maxSigBit = max(maxSigBit, mx.z);
		maxSigBit = max(maxSigBit, mx.w);
		msbScratch[0] = maxSigBit;
	}
	localMemoryFence();

	if (msbScratch[0] == -1)
		return;

	int startIndex = (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;

	////////////////////////////////////////////////////////////////////////////////////////////////
	// 2. CUP on MSB

	uint bp = msbScratch[0] + PIXEL_START_BITPOS;

	// i) set sigma_new for stripe column (sigma_old is zero)
	LOCAL uint* statePtr = state + startIndex;
	for (uint i = 0; i < 4; ++i) {
		uint current = statePtr[0];
		current |= BIT(current);	// set sigma_new
		*statePtr = current;
		statePtr += STATE_BUFFER_STRIDE;

	}
	localMemoryFence();

	// ii) set nbh for stripe column, and do CUP

	// Note: since this is the very first pass on this code block, the only possible significant pixels
	// following the scan pattern are the those in top, left top, left and left bottom positions.

	// RLC (which runs through all consecutive non-significant pixels in column)
	// ZC/SC on rest of pixels in column

	statePtr				= state + startIndex;
	uint current			= statePtr[0];
	uint top				= statePtr[TOP];
	uint leftTop			= statePtr[LEFT_TOP];
	uint left				= statePtr[LEFT];
	uint leftBottom			= statePtr[LEFT_BOTTOM];

	// first pixel in stripe column
	uint nbh =  SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left) | SIGMA_NEW(leftBottom);
	bool doRLC = !nbh && !SIGMA_NEW(current);
	if (!doRLC) {
		if (nbh) {
			//ZC
		}
		if (SIGMA_NEW(current) ){
			//SC
		}
	}

	for (int i = 0; i < 2; ++i) {
		statePtr += STATE_BUFFER_STRIDE;
		top			= current;
		current		= statePtr[0];
		leftTop		= left;
		left		= leftBottom;
		leftBottom	= statePtr[LEFT_BOTTOM];

		nbh =  SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left) | SIGMA_NEW(leftBottom);
		bool doRLCold = doRLC;
		doRLC = doRLC && !nbh && !SIGMA_NEW(current);
		if (!doRLC) {
		    if (doRLCold) {
				//RLC
			}
			if (nbh) {
				//ZC
			}
			if (SIGMA_NEW(current)) {
				//SC
			}
		}
	}

	// last pixel in stripe column -
	// ignore leftBottom pixel, because it is in the next
	// stripe and it hasn't been processed yet
	statePtr += STATE_BUFFER_STRIDE;
	top			= current;
	current		= statePtr[0];
	leftTop		= left;
	left		= leftBottom;

	nbh =  SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left);
	bool doRLCold = doRLC;
	doRLC = doRLC && !nbh && !SIGMA_NEW(current);
	if (!doRLC) {
	   if (doRLCold) {
		  //RLC
	   }

		if (nbh) {
			//ZC
		}
		if (SIGMA_NEW(current)) {
			//SC
		}
	}
	localMemoryFence();

	/////////////////////////////////////////////////////////////////////////////////////////////////
	// 3. Processs rest of bit planes
	bp--;
	LOCAL char blockVote;

	while (bp >= PIXEL_START_BITPOS) {
		blockVote = 0;

		/////////////////////////////
		// 4. pre-process bit plane

		// i) migrate sigma_new to sigma_old, and clear sigma new bit
		statePtr = state + startIndex;
		for (uint i = 0; i < 4; ++i) {
		    uint current = *statePtr;
			current |=  SIGMA_NEW(current) << SIGMA_OLD_BITPOS;
			CLEAR_SIGMA_NEW(current);

			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;

		}
		localMemoryFence();

		// ii) update nbh
		statePtr = state + startIndex;

		uint top			= statePtr[TOP];
		uint leftTop		= statePtr[LEFT_TOP];
		uint rightTop	= statePtr[RIGHT_TOP];
		uint left		= statePtr[LEFT];
		uint current		= statePtr[0];
		uint right		= statePtr[RIGHT];
		uint leftBottom  = statePtr[LEFT_BOTTOM];
		uint bottom		= statePtr[BOTTOM];
		uint rightBottom = statePtr[RIGHT_BOTTOM];

		current |=  (   SIGMA_OLD(leftTop) |
						SIGMA_OLD( top) |
						SIGMA_OLD( rightTop) |
						SIGMA_OLD( left) |
						SIGMA_OLD( right) |
						SIGMA_OLD( leftBottom) |
						SIGMA_OLD( bottom) |
						SIGMA_OLD( rightBottom)  ) << SIGMA_OLD_TO_NBH_SHIFT;

		if ( BIT(current) && NBH(current) && !SIGMA_OLD(current) ) {
			SET_SIGMA_NEW(current);
			blockVote = 1;

		}
		*statePtr = current;

		for (uint i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;

			top			 = current;
			leftTop		 = left;
			rightTop	 = right;
			left		 = leftBottom;
			current		 = statePtr[0];
			right		 = rightBottom;
			leftBottom   = statePtr[LEFT_BOTTOM];
			bottom		 = statePtr[BOTTOM];
			rightBottom  = statePtr[RIGHT_BOTTOM];

			current |=	   ( SIGMA_OLD(leftTop) |
							 SIGMA_OLD( top) |
							 SIGMA_OLD( rightTop) |
							 SIGMA_OLD( left) |
							 SIGMA_OLD( right) |
							 SIGMA_OLD( leftBottom) |
							 SIGMA_OLD( bottom) |
							 SIGMA_OLD( rightBottom)  ) << SIGMA_OLD_TO_NBH_SHIFT;

			if ( BIT(current) && NBH(current) && !SIGMA_OLD(current) ) {
			    SET_SIGMA_NEW(current);
				blockVote = 1;
			}
			*statePtr = current;

		}
		localMemoryFence();

		// iii) block vote on sigma new
		while (blockVote)
		{
		    blockVote = 0;
			localMemoryFence();

			statePtr = state + startIndex;

			//first pixel in stripe column
			uint top			= statePtr[TOP];
			uint leftTop		= statePtr[LEFT_TOP];
			uint left			= statePtr[LEFT];
			uint current		= statePtr[0];
			uint leftBottom		= statePtr[LEFT_BOTTOM];

			current  |= (   SIGMA_NEW(leftTop) |
							SIGMA_NEW( top) |
							SIGMA_NEW( left) |
							SIGMA_NEW( leftBottom) ) << NBH_BITPOS;

			if (  BIT(current) && !SIGMA_OLD(current) &&  NBH(current)  &&  !SIGMA_NEW(current)) {
				SET_SIGMA_NEW(current);
				blockVote = 1;

			}
			statePtr[0] = current;

			// next two pixels in stripe column
			for (uint i = 0; i < 2; ++i) {
				statePtr += STATE_BUFFER_STRIDE;

				top			= current;
				current		= statePtr[0];
				leftTop		= left;
				left		= leftBottom;
				leftBottom  = statePtr[LEFT_BOTTOM];

				current  |= (   SIGMA_NEW(leftTop) |
								SIGMA_NEW( top) |
								SIGMA_NEW( left) |
								SIGMA_NEW( leftBottom) ) << NBH_BITPOS;

				if (  BIT(current) && !SIGMA_OLD(current) && NBH(current)  && !SIGMA_NEW(current)) {
					SET_SIGMA_NEW(current);
					blockVote = 1;

				}
				statePtr[0] = current;
			}

			// last pixel in stripe column -
			// ignore leftBottom pixel, because it is in the next
			// stripe and it hasn't been processed yet
			statePtr += STATE_BUFFER_STRIDE;

			top			= current;
			current		= statePtr[0];
			leftTop		= left;
			left		= leftBottom;

			current |= ( SIGMA_NEW(leftTop) |
						 SIGMA_NEW( top) |
						 SIGMA_NEW( left) ) << NBH_BITPOS;
			if (  BIT(current) && !SIGMA_OLD(current) && NBH(current)  && !SIGMA_NEW(current)) {
				SET_SIGMA_NEW(current);
				blockVote = 1;
			}
			statePtr[0] = current;

			localMemoryFence();

		}

		/////////////////////////
		// 5. bit plane processing
		statePtr = state + startIndex;

		uint current0,current1,current2,current3;

		top = statePtr[TOP];
		leftTop			= statePtr[LEFT_TOP];
		rightTop		= statePtr[RIGHT_TOP];
		left			= statePtr[LEFT];
		current			= statePtr[0];
		right			= statePtr[RIGHT];
		leftBottom		= statePtr[LEFT_BOTTOM];
		bottom			= statePtr[BOTTOM];
		rightBottom		= statePtr[RIGHT_BOTTOM];

		bool doRLC = false;
		if (!doRLC && SIGMA_OLD(current)) {
			// MRC
		} else {
			int nbh = ( SIGMA_OLD_AND_NEW(leftTop) |
						SIGMA_OLD_AND_NEW( top) |
						SIGMA_OLD_AND_NEW( rightTop) |
						SIGMA_OLD_AND_NEW( left) |
						SIGMA_OLD_AND_NEW( right) |
						SIGMA_OLD_AND_NEW( leftBottom) |
						SIGMA_OLD_AND_NEW( bottom) |
						SIGMA_OLD_AND_NEW( rightBottom)  ) ;

			doRLC = !nbh && !SIGMA_NEW(current);
		}

		for (uint i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;

			top			= current;
			leftTop		= left;
			rightTop	= right;
			left		= leftBottom;
			current		= statePtr[0];
			right		= rightBottom;
			leftBottom  = statePtr[LEFT_BOTTOM];
			bottom		= statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];

			if (!doRLC && SIGMA_OLD(current)) {
				// MRC
			} else {
				int nbh = ( SIGMA_OLD_AND_NEW(leftTop) |
							SIGMA_OLD_AND_NEW( top) |
							SIGMA_OLD_AND_NEW( rightTop) |
							SIGMA_OLD_AND_NEW( left) |
							SIGMA_OLD_AND_NEW( right) |
							SIGMA_OLD_AND_NEW( leftBottom) |
							SIGMA_OLD_AND_NEW( bottom) |
							SIGMA_OLD_AND_NEW( rightBottom)  ) ;

				doRLC = doRLC && !nbh && !SIGMA_NEW(current);
			}
		}

		localMemoryFence();
		bp--;
	}
}

/*  Copyright 2014 Aaron Boxer

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

#include "ocl_platform.cl"

/*

Bit Plane Coder
===============

A. Scan Pattern

Xx4 stripes

The stripes are scanned from top to bottom, and the columns within a
stripe are scanned from left to right.

B. Serial State Variables

Pixels are stored in Sign + Magnitude format

sigma[y][x]			equal to 1 if at least one non-zero bit has been coded, otherwise zero

sigma_prime[y][x]   equal to 1 if magnitude refinement coding (MRC) has been applied to this pixel, otherwise zero

eta[y][x]		    equal to 1 if zero coding (ZC) has been applied in SPP for this bit plane, otherwise zero

Note: eta gets reset to zero at the start of each bit plane, while sigma and sigma_prime do not.

C. Concepts

Preferred Neighbourhood: at least one of the eight neighbours has sigma[y][x] == 1

Sign Coding (SC)
Zero Coding (ZC)
Magnitude Refinement Coding (MRC)
Run Length Encoding (RLC) (applied only to first pixel in stripe column)
	- if we are at the first pixel in the column, and all four pixels in column have sigma == 0 and eta == 0,
      then apply RLC to current pixel
	- occurrence of first 1 bit in column determines how RLC is coded

C. Serial Algorithm

Three passes per bit plane : Significance Propagation Pass (SPP)
                             Magnitude Refinement Pass	 (MRP)
						     Clean Up Pass				 (CUP)

Only the CUP is performed on the most significant bit plane (MSB)

Algorithm:

i. most significant bit plane is calculated

ii.  SPP
     // not significant, and in preferred nbhd
     if in preferred nbhd and sigma[y][x] == 0 then
	     ZC
		 eta[y][x] = 1
		 if (encoded bit == 1) then
			SC
			sigma[y][x] = 1
		 end if
	end if

iii. MRP
     // significant, but ZC has not been performed in SPP
	`if (sigma[y][x] == 1 and eta[y][x] == 0 then
		MRC
		sigma_prime = 1
	end if

iv. CUP
    // not significant, and not in preferred nbhd
	if sigma[y][x] == 0 and eta[y][x] == 0 then
		if pixel meets RLC conditions then
		   RLC
		else
		   ZC
		endif
*/

#define CODEBLOCKY_QUARTER 8

#define BOUNDARY 1
#define BOUNDARY_X2 2

#define CODEBLOCKY_X4 128

//////////////////////////////////////////////////////
// State Buffer

#define STATE_BUFFER_SIZE 1156
#define STATE_BOTTOM_BOUNDARY_OFFSET 1124

#define STATE_BUFFER_STRIDE 34
#define STATE_BUFFER_STRIDE_X2 68
#define STATE_BUFFER_STRIDE_X3 102
#define STATE_BUFFER_STRIDE_X4 136

#define LEFT_TOP -35
#define TOP  -34
#define RIGHT_TOP -33

#define LEFT   -1
#define RIGHT   1

#define LEFT_BOTTOM  33
#define BOTTOM  34
#define RIGHT_BOTTOM  35

////////////////////////////////////////////////////
// State Variables

// bit positions (0 based indices)
#define INPUT_SIGN_BITPOS  15  // assumes 16 bit signed input

#define SIGMA_NEW_BITPOS   0x0
#define SIGMA_OLD_BITPOS   0x1
#define NBH_BITPOS         0x2
#define PIXEL_START_BITPOS 0xA
#define PIXEL_END_BITPOS   0x18
#define SIGN_BITPOS        0x19

// bit flags
#define SIGMA_NEW_F			 0x1		  //position 0
#define SIGMA_OLD_F			 0x10		  //position  1
#define NBH_F				 0x20		  //position  2
#define PIXEL_F				 0x7FFF0000   //positions 10-24
#define SIGN_F				 0x2000000    //position  25

#define NOT_SIGMA_NEW_F      0xFFFFFFFE   // ~SIGMA_NEW
#define SIGMA_OLD_AND_NEW_F  0x11

#define INPUT_TO_SIGN_SHIFT 10
#define SIGMA_OLD_TO_NBH_SHIFT 0x1

#define BIT(pix) (((pix)>>(bp))&1)
#define NBH(pix) ((pix) & NBH_F)
#define SIGMA_OLD(pix) ((pix) & SIGMA_OLD_F)
#define SIGMA_NEW(pix) ((pix) & SIGMA_NEW_F)
#define SIGMA_OLD_AND_NEW(pix) ((pix) & SIGMA_OLD_AND_NEW_F)
#define SIGN(pix) ((pix) & SIGN_F)

#define SET_SIGMA_NEW(pix) ( (pix) |= SIGMA_NEW_F )
#define CLEAR_SIGMA_NEW(pix) ( (pix) &= NOT_SIGMA_NEW_F )

/////////////////////////////////////
// Context Variables

/*
0-2		(CX,D) pairs counter
3		SPP flag
4		MRP flag
5		SPP flag
8		D4
9-13	CX4
14		D3
15-19	CX3
20		D2
21-25	CX2
26		D1
27-31	CX1

*/

/////////////////////////////////////////////////////

// TABLES
/*

// ZERO CODING /////////////////////////////////////

// LL and LH sub-bands
values					Context
sumH	sumV	sumD	CX
2		x		x		8
1 		1		x		7
1		0 		1		6
1		0		0		5
0		2		x		4
0		1		x		3
0		0 		2		2
0		0		1		1
0		0		0		0

//HH sub-band

values					Context
sum(H + V )		sumD	CX
2 				3		8
1				2		7
1				2		6
1				1		5
0				1		4
0				1		3
0				0		2
0				0		1
0				0		0

// HL sub-band

 values Context
sumH	sumV	sumD	CX
x		2		x		8
1		1		x		7
0		1 		1		6
0		1		0		5
2		0		x		4
1		0		x		3
0		0 		2		2
0		0		1		1
0		0		0		0

// SIGN CODING ///////////////////////

H	V	X^	 CX
1	1	0	 13
1	0	0	 12
1	-1	0	 11
0	1	0	 10
0	0	0	 9
0	-1	1	 10
-1	1	1	 11
-1	0	1	 12
-1 -1	1	 13

// Magnitude Refinement //////////////////

sigm_prime [y; x]	(H + V + D)	CX
1					x			16
0 					1			15
0					0			14

*/

CONSTANT sampler_t sampler = CLK_NORMALIZED_COORDS_FALSE  | CLK_FILTER_NEAREST;

void KERNEL run(read_only image2d_t channel) {

	// state buffer
	LOCAL uint state[STATE_BUFFER_SIZE];
	LOCAL uint cxd[STATE_BUFFER_SIZE];

	///////////////////////////////////////////////////////////////////////////////////
	//1. Calculate MSB

	LOCAL int msbScratch[CODEBLOCKX]; // between -1 and 31 - negative one value indicates that this code block is identically zero

	int maxSigBit;
	if (getLocalId(1) == 0) {

		uint maxVal = 0;
		state[getLocalId(0)] = 0;   //top boundary
		LOCAL uint* statePtr = state + (BOUNDARY + getLocalId(0));
		int2 posIn = (int2)(getGlobalId(0),  (getGlobalId(1) >> 3)*CODEBLOCKY);

		for (uint i = 0; i < CODEBLOCKY; ++i) {
			int pixel = read_imagei(channel, sampler, posIn).x;
			uint absPixel = abs(pixel);
			maxVal = max(maxVal, absPixel);
			pixel = (absPixel << PIXEL_START_BITPOS) | SIGN((pixel << INPUT_TO_SIGN_SHIFT));
			statePtr[0] = pixel;

			posIn.y++;
			statePtr += STATE_BUFFER_STRIDE;
		}
		state[ getLocalId(0) + STATE_BOTTOM_BOUNDARY_OFFSET] = 0;		//bottom boundary

		//initialize full left and right boundary columns
		if (getLocalId(0) == 0 || getLocalId(0) == CODEBLOCKX-1) {
			int delta = -1 + ((getLocalId(0)/(CODEBLOCKX-1)) << 1); // -1 or +1
			statePtr = state + BOUNDARY + getLocalId(0) + delta;
			for (uint i = 0; i < CODEBLOCKY+ BOUNDARY_X2; ++i) {
				 *statePtr = 0;
				 statePtr += STATE_BUFFER_STRIDE;
			}
		}

		// calculate column-msb
		maxSigBit = 31 - clz(maxVal);
		msbScratch[getLocalId(0)] =maxSigBit;
	}
	localMemoryFence();

	//calculate global msb
	if (!getLocalId(0) && !getLocalId(1) ) {
		int4 mx = (int4)(maxSigBit);

		LOCAL int* scratchPtr = msbScratch;
		for(uint i=0; i < CODEBLOCKX; i+=4) {
			mx = max(mx,(int4)(msbScratch[i],msbScratch[i+1],msbScratch[i+2],msbScratch[i+3]));
		}
		maxSigBit = mx.x;
		maxSigBit = max(maxSigBit, mx.y);
		maxSigBit = max(maxSigBit, mx.z);
		maxSigBit = max(maxSigBit, mx.w);
		msbScratch[0] = maxSigBit;
	}
	localMemoryFence();

	if (msbScratch[0] == -1)
		return;

	int startIndex = (BOUNDARY + getLocalId(0)) + (BOUNDARY + getLocalId(1))*STATE_BUFFER_STRIDE_X4;

	////////////////////////////////////////////////////////////////////////////////////////////////
	// 2. CUP on MSB

	uint bp = msbScratch[0] + PIXEL_START_BITPOS;

	// i) set sigma_new for stripe column (sigma_old is zero)
	LOCAL uint* statePtr = state + startIndex;
	for (uint i = 0; i < 4; ++i) {
		uint current = statePtr[0];
		current |= BIT(current);	// set sigma_new
		*statePtr = current;
		statePtr += STATE_BUFFER_STRIDE;

	}
	localMemoryFence();

	// ii) set nbh for stripe column, and do CUP

	char rlcFirstSigBit = 0;	//y coordinate of first significant bit in strip
	uint current0,current1,current2,current3;

	statePtr = state + startIndex;
	current0				= statePtr[0];
	uint top				= statePtr[TOP];
	uint leftTop			= statePtr[LEFT_TOP];
	uint left				= statePtr[LEFT];
	uint leftBottom			= statePtr[LEFT_BOTTOM];

	// first pixel in stripe column
	uint nbh =  SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left) | SIGMA_NEW(leftBottom);
	bool doRLC = !nbh && !SIGMA_NEW(current0);

	statePtr += STATE_BUFFER_STRIDE;
	top			= current0;
	current1	= statePtr[0];
	leftTop		= left;
	left		= leftBottom;
	leftBottom	= statePtr[LEFT_BOTTOM];

	nbh =  SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left) | SIGMA_NEW(leftBottom);
	doRLC = doRLC && (!nbh && !SIGMA_NEW(current1));

	statePtr += STATE_BUFFER_STRIDE;
	top			= current1;
	current2	= statePtr[0];
	leftTop		= left;
	left		= leftBottom;
	leftBottom	= statePtr[LEFT_BOTTOM];

	nbh =  SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left) | SIGMA_NEW(leftBottom);
	doRLC = doRLC && (!nbh && !SIGMA_NEW(current2));

	// last pixel in stripe column -
	// ignore leftBottom pixel, because it is in the next
	// stripe and it hasn't been processed yet
	statePtr += STATE_BUFFER_STRIDE;
	top			= current2;
	current3	= statePtr[0];
	leftTop		= left;
	left		= leftBottom;

	nbh =  SIGMA_NEW(top) | SIGMA_NEW(leftTop) | SIGMA_NEW(left);
	doRLC = doRLC && (!nbh && !SIGMA_NEW(current3));

	localMemoryFence();

	/////////////////////////////////////////////////////////////////////////////////////////////////
	// 3. Processs rest of bit planes
	bp--;
	LOCAL char blockVote;

	while (bp >= PIXEL_START_BITPOS) {
		blockVote = 0;

		/////////////////////////////
		// 4. pre-process bit plane

		// i) migrate sigma_new to sigma_old, and clear sigma new bit
		statePtr = state + startIndex;
		for (uint i = 0; i < 4; ++i) {
		    uint current = *statePtr;
			current |=  SIGMA_NEW(current) << SIGMA_OLD_BITPOS;
			CLEAR_SIGMA_NEW(current);

			*statePtr = current;
			statePtr += STATE_BUFFER_STRIDE;

		}
		localMemoryFence();

		// ii) update nbh
		statePtr = state + startIndex;

		uint top			= statePtr[TOP];
		uint leftTop		= statePtr[LEFT_TOP];
		uint rightTop	= statePtr[RIGHT_TOP];
		uint left		= statePtr[LEFT];
		uint current		= statePtr[0];
		uint right		= statePtr[RIGHT];
		uint leftBottom  = statePtr[LEFT_BOTTOM];
		uint bottom		= statePtr[BOTTOM];
		uint rightBottom = statePtr[RIGHT_BOTTOM];

		current |=  (   SIGMA_OLD(leftTop) |
						SIGMA_OLD( top) |
						SIGMA_OLD( rightTop) |
						SIGMA_OLD( left) |
						SIGMA_OLD( right) |
						SIGMA_OLD( leftBottom) |
						SIGMA_OLD( bottom) |
						SIGMA_OLD( rightBottom)  ) << SIGMA_OLD_TO_NBH_SHIFT;

		if ( BIT(current) && NBH(current) && !SIGMA_OLD(current) ) {
			SET_SIGMA_NEW(current);
			blockVote = 1;

		}
		*statePtr = current;

		for (uint i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;

			top			 = current;
			leftTop		 = left;
			rightTop	 = right;
			left		 = leftBottom;
			current		 = statePtr[0];
			right		 = rightBottom;
			leftBottom   = statePtr[LEFT_BOTTOM];
			bottom		 = statePtr[BOTTOM];
			rightBottom  = statePtr[RIGHT_BOTTOM];

			current |=	   ( SIGMA_OLD(leftTop) |
							 SIGMA_OLD( top) |
							 SIGMA_OLD( rightTop) |
							 SIGMA_OLD( left) |
							 SIGMA_OLD( right) |
							 SIGMA_OLD( leftBottom) |
							 SIGMA_OLD( bottom) |
							 SIGMA_OLD( rightBottom)  ) << SIGMA_OLD_TO_NBH_SHIFT;

			if ( BIT(current) && NBH(current) && !SIGMA_OLD(current) ) {
			    SET_SIGMA_NEW(current);
				blockVote = 1;
			}
			*statePtr = current;

		}
		localMemoryFence();

		// iii) block vote on sigma new
		while (blockVote)
		{
		    blockVote = 0;
			localMemoryFence();

			statePtr = state + startIndex;

			//first pixel in stripe column
			uint top			= statePtr[TOP];
			uint leftTop		= statePtr[LEFT_TOP];
			uint left			= statePtr[LEFT];
			uint current		= statePtr[0];
			uint leftBottom		= statePtr[LEFT_BOTTOM];

			current  |= (   SIGMA_NEW(leftTop) |
							SIGMA_NEW( top) |
							SIGMA_NEW( left) |
							SIGMA_NEW( leftBottom) ) << NBH_BITPOS;

			if (  BIT(current) && !SIGMA_OLD(current) &&  NBH(current)  &&  !SIGMA_NEW(current)) {
				SET_SIGMA_NEW(current);
				blockVote = 1;

			}
			statePtr[0] = current;

			// next two pixels in stripe column
			for (uint i = 0; i < 2; ++i) {
				statePtr += STATE_BUFFER_STRIDE;

				top			= current;
				current		= statePtr[0];
				leftTop		= left;
				left		= leftBottom;
				leftBottom  = statePtr[LEFT_BOTTOM];

				current  |= (   SIGMA_NEW(leftTop) |
								SIGMA_NEW( top) |
								SIGMA_NEW( left) |
								SIGMA_NEW( leftBottom) ) << NBH_BITPOS;

				if (  BIT(current) && !SIGMA_OLD(current) && NBH(current)  && !SIGMA_NEW(current)) {
					SET_SIGMA_NEW(current);
					blockVote = 1;

				}
				statePtr[0] = current;
			}

			// last pixel in stripe column -
			// ignore leftBottom pixel, because it is in the next
			// stripe and it hasn't been processed yet
			statePtr += STATE_BUFFER_STRIDE;

			top			= current;
			current		= statePtr[0];
			leftTop		= left;
			left		= leftBottom;

			current |= ( SIGMA_NEW(leftTop) |
						 SIGMA_NEW( top) |
						 SIGMA_NEW( left) ) << NBH_BITPOS;
			if (  BIT(current) && !SIGMA_OLD(current) && NBH(current)  && !SIGMA_NEW(current)) {
				SET_SIGMA_NEW(current);
				blockVote = 1;
			}
			statePtr[0] = current;

			localMemoryFence();

		}

		/////////////////////////
		// 5. bit plane processing
		statePtr = state + startIndex;

		top = statePtr[TOP];
		leftTop			= statePtr[LEFT_TOP];
		rightTop		= statePtr[RIGHT_TOP];
		left			= statePtr[LEFT];
		current			= statePtr[0];
		right			= statePtr[RIGHT];
		leftBottom		= statePtr[LEFT_BOTTOM];
		bottom			= statePtr[BOTTOM];
		rightBottom		= statePtr[RIGHT_BOTTOM];

		doRLC = false;
		if (!doRLC && SIGMA_OLD(current)) {
			// MRC
		} else {
			int nbh = ( SIGMA_OLD_AND_NEW(leftTop) |
						SIGMA_OLD_AND_NEW( top) |
						SIGMA_OLD_AND_NEW( rightTop) |
						SIGMA_OLD_AND_NEW( left) |
						SIGMA_OLD_AND_NEW( right) |
						SIGMA_OLD_AND_NEW( leftBottom) |
						SIGMA_OLD_AND_NEW( bottom) |
						SIGMA_OLD_AND_NEW( rightBottom)  ) ;

			doRLC = !nbh && !SIGMA_NEW(current);
		}

		for (uint i = 0; i < 3; ++i) {
			statePtr += STATE_BUFFER_STRIDE;

			top			= current;
			leftTop		= left;
			rightTop	= right;
			left		= leftBottom;
			current		= statePtr[0];
			right		= rightBottom;
			leftBottom  = statePtr[LEFT_BOTTOM];
			bottom		= statePtr[BOTTOM];
			rightBottom = statePtr[RIGHT_BOTTOM];

			if (!doRLC && SIGMA_OLD(current)) {
				// MRC
			} else {
				int nbh = ( SIGMA_OLD_AND_NEW(leftTop) |
							SIGMA_OLD_AND_NEW( top) |
							SIGMA_OLD_AND_NEW( rightTop) |
							SIGMA_OLD_AND_NEW( left) |
							SIGMA_OLD_AND_NEW( right) |
							SIGMA_OLD_AND_NEW( leftBottom) |
							SIGMA_OLD_AND_NEW( bottom) |
							SIGMA_OLD_AND_NEW( rightBottom)  ) ;

				doRLC = doRLC && !nbh && !SIGMA_NEW(current);
			}
		}

		localMemoryFence();
		bp--;
	}
}


Use the correct y offset

Use the correct y offset