llama.cpp/ggml/src/ggml-opencl/kernels/ggml-opencl_mm.cl

//------------------------------------------------------------------------------
// This file is contains additional mulmat kernels
// (and potentially other kernels).
//------------------------------------------------------------------------------
#ifdef cl_khr_fp16
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#elif defined(cl_amd_fp16)
#pragma OPENCL EXTENSION cl_amd_fp16 : enable
#else
#error "Half precision floating point not supportedby OpenCL implementation on your device."
#endif

#ifdef cl_khr_subgroups
#pragma OPENCL EXTENSION cl_khr_subgroups : enable
#elif defined(cl_intel_subgroups)
#pragma OPENCL EXTENSION cl_intel_subgroups : enable
#else
#error "Subgroup not supported on your device."
#endif

#ifdef cl_intel_required_subgroup_size
// Always use subgroup size of 32 on Intel.
#pragma OPENCL EXTENSION cl_intel_required_subgroup_size : enable
#define INTEL_GPU 1
#define REQD_SUBGROUP_SIZE_16 __attribute__((intel_reqd_sub_group_size(16)))
#define REQD_SUBGROUP_SIZE_32 __attribute__((intel_reqd_sub_group_size(32)))
#elif defined(cl_qcom_reqd_sub_group_size)
// Always use subgroups size of 64 on Adreno.
#pragma OPENCL EXTENSION cl_qcom_reqd_sub_group_size : enable
#define ADRENO_GPU 1
#define REQD_SUBGROUP_SIZE_64  __attribute__((qcom_reqd_sub_group_size("half")))
#define REQD_SUBGROUP_SIZE_128 __attribute__((qcom_reqd_sub_group_size("full")))
#else
// TODO: do not know how to choose subgroup size on other GPUs.
#error "Selecting subgroup size is not supported on your device."
#endif

#define QK4_0                   32
#define QR4_0                   2
#define QK4_1                   32
#define QR4_1                   2
#define QK5_0                   32
#define QR5_0                   2
#define QK5_1                   32
#define QR5_1                   2
#define QK8_0                   32
#define QR8_0                   1
#define QK_K                    256
#define K_QUANTS_PER_ITERATION  2

typedef char int8_t;
typedef uchar uint8_t;
typedef short int16_t;
typedef ushort uint16_t;
typedef int int32_t;
typedef uint uint32_t;

//------------------------------------------------------------------------------
// block_q4_0
//------------------------------------------------------------------------------
struct block_q4_0
{
    half d;
    uint8_t qs[QK4_0 / 2];
};

//------------------------------------------------------------------------------
// block_q6_K
//------------------------------------------------------------------------------
// 6-bit quantization
// weight is represented as x = a * q
// 16 blocks of 16 elements each
// Effectively 6.5625 bits per weight
typedef struct {
    uint8_t ql[QK_K/2];      // quants, lower 4 bits
    uint8_t qh[QK_K/4];      // quants, upper 2 bits
    int8_t  scales[QK_K/16]; // scales, quantized with 8 bits
    half d;             // super-block scale
} block_q6_K;

//------------------------------------------------------------------------------
// These are the variant for matmatmul, based on the matvecmul kernel with
// flattened block_q4_0.
//------------------------------------------------------------------------------

// Common dot prod.
inline float mm_block_q_4_0_dot_y_flat(
        global uchar * x,
        global half  * dh,
        float sumy,
        float16 yl,
        int il
) {
    float           d   = *dh;
    global ushort * qs  = ((global ushort *)x + il/2);
    float           acc = 0.f;

    acc += yl.s0 * (qs[0] & 0x000F);
    acc += yl.s1 * (qs[0] & 0x0F00);
    acc += yl.s8 * (qs[0] & 0x00F0);
    acc += yl.s9 * (qs[0] & 0xF000);

    acc += yl.s2 * (qs[1] & 0x000F);
    acc += yl.s3 * (qs[1] & 0x0F00);
    acc += yl.sa * (qs[1] & 0x00F0);
    acc += yl.sb * (qs[1] & 0xF000);

    acc += yl.s4 * (qs[2] & 0x000F);
    acc += yl.s5 * (qs[2] & 0x0F00);
    acc += yl.sc * (qs[2] & 0x00F0);
    acc += yl.sd * (qs[2] & 0xF000);

    acc += yl.s6 * (qs[3] & 0x000F);
    acc += yl.s7 * (qs[3] & 0x0F00);
    acc += yl.se * (qs[3] & 0x00F0);
    acc += yl.sf * (qs[3] & 0xF000);

    return d * (sumy * -8.f + acc);
}

#undef N_DST
#undef N_SIMDGROUP
#undef N_SIMDWIDTH

#ifdef INTEL_GPU
#define N_DST 8 // each SIMD group works on 8 rows (in weights matrix)
#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
#define N_SIMDWIDTH 16 // assuming SIMD group size is 16
#elif defined (ADRENO_GPU)
#define N_DST 8
#define N_SIMDGROUP 1
#define N_SIMDWIDTH 64
#endif
//
// This variant performs 1d blocking with 8x output.
// Eeach simdgroup outputs 8 values on `n0` dim (row in the output matrix).
//
inline void mul_mat_q_n_f32_1d_8x_flat(
        global uchar * src0_q,
        global half  * src0_d,
        global float * src1,
        global float * dst,
        int ne00,
        int ne01,
        int ne02,
        int ne10,
        int ne12,
        int ne0,
        int ne1,
        int r2,
        int r3
) {
    const int nb = ne00/QK4_0;

    int r0 = get_group_id(0);
    int r1 = get_group_id(1);
    int im = get_group_id(2);

    // (r0 * N_SIMDGROUP + get_sub_group_id()) is the linear global id of
    // a SIMD group in the grid. Each SIMD group produces N_DST values in the
    // result, hence uses nb blocks, i.e., the offset becomes first_row*nb.
    // Currently with llama2 7B, im is always 0.
    // TODO: how to handle im/gqa*(nb*ne0)?
    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;

    int i12 = im%ne12;
    int i13 = im/ne12;

    // The number of scales is the same as the number of blocks.
    ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
    // Each block contains QK4_0/2 uchars, hence offset for qs is as follows.
    ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_0/2;

    global uchar * x = (global uchar *) src0_q + offset0_q;
    global half  * d = (global half  *) src0_d + offset0_d;
    global float * y = (global float *) src1   + r1*ne10 + im*ne00*ne1;

    float16 yl;
    float8 sumf = (float8)(0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f);

    int ix = get_sub_group_local_id()/2;
    int il = 8*(get_sub_group_local_id()%2);

    global float * yb = y + ix*QK4_0 + il;

    for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
        float sumy = 0.f;

        sumy += yb[0];
        sumy += yb[1];
        sumy += yb[2];
        sumy += yb[3];
        sumy += yb[4];
        sumy += yb[5];
        sumy += yb[6];
        sumy += yb[7];

        sumy += yb[16];
        sumy += yb[17];
        sumy += yb[18];
        sumy += yb[19];
        sumy += yb[20];
        sumy += yb[21];
        sumy += yb[22];
        sumy += yb[23];

        yl.s0 = yb[0];
        yl.s1 = yb[1]/256.f;

        yl.s2 = yb[2];
        yl.s3 = yb[3]/256.f;

        yl.s4 = yb[4];
        yl.s5 = yb[5]/256.f;

        yl.s6 = yb[6];
        yl.s7 = yb[7]/256.f;

        yl.s8 = yb[16]/16.f;
        yl.s9 = yb[17]/4096.f;

        yl.sa = yb[18]/16.f;
        yl.sb = yb[19]/4096.f;

        yl.sc = yb[20]/16.f;
        yl.sd = yb[21]/4096.f;

        yl.se = yb[22]/16.f;
        yl.sf = yb[23]/4096.f;

        sumf.s0 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 0*nb*QK4_0/2, d + ib + 0*nb, sumy, yl, il);
        sumf.s1 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 1*nb*QK4_0/2, d + ib + 1*nb, sumy, yl, il);
        sumf.s2 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 2*nb*QK4_0/2, d + ib + 2*nb, sumy, yl, il);
        sumf.s3 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 3*nb*QK4_0/2, d + ib + 3*nb, sumy, yl, il);

        sumf.s4 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 4*nb*QK4_0/2, d + ib + 4*nb, sumy, yl, il);
        sumf.s5 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 5*nb*QK4_0/2, d + ib + 5*nb, sumy, yl, il);
        sumf.s6 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 6*nb*QK4_0/2, d + ib + 6*nb, sumy, yl, il);
        sumf.s7 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 7*nb*QK4_0/2, d + ib + 7*nb, sumy, yl, il);

        yb += QK4_0 * (N_SIMDWIDTH/2);
    }

    float8 tot = (float8)(
        sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
        sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3),
        sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5),
        sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7)
    );

    if (get_sub_group_local_id() == 0) {
        if (first_row + 0 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
        }
        if (first_row + 1 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
        }
        if (first_row + 2 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
        }
        if (first_row + 3 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
        }

        if (first_row + 4 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4;
        }
        if (first_row + 5 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5;
        }
        if (first_row + 6 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6;
        }
        if (first_row + 7 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7;
        }
    }
}

#ifdef INTEL_GPU
REQD_SUBGROUP_SIZE_16
#elif defined (ADRENO_GPU)
REQD_SUBGROUP_SIZE_64
#endif
kernel void kernel_mul_mat_q4_0_f32_1d_8x_flat(
        global uchar * src0_q,
        global half  * src0_d,
        global float * src1,
        ulong offset1,
        global float * dst,
        ulong offsetd,
        int ne00,
        int ne01,
        int ne02,
        int ne10,
        int ne12,
        int ne0,
        int ne1,
        int r2,
        int r3
) {
    src1 = (global float*)((global char*)src1 + offset1);
    dst = (global float*)((global char*)dst + offsetd);

    mul_mat_q_n_f32_1d_8x_flat(src0_q, src0_d, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
}

#undef N_DST
#undef N_SIMDGROUP
#undef N_SIMDWIDTH

#ifdef INTEL_GPU
#define N_DST 16 // each SIMD group works on 8 rows (in weights matrix)
#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
#define N_SIMDWIDTH 16 // assuming SIMD group size is 16
#elif defined (ADRENO_GPU)
#define N_DST 16
#define N_SIMDGROUP 1
#define N_SIMDWIDTH 64
#endif
//
// This variant performs 1d blocking with 16x output.
// Eeach simdgroup outputs 16 values on `n0` dim (row in the output matrix).
//
inline void mul_mat_q_n_f32_1d_16x_flat(
        global uchar * src0_q,
        global half  * src0_d,
        global float * src1,
        global float * dst,
        int ne00,
        int ne01,
        int ne02,
        int ne10,
        int ne12,
        int ne0,
        int ne1,
        int r2,
        int r3
) {
    const int nb = ne00/QK4_0;

    int r0 = get_group_id(0);
    int r1 = get_group_id(1);
    int im = get_group_id(2);

    // (r0 * N_SIMDGROUP + get_sub_group_id()) is the linear global id of
    // a SIMD group in the grid. Each SIMD group produces N_DST values in the
    // result, hence uses nb blocks, i.e., the offset becomes first_row*nb.
    // Currently with llama2 7B, im is always 0.
    // TODO: how to handle im/gqa*(nb*ne0)?
    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;

    int i12 = im%ne12;
    int i13 = im/ne12;

    // The number of scales is the same as the number of blocks.
    ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
    // Each block contains QK4_0/2 uchars, hence offset for qs is as follows.
    ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_0/2;

    global uchar * x = (global uchar *) src0_q + offset0_q;
    global half  * d = (global half  *) src0_d + offset0_d;
    global float * y = (global float *) src1   + r1*ne10 + im*ne00*ne1;

    float16 yl;
    float16 sumf = (float16)(0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f,
                             0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f);

    int ix = get_sub_group_local_id()/2;
    int il = 8*(get_sub_group_local_id()%2);

    global float * yb = y + ix*QK4_0 + il;

    for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/2) {
        float sumy = 0.f;

        sumy += yb[0];
        sumy += yb[1];
        sumy += yb[2];
        sumy += yb[3];
        sumy += yb[4];
        sumy += yb[5];
        sumy += yb[6];
        sumy += yb[7];

        sumy += yb[16];
        sumy += yb[17];
        sumy += yb[18];
        sumy += yb[19];
        sumy += yb[20];
        sumy += yb[21];
        sumy += yb[22];
        sumy += yb[23];

        yl.s0 = yb[0];
        yl.s1 = yb[1]/256.f;

        yl.s2 = yb[2];
        yl.s3 = yb[3]/256.f;

        yl.s4 = yb[4];
        yl.s5 = yb[5]/256.f;

        yl.s6 = yb[6];
        yl.s7 = yb[7]/256.f;

        yl.s8 = yb[16]/16.f;
        yl.s9 = yb[17]/4096.f;

        yl.sa = yb[18]/16.f;
        yl.sb = yb[19]/4096.f;

        yl.sc = yb[20]/16.f;
        yl.sd = yb[21]/4096.f;

        yl.se = yb[22]/16.f;
        yl.sf = yb[23]/4096.f;

        sumf.s0 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 +  0*nb*QK4_0/2, d + ib +  0*nb, sumy, yl, il);
        sumf.s1 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 +  1*nb*QK4_0/2, d + ib +  1*nb, sumy, yl, il);
        sumf.s2 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 +  2*nb*QK4_0/2, d + ib +  2*nb, sumy, yl, il);
        sumf.s3 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 +  3*nb*QK4_0/2, d + ib +  3*nb, sumy, yl, il);

        sumf.s4 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 +  4*nb*QK4_0/2, d + ib +  4*nb, sumy, yl, il);
        sumf.s5 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 +  5*nb*QK4_0/2, d + ib +  5*nb, sumy, yl, il);
        sumf.s6 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 +  6*nb*QK4_0/2, d + ib +  6*nb, sumy, yl, il);
        sumf.s7 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 +  7*nb*QK4_0/2, d + ib +  7*nb, sumy, yl, il);

        sumf.s8 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 +  8*nb*QK4_0/2, d + ib +  8*nb, sumy, yl, il);
        sumf.s9 += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 +  9*nb*QK4_0/2, d + ib +  9*nb, sumy, yl, il);
        sumf.sa += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 10*nb*QK4_0/2, d + ib + 10*nb, sumy, yl, il);
        sumf.sb += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 11*nb*QK4_0/2, d + ib + 11*nb, sumy, yl, il);

        sumf.sc += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 12*nb*QK4_0/2, d + ib + 12*nb, sumy, yl, il);
        sumf.sd += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 13*nb*QK4_0/2, d + ib + 13*nb, sumy, yl, il);
        sumf.se += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 14*nb*QK4_0/2, d + ib + 14*nb, sumy, yl, il);
        sumf.sf += mm_block_q_4_0_dot_y_flat(x + ib*QK4_0/2 + 15*nb*QK4_0/2, d + ib + 15*nb, sumy, yl, il);

        yb += QK4_0 * (N_SIMDWIDTH/2);
    }

    float16 tot = (float16)(
        sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1),
        sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3),
        sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5),
        sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7),

        sub_group_reduce_add(sumf.s8), sub_group_reduce_add(sumf.s9),
        sub_group_reduce_add(sumf.sa), sub_group_reduce_add(sumf.sb),
        sub_group_reduce_add(sumf.sc), sub_group_reduce_add(sumf.sd),
        sub_group_reduce_add(sumf.se), sub_group_reduce_add(sumf.sf)
    );

    if (get_sub_group_local_id() == 0) {
        if (first_row + 0 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
        }
        if (first_row + 1 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
        }
        if (first_row + 2 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
        }
        if (first_row + 3 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
        }

        if (first_row + 4 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4;
        }
        if (first_row + 5 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5;
        }
        if (first_row + 6 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6;
        }
        if (first_row + 7 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7;
        }

        if (first_row + 8 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 8] = tot.s8;
        }
        if (first_row + 9 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 9] = tot.s9;
        }
        if (first_row + 10 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 10] = tot.sa;
        }
        if (first_row + 11 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 11] = tot.sb;
        }

        if (first_row + 12 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 12] = tot.sc;
        }
        if (first_row + 13 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 13] = tot.sd;
        }
        if (first_row + 14 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 14] = tot.se;
        }
        if (first_row + 15 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 15] = tot.sf;
        }
    }
}

#ifdef INTEL_GPU
REQD_SUBGROUP_SIZE_16
#elif defined (ADRENO_GPU)
REQD_SUBGROUP_SIZE_64
#endif
kernel void kernel_mul_mat_q4_0_f32_1d_16x_flat(
        global uchar * src0_q,
        global half  * src0_d,
        global float * src1,
        ulong offset1,
        global float * dst,
        ulong offsetd,
        int ne00,
        int ne01,
        int ne02,
        int ne10,
        int ne12,
        int ne0,
        int ne1,
        int r2,
        int r3
) {
    src1 = (global float*)((global char*)src1 + offset1);
    dst = (global float*)((global char*)dst + offsetd);

    mul_mat_q_n_f32_1d_16x_flat(src0_q, src0_d, src1, dst, ne00, ne01, ne02, ne10, ne12, ne0, ne1, r2, r3);
}

//------------------------------------------------------------------------------
// kernel_mul_mat_q4_0_f32_flat_v0
//------------------------------------------------------------------------------
inline float block_q_4_0_dot_y_flat_v2(
    half   x,
    half   d,
    float  sumy,
    float4 yl
) {
    uchar2 q = as_uchar2(x);
    float acc = 0.0f;

    acc += (q.s0 & 0x0F) * yl.s0;
    acc += (q.s1 & 0x0F) * yl.s1;

    acc += (q.s0 & 0xF0) * yl.s2;
    acc += (q.s1 & 0xF0) * yl.s3;

    return d * (sumy * -8.f + acc);;
}

inline float block_q_4_0_dot_y_flat_v4(
    float  x,
    half   d,
    float  sumy,
    float8 yl
) {
    uchar4 q = as_uchar4(x);
    float acc = 0.0f;

    acc += (q.s0 & 0x0F) * yl.s0;
    acc += (q.s1 & 0x0F) * yl.s1;
    acc += (q.s2 & 0x0F) * yl.s2;
    acc += (q.s3 & 0x0F) * yl.s3;

    acc += (q.s0 & 0xF0) * yl.s4;
    acc += (q.s1 & 0xF0) * yl.s5;
    acc += (q.s2 & 0xF0) * yl.s6;
    acc += (q.s3 & 0xF0) * yl.s7;

    return d * (sumy * -8.f + acc);;
}

inline float block_q_4_0_dot_y_flat_v8(
    float2  x,
    half    d,
    float   sumy,
    float16 yl
) {
    uchar8 q = as_uchar8(x);
    float acc = 0.0f;

    acc += (q.s0 & 0x0F) * yl.s0;
    acc += (q.s1 & 0x0F) * yl.s1;
    acc += (q.s2 & 0x0F) * yl.s2;
    acc += (q.s3 & 0x0F) * yl.s3;
    acc += (q.s4 & 0x0F) * yl.s4;
    acc += (q.s5 & 0x0F) * yl.s5;
    acc += (q.s6 & 0x0F) * yl.s6;
    acc += (q.s7 & 0x0F) * yl.s7;

    acc += (q.s0 & 0xF0) * yl.s8;
    acc += (q.s1 & 0xF0) * yl.s9;
    acc += (q.s2 & 0xF0) * yl.sa;
    acc += (q.s3 & 0xF0) * yl.sb;
    acc += (q.s4 & 0xF0) * yl.sc;
    acc += (q.s5 & 0xF0) * yl.sd;
    acc += (q.s6 & 0xF0) * yl.se;
    acc += (q.s7 & 0xF0) * yl.sf;

    return d * (sumy * -8.f + acc);;
}

#undef N_DST
#undef N_SIMDGROUP
#undef N_SIMDWIDTH

#ifdef INTEL_GPU
#define THREADS_PER_BLK 4   // Number of threads per block, or each thread process 1/THREADS_PER_BLK of a block
#define N_DST           4
#define N_SIMDGROUP     1
#define N_SIMDWIDTH     16
#elif defined (ADRENO_GPU)
#define THREADS_PER_BLK 4
#define N_DST           4
#define N_SIMDGROUP     1
#define N_SIMDWIDTH     64
#endif

#if THREADS_PER_BLK == 2                // Each thread processes 1/2 block
#   define ACT_TY                       float16
#   define Q_BLK_LD_TY                  float2
#   define block_q_4_0_dot_y_flat       block_q_4_0_dot_y_flat_v8
#elif THREADS_PER_BLK == 4              // Each thread processes 1/4 block
#   define ACT_TY                       float8
#   define Q_BLK_LD_TY                  float
#   define block_q_4_0_dot_y_flat       block_q_4_0_dot_y_flat_v4
#elif THREADS_PER_BLK == 8              // Each thread processes 1/8 block
#   define ACT_TY                       float4
#   define Q_BLK_LD_TY                  half
#   define block_q_4_0_dot_y_flat       block_q_4_0_dot_y_flat_v2
#endif

#define BTYES_PER_THREAD_IN_BLK         (QK4_0/2/THREADS_PER_BLK)

#if N_DST == 2
#   define  SUM_TY                      float2
#elif N_DST == 4
#   define  SUM_TY                      float4
#elif N_DST == 8
#   define  SUM_TY                      float8
#elif N_DST == 16
#   define  SUM_TY                      float16
#endif

#ifdef INTEL_GPU
REQD_SUBGROUP_SIZE_16
#elif defined (ADRENO_GPU)
REQD_SUBGROUP_SIZE_64
#endif
kernel void kernel_mul_mat_q4_0_f32_flat_v0(
        global uchar * src0_q,
        global half  * src0_d,
        global float * src1,
        ulong offset1,
        global float * dst,
        ulong offsetd,
        int ne00,
        int ne01,
        int ne02,
        int ne10,
        int ne12,
        int ne0,
        int ne1,
        int r2,
        int r3
) {
    src1 = (global float*)((global char*)src1 + offset1);
    dst = (global float*)((global char*)dst + offsetd);

    const int nb = ne00/QK4_0;

    int r0 = get_group_id(0);
    int r1 = get_group_id(1);
    int im = get_group_id(2);

    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;

    int i12 = im%ne12;
    int i13 = im/ne12;

    // The number of scales is the same as the number of blocks.
    ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
    // Each block contains QK4_0/2 uchars, hence offset for qs is as follows.
    ulong offset0_q = (first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02)) * QK4_0/2;

    global uchar * x = (global uchar *) src0_q + offset0_q;
    global half  * d = (global half  *) src0_d + offset0_d;
    global float * y = (global float *) src1   + r1*ne10 + im*ne00*ne1;

    int ix = get_sub_group_local_id()/THREADS_PER_BLK;
    int il = get_sub_group_local_id()%THREADS_PER_BLK;

    global float * yb = y + ix*QK4_0 + BTYES_PER_THREAD_IN_BLK*il;

    // Registers for caching activation
    ACT_TY yl = 0.f;

    // Registers for caching quants
    Q_BLK_LD_TY q_blk_0 = 0, q_blk_1 = 0;
#if N_DST == 4 || N_DST == 8 || N_DST == 16
    Q_BLK_LD_TY q_blk_2 = 0, q_blk_3 = 0;
#endif
#if N_DST == 8 || N_DST == 16
    Q_BLK_LD_TY q_blk_4 = 0, q_blk_5 = 0, q_blk_6 = 0, q_blk_7 = 0;
#endif

    // Partial sum
    SUM_TY sumf = 0.f;

    for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/THREADS_PER_BLK) {
        float sumy = 0.f;

        q_blk_0 = *(global Q_BLK_LD_TY*)(x + ib*QK4_0/2 + BTYES_PER_THREAD_IN_BLK*il + 0*nb*QK4_0/2);
        q_blk_1 = *(global Q_BLK_LD_TY*)(x + ib*QK4_0/2 + BTYES_PER_THREAD_IN_BLK*il + 1*nb*QK4_0/2);
#if N_DST == 4 || N_DST == 8 || N_DST == 16
        q_blk_2 = *(global Q_BLK_LD_TY*)(x + ib*QK4_0/2 + BTYES_PER_THREAD_IN_BLK*il + 2*nb*QK4_0/2);
        q_blk_3 = *(global Q_BLK_LD_TY*)(x + ib*QK4_0/2 + BTYES_PER_THREAD_IN_BLK*il + 3*nb*QK4_0/2);
#endif
#if N_DST == 8 || N_DST == 16
        q_blk_4 = (*(global Q_BLK_LD_TY*)(x + ib*QK4_0/2 + BTYES_PER_THREAD_IN_BLK*il + 4*nb*QK4_0/2));
        q_blk_5 = (*(global Q_BLK_LD_TY*)(x + ib*QK4_0/2 + BTYES_PER_THREAD_IN_BLK*il + 5*nb*QK4_0/2));
        q_blk_6 = (*(global Q_BLK_LD_TY*)(x + ib*QK4_0/2 + BTYES_PER_THREAD_IN_BLK*il + 6*nb*QK4_0/2));
        q_blk_7 = (*(global Q_BLK_LD_TY*)(x + ib*QK4_0/2 + BTYES_PER_THREAD_IN_BLK*il + 7*nb*QK4_0/2));
#endif

        // Load activation
#if THREADS_PER_BLK == 2    // Each thread processes 1/2 block
        yl.s01234567 = *(global float8 *)(yb);
        yl.s89abcdef = *(global float8 *)(yb + 16);

        sumy += yl.s0;
        sumy += yl.s1;
        sumy += yl.s2;
        sumy += yl.s3;
        sumy += yl.s4;
        sumy += yl.s5;
        sumy += yl.s6;
        sumy += yl.s7;
        sumy += yl.s8; yl.s8 /= 16.f;
        sumy += yl.s9; yl.s9 /= 16.f;
        sumy += yl.sa; yl.sa /= 16.f;
        sumy += yl.sb; yl.sb /= 16.f;
        sumy += yl.sc; yl.sc /= 16.f;
        sumy += yl.sd; yl.sd /= 16.f;
        sumy += yl.se; yl.se /= 16.f;
        sumy += yl.sf; yl.sf /= 16.f;
#elif THREADS_PER_BLK == 4  // Each thread processes 1/4 block
        yl.s0123 = *(global float4 *)(yb);
        yl.s4567 = *(global float4 *)(yb + 16);

        sumy += yl.s0;
        sumy += yl.s1;
        sumy += yl.s2;
        sumy += yl.s3;
        sumy += yl.s4; yl.s4 /= 16.f;
        sumy += yl.s5; yl.s5 /= 16.f;
        sumy += yl.s6; yl.s6 /= 16.f;
        sumy += yl.s7; yl.s7 /= 16.f;
#elif THREADS_PER_BLK == 8  // Each thread processes 1/8 block
        yl.s01 = *(global float2 *)(yb);
        yl.s23 = *(global float2 *)(yb + 16);

        sumy += yl.s0;
        sumy += yl.s1;
        sumy += yl.s2; yl.s2 /= 16.f;
        sumy += yl.s3; yl.s3 /= 16.f;
#endif

        sumf.s0 += block_q_4_0_dot_y_flat(q_blk_0, *(d + ib + 0*nb), sumy, yl);
        sumf.s1 += block_q_4_0_dot_y_flat(q_blk_1, *(d + ib + 1*nb), sumy, yl);
#if N_DST == 4 || N_DST == 8 || N_DST == 16
        sumf.s2 += block_q_4_0_dot_y_flat(q_blk_2, *(d + ib + 2*nb), sumy, yl);
        sumf.s3 += block_q_4_0_dot_y_flat(q_blk_3, *(d + ib + 3*nb), sumy, yl);
#endif
#if N_DST == 8 || N_DST == 16
        sumf.s4 += block_q_4_0_dot_y_flat(q_blk_4, *(d + ib + 4*nb), sumy, yl);
        sumf.s5 += block_q_4_0_dot_y_flat(q_blk_5, *(d + ib + 5*nb), sumy, yl);
        sumf.s6 += block_q_4_0_dot_y_flat(q_blk_6, *(d + ib + 6*nb), sumy, yl);
        sumf.s7 += block_q_4_0_dot_y_flat(q_blk_7, *(d + ib + 7*nb), sumy, yl);
#endif

        yb += QK4_0 * (N_SIMDWIDTH/THREADS_PER_BLK);
    }

    SUM_TY tot = (SUM_TY)(
          sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1)
#if N_DST == 4 || N_DST == 8 || N_DST == 16
        , sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3)
#endif
#if N_DST == 8 || N_DST == 16
        , sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5)
        , sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7)
#endif
    );

    if (get_sub_group_local_id() == 0) {
        if (first_row + 0 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
        }
        if (first_row + 1 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
        }
#if N_DST == 4 || N_DST == 8 || N_DST == 16
        if (first_row + 2 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
        }
        if (first_row + 3 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
        }
#endif
#if N_DST == 8 || N_DST == 16
        if (first_row + 4 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4;
        }
        if (first_row + 5 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5;
        }
        if (first_row + 6 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6;
        }
        if (first_row + 7 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7;
        }
#endif
    }
}

//------------------------------------------------------------------------------
// Using image1d_buffer_t

#if defined(cl_qcom_subgroup_shuffle)
#pragma OPENCL EXTENSION cl_qcom_subgroup_shuffle : enable
float qcom_sub_group_reduce_add(float sum) {
    sum += qcom_sub_group_shuffle_down(sum, 32, CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.f);
    sum += qcom_sub_group_shuffle_down(sum, 16, CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.f);
    sum += qcom_sub_group_shuffle_down(sum,  8, CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.f);
    sum += qcom_sub_group_shuffle_down(sum,  4, CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.f);
    sum += qcom_sub_group_shuffle_down(sum,  2, CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.f);
    sum += qcom_sub_group_shuffle_down(sum,  1, CLK_SUB_GROUP_SHUFFLE_WIDTH_WAVE_SIZE_QCOM, 0.f);
    return sum;
}
#define sub_group_reduce_add qcom_sub_group_reduce_add
#else
#define sub_group_reduce_add sub_group_reduce_add
#endif

#undef THREADS_PER_BLK
#undef N_DST
#undef N_SIMDGROUP
#undef N_SIMDWIDTH

#ifdef INTEL_GPU
#define THREADS_PER_BLK 4   // Number of threads per block, or each thread process 1/THREADS_PER_BLK of a block
#define N_DST           4
#define N_SIMDGROUP     1
#define N_SIMDWIDTH     16
#elif defined (ADRENO_GPU)
#define THREADS_PER_BLK 4
#define N_DST           4
#define N_SIMDGROUP     1
#define N_SIMDWIDTH     64
#endif

#if THREADS_PER_BLK == 2                // Each thread processes 1/2 block
#   define ACT_TY                       float16
#   define Q_BLK_LD_TY                  float2
#   define EXTRACT_BLK_DATA(tmp, part)  *((float2*)&tmp + part)
#   define block_q_4_0_dot_y_flat       block_q_4_0_dot_y_flat_v8
#elif THREADS_PER_BLK == 4              // Each thread processes 1/4 block
#   define ACT_TY                       float8
#   define Q_BLK_LD_TY                  float
#   define EXTRACT_BLK_DATA(tmp, part)  *((float*)&tmp + part)
#   define block_q_4_0_dot_y_flat       block_q_4_0_dot_y_flat_v4
#elif THREADS_PER_BLK == 8              // Each thread processes 1/8 block
#   define ACT_TY                       float4
#   define Q_BLK_LD_TY                  half
#   define EXTRACT_BLK_DATA(tmp, part)  *((half*)&tmp + part)
#   define block_q_4_0_dot_y_flat       block_q_4_0_dot_y_flat_v2
#endif

#define BTYES_PER_THREAD_IN_BLK         (QK4_0/2/THREADS_PER_BLK)

#if N_DST == 2
#   define  SUM_TY                      float2
#elif N_DST == 4
#   define  SUM_TY                      float4
#elif N_DST == 8
#   define  SUM_TY                      float8
#elif N_DST == 16
#   define  SUM_TY                      float16
#endif

#ifdef INTEL_GPU
REQD_SUBGROUP_SIZE_16
#elif defined (ADRENO_GPU)
REQD_SUBGROUP_SIZE_64
#endif
kernel void kernel_mul_mat_q4_0_f32_flat_img_v0(
        read_only image1d_buffer_t src0_q,
        read_only image1d_buffer_t src0_d,
        global float * src1,
        ulong offset1,
        global float * dst,
        ulong offsetd,
        int ne00,
        int ne01,
        int ne02,
        int ne10,
        int ne12,
        int ne0,
        int ne1,
        int r2,
        int r3
) {
    src1 = (global float*)((global char*)src1 + offset1);
    dst = (global float*)((global char*)dst + offsetd);

    const int nb = ne00/QK4_0;

    int r0 = get_group_id(0);
    int r1 = get_group_id(1);
    int im = get_group_id(2);

    int first_row = (r0 * N_SIMDGROUP + get_sub_group_id()) * N_DST;

    int i12 = im%ne12;
    int i13 = im/ne12;

    // The number of scales is the same as the number of blocks.
    ulong offset0_d = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);
    // Each block contains QK4_0/2 uchars, hence offset for qs is as follows.
    ulong offset0_q = first_row * nb + (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);

    global float * y = (global float *) src1   + r1*ne10 + im*ne00*ne1;

    int ix = get_sub_group_local_id()/THREADS_PER_BLK;
    int il = get_sub_group_local_id()%THREADS_PER_BLK;

    global float * yb = y + ix*QK4_0 + BTYES_PER_THREAD_IN_BLK*il;

    // Registers for caching activation
    ACT_TY yl = 0.f;

    // Registers for caching quants
    Q_BLK_LD_TY q_blk_0 = 0, q_blk_1 = 0;
#if N_DST == 4 || N_DST == 8 || N_DST == 16
    Q_BLK_LD_TY q_blk_2 = 0, q_blk_3 = 0;
#endif
#if N_DST == 8 || N_DST == 16
    Q_BLK_LD_TY q_blk_4 = 0, q_blk_5 = 0, q_blk_6 = 0, q_blk_7 = 0;
#endif

    // Partial sum
    SUM_TY sumf = 0.f;

    for (int ib = ix; ib < nb; ib += N_SIMDWIDTH/THREADS_PER_BLK) {
        float sumy = 0.f;;

        float4 tmp;
        tmp = read_imagef(src0_q, offset0_q + ib + 0*nb);
        q_blk_0 = EXTRACT_BLK_DATA(tmp, il);
        tmp = read_imagef(src0_q, offset0_q + ib + 1*nb);
        q_blk_1 = EXTRACT_BLK_DATA(tmp, il);
#if N_DST == 4 || N_DST == 8 || N_DST == 16
        tmp = read_imagef(src0_q, offset0_q + ib + 2*nb);
        q_blk_2 = EXTRACT_BLK_DATA(tmp, il);
        tmp = read_imagef(src0_q, offset0_q + ib + 3*nb);
        q_blk_3 = EXTRACT_BLK_DATA(tmp, il);
#endif
#if N_DST == 8 || N_DST == 16
        tmp = read_imagef(src0_q, offset0_q + ib + 4*nb);
        q_blk_4 = EXTRACT_BLK_DATA(tmp, il);
        tmp = read_imagef(src0_q, offset0_q + ib + 5*nb);
        q_blk_5 = EXTRACT_BLK_DATA(tmp, il);
        tmp = read_imagef(src0_q, offset0_q + ib + 6*nb);
        q_blk_6 = EXTRACT_BLK_DATA(tmp, il);
        tmp = read_imagef(src0_q, offset0_q + ib + 7*nb);
        q_blk_7 = EXTRACT_BLK_DATA(tmp, il);
#endif

        // Load activation
#if THREADS_PER_BLK == 2    // Each thread processes 1/2 block
        yl.s01234567 = *(global float8 *)(yb);
        yl.s89abcdef = *(global float8 *)(yb + 16);

        sumy += yl.s0;
        sumy += yl.s1;
        sumy += yl.s2;
        sumy += yl.s3;
        sumy += yl.s4;
        sumy += yl.s5;
        sumy += yl.s6;
        sumy += yl.s7;
        sumy += yl.s8; yl.s8 /= 16.f;
        sumy += yl.s9; yl.s9 /= 16.f;
        sumy += yl.sa; yl.sa /= 16.f;
        sumy += yl.sb; yl.sb /= 16.f;
        sumy += yl.sc; yl.sc /= 16.f;
        sumy += yl.sd; yl.sd /= 16.f;
        sumy += yl.se; yl.se /= 16.f;
        sumy += yl.sf; yl.sf /= 16.f;
#elif THREADS_PER_BLK == 4  // Each thread processes 1/4 block
        yl.s0123 = *(global float4 *)(yb);
        yl.s4567 = *(global float4 *)(yb + 16);

        sumy += yl.s0;
        sumy += yl.s1;
        sumy += yl.s2;
        sumy += yl.s3;
        sumy += yl.s4; yl.s4 /= 16.f;
        sumy += yl.s5; yl.s5 /= 16.f;
        sumy += yl.s6; yl.s6 /= 16.f;
        sumy += yl.s7; yl.s7 /= 16.f;
#elif THREADS_PER_BLK == 8  // Each thread processes 1/8 block
        yl.s01 = *(global float2 *)(yb);
        yl.s23 = *(global float2 *)(yb + 16);

        sumy += yl.s0;
        sumy += yl.s1;
        sumy += yl.s2; yl.s2 /= 16.f;
        sumy += yl.s3; yl.s3 /= 16.f;
#endif

        sumf.s0 += block_q_4_0_dot_y_flat(q_blk_0, read_imageh(src0_d, offset0_d + ib + 0*nb).s0, sumy, yl);
        sumf.s1 += block_q_4_0_dot_y_flat(q_blk_1, read_imageh(src0_d, offset0_d + ib + 1*nb).s0, sumy, yl);
#if N_DST == 4 || N_DST == 8 || N_DST == 16
        sumf.s2 += block_q_4_0_dot_y_flat(q_blk_2, read_imageh(src0_d, offset0_d + ib + 2*nb).s0, sumy, yl);
        sumf.s3 += block_q_4_0_dot_y_flat(q_blk_3, read_imageh(src0_d, offset0_d + ib + 3*nb).s0, sumy, yl);
#endif
#if N_DST == 8 || N_DST == 16
        sumf.s4 += block_q_4_0_dot_y_flat(q_blk_4, read_imageh(src0_d, offset0_d + ib + 4*nb).s0, sumy, yl);
        sumf.s5 += block_q_4_0_dot_y_flat(q_blk_5, read_imageh(src0_d, offset0_d + ib + 5*nb).s0, sumy, yl);
        sumf.s6 += block_q_4_0_dot_y_flat(q_blk_6, read_imageh(src0_d, offset0_d + ib + 6*nb).s0, sumy, yl);
        sumf.s7 += block_q_4_0_dot_y_flat(q_blk_7, read_imageh(src0_d, offset0_d + ib + 7*nb).s0, sumy, yl);
#endif

        yb += QK4_0 * (N_SIMDWIDTH/THREADS_PER_BLK);
    }

    SUM_TY tot = (SUM_TY)(
          sub_group_reduce_add(sumf.s0), sub_group_reduce_add(sumf.s1)
#if N_DST == 4 || N_DST == 8 || N_DST == 16
        , sub_group_reduce_add(sumf.s2), sub_group_reduce_add(sumf.s3)
#endif
#if N_DST == 8 || N_DST == 16
        , sub_group_reduce_add(sumf.s4), sub_group_reduce_add(sumf.s5)
        , sub_group_reduce_add(sumf.s6), sub_group_reduce_add(sumf.s7)
#endif
    );

    if (get_sub_group_local_id() == 0) {
        if (first_row + 0 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 0] = tot.s0;
        }
        if (first_row + 1 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 1] = tot.s1;
        }
#if N_DST == 4 || N_DST == 8 || N_DST == 16
        if (first_row + 2 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 2] = tot.s2;
        }
        if (first_row + 3 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 3] = tot.s3;
        }
#endif
#if N_DST == 8 || N_DST == 16
        if (first_row + 4 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 4] = tot.s4;
        }
        if (first_row + 5 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 5] = tot.s5;
        }
        if (first_row + 6 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 6] = tot.s6;
        }
        if (first_row + 7 < ne01) {
            dst[r1*ne0 + im*ne0*ne1 + first_row + 7] = tot.s7;
        }
#endif
    }
}

//------------------------------------------------------------------------------
// kernel_mul_mv_q6_K_f32
//------------------------------------------------------------------------------

#undef N_DST
#undef N_SIMDGROUP
#undef N_SIMDWIDTH

#ifdef INTEL_GPU
#define N_DST 1 // number of rows each SIMD group works on
#define N_SIMDGROUP 2 // number of SIMD groups in a thread group
#define N_SIMDWIDTH 16 // SIMD group size
#elif defined (ADRENO_GPU)
#define N_DST 1
#define N_SIMDGROUP 2
#define N_SIMDWIDTH 64
#endif

#define BLOCK_STRIDE (N_SIMDWIDTH/16) // number of blocks each subgroup processes

#ifdef INTEL_GPU
REQD_SUBGROUP_SIZE_16
#elif defined (ADRENO_GPU)
REQD_SUBGROUP_SIZE_64
#endif
kernel void kernel_mul_mv_q6_K_f32(
        global void * src0,
        ulong offset0,
        global float * src1,
        ulong offset1,
        global float * dst,
        ulong offsetd,
        int ne00,
        int ne01,
        int ne02,
        int ne10,
        int ne12,
        int ne0,
        int ne1,
        int r2,
        int r3
) {
    src0 = (global void*)((global char*)src0 + offset0);
    src1 = (global float*)((global char*)src1 + offset1);
    dst = (global float*)((global char*)dst + offsetd);

    uchar kmask1 = 0x03;
    uchar kmask2 = 0x0C;
    uchar kmask3 = 0x30;
    uchar kmask4 = 0xC0;

    int nb = ne00/QK_K;

    int r0 = get_group_id(0);
    int r1 = get_group_id(1);
    int im = get_group_id(2);

    int row = N_SIMDGROUP * r0 + get_sub_group_id();

    int i12 = im%ne12;
    int i13 = im/ne12;

    ulong offset_src0 = (i12/r2)*(nb*ne01) + (i13/r3)*(nb*ne01*ne02);

    global block_q6_K * x = (global block_q6_K *) src0 + row*nb + offset_src0;
    global float      * yy = (global float     *) src1 + r1*ne10 + im*ne00*ne1;

    float sumf = 0;

    // For Q6_K quantization, 16 values forms a subblock, 16 subblock forms a
    // block. Values in a subblock shares a scale that is quantized with 8 bits;
    // the entire block shares a single floating point scale.
    // For work distribution, each thread processes a subblock (16 weights), hence
    // 16 threads process a (super) block -- a subgroup thus handles SIMDWIDTH/16
    // (super) blocks -- this is the block stride.
    // The 16 threads that process a (super) block are split into 2 portions, each has
    // 8 threads; each portion works on 8 subblocks.
    // For subgroup of 16 threads, the entire subgroup works on a single (super) block
    // before moving to the next (super) block. Thread0 - thread7 work on the
    // first 8 subblocks; thread8 - thread15 works on the last 8 subblocks.
    // Thread0 - thread3 work on subblocks 0, 2, 4, 6; thread4 - thread7 work on
    // subblocks 1, 3, 5, 7. Each thread does not work on an entire subblock, but
    // works on a total of 16 weight values.
    int tid  = get_sub_group_local_id()/BLOCK_STRIDE; // first block_stride groups have tid=0
    int ix   = get_sub_group_local_id()%BLOCK_STRIDE; // first block is 0..block_stride-1
    int ip   = tid/8;   // first or second half of (super) block (0 or 1)
    int il   = tid%8;   // each half has 8 parts, one per scale
    int n    = 4;       // 4 scales at a time (and 4 sums)
    int l0   = n*il;    // offset into half-block, 0..28
    int is   = 8*ip + l0/16; // 0, 1, 8, 9

    int y_offset = 128*ip + l0;
    int q_offset_l = 64*ip + l0;
    int q_offset_h = 32*ip + l0;

    for (int i = ix; i < nb; i += BLOCK_STRIDE) {

        global uint8_t * q1 = x[i].ql + q_offset_l;
        global uint8_t * q2 = q1 + QK_K/8;
        global uint8_t * qh = x[i].qh + q_offset_h;
        global int8_t  * sc = x[i].scales + is;

        global float * y = yy + i * QK_K + y_offset;

        float dall = x[i].d;

        float4 sums = {0.f, 0.f, 0.f, 0.f};

        sums.s0 += y[0+ 0] * ((float)((q1[0] & 0xF) | ((qh[0] & kmask1) << 4)) - 32.f);
        sums.s1 += y[0+32] * ((float)((q2[0] & 0xF) | ((qh[0] & kmask2) << 2)) - 32.f);
        sums.s2 += y[0+64] * ((float)((q1[0]  >> 4) | ((qh[0] & kmask3) << 0)) - 32.f);
        sums.s3 += y[0+96] * ((float)((q2[0]  >> 4) | ((qh[0] & kmask4) >> 2)) - 32.f);

        sums.s0 += y[1+ 0] * ((float)((q1[1] & 0xF) | ((qh[1] & kmask1) << 4)) - 32.f);
        sums.s1 += y[1+32] * ((float)((q2[1] & 0xF) | ((qh[1] & kmask2) << 2)) - 32.f);
        sums.s2 += y[1+64] * ((float)((q1[1]  >> 4) | ((qh[1] & kmask3) << 0)) - 32.f);
        sums.s3 += y[1+96] * ((float)((q2[1]  >> 4) | ((qh[1] & kmask4) >> 2)) - 32.f);

        sums.s0 += y[2+ 0] * ((float)((q1[2] & 0xF) | ((qh[2] & kmask1) << 4)) - 32.f);
        sums.s1 += y[2+32] * ((float)((q2[2] & 0xF) | ((qh[2] & kmask2) << 2)) - 32.f);
        sums.s2 += y[2+64] * ((float)((q1[2]  >> 4) | ((qh[2] & kmask3) << 0)) - 32.f);
        sums.s3 += y[2+96] * ((float)((q2[2]  >> 4) | ((qh[2] & kmask4) >> 2)) - 32.f);

        sums.s0 += y[3+ 0] * ((float)((q1[3] & 0xF) | ((qh[3] & kmask1) << 4)) - 32.f);
        sums.s1 += y[3+32] * ((float)((q2[3] & 0xF) | ((qh[3] & kmask2) << 2)) - 32.f);
        sums.s2 += y[3+64] * ((float)((q1[3]  >> 4) | ((qh[3] & kmask3) << 0)) - 32.f);
        sums.s3 += y[3+96] * ((float)((q2[3]  >> 4) | ((qh[3] & kmask4) >> 2)) - 32.f);

        sumf += dall * (sums.s0 * sc[0] + sums.s1 * sc[2] + sums.s2 * sc[4] + sums.s3 * sc[6]);
    }

    float tot = sub_group_reduce_add(sumf);
    if (get_sub_group_local_id() == 0) {
        dst[r1*ne0 + im*ne0*ne1 + row] = tot;
    }
}