Fix code conformance in BDPT kernels

Fix code conformance in BDPT kernels

/**********************************************************************
Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
********************************************************************/
#include <../App/CL/common.cl>
#include <../App/CL/ray.cl>
#include <../App/CL/isect.cl>
#include <../App/CL/utils.cl>
#include <../App/CL/payload.cl>
#include <../App/CL/texture.cl>
#include <../App/CL/sampling.cl>
#include <../App/CL/normalmap.cl>
#include <../App/CL/bxdf.cl>
#include <../App/CL/light.cl>
#include <../App/CL/camera.cl>
#include <../App/CL/scene.cl>
#include <../App/CL/material.cl>
#include <../App/CL/volumetrics.cl>
#include <../App/CL/path.cl>
#include <../App/CL/vertex.cl>

// Convert PDF of sampling point po from point p from solid angle measure to area measure
INLINE
float Pdf_ConvertSolidAngleToArea(float pdf, float3 po, float3 p, float3 n)
{
    float3 v = po - p;
    float dist = length(v);
    return pdf * fabs(dot(normalize(v), n)) / (dist * dist);
}

KERNEL void GenerateLightVertices(
    // Number of subpaths to generate
    int num_subpaths,
    // Vertices
    GLOBAL float3 const* restrict vertices,
    // Normals
    GLOBAL float3 const* restrict normals,
    // UVs
    GLOBAL float2 const* restrict uvs,
    // Indices
    GLOBAL int const* restrict indices,
    // Shapes
    GLOBAL Shape const* restrict shapes,
    // Material IDs
    GLOBAL int const* restrict material_ids,
    // Materials
    GLOBAL Material const* restrict materials,
    // Textures
    TEXTURE_ARG_LIST,
    // Environment texture index
    int env_light_idx,
    // Emissives
    GLOBAL Light const* restrict lights,
    // Number of emissive objects
    int num_lights,
    // RNG seed value
    uint rngseed,
    int frame,
    // RNG data
    GLOBAL uint const* restrict random,
    GLOBAL uint const* restrict sobolmat,
    // Output rays
    GLOBAL ray* restrict rays,
    // Light subpath
    GLOBAL PathVertex* restrict light_subpath,
    // Light subpath length
    GLOBAL int* restrict light_subpath_length,
    // Path buffer
    GLOBAL Path* restrict paths
)
{
    Scene scene =
    {
        vertices,
        normals,
        uvs,
        indices,
        shapes,
        material_ids,
        materials,
        lights,
        env_light_idx,
        num_lights
    };

    int global_id = get_global_id(0);

    // Check borders
    if (global_id < num_subpaths)
    {
        // Get pointer to ray to handle
        GLOBAL ray* my_ray = rays + global_id;
        GLOBAL PathVertex* my_vertex = light_subpath + BDPT_MAX_SUBPATH_LEN * global_id;
        GLOBAL int* my_count = light_subpath_length + global_id;
        GLOBAL Path* my_path = paths + global_id;

        Sampler sampler;
#if SAMPLER == SOBOL
        uint scramble = random[global_id] * 0x1fe3434f;

        if (frame & 0xF)
        {
            random[global_id] = WangHash(scramble);
        }

        Sampler_Init(&sampler, frame, SAMPLE_DIM_CAMERA_OFFSET, scramble);
#elif SAMPLER == RANDOM
        uint scramble = global_id * rngseed;
        Sampler_Init(&sampler, scramble);
#elif SAMPLER == CMJ
        uint rnd = random[global_id];
        uint scramble = rnd * 0x1fe3434f * ((frame + 133 * rnd) / (CMJ_DIM * CMJ_DIM));
        Sampler_Init(&sampler, frame % (CMJ_DIM * CMJ_DIM), SAMPLE_DIM_CAMERA_OFFSET, scramble);
#endif

        // Generate sample
        float2 sample0 = Sampler_Sample2D(&sampler, SAMPLER_ARGS);
        float2 sample1 = Sampler_Sample2D(&sampler, SAMPLER_ARGS);

        float selection_pdf;
        int idx = Scene_SampleLight(&scene, Sampler_Sample1D(&sampler, SAMPLER_ARGS), &selection_pdf);

        float3 p, n, wo;
        float light_pdf;
        float3 ke = Light_SampleVertex(idx, &scene, TEXTURE_ARGS, sample0, sample1, &p, &n, &wo, &light_pdf);

        // Calculate direction to image plane
        my_ray->d.xyz = normalize(wo);
        // Origin == camera position + nearz * d
        my_ray->o.xyz = p + CRAZY_LOW_DISTANCE * n;
        // Max T value = zfar - znear since we moved origin to znear
        my_ray->o.w = CRAZY_HIGH_DISTANCE;
        // Generate random time from 0 to 1
        my_ray->d.w = sample0.x;
        // Set ray max
        my_ray->extra.x = 0xFFFFFFFF;
        my_ray->extra.y = 0xFFFFFFFF;
        Ray_SetExtra(my_ray, 1.f);

        PathVertex v;
        PathVertex_Init(&v,
            p,
            n,
            n,
            0.f,
            1.f,
            1.f,
            ke / (selection_pdf * light_pdf),
            kLight,
            -1);

        *my_count = 1;
        *my_vertex = v;

        // Initlize path data
        my_path->throughput = make_float3(1.f, 1.f, 1.f);
        my_path->volume = -1;
        my_path->flags = 0;
        my_path->active = 0xFF;
    }
}

KERNEL void SampleSurface(
    // Ray batch
    GLOBAL ray const* rays,
    // Intersection data
    GLOBAL Intersection const* intersections,
    // Hit indices
    GLOBAL int const* hit_indices,
    // Pixel indices
    GLOBAL int const* pixel_indices,
    // Number of rays
    GLOBAL int const* num_hits,
    // Vertices
    GLOBAL float3 const* vertices,
    // Normals
    GLOBAL float3 const* normals,
    // UVs
    GLOBAL float2 const* uvs,
    // Indices
    GLOBAL int const* indices,
    // Shapes
    GLOBAL Shape const* shapes,
    // Material IDs
    GLOBAL int const* material_ids,
    // Materials
    GLOBAL Material const* materials,
    // Textures
    TEXTURE_ARG_LIST,
    // Environment texture index
    int env_light_idx,
    // Emissives
    GLOBAL Light const* lights,
    // Number of emissive objects
    int num_lights,
    // RNG seed
    uint rngseed,
    // Sampler states
    GLOBAL uint* random,
    // Sobol matrices
    GLOBAL uint const* sobolmat,
    // Current bounce
    int bounce,
    // Frame
    int frame,
    // Radiance or importance
    int transfer_mode,
    // Path throughput
    GLOBAL Path* paths,
    // Indirect rays
    GLOBAL ray* extension_rays,
    // Vertices
    GLOBAL PathVertex* subpath,
    GLOBAL int* subpath_length
)
    {
        int global_id = get_global_id(0);

        Scene scene =
        {
            vertices,
            normals,
            uvs,
            indices,
            shapes,
            material_ids,
            materials,
            lights,
            env_light_idx,
            num_lights
        };

        // Only applied to active rays after compaction
        if (global_id < *num_hits)
        {
            // Fetch index
            int hit_idx = hit_indices[global_id];
            int pixel_idx = pixel_indices[global_id];
            Intersection isect = intersections[hit_idx];

            GLOBAL Path* path = paths + pixel_idx;

            // Fetch incoming ray direction
            float3 wi = -normalize(rays[hit_idx].d.xyz);

            Sampler sampler;
#if SAMPLER == SOBOL
            uint scramble = random[pixel_idx] * 0x1fe3434f;
            Sampler_Init(&sampler, frame, SAMPLE_DIM_SURFACE_OFFSET + bounce * SAMPLE_DIMS_PER_BOUNCE, scramble);
#elif SAMPLER == RANDOM
            uint scramble = pixel_idx * rngseed;
            Sampler_Init(&sampler, scramble);
#elif SAMPLER == CMJ
            uint rnd = random[pixel_idx];
            uint scramble = rnd * 0x1fe3434f * ((frame + 331 * rnd) / (CMJ_DIM * CMJ_DIM));
            Sampler_Init(&sampler, frame % (CMJ_DIM * CMJ_DIM), SAMPLE_DIM_SURFACE_OFFSET + bounce * SAMPLE_DIMS_PER_BOUNCE, scramble);
#endif

            // Fill surface data
            DifferentialGeometry diffgeo;
            Scene_FillDifferentialGeometry(&scene, &isect, &diffgeo);
            diffgeo.transfer_mode = transfer_mode;

            // Check if we are hitting from the inside
            float backfacing = dot(diffgeo.ng, wi) < 0.f;
            int twosided = false; // diffgeo.mat.twosided;
            if (twosided && backfacing)
            {
                // Reverse normal and tangents in this case
                // but not for BTDFs, since BTDFs rely
                // on normal direction in order to arrange
                // indices of refraction
                diffgeo.n = -diffgeo.n;
                diffgeo.dpdu = -diffgeo.dpdu;
                diffgeo.dpdv = -diffgeo.dpdv;
            }

            float ndotwi = dot(diffgeo.n, wi);

            // Select BxDF
            Material_Select(&scene, wi, &sampler, TEXTURE_ARGS, SAMPLER_ARGS, &diffgeo);

            // Terminate if emissive
            if (Bxdf_IsEmissive(&diffgeo))
            {
                Path_Kill(path);
                return;
            }

            float s = Bxdf_IsBtdf(&diffgeo) ? (-sign(ndotwi)) : 1.f;
            if (!twosided && backfacing && !Bxdf_IsBtdf(&diffgeo))
            {
                //Reverse normal and tangents in this case
                //but not for BTDFs, since BTDFs rely
                //on normal direction in order to arrange
                //indices of refraction
                diffgeo.n = -diffgeo.n;
                diffgeo.dpdu = -diffgeo.dpdu;
                diffgeo.dpdv = -diffgeo.dpdv;
            }

            // Check if we need to apply normal map
            DifferentialGeometry_ApplyBumpNormalMap(&diffgeo, TEXTURE_ARGS);
            DifferentialGeometry_CalculateTangentTransforms(&diffgeo);

            float lightpdf = 0.f;
            float bxdflightpdf = 0.f;
            float bxdfpdf = 0.f;
            float lightbxdfpdf = 0.f;
            float selection_pdf = 0.f;
            float3 radiance = 0.f;
            float3 lightwo;
            float3 bxdfwo;
            float3 wo;
            float bxdfweight = 1.f;
            float lightweight = 1.f;

            float3 throughput = Path_GetThroughput(path);

            // Sample bxdf
            float3 bxdf = Bxdf_Sample(&diffgeo, wi, TEXTURE_ARGS, Sampler_Sample2D(&sampler, SAMPLER_ARGS), &bxdfwo, &bxdfpdf);

            bxdfwo = normalize(bxdfwo);
            float3 t = bxdf * fabs(dot(diffgeo.n, bxdfwo));

            // Write path vertex
            GLOBAL int* my_counter = subpath_length + pixel_idx;
            int idx = atom_inc(my_counter);

            if (idx < BDPT_MAX_SUBPATH_LEN)
            {
                GLOBAL PathVertex* my_vertex = subpath + BDPT_MAX_SUBPATH_LEN * pixel_idx + idx;
                GLOBAL PathVertex* my_prev_vertex = subpath + BDPT_MAX_SUBPATH_LEN * pixel_idx + idx - 1;
                // Fill vertex
                // Calculate forward PDF of a current vertex
                float pdf_solid_angle = Ray_GetExtra(&rays[hit_idx]).x;
                float pdf_fwd = Pdf_ConvertSolidAngleToArea(pdf_solid_angle, rays[hit_idx].o.xyz, diffgeo.p, diffgeo.n);

                PathVertex v;
                PathVertex_Init(&v,
                    diffgeo.p,
                    diffgeo.n,
                    diffgeo.ng,
                    diffgeo.uv,
                    pdf_fwd,
                    0.f,
                    throughput * t / bxdfpdf,
                    kSurface,
                    diffgeo.material_index);

                *my_vertex = v;

                // TODO: wrong
                if (my_prev_vertex->type == kSurface)
                {
                    float pdf_bwd = Pdf_ConvertSolidAngleToArea(Bxdf_GetPdf(&diffgeo, bxdfwo, wi, TEXTURE_ARGS), diffgeo.p, my_prev_vertex->position, my_prev_vertex->shading_normal);
                    my_prev_vertex->pdf_backward = pdf_bwd;
                }
            }
            else
            {
                atom_dec(my_counter);
                Path_Kill(path);
                Ray_SetInactive(extension_rays + global_id);
                return;
            }

            // Only continue if we have non-zero throughput & pdf
            if (NON_BLACK(t) && bxdfpdf > 0.f)
            {
                // Update the throughput
                Path_MulThroughput(path, t / bxdfpdf);

                // Generate ray
                float3 new_ray_dir = bxdfwo;
                float3 new_ray_o = diffgeo.p + CRAZY_LOW_DISTANCE * s * diffgeo.n;

                Ray_Init(extension_rays + global_id, new_ray_o, new_ray_dir, CRAZY_HIGH_DISTANCE, 0.f, 0xFFFFFFFF);
                Ray_SetExtra(extension_rays + global_id, make_float2(bxdfpdf, 0.f));
            }
            else
            {
                // Otherwise kill the path
                Path_Kill(path);
                Ray_SetInactive(extension_rays + global_id);
            }
        }
    }

    KERNEL void ConnectDirect(
        int eye_vertex_index,
        int num_rays,
        GLOBAL int const* pixel_indices,
        GLOBAL PathVertex const* restrict eye_subpath,
        GLOBAL int const* restrict eye_subpath_length,
        GLOBAL float3 const* restrict vertices,
        GLOBAL float3 const* restrict normals,
        GLOBAL float2 const* restrict uvs,
        GLOBAL int const* restrict indices,
        GLOBAL Shape const* restrict shapes,
        GLOBAL int const* restrict materialids,
        GLOBAL Material const* restrict materials,
        TEXTURE_ARG_LIST,
        int env_light_idx,
        GLOBAL Light const* restrict lights,
        int num_lights,
        uint rngseed,
        GLOBAL uint const* restrict random,
        GLOBAL uint const* restrict sobolmat,
        int frame,
        GLOBAL ray* restrict shadowrays,
        GLOBAL float3* restrict lightsamples
    )
    {
        int global_id = get_global_id(0);

        if (global_id < num_rays)
        {
            int pixel_idx = pixel_indices[global_id];
            int num_eye_vertices = *(eye_subpath_length + pixel_idx);

            GLOBAL PathVertex const* my_eye_vertex = eye_subpath + BDPT_MAX_SUBPATH_LEN * pixel_idx + eye_vertex_index;
            GLOBAL PathVertex const* my_prev_eye_vertex = eye_subpath + BDPT_MAX_SUBPATH_LEN * pixel_idx + eye_vertex_index - 1;

            DifferentialGeometry diffgeo;
            diffgeo.p = my_eye_vertex->position;
            diffgeo.n = my_eye_vertex->shading_normal;
            diffgeo.ng = my_eye_vertex->geometric_normal;
            diffgeo.uv = my_eye_vertex->uv;
            diffgeo.dpdu = GetOrthoVector(diffgeo.n);
            diffgeo.dpdv = cross(diffgeo.n, diffgeo.dpdu);
            diffgeo.mat = materials[my_eye_vertex->material_index];
            diffgeo.mat.fresnel = 1.f;
            DifferentialGeometry_CalculateTangentTransforms(&diffgeo);

            float3 wi = normalize(my_prev_eye_vertex->position - diffgeo.p);
            float3 throughput = my_prev_eye_vertex->flow;

            if (eye_vertex_index >= num_eye_vertices)
            {
                lightsamples[global_id].xyz = 0.f;
                Ray_SetInactive(shadowrays + global_id);
                return;
            }

            Scene scene =
            {
                vertices,
                normals,
                uvs,
                indices,
                shapes,
                materialids,
                materials,
                lights,
                env_light_idx,
                num_lights
            };

            Sampler sampler;
#if SAMPLER == SOBOL
            uint scramble = random[global_id] * 0x1fe3434f;
            Sampler_Init(&sampler, frame, SAMPLE_DIM_SURFACE_OFFSET + eye_vertex_index * SAMPLE_DIMS_PER_BOUNCE, scramble);
#elif SAMPLER == RANDOM
            uint scramble = global_id * rngseed;
            Sampler_Init(&sampler, scramble);
#elif SAMPLER == CMJ
            uint rnd = random[global_id];
            uint scramble = rnd * 0x1fe3434f * ((frame + 331 * rnd) / (CMJ_DIM * CMJ_DIM));
            Sampler_Init(&sampler, frame % (CMJ_DIM * CMJ_DIM), SAMPLE_DIM_SURFACE_OFFSET + eye_vertex_index * SAMPLE_DIMS_PER_BOUNCE, scramble);
#endif

            float selection_pdf;
            int light_idx = Scene_SampleLight(&scene, Sampler_Sample1D(&sampler, SAMPLER_ARGS), &selection_pdf);

            // Sample light
            float lightpdf = 0.f;
            float lightbxdfpdf = 0.f;
            float bxdfpdf = 0.f;
            float bxdflightpdf = 0.f;
            float lightweight = 0.f;
            float bxdfweight = 0.f;

            float3 lightwo;
            float3 bxdfwo;
            float3 le = Light_Sample(light_idx, &scene, &diffgeo, TEXTURE_ARGS, Sampler_Sample2D(&sampler, SAMPLER_ARGS), &lightwo, &lightpdf);
            float3 bxdf = Bxdf_Sample(&diffgeo, normalize(wi), TEXTURE_ARGS, Sampler_Sample2D(&sampler, SAMPLER_ARGS), &bxdfwo, &bxdfpdf);
            lightbxdfpdf = Bxdf_GetPdf(&diffgeo, normalize(wi), normalize(lightwo), TEXTURE_ARGS);
            bxdflightpdf = Light_GetPdf(light_idx, &scene, &diffgeo, normalize(bxdfwo), TEXTURE_ARGS);

            bool singular_light = Light_IsSingular(&scene.lights[light_idx]);
            bool singular_bxdf = Bxdf_IsSingular(&diffgeo);
            lightweight = singular_light ? 1.f : BalanceHeuristic(1, lightpdf, 1, lightbxdfpdf);
            bxdfweight = singular_bxdf ? 1.f : BalanceHeuristic(1, bxdfpdf, 1, bxdflightpdf);

            float3 radiance = 0.f;
            float3 wo;

            if (singular_light && singular_bxdf)
            {
                // Otherwise save some intersector cycles
                Ray_SetInactive(shadowrays + global_id);
                lightsamples[global_id] = 0;
                return;
            }

            float split = Sampler_Sample1D(&sampler, SAMPLER_ARGS);

            if (((singular_bxdf && !singular_light) || split < 0.5f) && bxdfpdf > 0.f)
            {
                wo = CRAZY_HIGH_DISTANCE * bxdfwo;
                le = Light_GetLe(light_idx, &scene, &diffgeo, &wo, TEXTURE_ARGS);
                float ndotwo = fabs(dot(diffgeo.n, normalize(wo)));
                radiance = 2.f * le * bxdfweight * throughput * bxdf * ndotwo / bxdfpdf;
            }

            if (((!singular_bxdf && singular_light) || split >= 0.5f) && lightpdf > 0.f)
            {
                wo = lightwo;
                float ndotwo = fabs(dot(diffgeo.n, normalize(wo)));
                radiance = 2.f * lightweight * le * Bxdf_Evaluate(&diffgeo, wi, normalize(wo), TEXTURE_ARGS) * throughput * ndotwo / lightpdf / selection_pdf;
            }

            // If we have some light here generate a shadow ray
            if (NON_BLACK(radiance))
            {
                // Generate shadow ray
                float3 shadow_ray_o = diffgeo.p + CRAZY_LOW_DISTANCE * diffgeo.n;
                float3 temp = diffgeo.p + wo - shadow_ray_o;
                float3 shadow_ray_dir = normalize(temp);
                float shadow_ray_length = length(temp);
                int shadow_ray_mask = 0xFFFFFFFF;

                Ray_Init(shadowrays + global_id, shadow_ray_o, shadow_ray_dir, shadow_ray_length, 0.f, shadow_ray_mask);

                // And write the light sample
                lightsamples[global_id] = REASONABLE_RADIANCE(radiance);
            }
            else
            {
                // Otherwise save some intersector cycles
                Ray_SetInactive(shadowrays + global_id);
                lightsamples[global_id] = 0;
            }
        }
    }

    KERNEL void Connect(
        int num_rays,
        // Index of eye vertex we are trying to connect
        int eye_vertex_index,
        // Index of light vertex we are trying to connect
        int light_vertex_index,
        GLOBAL int const* pixel_indices,
        // Vertex arrays
        GLOBAL PathVertex const* restrict eye_subpath,
        GLOBAL int const* restrict eye_subpath_length,
        GLOBAL PathVertex const* restrict light_subpath,
        GLOBAL int const* restrict light_subpath_length,
        GLOBAL Material const* restrict materials,
        // Textures
        TEXTURE_ARG_LIST,
        GLOBAL ray* connection_rays,
        // Path contriburions to final image
        GLOBAL float4* contributions
    )
    {
        int global_id = get_global_id(0);

        if (global_id >= num_rays)
        {
            return;
        }

        int pixel_idx = pixel_indices[global_id];

        // Check if we our indices are within subpath index range
        if (eye_vertex_index >= eye_subpath_length[pixel_idx] ||
            light_vertex_index >= light_subpath_length[global_id])
        {
            contributions[global_id].xyz = 0.f;
            Ray_SetInactive(&connection_rays[global_id]);
            return;
        }

        // Prepare data pointers
        GLOBAL PathVertex const* my_eye_vertex = eye_subpath + BDPT_MAX_SUBPATH_LEN * pixel_idx + eye_vertex_index;
        GLOBAL PathVertex const* my_prev_eye_vertex = eye_subpath + BDPT_MAX_SUBPATH_LEN * pixel_idx + eye_vertex_index - 1;

        GLOBAL PathVertex const* my_light_vertex = light_subpath + BDPT_MAX_SUBPATH_LEN * global_id + light_vertex_index;
        GLOBAL PathVertex const* my_prev_light_vertex = light_subpath + BDPT_MAX_SUBPATH_LEN * global_id + light_vertex_index - 1;

        // Fill differential geometries for both eye and light vertices
        DifferentialGeometry eye_dg;
        eye_dg.p = my_eye_vertex->position;
        eye_dg.n = my_eye_vertex->shading_normal;
        eye_dg.ng = my_eye_vertex->geometric_normal;
        eye_dg.uv = my_eye_vertex->uv;
        eye_dg.dpdu = GetOrthoVector(eye_dg.n);
        eye_dg.dpdv = cross(eye_dg.n, eye_dg.dpdu);
        eye_dg.mat = materials[my_eye_vertex->material_index];
        eye_dg.mat.fresnel = 1.f;
        DifferentialGeometry_CalculateTangentTransforms(&eye_dg);

        DifferentialGeometry light_dg;
        light_dg.p = my_light_vertex->position;
        light_dg.n = my_light_vertex->shading_normal;
        light_dg.ng = my_light_vertex->geometric_normal;
        light_dg.uv = my_light_vertex->uv;
        light_dg.dpdu = GetOrthoVector(light_dg.n);
        light_dg.dpdv = cross(light_dg.dpdu, light_dg.n);
        light_dg.mat = materials[my_light_vertex->material_index];
        light_dg.mat.fresnel = 1.f;
        DifferentialGeometry_CalculateTangentTransforms(&light_dg);

        // Vector from eye subpath vertex to previous eye subpath vertex (incoming vector)
        float3 eye_wi = normalize(my_prev_eye_vertex->position - eye_dg.p);
        // Vector from camera subpath vertex to light subpath vertex (connection vector)
        float3 eye_wo = normalize(light_dg.p - eye_dg.p);
        // Distance between vertices we are connecting
        float dist = length(light_dg.p - eye_dg.p);
        // BXDF at eye subpath vertex
        float3 eye_bxdf = Bxdf_Evaluate(&eye_dg, eye_wi, eye_wo, TEXTURE_ARGS);
        // Eye subpath vertex cosine factor
        float eye_dotnl = fabs(dot(eye_dg.n, eye_wo));

        // Calculate througput of a segment eye->eyevetex0->...->eyevertexN->lightvertexM->lightvertexM-1...->light
        // First eye subpath throughput eye->eyevertex0->...eyevertexN
        float3 eye_contribution = my_prev_eye_vertex->flow;
        eye_contribution *= (eye_bxdf * eye_dotnl);

        // Vector from light subpath vertex to prev light subpath vertex (incoming light vector)
        float3 light_wi = normalize(my_prev_light_vertex->position - light_dg.p);
        // Outgoing light subpath vector
        float3 light_wo = -eye_wo;
        // BXDF at light subpath vertex
        float3 light_bxdf = Bxdf_Evaluate(&light_dg, light_wi, light_wo, TEXTURE_ARGS);
        // Light subpath vertex cosine factor
        float light_dotnl = fabs(dot(light_dg.n, light_wo));

        // Light subpath throughput lightvertexM->lightvertexM-1->light
        float3 light_contribution = my_prev_light_vertex->flow;
        light_contribution *= (light_bxdf * light_dotnl);

        // Now we need to figure out MIS weight for the path we are trying to build
        // connecting these vertices.
        // MIS weight should account for all possible paths of the same length using
        // same vertices, but generated using different subpath lengths, for example
        // Original: eye0 eye1 eye2 eye3 | connect | light3 light2 light1 light0
        // Variant 1: eye0 eye1 eye2 | connect | eye3 light3 light2 light1 light0
        // as if only eye0->eye1->eye2 was generated using eye walk,
        // and eye3->light3->light2->light1->light0 was generated using light walk.

        // This is PDF of sampling eye subpath vertex from previous eye subpath vertex using eye walk strategy.
        float eye_vertex_pdf_fwd = my_eye_vertex->pdf_forward;
        // This is PDF of sampling light subpath vertex from previous light subpath vertex using light walk strategy.
        float light_vertex_pdf_fwd = my_light_vertex->pdf_forward;

        // Important: this is PDF of sampling eye subpath vertex from light subpath vertex using light walk strategy.
        float eye_vertex_pdf_bwd = Pdf_ConvertSolidAngleToArea(Bxdf_GetPdf(&light_dg, light_wi, light_wo, TEXTURE_ARGS),
            eye_dg.p, light_dg.p, light_dg.n);
        // Important: this is PDF of sampling prev eye subpath vertex from eye subpath vertex using light walk strategy.
        float eye_prev_vertex_pdf_bwd = Pdf_ConvertSolidAngleToArea(Bxdf_GetPdf(&eye_dg, eye_wi, eye_wo, TEXTURE_ARGS),
            my_prev_eye_vertex->position, eye_dg.p, eye_dg.n);

        // Important: this is PDF of sampling light subpath vertex from eye subpath vertex using eye walk strategy.
        float light_vertex_pdf_bwd = Pdf_ConvertSolidAngleToArea(Bxdf_GetPdf(&eye_dg, eye_wo, eye_wi, TEXTURE_ARGS),
            light_dg.p, eye_dg.p, eye_dg.n);
        // Important: this is PDF of sampling prev light subpath vertex from light subpath vertex using eye walk strategy.
        float light_prev_vertex_pdf_bwd = Pdf_ConvertSolidAngleToArea(Bxdf_GetPdf(&light_dg, light_wo, light_wi, TEXTURE_ARGS),
            my_prev_light_vertex->position, light_dg.p, light_dg.n);

        // We need sum of all weights
        float weights_sum = (eye_vertex_pdf_bwd / eye_vertex_pdf_fwd);
        float weights_eye = (eye_vertex_pdf_bwd / eye_vertex_pdf_fwd) * (eye_prev_vertex_pdf_bwd / my_prev_eye_vertex->pdf_forward);
        weights_sum += weights_eye;

        weights_sum += (light_vertex_pdf_bwd / light_vertex_pdf_fwd);
        float weights_light = (light_vertex_pdf_bwd / light_vertex_pdf_fwd) * (light_prev_vertex_pdf_bwd / my_prev_light_vertex->pdf_forward);
        weights_sum += weights_light;

        // Iterate eye subpath
        // (we already accounted for prev vertex, so start from one before, that's why -2)
        for (int i = eye_vertex_index - 2; i >= 0; --i)
        {
            GLOBAL PathVertex const* v = &eye_subpath[BDPT_MAX_SUBPATH_LEN * pixel_idx + i];
            weights_eye *= (v->pdf_backward / v->pdf_forward);
            weights_sum += weights_eye;
        }

        // Iterate light subpath
        // (we already accounted for prev vertex, so start from one before, that's why -2)
        for (int i = light_vertex_index - 2; i >= 0; --i)
        {
            GLOBAL PathVertex const* v = &light_subpath[BDPT_MAX_SUBPATH_LEN * global_id + i];
            weights_light *= (v->pdf_backward / v->pdf_forward);
            weights_sum += weights_light;
        }

        float mis_weight = 1.f / (1.f + weights_sum);
        contributions[global_id].xyz = REASONABLE_RADIANCE(mis_weight * (eye_contribution * light_contribution) / (dist * dist));

        float3 connect_ray_o = eye_dg.p + CRAZY_LOW_DISTANCE * eye_dg.n;
        float3 temp = light_dg.p - connect_ray_o;
        float  connect_ray_length = length(temp);
        float3 connect_ray_dir = eye_wo;

        Ray_Init(&connection_rays[global_id], connect_ray_o, connect_ray_dir, connect_ray_length, 0.f, 0xFFFFFFFF);
    }

// Generate pixel indices withing a tile
KERNEL void GenerateTileDomain(
    // Output buffer width
    int output_width,
    // Output buffer height
    int output_height,
    // Tile offset within an output buffer
    int tile_offset_x,
    int tile_offset_y,
    // Tile size
    int tile_width,
    int tile_height,
    // Subtile size (tile can be organized in subtiles)
    int subtile_width,
    int subtile_height,
    // Pixel indices
    GLOBAL int* restrict pixel_idx0,
    GLOBAL int* restrict pixel_idx1,
    // Pixel count
    GLOBAL int* restrict count
)
{
    int tile_x = get_global_id(0);
    int tile_y = get_global_id(1);
    int tile_start_idx = output_width * tile_offset_y + tile_offset_x;

    if (tile_x < tile_width && tile_y < tile_height)
    {
        // TODO: implement subtile support
        int idx = tile_start_idx + tile_y * output_width + tile_x;
        pixel_idx0[tile_y * tile_width + tile_x] = idx;
        pixel_idx1[tile_y * tile_width + tile_x] = idx;
    }

    if (tile_x == 0 && tile_y == 0)
    {
        *count = tile_width * tile_height;
    }
}

///< Restore pixel indices after compaction
KERNEL void FilterPathStream(
    GLOBAL Intersection const* restrict intersections,
    // Number of compacted indices
    GLOBAL int const* restrict num_items,
    GLOBAL int const* restrict pixel_indices,
    GLOBAL Path* restrict paths,
    // 1 or 0 for each item (include or exclude)
    GLOBAL int* restrict predicate
)
{
    int global_id = get_global_id(0);

    // Handle only working subset
    if (global_id < *num_items)
    {
        int pixelidx = pixel_indices[global_id];

        GLOBAL Path* path = paths + pixelidx;

        if (Path_IsAlive(path))
        {
            bool kill = (length(Path_GetThroughput(path)) < CRAZY_LOW_THROUGHPUT);

            if (!kill)
            {
                predicate[global_id] = intersections[global_id].shapeid >= 0 ? 1 : 0;
            }
            else
            {
                Path_Kill(path);
                predicate[global_id] = 0;
            }
        }
        else
        {
            predicate[global_id] = 0;
        }
    }
}

KERNEL void IncrementSampleCounter(
    // Number of rays
    int num_rays,
    GLOBAL int const* restrict pixel_indices,
    // Radiance sample buffer
    GLOBAL float4* restrict output
)
{
    int global_id = get_global_id(0);

    if (global_id < num_rays)
    {
        int pixel_idx = pixel_indices[global_id];
        // Start collecting samples
        {
            output[pixel_idx].w += 1.f;
        }
    }
}

KERNEL void GatherContributions(
    // Number of rays
    int num_rays,
    GLOBAL int const* restrict pixel_indices,
    // Shadow rays hits
    GLOBAL int const* restrict shadow_hits,
    // Light samples
    GLOBAL float3 const* restrict contributions,
    // Radiance sample buffer
    GLOBAL float4* output
)
{
    int global_id = get_global_id(0);

    if (global_id < num_rays)
    {
        int pixel_idx = pixel_indices[global_id];
        // Prepare accumulator variable
        float3 radiance = make_float3(0.f, 0.f, 0.f);

        // Start collecting samples
        {
            // If shadow ray global_id't hit anything and reached skydome
            if (shadow_hits[global_id] == -1)
            {
                // Add its contribution to radiance accumulator
                radiance += contributions[global_id];
                output[pixel_idx].xyz += radiance;
            }
        }
    }
}

// Restore pixel indices after compaction
KERNEL void RestorePixelIndices(
    // Compacted indices
    GLOBAL int const* restrict compacted_indices,
    // Number of compacted indices
    GLOBAL int* restrict num_items,
    // Previous pixel indices
    GLOBAL int const* restrict prev_indices,
    // New pixel indices
    GLOBAL int* restrict new_indices
)
{
    int global_id = get_global_id(0);

    // Handle only working subset
    if (global_id < *num_items)
    {
        new_indices[global_id] = prev_indices[compacted_indices[global_id]];
    }
}

// Copy data to interop texture if supported
KERNEL void ApplyGammaAndCopyData(
    //
    GLOBAL float4 const* data,
    int width,
    int height,
    float gamma,
    write_only image2d_t img
)
{
    int global_id = get_global_id(0);
    int x = global_id % width;
    int y = global_id / width;

    if (x < width && y < height)
    {
        float4 v = data[global_id];
        float4 val = clamp(native_powr(v / v.w, 1.f / gamma), 0.f, 1.f);
        write_imagef(img, make_int2(x, y), val);
    }
}

/**********************************************************************
Copyright (c) 2016 Advanced Micro Devices, Inc. All rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
********************************************************************/
#include <../App/CL/common.cl>
#include <../App/CL/ray.cl>
#include <../App/CL/isect.cl>
#include <../App/CL/utils.cl>
#include <../App/CL/payload.cl>
#include <../App/CL/texture.cl>
#include <../App/CL/sampling.cl>
#include <../App/CL/normalmap.cl>
#include <../App/CL/bxdf.cl>
#include <../App/CL/light.cl>
#include <../App/CL/camera.cl>
#include <../App/CL/scene.cl>
#include <../App/CL/material.cl>
#include <../App/CL/volumetrics.cl>
#include <../App/CL/path.cl>
#include <../App/CL/vertex.cl>

// Convert PDF of sampling point po from point p from solid angle measure to area measure
INLINE
float Pdf_ConvertSolidAngleToArea(float pdf, float3 po, float3 p, float3 n)
{
    float3 v = po - p;
    float dist = length(v);
    return pdf * fabs(dot(normalize(v), n)) / (dist * dist);
}

KERNEL void GenerateLightVertices(
    // Number of subpaths to generate
    int num_subpaths,
    // Vertices
    GLOBAL float3 const* restrict vertices,
    // Normals
    GLOBAL float3 const* restrict normals,
    // UVs
    GLOBAL float2 const* restrict uvs,
    // Indices
    GLOBAL int const* restrict indices,
    // Shapes
    GLOBAL Shape const* restrict shapes,
    // Material IDs
    GLOBAL int const* restrict material_ids,
    // Materials
    GLOBAL Material const* restrict materials,
    // Textures
    TEXTURE_ARG_LIST,
    // Environment texture index
    int env_light_idx,
    // Emissives
    GLOBAL Light const* restrict lights,
    // Number of emissive objects
    int num_lights,
    // RNG seed value
    uint rngseed,
    int frame,
    // RNG data
    GLOBAL uint const* restrict random,
    GLOBAL uint const* restrict sobolmat,
    // Output rays
    GLOBAL ray* restrict rays,
    // Light subpath
    GLOBAL PathVertex* restrict light_subpath,
    // Light subpath length
    GLOBAL int* restrict light_subpath_length,
    // Path buffer
    GLOBAL Path* restrict paths
)
{
    Scene scene =
    {
        vertices,
        normals,
        uvs,
        indices,
        shapes,
        material_ids,
        materials,
        lights,
        env_light_idx,
        num_lights
    };

    int global_id = get_global_id(0);

    // Check borders
    if (global_id < num_subpaths)
    {
        // Get pointer to ray to handle
        GLOBAL ray* my_ray = rays + global_id;
        GLOBAL PathVertex* my_vertex = light_subpath + BDPT_MAX_SUBPATH_LEN * global_id;
        GLOBAL int* my_count = light_subpath_length + global_id;
        GLOBAL Path* my_path = paths + global_id;

        Sampler sampler;
#if SAMPLER == SOBOL
        uint scramble = random[global_id] * 0x1fe3434f;

        if (frame & 0xF)
        {
            random[global_id] = WangHash(scramble);
        }

        Sampler_Init(&sampler, frame, SAMPLE_DIM_CAMERA_OFFSET, scramble);
#elif SAMPLER == RANDOM
        uint scramble = global_id * rngseed;
        Sampler_Init(&sampler, scramble);
#elif SAMPLER == CMJ
        uint rnd = random[global_id];
        uint scramble = rnd * 0x1fe3434f * ((frame + 133 * rnd) / (CMJ_DIM * CMJ_DIM));
        Sampler_Init(&sampler, frame % (CMJ_DIM * CMJ_DIM), SAMPLE_DIM_CAMERA_OFFSET, scramble);
#endif

        // Generate sample
        float2 sample0 = Sampler_Sample2D(&sampler, SAMPLER_ARGS);
        float2 sample1 = Sampler_Sample2D(&sampler, SAMPLER_ARGS);

        float selection_pdf;
        int idx = Scene_SampleLight(&scene, Sampler_Sample1D(&sampler, SAMPLER_ARGS), &selection_pdf);

        float3 p, n, wo;
        float light_pdf;
        float3 ke = Light_SampleVertex(idx, &scene, TEXTURE_ARGS, sample0, sample1, &p, &n, &wo, &light_pdf);

        // Calculate direction to image plane
        my_ray->d.xyz = normalize(wo);
        // Origin == camera position + nearz * d
        my_ray->o.xyz = p + CRAZY_LOW_DISTANCE * n;
        // Max T value = zfar - znear since we moved origin to znear
        my_ray->o.w = CRAZY_HIGH_DISTANCE;
        // Generate random time from 0 to 1
        my_ray->d.w = sample0.x;
        // Set ray max
        my_ray->extra.x = 0xFFFFFFFF;
        my_ray->extra.y = 0xFFFFFFFF;
        Ray_SetExtra(my_ray, 1.f);

        PathVertex v;
        PathVertex_Init(&v,
            p,
            n,
            n,
            0.f,
            1.f,
            1.f,
            ke / (selection_pdf * light_pdf),
            kLight,
            -1);

        *my_count = 1;
        *my_vertex = v;

        // Initlize path data
        my_path->throughput = make_float3(1.f, 1.f, 1.f);
        my_path->volume = -1;
        my_path->flags = 0;
        my_path->active = 0xFF;
    }
}

KERNEL void SampleSurface(
    // Ray batch
    GLOBAL ray const* rays,
    // Intersection data
    GLOBAL Intersection const* isects,
    // Hit indices
    GLOBAL int const* hit_indices,
    // Pixel indices
    GLOBAL int const* pixel_indices,
    // Number of rays
    GLOBAL int const* num_hits,
    // Vertices
    GLOBAL float3 const* vertices,
    // Normals
    GLOBAL float3 const* normals,
    // UVs
    GLOBAL float2 const* uvs,
    // Indices
    GLOBAL int const* indices,
    // Shapes
    GLOBAL Shape const* shapes,
    // Material IDs
    GLOBAL int const* material_ids,
    // Materials
    GLOBAL Material const* materials,
    // Textures
    TEXTURE_ARG_LIST,
    // Environment texture index
    int env_light_idx,
    // Emissives
    GLOBAL Light const* lights,
    // Number of emissive objects
    int num_lights,
    // RNG seed
    uint rngseed,
    // Sampler states
    GLOBAL uint* random,
    // Sobol matrices
    GLOBAL uint const* sobolmat,
    // Current bounce
    int bounce,
    // Frame
    int frame,
    // Radiance or importance
    int transfer_mode,
    // Path throughput
    GLOBAL Path* paths,
    // Indirect rays
    GLOBAL ray* extension_rays,
    // Vertices
    GLOBAL PathVertex* subpath,
    GLOBAL int* subpath_length
)
    {
        int global_id = get_global_id(0);

        Scene scene =
        {
            vertices,
            normals,
            uvs,
            indices,
            shapes,
            material_ids,
            materials,
            lights,
            env_light_idx,
            num_lights
        };

        // Only applied to active rays after compaction
        if (global_id < *num_hits)
        {
            // Fetch index
            int hit_idx = hit_indices[global_id];
            int pixel_idx = pixel_indices[global_id];
            Intersection isect = isects[hit_idx];

            GLOBAL Path* path = paths + pixel_idx;

            // Fetch incoming ray direction
            float3 wi = -normalize(rays[hit_idx].d.xyz);

            Sampler sampler;
#if SAMPLER == SOBOL
            uint scramble = random[pixel_idx] * 0x1fe3434f;
            Sampler_Init(&sampler, frame, SAMPLE_DIM_SURFACE_OFFSET + bounce * SAMPLE_DIMS_PER_BOUNCE, scramble);
#elif SAMPLER == RANDOM
            uint scramble = pixel_idx * rngseed;
            Sampler_Init(&sampler, scramble);
#elif SAMPLER == CMJ
            uint rnd = random[pixel_idx];
            uint scramble = rnd * 0x1fe3434f * ((frame + 331 * rnd) / (CMJ_DIM * CMJ_DIM));
            Sampler_Init(&sampler, frame % (CMJ_DIM * CMJ_DIM), SAMPLE_DIM_SURFACE_OFFSET + bounce * SAMPLE_DIMS_PER_BOUNCE, scramble);
#endif

            // Fill surface data
            DifferentialGeometry diffgeo;
            Scene_FillDifferentialGeometry(&scene, &isect, &diffgeo);
            diffgeo.transfer_mode = transfer_mode;

            // Check if we are hitting from the inside
            float backfacing = dot(diffgeo.ng, wi) < 0.f;
            int twosided = false; // diffgeo.mat.twosided;
            if (twosided && backfacing)
            {
                // Reverse normal and tangents in this case
                // but not for BTDFs, since BTDFs rely
                // on normal direction in order to arrange
                // indices of refraction
                diffgeo.n = -diffgeo.n;
                diffgeo.dpdu = -diffgeo.dpdu;
                diffgeo.dpdv = -diffgeo.dpdv;
            }

            float ndotwi = dot(diffgeo.n, wi);

            // Select BxDF
            Material_Select(&scene, wi, &sampler, TEXTURE_ARGS, SAMPLER_ARGS, &diffgeo);

            // Terminate if emissive
            if (Bxdf_IsEmissive(&diffgeo))
            {
                Path_Kill(path);
                return;
            }

            float s = Bxdf_IsBtdf(&diffgeo) ? (-sign(ndotwi)) : 1.f;
            if (!twosided && backfacing && !Bxdf_IsBtdf(&diffgeo))
            {
                //Reverse normal and tangents in this case
                //but not for BTDFs, since BTDFs rely
                //on normal direction in order to arrange
                //indices of refraction
                diffgeo.n = -diffgeo.n;
                diffgeo.dpdu = -diffgeo.dpdu;
                diffgeo.dpdv = -diffgeo.dpdv;
            }

            // Check if we need to apply normal map
            DifferentialGeometry_ApplyBumpNormalMap(&diffgeo, TEXTURE_ARGS);
            DifferentialGeometry_CalculateTangentTransforms(&diffgeo);

            float lightpdf = 0.f;
            float bxdflightpdf = 0.f;
            float bxdfpdf = 0.f;
            float lightbxdfpdf = 0.f;
            float selection_pdf = 0.f;
            float3 radiance = 0.f;
            float3 lightwo;
            float3 bxdfwo;
            float3 wo;
            float bxdfweight = 1.f;
            float lightweight = 1.f;

            float3 throughput = Path_GetThroughput(path);

            // Sample bxdf
            float3 bxdf = Bxdf_Sample(&diffgeo, wi, TEXTURE_ARGS, Sampler_Sample2D(&sampler, SAMPLER_ARGS), &bxdfwo, &bxdfpdf);

            bxdfwo = normalize(bxdfwo);
            float3 t = bxdf * fabs(dot(diffgeo.n, bxdfwo));

            // Write path vertex
            GLOBAL int* my_counter = subpath_length + pixel_idx;
            int idx = atom_inc(my_counter);

            if (idx < BDPT_MAX_SUBPATH_LEN)
            {
                GLOBAL PathVertex* my_vertex = subpath + BDPT_MAX_SUBPATH_LEN * pixel_idx + idx;
                GLOBAL PathVertex* my_prev_vertex = subpath + BDPT_MAX_SUBPATH_LEN * pixel_idx + idx - 1;
                // Fill vertex
                // Calculate forward PDF of a current vertex
                float pdf_solid_angle = Ray_GetExtra(&rays[hit_idx]).x;
                float pdf_fwd = Pdf_ConvertSolidAngleToArea(pdf_solid_angle, rays[hit_idx].o.xyz, diffgeo.p, diffgeo.n);

                PathVertex v;
                PathVertex_Init(&v,
                    diffgeo.p,
                    diffgeo.n,
                    diffgeo.ng,
                    diffgeo.uv,
                    pdf_fwd,
                    0.f,
                    throughput * t / bxdfpdf,
                    kSurface,
                    diffgeo.material_index);

                *my_vertex = v;

                // TODO: wrong
                if (my_prev_vertex->type == kSurface)
                {
                    float pdf_bwd = Pdf_ConvertSolidAngleToArea(Bxdf_GetPdf(&diffgeo, bxdfwo, wi, TEXTURE_ARGS), diffgeo.p, my_prev_vertex->position, my_prev_vertex->shading_normal);
                    my_prev_vertex->pdf_backward = pdf_bwd;
                }
            }
            else
            {
                atom_dec(my_counter);
                Path_Kill(path);
                Ray_SetInactive(extension_rays + global_id);
                return;
            }

            // Only continue if we have non-zero throughput & pdf
            if (NON_BLACK(t) && bxdfpdf > 0.f)
            {
                // Update the throughput
                Path_MulThroughput(path, t / bxdfpdf);

                // Generate ray
                float3 new_ray_dir = bxdfwo;
                float3 new_ray_o = diffgeo.p + CRAZY_LOW_DISTANCE * s * diffgeo.n;

                Ray_Init(extension_rays + global_id, new_ray_o, new_ray_dir, CRAZY_HIGH_DISTANCE, 0.f, 0xFFFFFFFF);
                Ray_SetExtra(extension_rays + global_id, make_float2(bxdfpdf, 0.f));
            }
            else
            {
                // Otherwise kill the path
                Path_Kill(path);
                Ray_SetInactive(extension_rays + global_id);
            }
        }
    }

    __kernel void ConnectDirect(
        int eye_vertex_index,
        int num_rays,
        GLOBAL int const* pixel_indices,
        GLOBAL PathVertex const* restrict eye_subpath,
        GLOBAL int const* restrict eye_subpath_length,
        GLOBAL float3 const* restrict vertices,
        GLOBAL float3 const* restrict normals,
        GLOBAL float2 const* restrict uvs,
        GLOBAL int const* restrict indices,
        GLOBAL Shape const* restrict shapes,
        GLOBAL int const* restrict materialids,
        GLOBAL Material const* restrict materials,
        TEXTURE_ARG_LIST,
        int env_light_idx,
        GLOBAL Light const* restrict lights,
        int num_lights,
        uint rngseed,
        GLOBAL uint const* restrict random,
        GLOBAL uint const* restrict sobolmat,
        int frame,
        GLOBAL ray* restrict shadowrays,
        GLOBAL float3* restrict lightsamples
    )
    {
        int global_id = get_global_id(0);

        if (global_id < num_rays)
        {
            int pixel_idx = pixel_indices[global_id];
            int num_eye_vertices = *(eye_subpath_length + pixel_idx);

            GLOBAL PathVertex const* my_eye_vertex = eye_subpath + BDPT_MAX_SUBPATH_LEN * pixel_idx + eye_vertex_index;
            GLOBAL PathVertex const* my_prev_eye_vertex = eye_subpath + BDPT_MAX_SUBPATH_LEN * pixel_idx + eye_vertex_index - 1;

            DifferentialGeometry diffgeo;
            diffgeo.p = my_eye_vertex->position;
            diffgeo.n = my_eye_vertex->shading_normal;
            diffgeo.ng = my_eye_vertex->geometric_normal;
            diffgeo.uv = my_eye_vertex->uv;
            diffgeo.dpdu = GetOrthoVector(diffgeo.n);
            diffgeo.dpdv = cross(diffgeo.n, diffgeo.dpdu);
            diffgeo.mat = materials[my_eye_vertex->material_index];
            diffgeo.mat.fresnel = 1.f;
            DifferentialGeometry_CalculateTangentTransforms(&diffgeo);

            float3 wi = normalize(my_prev_eye_vertex->position - diffgeo.p);
            float3 throughput = my_prev_eye_vertex->flow;

            if (eye_vertex_index >= num_eye_vertices)
            {
                lightsamples[global_id].xyz = 0.f;
                Ray_SetInactive(shadowrays + global_id);
                return;
            }

            Scene scene =
            {
                vertices,
                normals,
                uvs,
                indices,
                shapes,
                materialids,
                materials,
                lights,
                env_light_idx,
                num_lights
            };

            Sampler sampler;
#if SAMPLER == SOBOL
            uint scramble = random[global_id] * 0x1fe3434f;
            Sampler_Init(&sampler, frame, SAMPLE_DIM_SURFACE_OFFSET + eye_vertex_index * SAMPLE_DIMS_PER_BOUNCE, scramble);
#elif SAMPLER == RANDOM
            uint scramble = global_id * rngseed;
            Sampler_Init(&sampler, scramble);
#elif SAMPLER == CMJ
            uint rnd = random[global_id];
            uint scramble = rnd * 0x1fe3434f * ((frame + 331 * rnd) / (CMJ_DIM * CMJ_DIM));
            Sampler_Init(&sampler, frame % (CMJ_DIM * CMJ_DIM), SAMPLE_DIM_SURFACE_OFFSET + eye_vertex_index * SAMPLE_DIMS_PER_BOUNCE, scramble);
#endif

            float selection_pdf;
            int light_idx = Scene_SampleLight(&scene, Sampler_Sample1D(&sampler, SAMPLER_ARGS), &selection_pdf);

            // Sample light
            float lightpdf = 0.f;
            float lightbxdfpdf = 0.f;
            float bxdfpdf = 0.f;
            float bxdflightpdf = 0.f;
            float lightweight = 0.f;
            float bxdfweight = 0.f;

            float3 lightwo;
            float3 bxdfwo;
            float3 le = Light_Sample(light_idx, &scene, &diffgeo, TEXTURE_ARGS, Sampler_Sample2D(&sampler, SAMPLER_ARGS), &lightwo, &lightpdf);
            float3 bxdf = Bxdf_Sample(&diffgeo, normalize(wi), TEXTURE_ARGS, Sampler_Sample2D(&sampler, SAMPLER_ARGS), &bxdfwo, &bxdfpdf);
            lightbxdfpdf = Bxdf_GetPdf(&diffgeo, normalize(wi), normalize(lightwo), TEXTURE_ARGS);
            bxdflightpdf = Light_GetPdf(light_idx, &scene, &diffgeo, normalize(bxdfwo), TEXTURE_ARGS);

            bool singular_light = Light_IsSingular(&scene.lights[light_idx]);
            bool singular_bxdf = Bxdf_IsSingular(&diffgeo);
            lightweight = 1;// singular_light ? 1.f : BalanceHeuristic(1, lightpdf, 1, lightbxdfpdf);
            bxdfweight = 1;// singular_bxdf ? 1.f : BalanceHeuristic(1, bxdfpdf, 1, bxdflightpdf);

            float3 radiance = 0.f;
            float3 wo;

            if (singular_light && singular_bxdf)
            {
                // Otherwise save some intersector cycles
                Ray_SetInactive(shadowrays + global_id);
                lightsamples[global_id] = 0;
                return;
            }

            float split = Sampler_Sample1D(&sampler, SAMPLER_ARGS);

            if (((singular_bxdf && !singular_light) || split < 0.5f) && bxdfpdf > 0.f)
            {
                wo = CRAZY_HIGH_DISTANCE * bxdfwo;
                le = Light_GetLe(light_idx, &scene, &diffgeo, &wo, TEXTURE_ARGS);
                float ndotwo = fabs(dot(diffgeo.n, normalize(wo)));
                radiance = 2.f * le * bxdfweight * throughput * bxdf * ndotwo / bxdfpdf;
            }

            if (((!singular_bxdf && singular_light) || split >= 0.5f) && lightpdf > 0.f)
            {
                wo = lightwo;
                float ndotwo = fabs(dot(diffgeo.n, normalize(wo)));
                radiance = 2.f * lightweight * le * Bxdf_Evaluate(&diffgeo, wi, normalize(wo), TEXTURE_ARGS) * throughput * ndotwo / lightpdf / selection_pdf;
            }

            // If we have some light here generate a shadow ray
            if (NON_BLACK(radiance))
            {
                // Generate shadow ray
                float3 shadow_ray_o = diffgeo.p + CRAZY_LOW_DISTANCE * diffgeo.n;
                float3 temp = diffgeo.p + wo - shadow_ray_o;
                float3 shadow_ray_dir = normalize(temp);
                float shadow_ray_length = length(temp);
                int shadow_ray_mask = 0xFFFFFFFF;

                Ray_Init(shadowrays + global_id, shadow_ray_o, shadow_ray_dir, shadow_ray_length, 0.f, shadow_ray_mask);

                // And write the light sample
                lightsamples[global_id] = REASONABLE_RADIANCE(radiance);
            }
            else
            {
                // Otherwise save some intersector cycles
                Ray_SetInactive(shadowrays + global_id);
                lightsamples[global_id] = 0;
            }
        }
    }

    KERNEL void Connect(
        int num_rays,
        // Index of eye vertex we are trying to connect
        int eye_vertex_index,
        // Index of light vertex we are trying to connect
        int light_vertex_index,
        GLOBAL int const* pixel_indices,
        // Vertex arrays
        GLOBAL PathVertex const* restrict eye_subpath,
        GLOBAL int const* restrict eye_subpath_length,
        GLOBAL PathVertex const* restrict light_subpath,
        GLOBAL int const* restrict light_subpath_length,
        GLOBAL Material const* restrict materials,
        // Textures
        TEXTURE_ARG_LIST,
        GLOBAL ray* connection_rays,
        // Path contriburions to final image
        GLOBAL float4* contributions
    )
    {
        int global_id = get_global_id(0);

        if (global_id >= num_rays)
        {
            return;
        }

        int pixel_idx = pixel_indices[global_id];

        // Check if we our indices are within subpath index range
        if (eye_vertex_index >= eye_subpath_length[pixel_idx] ||
            light_vertex_index >= light_subpath_length[global_id])
        {
            contributions[global_id].xyz = 0.f;
            Ray_SetInactive(&connection_rays[global_id]);
            return;
        }

        // Prepare data pointers
        GLOBAL PathVertex const* my_eye_vertex = eye_subpath + BDPT_MAX_SUBPATH_LEN * pixel_idx + eye_vertex_index;
        GLOBAL PathVertex const* my_prev_eye_vertex = eye_subpath + BDPT_MAX_SUBPATH_LEN * pixel_idx + eye_vertex_index - 1;

        GLOBAL PathVertex const* my_light_vertex = light_subpath + BDPT_MAX_SUBPATH_LEN * global_id + light_vertex_index;
        GLOBAL PathVertex const* my_prev_light_vertex = light_subpath + BDPT_MAX_SUBPATH_LEN * global_id + light_vertex_index - 1;

        // Fill differential geometries for both eye and light vertices
        DifferentialGeometry eye_dg;
        eye_dg.p = my_eye_vertex->position;
        eye_dg.n = my_eye_vertex->shading_normal;
        eye_dg.ng = my_eye_vertex->geometric_normal;
        eye_dg.uv = my_eye_vertex->uv;
        eye_dg.dpdu = GetOrthoVector(eye_dg.n);
        eye_dg.dpdv = cross(eye_dg.n, eye_dg.dpdu);
        eye_dg.mat = materials[my_eye_vertex->material_index];
        eye_dg.mat.fresnel = 1.f;
        DifferentialGeometry_CalculateTangentTransforms(&eye_dg);

        DifferentialGeometry light_dg;
        light_dg.p = my_light_vertex->position;
        light_dg.n = my_light_vertex->shading_normal;
        light_dg.ng = my_light_vertex->geometric_normal;
        light_dg.uv = my_light_vertex->uv;
        light_dg.dpdu = GetOrthoVector(light_dg.n);
        light_dg.dpdv = cross(light_dg.dpdu, light_dg.n);
        light_dg.mat = materials[my_light_vertex->material_index];
        light_dg.mat.fresnel = 1.f;
        DifferentialGeometry_CalculateTangentTransforms(&light_dg);

        // Vector from eye subpath vertex to previous eye subpath vertex (incoming vector)
        float3 eye_wi = normalize(my_prev_eye_vertex->position - eye_dg.p);
        // Vector from camera subpath vertex to light subpath vertex (connection vector)
        float3 eye_wo = normalize(light_dg.p - eye_dg.p);
        // Distance between vertices we are connecting
        float dist = length(light_dg.p - eye_dg.p);
        // BXDF at eye subpath vertex
        float3 eye_bxdf = Bxdf_Evaluate(&eye_dg, eye_wi, eye_wo, TEXTURE_ARGS);
        // Eye subpath vertex cosine factor
        float eye_dotnl = fabs(dot(eye_dg.n, eye_wo));

        // Calculate througput of a segment eye->eyevetex0->...->eyevertexN->lightvertexM->lightvertexM-1...->light
        // First eye subpath throughput eye->eyevertex0->...eyevertexN
        float3 eye_contribution = my_prev_eye_vertex->flow;
        eye_contribution *= (eye_bxdf * eye_dotnl);

        // Vector from light subpath vertex to prev light subpath vertex (incoming light vector)
        float3 light_wi = normalize(my_prev_light_vertex->position - light_dg.p);
        // Outgoing light subpath vector
        float3 light_wo = -eye_wo;
        // BXDF at light subpath vertex
        float3 light_bxdf = Bxdf_Evaluate(&light_dg, light_wi, light_wo, TEXTURE_ARGS);
        // Light subpath vertex cosine factor
        float light_dotnl = fabs(dot(light_dg.n, light_wo));

        // Light subpath throughput lightvertexM->lightvertexM-1->light
        float3 light_contribution = my_prev_light_vertex->flow;
        light_contribution *= (light_bxdf * light_dotnl);

        // Now we need to figure out MIS weight for the path we are trying to build
        // connecting these vertices.
        // MIS weight should account for all possible paths of the same length using
        // same vertices, but generated using different subpath lengths, for example
        // Original: eye0 eye1 eye2 eye3 | connect | light3 light2 light1 light0
        // Variant 1: eye0 eye1 eye2 | connect | eye3 light3 light2 light1 light0
        // as if only eye0->eye1->eye2 was generated using eye walk,
        // and eye3->light3->light2->light1->light0 was generated using light walk.

        // This is PDF of sampling eye subpath vertex from previous eye subpath vertex using eye walk strategy.
        float eye_vertex_pdf_fwd = my_eye_vertex->pdf_forward;
        // This is PDF of sampling light subpath vertex from previous light subpath vertex using light walk strategy.
        float light_vertex_pdf_fwd = my_light_vertex->pdf_forward;

        // Important: this is PDF of sampling eye subpath vertex from light subpath vertex using light walk strategy.
        float eye_vertex_pdf_bwd = Pdf_ConvertSolidAngleToArea(Bxdf_GetPdf(&light_dg, light_wi, light_wo, TEXTURE_ARGS),
            eye_dg.p, light_dg.p, light_dg.n);
        // Important: this is PDF of sampling prev eye subpath vertex from eye subpath vertex using light walk strategy.
        float eye_prev_vertex_pdf_bwd = Pdf_ConvertSolidAngleToArea(Bxdf_GetPdf(&eye_dg, eye_wi, eye_wo, TEXTURE_ARGS),
            my_prev_eye_vertex->position, eye_dg.p, eye_dg.n);

        // Important: this is PDF of sampling light subpath vertex from eye subpath vertex using eye walk strategy.
        float light_vertex_pdf_bwd = Pdf_ConvertSolidAngleToArea(Bxdf_GetPdf(&eye_dg, eye_wo, eye_wi, TEXTURE_ARGS),
            light_dg.p, eye_dg.p, eye_dg.n);
        // Important: this is PDF of sampling prev light subpath vertex from light subpath vertex using eye walk strategy.
        float light_prev_vertex_pdf_bwd = Pdf_ConvertSolidAngleToArea(Bxdf_GetPdf(&light_dg, light_wo, light_wi, TEXTURE_ARGS),
            my_prev_light_vertex->position, light_dg.p, light_dg.n);

        // We need sum of all weights
        float weights_sum = (eye_vertex_pdf_bwd / eye_vertex_pdf_fwd);
        float weights_eye = (eye_vertex_pdf_bwd / eye_vertex_pdf_fwd) * (eye_prev_vertex_pdf_bwd / my_prev_eye_vertex->pdf_forward);
        weights_sum += weights_eye;

        weights_sum += (light_vertex_pdf_bwd / light_vertex_pdf_fwd);
        float weights_light = (light_vertex_pdf_bwd / light_vertex_pdf_fwd) * (light_prev_vertex_pdf_bwd / my_prev_light_vertex->pdf_forward);
        weights_sum += weights_light;

        // Iterate eye subpath
        // (we already accounted for prev vertex, so start from one before, that's why -2)
        for (int i = eye_vertex_index - 2; i >= 0; --i)
        {
            GLOBAL PathVertex const* v = &eye_subpath[BDPT_MAX_SUBPATH_LEN * pixel_idx + i];
            weights_eye *= (v->pdf_backward / v->pdf_forward);
            weights_sum += weights_eye;
        }

        // Iterate light subpath
        // (we already accounted for prev vertex, so start from one before, that's why -2)
        for (int i = light_vertex_index - 2; i >= 0; --i)
        {
            GLOBAL PathVertex const* v = &light_subpath[BDPT_MAX_SUBPATH_LEN * global_id + i];
            weights_light *= (v->pdf_backward / v->pdf_forward);
            weights_sum += weights_light;
        }

        float mis_weight = 1.f / (1.f + weights_sum);
        contributions[global_id].xyz = REASONABLE_RADIANCE(mis_weight * (eye_contribution * light_contribution) / (dist * dist));

        float3 connect_ray_o = eye_dg.p + CRAZY_LOW_DISTANCE * eye_dg.n;
        float3 temp = light_dg.p - connect_ray_o;
        float  connect_ray_length = length(temp);
        float3 connect_ray_dir = eye_wo;

        Ray_Init(&connection_rays[global_id], connect_ray_o, connect_ray_dir, connect_ray_length, 0.f, 0xFFFFFFFF);
    }

// Generate pixel indices withing a tile
KERNEL void GenerateTileDomain(
    // Output buffer width
    int output_width,
    // Output buffer height
    int output_height,
    // Tile offset within an output buffer
    int tile_offset_x,
    int tile_offset_y,
    // Tile size
    int tile_width,
    int tile_height,
    // Subtile size (tile can be organized in subtiles)
    int subtile_width,
    int subtile_height,
    // Pixel indices
    GLOBAL int* restrict pixel_idx0,
    GLOBAL int* restrict pixel_idx1,
    // Pixel count
    GLOBAL int* restrict count
)
{
    int tile_x = get_global_id(0);
    int tile_y = get_global_id(1);
    int tile_start_idx = output_width * tile_offset_y + tile_offset_x;

    if (tile_x < tile_width && tile_y < tile_height)
    {
        // TODO: implement subtile support
        int idx = tile_start_idx + tile_y * output_width + tile_x;
        pixel_idx0[tile_y * tile_width + tile_x] = idx;
        pixel_idx1[tile_y * tile_width + tile_x] = idx;
    }

    if (tile_x == 0 && tile_y == 0)
    {
        *count = tile_width * tile_height;
    }
}

///< Restore pixel indices after compaction
__kernel void FilterPathStream(
    // Intersections
    __global Intersection const* isects,
    // Number of compacted indices
    __global int const* numitems,
    // Pixel indices
    __global int const* pixelindices,
    // Paths
    __global Path* paths,
    // Predicate
    __global int* predicate
)
{
    int globalid = get_global_id(0);

    // Handle only working subset
    if (globalid < *numitems)
    {
        int pixelidx = pixelindices[globalid];

        __global Path* path = paths + pixelidx;

        if (Path_IsAlive(path))
        {
            bool kill = (length(Path_GetThroughput(path)) < CRAZY_LOW_THROUGHPUT);

            if (!kill)
            {
                predicate[globalid] = isects[globalid].shapeid >= 0 ? 1 : 0;
            }
            else
            {
                Path_Kill(path);
                predicate[globalid] = 0;
            }
        }
        else
        {
            predicate[globalid] = 0;
        }
    }
}

KERNEL void IncrementSampleCounter(
    // Number of rays
    int num_rays,
    GLOBAL int const* restrict pixel_indices,
    // Radiance sample buffer
    GLOBAL float4* restrict output
)
{
    int global_id = get_global_id(0);

    if (global_id < num_rays)
    {
        int pixel_idx = pixel_indices[global_id];
        // Start collecting samples
        {
            output[pixel_idx].w += 1.f;
        }
    }
}

KERNEL void GatherContributions(
    // Number of rays
    int num_rays,
    GLOBAL int const* restrict pixel_indices,
    // Shadow rays hits
    GLOBAL int const* restrict shadowhits,
    // Light samples
    GLOBAL float3 const* restrict contributions,
    // Radiance sample buffer
    GLOBAL float4* output
)
{
    int global_id = get_global_id(0);

    if (global_id < num_rays)
    {
        int pixel_idx = pixel_indices[global_id];
        // Prepare accumulator variable
        float3 radiance = make_float3(0.f, 0.f, 0.f);

        // Start collecting samples
        {
            // If shadow ray global_id't hit anything and reached skydome
            if (shadowhits[global_id] == -1)
            {
                // Add its contribution to radiance accumulator
                radiance += contributions[global_id];
                output[pixel_idx].xyz += radiance;
            }
        }
    }
}

///< Restore pixel indices after compaction
__kernel void RestorePixelIndices(
    // Compacted indices
    __global int const* compacted_indices,
    // Number of compacted indices
    __global int* numitems,
    // Previous pixel indices
    __global int const* previndices,
    // New pixel indices
    __global int* newindices
)
{
    int globalid = get_global_id(0);

    // Handle only working subset
    if (globalid < *numitems)
    {
        newindices[globalid] = previndices[compacted_indices[globalid]];
    }
}

// Copy data to interop texture if supported
__kernel void ApplyGammaAndCopyData(
    __global float4 const* data,
    int imgwidth,
    int imgheight,
    float gamma,
    write_only image2d_t img
)
{
    int gid = get_global_id(0);

    int gidx = gid % imgwidth;
    int gidy = gid / imgwidth;

    if (gidx < imgwidth && gidy < imgheight)
    {
        float4 v = data[gid];
        float4 val = clamp(native_powr(v / v.w, 1.f / gamma), 0.f, 1.f);
        write_imagef(img, make_int2(gidx, gidy), val);
    }
}


HSA: native_recip.cl: there is no recip() builtin.

/* OpenCL built-in library: native_recip()

   Copyright (c) 2015 Michal Babej / Tampere University of Technology

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/
#include "hsail_templates.h"

DEFINE_LLVM_INTRIN_FP32_FP64(native_recip, V_V, hsail.nrcp, ((vtype)(1.0) / a))

/* OpenCL built-in library: native_recip()

   Copyright (c) 2015 Michal Babej / Tampere University of Technology

   Permission is hereby granted, free of charge, to any person obtaining a copy
   of this software and associated documentation files (the "Software"), to deal
   in the Software without restriction, including without limitation the rights
   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
   copies of the Software, and to permit persons to whom the Software is
   furnished to do so, subject to the following conditions:

   The above copyright notice and this permission notice shall be included in
   all copies or substantial portions of the Software.

   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
   THE SOFTWARE.
*/
#include "hsail_templates.h"

DEFINE_LLVM_INTRIN_FP32_FP64(native_recip, V_V, hsail.nrcp, recip(a))


DEVICE kernel Echo_8way

DEVICE kernel Echo_8way

Now works with new drivers. Clean up some shit which didn't make sense
(how did I end up writing this shit)?
It also ends up being faster... 8-10%

/*
 * Copyright (C) 2014 Massimo Del Zotto
 * This code is released under the MIT license.
 * For conditions of distribution and use, see the LICENSE or hit the web.
 */
void AESRoundLDS(local uint *o0, local uint *o1, local uint *o2, local uint *o3, uint k0, uint k1, uint k2, uint k3,
                 local uint *lut0, local uint *lut1, local uint *lut2, local uint *lut3) {
#if __ENDIAN_LITTLE__
#define LUT(li, val)  (lut##li != 0? lut##li[(val >> (8 * li)) & 0xFF] : rotate(lut0[(val >> (8 * li)) & 0xFF], (8u * li##u)))

	uint i0 = *o0;
	uint i1 = *o1;
	uint i2 = *o2;
	uint i3 = *o3;
	*o0 = lut0[i0 & 0xFF] ^ LUT(1, i1) ^ LUT(2, i2) ^ LUT(3, i3) ^ k0;
	*o1 = lut0[i1 & 0xFF] ^ LUT(1, i2) ^ LUT(2, i3) ^ LUT(3, i0) ^ k1;
	*o2 = lut0[i2 & 0xFF] ^ LUT(1, i3) ^ LUT(2, i0) ^ LUT(3, i1) ^ k2;
	*o3 = lut0[i3 & 0xFF] ^ LUT(1, i0) ^ LUT(2, i1) ^ LUT(3, i2) ^ k3;

#undef LUT
#else
#error Endianness?
#endif
}

void AESRoundNoKeyLDS(local uint *o0, local uint *o1, local uint *o2, local uint *o3,
                   local uint *lut0, local uint *lut1, local uint *lut2, local uint *lut3) {
	AESRoundLDS(o0, o1, o2, o3, 0, 0, 0, 0, lut0, lut1, lut2, lut3);
}

/* stop of repeated code, below is echo specific */

void Increment(uint4 *modify, uint add) {
	uint4 inc = *modify;
	inc.x += add;
	if(inc.x < (*modify).x) { // wrapped
		inc.y++;
		if(inc.y == 0) {
			inc.z++;
			if(inc.z == 0) inc.w++;
		}
	}
	*modify = inc;
}

uint2 hilo(uint2 val) {
#if __ENDIAN_LITTLE__
	return val.yx;
#else
#error Endianness?
#endif
}

uint2 Echo_RoundMangle(uint2 vec) {
	ulong big = (convert_ulong(vec.x) << 32) | vec.y;
	const ulong a = (big & 0x8080808080808080ul) >> 7u;
	const ulong b = (big & 0x7F7F7F7F7F7F7F7Ful) << 1u;
	big = (a * 27u) ^ b;
	return (uint2)(convert_uint(big >> 32), convert_uint(big));
}

uint Wrap(uint offset) {
	uint displaced = offset + (get_local_id(0) % 4);
	displaced %= 4;
	if(offset % 2) displaced += (get_local_id(0) < 4? 4 : -4);
	displaced += offset * 64; // offset is also row index
	return displaced;
}

#if defined ECHO_IS_LAST
__attribute__((reqd_work_group_size(8, 8, 1)))
kernel void Echo_8way(global uint2 *input, global uint *found, global uint *dispatchData, global uint *aes_round_luts) {
#else
__attribute__((reqd_work_group_size(8, 8, 1)))
kernel void Echo_8way(global uint2 *input, global uint2 *hashOut, global uint *aes_round_luts, global uint *debug) {
#endif
    input += (get_global_id(1) - get_global_offset(1)) * 8;
    local uint aesLUT0[256];
	event_t ldsReady = async_work_group_copy(aesLUT0, aes_round_luts + 256 * 0, 256, 0);
#if defined AES_TABLE_ROW_1
    local uint aesLUT1[256];
	async_work_group_copy(aesLUT1, aes_round_luts + 256 * 1, 256, ldsReady);
#else
	local uint *aesLUT1 = 0;
#endif
#if defined AES_TABLE_ROW_2
    local uint aesLUT2[256];
	async_work_group_copy(aesLUT2, aes_round_luts + 256 * 2, 256, ldsReady);
#else
	local uint *aesLUT2 = 0;
#endif
#if defined AES_TABLE_ROW_3
    local uint aesLUT3[256];
	async_work_group_copy(aesLUT3, aes_round_luts + 256 * 3, 256, ldsReady);
#else
	local uint *aesLUT3 = 0;
#endif

	/* Legacy 1-way kernels here have a boatload of registers here. Think at them as
		ulong W[16][2], Vb[8][2];
	If you look carefully, you'll see Vb is constants.
	As for W, each thread here works "on a column", so across the work items registers are
	WI0 using 00, 10, 20, 30    	WI [4-7] use 01,11,21,31 instead.
	  1       40  50  60  70
	  2       80  90  A0  B0
	  3       C0  D0  E0  F0
	This allows to keep the MIX_COLUMN operation inside a single WI, which is good for perf
	because it would require re-sync before next iteration.
	Those registers are workN, the other index is 0 if localindex(0)<4, otherwise 1. */
	uint2 work0, work1, work2, work3;
    uint4 notSoK = (uint4)(512, 0, 0, 0);
	notSoK.x += (get_local_id(0) % 4) * 4 + (get_local_id(0) < 4? 0 : 1);

	switch(get_local_id(0)) {
	case 0:
	case 1:
		work0 = work1 = work2 = work3 = (uint2)(0, 512);
		break;
	case 3:
	case 7:
		work0 = get_local_id(0) < 4? (uint2)(0, 0x80)   : (uint2)(0);
		work1 = get_local_id(0) < 4? (uint2)(0)         : (uint2)(0);
		work2 = get_local_id(0) < 4? (uint2)(0)         : (uint2)(0x02000000, 0);
		work3 = get_local_id(0) < 4? (uint2)(0, 0x0200) : (uint2)(0);
		break;
	case 4:
	case 5:
		work0 = work1 = work2 = work3 = (uint2)(0);
		break;
	case 2:
	case 6:
		work0 = hilo(input[0 + (get_local_id(0) < 4? 0 : 1)]);
		work1 = hilo(input[2 + (get_local_id(0) < 4? 0 : 1)]);
		work2 = hilo(input[4 + (get_local_id(0) < 4? 0 : 1)]);
		work3 = hilo(input[6 + (get_local_id(0) < 4? 0 : 1)]);
		break;
	}

	wait_group_events(1, &ldsReady); // this is really necessary only a few lines below but here is nicer for structure
	local uint passhi[4 * 8 * 8];
	local uint passlo[4 * 8 * 8];

	uint evnSlot = get_local_id(0) + get_local_id(1) * 8;
	uint oddSlot = get_local_id(0) + get_local_id(1) * 8 + (get_local_id(0) < 4? 4 : -4);
	passhi[64 * 0 + evnSlot] = work0.hi;    passlo[64 * 0 + evnSlot] = work0.lo;
	passhi[64 * 1 + oddSlot] = work1.hi;    passlo[64 * 1 + oddSlot] = work1.lo;
	passhi[64 * 2 + evnSlot] = work2.hi;    passlo[64 * 2 + evnSlot] = work2.lo;
	passhi[64 * 3 + oddSlot] = work3.hi;    passlo[64 * 3 + oddSlot] = work3.lo;

	for (unsigned u = 0; u < 10; u ++) {
		{ /* Legacy kernel performs 16 passes of two AES rounds on the various registers. WorkNo, WorkNi.
			I do that explicitly as I already "unrolled" those 16 passes across 8 WI. Note I need to increment
			the K values differently. */
			uint slot = get_local_id(1) * 8; // location of W00 for this group of 8 threads
			slot += get_local_id(0) % 4; // selected a column
			slot += get_local_id(0) < 4? 0 : 64; // go down a line skipping values not yours
			const int toi = get_local_id(0) < 4? 4 : 0;
			const int too = get_local_id(0) < 4? 0 : 4;
			{
				local uint *x0 = passhi + slot + too, *x1 = passlo + slot + too;
				local uint *x2 = passhi + slot + toi, *x3 = passlo + slot + toi;
				barrier(CLK_LOCAL_MEM_FENCE);
				AESRoundLDS(x0, x1, x2, x3, notSoK.x, notSoK.y, notSoK.z, notSoK.w, aesLUT0, aesLUT1, aesLUT2, aesLUT3);
				barrier(CLK_LOCAL_MEM_FENCE);
				AESRoundNoKeyLDS(x0, x1, x2, x3, aesLUT0, aesLUT1, aesLUT2, aesLUT3);
				Increment(&notSoK, 2);
			}
			slot += 8 * 8 * 2;
			{
				local uint *x0 = passhi + slot + too, *x1 = passlo + slot + too;
				local uint *x2 = passhi + slot + toi, *x3 = passlo + slot + toi;
				barrier(CLK_LOCAL_MEM_FENCE);
				AESRoundLDS(x0, x1, x2, x3, notSoK.x, notSoK.y, notSoK.z, notSoK.w, aesLUT0, aesLUT1, aesLUT2, aesLUT3);
				barrier(CLK_LOCAL_MEM_FENCE);
				AESRoundNoKeyLDS(x0, x1, x2, x3, aesLUT0, aesLUT1, aesLUT2, aesLUT3);
				Increment(&notSoK, 16 - 2);
			}
		}
		/* I somehow managed to write shift-rows merged with mix-column... but it was fairly complicated
		and it broke (I'm surprised it worked). It's just better to go with something easier and do it as standard.
		Instead of rotating the rows, I just fetch the values to registers and be done with it. */
		{
			barrier(CLK_LOCAL_MEM_FENCE);
			local uint *halfhi = passhi + (get_local_id(0) < 4? 0 : 4) + get_local_id(1) * 8;
			local uint *halflo = passlo + (get_local_id(0) < 4? 0 : 4) + get_local_id(1) * 8;
			work0.hi = halfhi[Wrap(0)];    work0.lo = halflo[Wrap(0)];
			work1.hi = halfhi[Wrap(1)];    work1.lo = halflo[Wrap(1)];
			work2.hi = halfhi[Wrap(2)];    work2.lo = halflo[Wrap(2)];
			work3.hi = halfhi[Wrap(3)];    work3.lo = halflo[Wrap(3)];
		}
		{ // MIX_COLUMN by registers is super easy, now almost identical to SPH
			uint2 a = work0, b = work1, c = work2, d = work3; // I got plenty registers anyway!
			uint2 ab = a ^ b;
			uint2 bc = b ^ c;
			uint2 cd = c ^ d;
			uint2 abx = Echo_RoundMangle(ab);
			uint2 bcx = Echo_RoundMangle(bc);
			uint2 cdx = Echo_RoundMangle(cd);
			work0 = abx ^ bc ^ d;
			work1 = bcx ^ a ^ cd;
			work2 = cdx ^ ab ^ d;
			work3 = abx ^ bcx ^ cdx ^ ab ^ c;
		}
		passhi[64 * 0 + evnSlot] = work0.hi;    passlo[64 * 0 + evnSlot] = work0.lo;
		passhi[64 * 1 + oddSlot] = work1.hi;    passlo[64 * 1 + oddSlot] = work1.lo;
		passhi[64 * 2 + evnSlot] = work2.hi;    passlo[64 * 2 + evnSlot] = work2.lo;
		passhi[64 * 3 + oddSlot] = work3.hi;    passlo[64 * 3 + oddSlot] = work3.lo;
	}
	barrier(CLK_LOCAL_MEM_FENCE);
	// End, some bank collisions here but not much of a problem.
	const uint row = get_local_id(0) / 2; // 00,01,10,11,20,21,30,31 ^ 80,81,90,91,A0,A1,B0,B1
	const int rowbeg = get_local_id(1) * 8 + row * 64;
	int no = rowbeg + ((row + get_local_id(0)) % 2 == 0? 0 : 4);
	int ni = no + 2;
	const uint2 noval = (uint2)(passhi[no], passlo[no]);
	const uint2 nival = (uint2)(passhi[ni], passlo[ni]);
	const uint2 xorv = get_local_id(0) % 2 == 0? (uint2)(512, 0) : (uint2)(0);
	const uint2 myHash = xorv ^ input[get_local_id(0)] ^ noval ^ nival;

#if defined ECHO_IS_LAST
	if(get_local_id(0) == 3) {
		ulong magic = (((ulong)myHash.y) << 32) | myHash.x;
		ulong target = (((ulong)dispatchData[1]) << 32) | dispatchData[2]; // watch out for endianess!
		if(magic <= target) {
			uint storage = atomic_inc(found);
			found[storage + 1] = as_uint(as_char4(get_global_id(1)).wzyx); // watch out for endianess!
		}
	}
#else
#error to be tested!
	hashOut += (get_global_id(1) - get_global_offset(1)) * 8;
	hashOut[get_local_id(0)] = myHash; // maybe swap uints
#endif
}

/*
 * Copyright (C) 2014 Massimo Del Zotto
 * This code is released under the MIT license.
 * For conditions of distribution and use, see the LICENSE or hit the web.
 */
void AESRoundLDS(local uint *o0, local uint *o1, local uint *o2, local uint *o3, uint k0, uint k1, uint k2, uint k3,
                 local uint *lut0, local uint *lut1, local uint *lut2, local uint *lut3) {
#if __ENDIAN_LITTLE__
#define LUT(li, val)  (lut##li != 0? lut##li[(val >> (8 * li)) & 0xFF] : rotate(lut0[(val >> (8 * li)) & 0xFF], (8u * li##u)))

	uint i0 = *o0;
	uint i1 = *o1;
	uint i2 = *o2;
	uint i3 = *o3;
	*o0 = lut0[i0 & 0xFF] ^ LUT(1, i1) ^ LUT(2, i2) ^ LUT(3, i3) ^ k0;
	*o1 = lut0[i1 & 0xFF] ^ LUT(1, i2) ^ LUT(2, i3) ^ LUT(3, i0) ^ k1;
	*o2 = lut0[i2 & 0xFF] ^ LUT(1, i3) ^ LUT(2, i0) ^ LUT(3, i1) ^ k2;
	*o3 = lut0[i3 & 0xFF] ^ LUT(1, i0) ^ LUT(2, i1) ^ LUT(3, i2) ^ k3;

#undef LUT
#else
#error Endianness?
#endif
}

void AESRoundNoKeyLDS(local uint *o0, local uint *o1, local uint *o2, local uint *o3,
                   local uint *lut0, local uint *lut1, local uint *lut2, local uint *lut3) {
	AESRoundLDS(o0, o1, o2, o3, 0, 0, 0, 0, lut0, lut1, lut2, lut3);
}

/* stop of repeated code, below is echo specific */

void Increment(uint4 *modify, uint add) {
	uint4 inc = *modify;
	inc.x += add;
	if(inc.x < (*modify).x) { // wrapped
		inc.y++;
		if(inc.y == 0) {
			inc.z++;
			if(inc.z == 0) inc.w++;
		}
	}
	*modify = inc;
}

uint2 hilo(uint2 val) {
#if __ENDIAN_LITTLE__
	return val.yx;
#else
#error Endianness?
#endif
}

uint2 Echo_RoundMangle(uint2 vec) {
	ulong big = (convert_ulong(vec.x) << 32) | vec.y;
	const ulong a = (big & 0x8080808080808080ul) >> 7u;
	const ulong b = (big & 0x7F7F7F7F7F7F7F7Ful) << 1u;
	big = (a * 27u) ^ b;
	return (uint2)(convert_uint(big >> 32), convert_uint(big));
}

// Returns the index of a given uint relative to address of W00 for a WI so &W00 + RegIndex(5, 0) points W50.
uint Echo_RegSlot(uint major, bool minor) {
	uint col = major / 4;
	uint row = major % 4;
	bool odd = row % 2 != 0;
	uint base = minor ^ odd? 4 : 0;
	return row * 64 + base + col;
}

// Row shifting is performed in parallel across WNo and WNi so this is fairly simple:
uint Echo_ShiftRow(uint maj) {
	return (maj + maj * 4) % 16;
}

// Helper func for even easier looping in mix column op
uint2 ECHO_SHIFTVEC(local uint *hi, local uint *lo, uint i) {
	uint off = Echo_RegSlot(Echo_ShiftRow(((get_local_id(0) % 4) * 4) + i), get_local_id(0) >= 4);
	off += get_local_id(1) * 8;
	return (uint2)(hi[off], lo[off]);
}

#if defined ECHO_IS_LAST
__attribute__((reqd_work_group_size(8, 8, 1)))
kernel void Echo_8way(global uint2 *input, global uint *found, global uint *dispatchData, global uint *aes_round_luts) {
#else
__attribute__((reqd_work_group_size(8, 8, 1)))
kernel void Echo_8way(global uint2 *input, global uint2 *hashOut, global uint *aes_round_luts, global uint *debug) {
#endif
    input += (get_global_id(1) - get_global_offset(1)) * 8;
    local uint aesLUT0[256];
	event_t ldsReady = async_work_group_copy(aesLUT0, aes_round_luts + 256 * 0, 256, 0);
#if defined AES_TABLE_ROW_1
    local uint aesLUT1[256];
	async_work_group_copy(aesLUT1, aes_round_luts + 256 * 1, 256, ldsReady);
#else
	local uint *aesLUT1 = 0;
#endif
#if defined AES_TABLE_ROW_2
    local uint aesLUT2[256];
	async_work_group_copy(aesLUT2, aes_round_luts + 256 * 2, 256, ldsReady);
#else
	local uint *aesLUT2 = 0;
#endif
#if defined AES_TABLE_ROW_3
    local uint aesLUT3[256];
	async_work_group_copy(aesLUT3, aes_round_luts + 256 * 3, 256, ldsReady);
#else
	local uint *aesLUT3 = 0;
#endif

	/* Legacy 1-way kernels here have a boatload of registers here. Think at them as
		ulong W[16][2], Vb[8][2];
	If you look carefully, you'll see Vb is constants.
	As for W, each thread here works "on a column", so across the work items registers are
	WI0 using 00, 10, 20, 30    	WI [4-7] use 01,11,21,31 instead.
	  1       40  50  60  70
	  2       80  90  A0  B0
	  3       C0  D0  E0  F0
	This allows to keep the MIX_COLUMN operation inside a single WI, which is good for perf
	because it would require re-sync before next iteration.
	Those registers are workN, the other index is 0 if localindex(0)<4, otherwise 1. */
	uint2 work0, work1, work2, work3;
    uint4 notSoK = (uint4)(512, 0, 0, 0);
	notSoK.x += (get_local_id(0) % 4) * 4 + (get_local_id(0) < 4? 0 : 1);

	switch(get_local_id(0)) {
	case 0:
	case 1:
		work0 = work1 = work2 = work3 = (uint2)(0, 512);
		break;
	case 3:
	case 7:
		work0 = get_local_id(0) < 4? (uint2)(0, 0x80)   : (uint2)(0);
		work1 = get_local_id(0) < 4? (uint2)(0)         : (uint2)(0);
		work2 = get_local_id(0) < 4? (uint2)(0)         : (uint2)(0x02000000, 0);
		work3 = get_local_id(0) < 4? (uint2)(0, 0x0200) : (uint2)(0);
		break;
	case 4:
	case 5:
		work0 = work1 = work2 = work3 = (uint2)(0);
		break;
	case 2:
	case 6:
		work0 = hilo(input[0 + (get_local_id(0) < 4? 0 : 1)]);
		work1 = hilo(input[2 + (get_local_id(0) < 4? 0 : 1)]);
		work2 = hilo(input[4 + (get_local_id(0) < 4? 0 : 1)]);
		work3 = hilo(input[6 + (get_local_id(0) < 4? 0 : 1)]);
		break;
	}

	wait_group_events(1, &ldsReady); // this is really necessary only a few lines below but here is nicer for structure
	local uint passhi[4 * 8 * 8];
	local uint passlo[4 * 8 * 8];

	uint evnSlot = get_local_id(0) + get_local_id(1) * 8;
	uint oddSlot = get_local_id(0) + get_local_id(1) * 8 + (get_local_id(0) < 4? 4 : -4);
	passhi[64 * 0 + evnSlot] = work0.hi;    passlo[64 * 0 + evnSlot] = work0.lo;
	passhi[64 * 1 + oddSlot] = work1.hi;    passlo[64 * 1 + oddSlot] = work1.lo;
	passhi[64 * 2 + evnSlot] = work2.hi;    passlo[64 * 2 + evnSlot] = work2.lo;
	passhi[64 * 3 + oddSlot] = work3.hi;    passlo[64 * 3 + oddSlot] = work3.lo;

	for (unsigned u = 0; u < 10; u ++) {
		barrier(CLK_LOCAL_MEM_FENCE);
		{ /* Legacy kernel performs 16 passes of two AES rounds on the various registers. WorkNo, WorkNi.
			I do that explicitly as I already "unrolled" those 16 passes across 8 WI. Note I need to increment
			the K values differently. */
			uint slot = get_local_id(1) * 8; // location of W00 for this group of 8 threads
			slot += get_local_id(0) % 4; // selected a column
			slot += get_local_id(0) < 4? 0 : 64; // go down a line skipping values not yours
			const int toi = get_local_id(0) < 4? 4 : 0;
			const int too = get_local_id(0) < 4? 0 : 4;
			{
				local uint *x0 = passhi + slot + too, *x1 = passlo + slot + too;
				local uint *x2 = passhi + slot + toi, *x3 = passlo + slot + toi;
				AESRoundLDS(x0, x1, x2, x3, notSoK.x, notSoK.y, notSoK.z, notSoK.w, aesLUT0, aesLUT1, aesLUT2, aesLUT3);
				AESRoundNoKeyLDS(x0, x1, x2, x3, aesLUT0, aesLUT1, aesLUT2, aesLUT3);
				Increment(&notSoK, 2);
			}
			slot += 8 * 8 * 2;
			{
				local uint *x0 = passhi + slot + too, *x1 = passlo + slot + too;
				local uint *x2 = passhi + slot + toi, *x3 = passlo + slot + toi;
				AESRoundLDS(x0, x1, x2, x3, notSoK.x, notSoK.y, notSoK.z, notSoK.w, aesLUT0, aesLUT1, aesLUT2, aesLUT3);
				AESRoundNoKeyLDS(x0, x1, x2, x3, aesLUT0, aesLUT1, aesLUT2, aesLUT3);
				Increment(&notSoK, 16 - 2);
			}
		}
		/* Legacy kernels have a shift-rows operations there. It looks compliceted because they use registers.
		Registers cannot be accessed dynamically, but LDS can so the real deal is the following:
			register index to fetch = (N + N * 4) % 16
		where N is the major index (0-F) of the register I am fetching.
		LDS matrix is in the form of 0o4o8oCo0i4i8iCi... with odd lines having Is first and Os later.
		So we just access the correct index when accessing and this is basically NOP as we do it "inline" */
		{ // Column mixing is very easy now I am in column mode, prev value already in LDS :)
			uint2 tmp[3];
			uint2 fin[3];
			for(uint i = 0; i < 3; i++) {
				tmp[i]  = ECHO_SHIFTVEC(passhi, passlo, i + 0);
				tmp[i] ^= ECHO_SHIFTVEC(passhi, passlo, i + 1);
				fin[i] = Echo_RoundMangle(tmp[i]);
			}
			work0 = hilo(fin[0] ^ tmp[1] ^ ECHO_SHIFTVEC(passhi, passlo, 3));
			work1 = hilo(fin[1] ^ tmp[2] ^ ECHO_SHIFTVEC(passhi, passlo, 0));
			work2 = hilo(fin[2] ^ tmp[0] ^ ECHO_SHIFTVEC(passhi, passlo, 3));
			work3 = hilo(fin[0] ^ fin[1] ^ fin[2] ^ tmp[0] ^ ECHO_SHIFTVEC(passhi, passlo, 2));
		}
		passhi[64 * 0 + evnSlot] = work0.hi;    passlo[64 * 0 + evnSlot] = work0.lo;
		passhi[64 * 1 + oddSlot] = work1.hi;    passlo[64 * 1 + oddSlot] = work1.lo;
		passhi[64 * 2 + evnSlot] = work2.hi;    passlo[64 * 2 + evnSlot] = work2.lo;
		passhi[64 * 3 + oddSlot] = work3.hi;    passlo[64 * 3 + oddSlot] = work3.lo;
	}
	barrier(CLK_LOCAL_MEM_FENCE);
	// End, some bank collisions here but not much of a problem.
	const uint row = get_local_id(0) / 2; // 00,01,10,11,20,21,30,31 ^ 80,81,90,91,A0,A1,B0,B1
	const int rowbeg = get_local_id(1) * 8 + row * 64;
	int no = rowbeg + ((row + get_local_id(0)) % 2 == 0? 0 : 4);
	int ni = no + 2;
	const uint2 noval = (uint2)(passhi[no], passlo[no]);
	const uint2 nival = (uint2)(passhi[ni], passlo[ni]);
	const uint2 xorv = get_local_id(0) % 2 == 0? (uint2)(512, 0) : (uint2)(0);
	const uint2 myHash = xorv ^ input[get_local_id(0)] ^ noval ^ nival;

#if defined ECHO_IS_LAST
	if(get_local_id(0) == 3) {
		ulong magic = (((ulong)myHash.y) << 32) | myHash.x;
		ulong target = (((ulong)dispatchData[1]) << 32) | dispatchData[2]; // watch out for endianess!
		if(magic <= target) {
			uint storage = atomic_inc(found);
			found[storage + 1] = as_uint(as_char4(get_global_id(1)).wzyx); // watch out for endianess!
		}
	}
#else
#error to be tested!
	hashOut += (get_global_id(1) - get_global_offset(1)) * 8;
	hashOut[get_local_id(0)] = myHash; // maybe swap uints
#endif
}


[OpenCL] Added extended tests on metadata generation for half data type and arrays.

Reviewers: Anastasia

Reviewed By: Anastasia

Subscribers: bader, cfe-commits, yaxunl

Differential Revision: https://reviews.llvm.org/D35000

git-svn-id: ffe668792ed300d6c2daa1f6eba2e0aa28d7ec6c@308266 91177308-0d34-0410-b5e6-96231b3b80d8

// RUN: %clang_cc1 %s -cl-std=CL2.0 -emit-llvm -o - -triple spir-unknown-unknown | FileCheck %s
// RUN: %clang_cc1 %s -cl-std=CL2.0 -emit-llvm -o - -triple spir-unknown-unknown -cl-kernel-arg-info | FileCheck %s -check-prefix ARGINFO

kernel void foo(global int * globalintp, global int * restrict globalintrestrictp,
                global const int * globalconstintp,
                global const int * restrict globalconstintrestrictp,
                constant int * constantintp, constant int * restrict constantintrestrictp,
                global const volatile int * globalconstvolatileintp,
                global const volatile int * restrict globalconstvolatileintrestrictp,
                global volatile int * globalvolatileintp,
                global volatile int * restrict globalvolatileintrestrictp,
                local int * localintp, local int * restrict localintrestrictp,
                local const int * localconstintp,
                local const int * restrict localconstintrestrictp,
                local const volatile int * localconstvolatileintp,
                local const volatile int * restrict localconstvolatileintrestrictp,
                local volatile int * localvolatileintp,
                local volatile int * restrict localvolatileintrestrictp,
                int X, const int constint, const volatile int constvolatileint,
                volatile int volatileint) {
  *globalintrestrictp = constint + volatileint;
}
// CHECK: define spir_kernel void @foo{{[^!]+}}
// CHECK: !kernel_arg_addr_space ![[MD11:[0-9]+]]
// CHECK: !kernel_arg_access_qual ![[MD12:[0-9]+]]
// CHECK: !kernel_arg_type ![[MD13:[0-9]+]]
// CHECK: !kernel_arg_base_type ![[MD13]]
// CHECK: !kernel_arg_type_qual ![[MD14:[0-9]+]]
// CHECK-NOT: !kernel_arg_name
// ARGINFO: !kernel_arg_name ![[MD15:[0-9]+]]

kernel void foo2(read_only image1d_t img1, image2d_t img2, write_only image2d_array_t img3, read_write image1d_t img4) {
}
// CHECK: define spir_kernel void @foo2{{[^!]+}}
// CHECK: !kernel_arg_addr_space ![[MD21:[0-9]+]]
// CHECK: !kernel_arg_access_qual ![[MD22:[0-9]+]]
// CHECK: !kernel_arg_type ![[MD23:[0-9]+]]
// CHECK: !kernel_arg_base_type ![[MD23]]
// CHECK: !kernel_arg_type_qual ![[MD24:[0-9]+]]
// CHECK-NOT: !kernel_arg_name
// ARGINFO: !kernel_arg_name ![[MD25:[0-9]+]]

kernel void foo3(__global half * X) {
}
// CHECK: define spir_kernel void @foo3{{[^!]+}}
// CHECK: !kernel_arg_addr_space ![[MD31:[0-9]+]]
// CHECK: !kernel_arg_access_qual ![[MD32:[0-9]+]]
// CHECK: !kernel_arg_type ![[MD33:[0-9]+]]
// CHECK: !kernel_arg_base_type ![[MD33]]
// CHECK: !kernel_arg_type_qual ![[MD34:[0-9]+]]
// CHECK-NOT: !kernel_arg_name
// ARGINFO: !kernel_arg_name ![[MD35:[0-9]+]]

typedef unsigned int myunsignedint;
kernel void foo4(__global unsigned int * X, __global myunsignedint * Y) {
}
// CHECK: define spir_kernel void @foo4{{[^!]+}}
// CHECK: !kernel_arg_addr_space ![[MD41:[0-9]+]]
// CHECK: !kernel_arg_access_qual ![[MD42:[0-9]+]]
// CHECK: !kernel_arg_type ![[MD43:[0-9]+]]
// CHECK: !kernel_arg_base_type ![[MD44:[0-9]+]]
// CHECK: !kernel_arg_type_qual ![[MD45:[0-9]+]]
// CHECK-NOT: !kernel_arg_name
// ARGINFO: !kernel_arg_name ![[MD46:[0-9]+]]

typedef image1d_t myImage;
kernel void foo5(myImage img1, write_only image1d_t img2) {
}
// CHECK: define spir_kernel void @foo5{{[^!]+}}
// CHECK: !kernel_arg_addr_space ![[MD41:[0-9]+]]
// CHECK: !kernel_arg_access_qual ![[MD51:[0-9]+]]
// CHECK: !kernel_arg_type ![[MD52:[0-9]+]]
// CHECK: !kernel_arg_base_type ![[MD53:[0-9]+]]
// CHECK: !kernel_arg_type_qual ![[MD45]]
// CHECK-NOT: !kernel_arg_name
// ARGINFO: !kernel_arg_name ![[MD54:[0-9]+]]

typedef char char16 __attribute__((ext_vector_type(16)));
__kernel void foo6(__global char16 arg[]) {}
// CHECK: !kernel_arg_type ![[MD61:[0-9]+]]

// CHECK: ![[MD11]] = !{i32 1, i32 1, i32 1, i32 1, i32 2, i32 2, i32 1, i32 1, i32 1, i32 1, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 0, i32 0, i32 0, i32 0}
// CHECK: ![[MD12]] = !{!"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none", !"none"}
// CHECK: ![[MD13]] = !{!"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int*", !"int", !"int", !"int", !"int"}
// CHECK: ![[MD14]] = !{!"", !"restrict", !"const", !"restrict const", !"const", !"restrict const", !"const volatile", !"restrict const volatile", !"volatile", !"restrict volatile", !"", !"restrict", !"const", !"restrict const", !"const volatile", !"restrict const volatile", !"volatile", !"restrict volatile", !"", !"", !"", !""}
// ARGINFO: ![[MD15]] = !{!"globalintp", !"globalintrestrictp", !"globalconstintp", !"globalconstintrestrictp", !"constantintp", !"constantintrestrictp", !"globalconstvolatileintp", !"globalconstvolatileintrestrictp", !"globalvolatileintp", !"globalvolatileintrestrictp", !"localintp", !"localintrestrictp", !"localconstintp", !"localconstintrestrictp", !"localconstvolatileintp", !"localconstvolatileintrestrictp", !"localvolatileintp", !"localvolatileintrestrictp", !"X", !"constint", !"constvolatileint", !"volatileint"}
// CHECK: ![[MD21]] = !{i32 1, i32 1, i32 1, i32 1}
// CHECK: ![[MD22]] = !{!"read_only", !"read_only", !"write_only", !"read_write"}
// CHECK: ![[MD23]] = !{!"image1d_t", !"image2d_t", !"image2d_array_t", !"image1d_t"}
// CHECK: ![[MD24]] = !{!"", !"", !"", !""}
// ARGINFO: ![[MD25]] = !{!"img1", !"img2", !"img3", !"img4"}
// CHECK: ![[MD31]] = !{i32 1}
// CHECK: ![[MD32]] = !{!"none"}
// CHECK: ![[MD33]] = !{!"half*"}
// CHECK: ![[MD34]] = !{!""}
// ARGINFO: ![[MD35]] = !{!"X"}
// CHECK: ![[MD41]] = !{i32 1, i32 1}
// CHECK: ![[MD42]] = !{!"none", !"none"}
// CHECK: ![[MD43]] = !{!"uint*", !"myunsignedint*"}
// CHECK: ![[MD44]] = !{!"uint*", !"uint*"}
// CHECK: ![[MD45]] = !{!"", !""}
// ARGINFO: ![[MD46]] = !{!"X", !"Y"}
// CHECK: ![[MD51]] = !{!"read_only", !"write_only"}
// CHECK: ![[MD52]] = !{!"myImage", !"image1d_t"}
// CHECK: ![[MD53]] = !{!"image1d_t", !"image1d_t"}
// ARGINFO: ![[MD54]] = !{!"img1", !"img2"}
// CHECK: ![[MD61]] = !{!"char16*"}

// RUN: %clang_cc1 %s -cl-std=CL2.0 -emit-llvm -o - -triple spir-unknown-unknown | FileCheck %s
// RUN: %clang_cc1 %s -cl-std=CL2.0 -emit-llvm -o - -triple spir-unknown-unknown -cl-kernel-arg-info | FileCheck %s -check-prefix ARGINFO

kernel void foo(__global int * restrict X, const int Y,
                volatile int anotherArg, __constant float * restrict Z,
                __global volatile int * V, __global const int * C) {
  *X = Y + anotherArg;
}
// CHECK: define spir_kernel void @foo{{[^!]+}}
// CHECK: !kernel_arg_addr_space ![[MD11:[0-9]+]]
// CHECK: !kernel_arg_access_qual ![[MD12:[0-9]+]]
// CHECK: !kernel_arg_type ![[MD13:[0-9]+]]
// CHECK: !kernel_arg_base_type ![[MD13]]
// CHECK: !kernel_arg_type_qual ![[MD14:[0-9]+]]
// CHECK-NOT: !kernel_arg_name
// ARGINFO: !kernel_arg_name ![[MD15:[0-9]+]]

kernel void foo2(read_only image1d_t img1, image2d_t img2, write_only image2d_array_t img3, read_write image1d_t img4) {
}
// CHECK: define spir_kernel void @foo2{{[^!]+}}
// CHECK: !kernel_arg_addr_space ![[MD21:[0-9]+]]
// CHECK: !kernel_arg_access_qual ![[MD22:[0-9]+]]
// CHECK: !kernel_arg_type ![[MD23:[0-9]+]]
// CHECK: !kernel_arg_base_type ![[MD23]]
// CHECK: !kernel_arg_type_qual ![[MD24:[0-9]+]]
// CHECK-NOT: !kernel_arg_name
// ARGINFO: !kernel_arg_name ![[MD25:[0-9]+]]

kernel void foo3(__global half * X) {
}
// CHECK: define spir_kernel void @foo3{{[^!]+}}
// CHECK: !kernel_arg_addr_space ![[MD31:[0-9]+]]
// CHECK: !kernel_arg_access_qual ![[MD32:[0-9]+]]
// CHECK: !kernel_arg_type ![[MD33:[0-9]+]]
// CHECK: !kernel_arg_base_type ![[MD33]]
// CHECK: !kernel_arg_type_qual ![[MD34:[0-9]+]]
// CHECK-NOT: !kernel_arg_name
// ARGINFO: !kernel_arg_name ![[MD35:[0-9]+]]

typedef unsigned int myunsignedint;
kernel void foo4(__global unsigned int * X, __global myunsignedint * Y) {
}
// CHECK: define spir_kernel void @foo4{{[^!]+}}
// CHECK: !kernel_arg_addr_space ![[MD41:[0-9]+]]
// CHECK: !kernel_arg_access_qual ![[MD42:[0-9]+]]
// CHECK: !kernel_arg_type ![[MD43:[0-9]+]]
// CHECK: !kernel_arg_base_type ![[MD44:[0-9]+]]
// CHECK: !kernel_arg_type_qual ![[MD45:[0-9]+]]
// CHECK-NOT: !kernel_arg_name
// ARGINFO: !kernel_arg_name ![[MD46:[0-9]+]]

typedef image1d_t myImage;
kernel void foo5(myImage img1, write_only image1d_t img2) {
}
// CHECK: define spir_kernel void @foo5{{[^!]+}}
// CHECK: !kernel_arg_addr_space ![[MD41:[0-9]+]]
// CHECK: !kernel_arg_access_qual ![[MD51:[0-9]+]]
// CHECK: !kernel_arg_type ![[MD52:[0-9]+]]
// CHECK: !kernel_arg_base_type ![[MD53:[0-9]+]]
// CHECK: !kernel_arg_type_qual ![[MD45]]
// CHECK-NOT: !kernel_arg_name
// ARGINFO: !kernel_arg_name ![[MD54:[0-9]+]]

// CHECK: ![[MD11]] = !{i32 1, i32 0, i32 0, i32 2, i32 1, i32 1}
// CHECK: ![[MD12]] = !{!"none", !"none", !"none", !"none", !"none", !"none"}
// CHECK: ![[MD13]] = !{!"int*", !"int", !"int", !"float*", !"int*", !"int*"}
// CHECK: ![[MD14]] = !{!"restrict", !"", !"", !"restrict const", !"volatile", !"const"}
// ARGINFO: ![[MD15]] = !{!"X", !"Y", !"anotherArg", !"Z", !"V", !"C"}
// CHECK: ![[MD21]] = !{i32 1, i32 1, i32 1, i32 1}
// CHECK: ![[MD22]] = !{!"read_only", !"read_only", !"write_only", !"read_write"}
// CHECK: ![[MD23]] = !{!"image1d_t", !"image2d_t", !"image2d_array_t", !"image1d_t"}
// CHECK: ![[MD24]] = !{!"", !"", !"", !""}
// ARGINFO: ![[MD25]] = !{!"img1", !"img2", !"img3", !"img4"}
// CHECK: ![[MD31]] = !{i32 1}
// CHECK: ![[MD32]] = !{!"none"}
// CHECK: ![[MD33]] = !{!"half*"}
// CHECK: ![[MD34]] = !{!""}
// ARGINFO: ![[MD35]] = !{!"X"}
// CHECK: ![[MD41]] = !{i32 1, i32 1}
// CHECK: ![[MD42]] = !{!"none", !"none"}
// CHECK: ![[MD43]] = !{!"uint*", !"myunsignedint*"}
// CHECK: ![[MD44]] = !{!"uint*", !"uint*"}
// CHECK: ![[MD45]] = !{!"", !""}
// ARGINFO: ![[MD46]] = !{!"X", !"Y"}
// CHECK: ![[MD51]] = !{!"read_only", !"write_only"}
// CHECK: ![[MD52]] = !{!"myImage", !"image1d_t"}
// CHECK: ![[MD53]] = !{!"image1d_t", !"image1d_t"}
// ARGINFO: ![[MD54]] = !{!"img1", !"img2"}