OpenCL: Updated gradient computation in shading.cl

/*********************************************************************************
 *
 * Inviwo - Interactive Visualization Workshop
 *
 * Copyright (c) 2013-2015 Inviwo Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *********************************************************************************/

#ifndef SHADING_CL
#define SHADING_CL

#include "shading/phasefunctions.cl"
#include "shading/blinnphongbrdf.cl"
#include "shading/cooktorrancebrdf.cl"
#include "shading/ashikiminbrdf.cl"
#include "shading/wardbrdf.cl"
#include "shading/abcbrdf.cl"
#include "datastructures/lightsource.cl"

#include "gradients.cl"
#include "samplers.cl"
// Take a look at this pdf, it explains many of the models used:
// http://www.cs.ucl.ac.uk/staff/j.kautz/GameCourse/04_PointLights.pdf

// Helper functions to calculate the shading
float3 shadeDiffuseCalculation(LightParameters light_, float3 materialDiffuseColor, float3 normal,
                             float3 toLightDir) {
    return materialDiffuseColor * light_.diffuseColor * max(dot(normal, toLightDir), 0.0f);
}

float3 shadeSpecularBlinnPhongCalculation(LightParameters light_, float3 materialSpecularColor,
                                        float3 normal, float3 toLightDir, float3 toCameraDir) {
    float3 halfway = toCameraDir + toLightDir;

    // check for special case where the light source is exactly opposite
    // to the view direction, i.e. the length of the halfway vector is zero
    if (dot(halfway, halfway) < 1.e-6f) {  // check for squared length
        return (float3)(0.0f);
    } else {
        halfway = normalize(halfway);
        return materialSpecularColor * light_.specularColor *
               native_powr(max(dot(normal, halfway), 0.f), light_.specularExponent);
    }
}

float3 shadeSpecularPhongCalculation(LightParameters light_, float3 materialSpecularColor, float3 normal,
                                   float3 toLightDir, float3 toCameraDir) {
    // Compute reflection (not that glsl uses incident direction)
    // Equivalent to: 2.0*dot(toLightDir, normal)*normal - toLightDir;
    float3 r = reflect(-toLightDir, normal);

    return materialSpecularColor * light_.specularColor *
           native_powr(max(dot(r, toCameraDir), 0.f), light_.specularExponent * 0.25f);
}

// Functions to apply different shading modes.
// All positions and directions should be in world space!
float3 shadeAmbient(LightParameters light_, float3 materialAmbientColor) {
    return materialAmbientColor * light_.ambientColor;
}

float3 shadeDiffuse(LightParameters light_, float3 materialDiffuseColor, float3 position, float3 normal) {
    return shadeDiffuseCalculation(light_, materialDiffuseColor, normal,
                                   normalize(light_.position - position));
}

float3 shadeSpecular(LightParameters light_, float3 materialSpecularColor, float3 position, float3 normal,
                   float3 toCameraDir) {
    return shadeSpecularPhongCalculation(light_, materialSpecularColor, normal,
                                         normalize(light_.position - position), toCameraDir);
}

float3 shadeBlinnPhong(LightParameters light_, float3 materialAmbientColor, float3 materialDiffuseColor,
                     float3 materialSpecularColor, float3 position, float3 normal, float3 toCameraDir) {
    float3 toLightDir = normalize(light_.position - position);
    float3 resAmb = shadeAmbient(light_, materialAmbientColor);
    float3 resDiff = shadeDiffuseCalculation(light_, materialDiffuseColor, normal, toLightDir);
    float3 resSpec = shadeSpecularBlinnPhongCalculation(light_, materialSpecularColor, normal,
                                                      toLightDir, toCameraDir);
    return resAmb + resDiff + resSpec;
}

float3 shadePhong(LightParameters light_, float3 materialAmbientColor, float3 materialDiffuseColor,
                float3 materialSpecularColor, float3 position, float3 normal, float3 toCameraDir) {
    float3 toLightDir = normalize(light_.position - position);
    float3 resAmb = shadeAmbient(light_, materialAmbientColor);
    float3 resDiff = shadeDiffuseCalculation(light_, materialDiffuseColor, normal, toLightDir);
    float3 resSpec = shadeSpecularPhongCalculation(light_, materialSpecularColor, normal, toLightDir,
                                                 toCameraDir);
    return resAmb + resDiff + resSpec;
}

/////// Physically based shading //////////
#define PHASE_FUNCTION_HENYEY_GREENSTEIN 0
#define PHASE_FUNCTION_SCHLICK 1
#define PHASE_FUNCTION_BLINN_PHONG 2
#define PHASE_FUNCTION_WARD 3
#define PHASE_FUNCTION_COOK_TORRANCE 4
#define PHASE_FUNCTION_ABC_MICROFACET 5
#define PHASE_FUNCTION_ASHIKHMIN 6
#define PHASE_FUNCTION_MIX 7
#define PHASE_FUNCTION_ISOTROPIC 8

typedef enum ShadingType {
    HENYEY_GREENSTEIN=0,
    SCHLICK,
    BLINN_PHONG,
    WARD,
    COOK_TORRANCE,
    ABC_MICROFACET,
    ASHIKHMIN,
    MIX,
    ISOTROPIC
} ShadingType;

#define BLINN_EXPONENT 10.0f
#define GRADIENT_MAGNITUDE_THRESHOLD (MAX_GRAD_MAG*0.5f)
    // Material properties:
    // x = fresnel factor f_0 = ((1-n)/(1+n))^2,
    // y = exponent (blinn)
    // z = phase function scattering
    // w = Mix between phase function and BRDF (MIX shadingtype)
#define EXPONENT_SCALE 100.f

// Maps parameter x from 0-1. Used for exposure in Blinn-Phong and other parameters.
inline float vrayParameterMapping(float x) {
    return 1.f/native_powr(1.f - x, 3.5f) - 1.f;
}
inline float WardParameterMapping(float x) {
    return fmax(x,1e-4f);
}
// Convert between the roughness m in Beckmann distribution and shininess exponent in Blinn-Phong distribution
float shininessToRoughness(float exponent) {
    return native_sqrt( 2.f/(exponent+2.f) );
}
float roughnessToShininess(float m) {
    return 2.f/(fmax(1e-4f, m*m))-2.f;
}
float2 BCParameterMapping(float2 bc) {
    return bc*(float2)(13000.f,1.f);
}
float HenyehGreensteinToSchlick(float g) {
    // Should work for intermediate values of k in Schlick
    return 1.55f*g-0.55f*pown(g, 3);
    //return 1.5f*g-(1.f-1.5f)*g*g*g;
}

inline float4 scaleShadingParameter(float4 material, const ShadingType shadingType) {
    float4 scaledMaterial = material;
#ifdef PERFORMANCE
#if PHASE_FUNCTION == PHASE_FUNCTION_HENYEY_GREENSTEIN
    scaledMaterial.z = -1.f+2.f*material.z;
#elif PHASE_FUNCTION == PHASE_FUNCTION_SCHLICK
    scaledMaterial.z = HenyehGreensteinToSchlick(-1.f+2.f*material.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_BLINN_PHONG
    scaledMaterial.y = roughnessToShininess(material.y);
#elif PHASE_FUNCTION == PHASE_FUNCTION_WARD
    scaledMaterial.y = WardParameterMapping(material.y);
#elif PHASE_FUNCTION == PHASE_FUNCTION_COOK_TORRANCE
    scaledMaterial.y = WardParameterMapping(material.y);
#elif PHASE_FUNCTION == PHASE_FUNCTION_ABC_MICROFACET
    scaledMaterial.yz = BCParameterMapping(material.yz);
#elif PHASE_FUNCTION == PHASE_FUNCTION_ASHIKHMIN
    scaledMaterial.y = roughnessToShininess(material.y);
#elif PHASE_FUNCTION == PHASE_FUNCTION_MIX
    scaledMaterial.y = roughnessToShininess(material.y);
#else
    scaledMaterial.y = material.y;
#endif
#else
    if ( shadingType == HENYEY_GREENSTEIN ) {
        scaledMaterial.z = -1.f+2.f*material.z;
    }else if ( shadingType == SCHLICK ) {
        scaledMaterial.z = HenyehGreensteinToSchlick(-1.f+2.f*material.z);
    }else if ( shadingType == BLINN_PHONG ) {
        scaledMaterial.y = roughnessToShininess(material.y);
    } else if ( shadingType == WARD ) {
        scaledMaterial.y = WardParameterMapping(material.y);
    } else if ( shadingType == COOK_TORRANCE ) {
        //scaledMaterial.y = roughnessToShininess(material.y);
        scaledMaterial.y = WardParameterMapping(material.y);
    } else if ( shadingType == ABC_MICROFACET ) {
        scaledMaterial.yz = BCParameterMapping(material.yz);
    } else if ( shadingType == ASHIKHMIN ) {
        scaledMaterial.y = roughnessToShininess(material.y);
    } else if ( shadingType == MIX ) {
        scaledMaterial.y = roughnessToShininess(material.y);
    } else {
        scaledMaterial.y = material.y;
    }
#endif
    // Scale index of refraction (IOR) f0 = ((n1-n2)/(n1+n2))^2, where n1 = 1
    // n2=20 gives f0=0.8185
    scaledMaterial.x *= 0.81859410430839002267573696145125f;
    //scaledMaterial.x *= 4.f;
    return scaledMaterial;
}

void sampleShadingFunction(read_only image3d_t volumeTex, __constant VolumeParameters* volumeParams, float volumeSample, float4 material,
                 float3 pos,
                 float3* direction, float2 rnd, const ShadingType shadingType) {

    // Material properties:
    // x = fresnel factor f_0 = ((1-n)/(1+n))^2,
    // y = exponent (blinn)
    // z = phase function scattering

    if ( shadingType == HENYEY_GREENSTEIN ) {
        *direction = HenyeyGreensteinSample(-*direction, material.z, rnd);
    } else if ( shadingType == SCHLICK ) {
         *direction = SchlickSample(-*direction, material.z, rnd);
    } else if ( shadingType == BLINN_PHONG ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = -*direction;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        *direction = shadingToWorld(Nu, Nv, gradient, BlinnSample(wo, gradient, material.y, rnd));
    } else if ( shadingType == WARD ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = -*direction;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        *direction = shadingToWorld(Nu, Nv, gradient, WardSample(wo, gradient, material.y,material.y, rnd));
    } else if ( shadingType == COOK_TORRANCE ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = -*direction;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        *direction = shadingToWorld(Nu, Nv, gradient, CookTorranceSample(wo, material.y, rnd));
    } else if ( shadingType == ABC_MICROFACET ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = -*direction;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        *direction = shadingToWorld(Nu, Nv, gradient, ABCMicrofacetSample(wo, material.z, material.w, rnd));
    } else if ( shadingType == ASHIKHMIN ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = -*direction;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        *direction = shadingToWorld(Nu, Nv, gradient, AshikhminSample(wo, gradient, material.y, rnd));
    } else if ( shadingType == MIX ) {
        float3 gradient = (gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float gradientMagnitude = length(gradient);
        if(material.w < gradientMagnitude) {
            *direction = uniformSampleSphere(rnd);
        } else {
            gradient = normalize(gradient);
            float3 wo = -*direction;
            float3 Nu = normalize(cross(gradient, wo));
            float3 Nv = normalize(cross(gradient, Nu));
            wo = worldToShading(Nu, Nv, gradient, wo);
            *direction = shadingToWorld(Nu, Nv, gradient, AshikhminSample(wo, gradient, material.y, rnd));
        }
    } else { // Isotropic phase function
        *direction = uniformSampleSphere(rnd);
    }

}

// directionInOut should contain toCameraDirection upon input and will contain toLightDirection upon output.
void sampleShadingFunctionPdf(read_only image3d_t volumeTex, __constant VolumeParameters* volumeParams, float volumeSample, float4 material, float3 pos,
                 float3* directionInOut, float *pdf, float2 rnd, const ShadingType shadingType) {
    // directionInOut : A direction towards light source or camera upon input. Will be overwritten to new sample direction
    if ( shadingType == HENYEY_GREENSTEIN ) {
        float3 newDir = HenyeyGreensteinSample(-*directionInOut, material.z, rnd);
        *pdf = HenyeyGreensteinPhaseFunction(-*directionInOut, newDir, material.z);
        *directionInOut = newDir;
    } else if ( shadingType == SCHLICK ) {
         float3 newDir = SchlickSample(-*directionInOut, material.z, rnd);
          *pdf = SchlickPhaseFunction(-*directionInOut, newDir, material.z);
          *directionInOut = newDir;
    } else if ( shadingType == BLINN_PHONG ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = -*directionInOut;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = BlinnSample(wo, gradient, material.y, rnd);
        *pdf = BlinnPdf(wo, wi, material.y);
        *directionInOut = shadingToWorld(Nu, Nv, gradient, wi);
    } else if ( shadingType == WARD ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = -*directionInOut;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = WardSample(wo, gradient, material.y,material.y, rnd);
        *pdf = WardPdf(wo, wi, gradient, material.y, material.y);
        *directionInOut = shadingToWorld(Nu, Nv, gradient, wi);
    } else if ( shadingType == COOK_TORRANCE ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = -*directionInOut;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = CookTorranceSample(wo, material.y, rnd);
        *pdf = CookTorrancePdf(wo, wi, gradient, material.y);
        *directionInOut = shadingToWorld(Nu, Nv, gradient, wi);
    } else if ( shadingType == ABC_MICROFACET ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = -*directionInOut;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = ABCMicrofacetSample(wo, material.z, material.w, rnd);
        // TODO: Implement ABC pdf
        *pdf = CookTorrancePdf(wo, wi, gradient, material.w);
        *directionInOut = shadingToWorld(Nu, Nv, gradient, wi);
    } else if ( shadingType == ASHIKHMIN ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = -*directionInOut;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = AshikhminSample(wo, gradient, material.y, rnd);
        *pdf = AshikhminPdf(wo, wi, material.y);
        *directionInOut = shadingToWorld(Nu, Nv, gradient, wi);
    } else if ( shadingType == MIX ) {
        float3 gradient = (gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float gradientMagnitude = length(gradient);
        if(material.w < gradientMagnitude) {
            *directionInOut = uniformSampleSphere(rnd);
            *pdf = uniformSpherePdf();
        } else {
            gradient = normalize(gradient);
            float3 wo = -*directionInOut;
            float3 Nu = normalize(cross(gradient, wo));
            float3 Nv = normalize(cross(gradient, Nu));
            wo = worldToShading(Nu, Nv, gradient, wo);
            float3 wi = AshikhminSample(wo, gradient, material.y, rnd);
            *pdf = AshikhminPdf(wo, wi, material.y);
            *directionInOut = shadingToWorld(Nu, Nv, gradient, wi);
        }
    } else {
        *directionInOut = normalize(uniformSampleSphere(rnd));
        *pdf = uniformSpherePdf();
    }

}
float shadingFunctionPdf(read_only image3d_t volumeTex, __constant VolumeParameters* volumeParams, float volumeSample, float4 material,
                 float3 pos, float3 toCameraDir,
                 float3 toLightDirection, const ShadingType shadingType) {

    float pdf;
    if ( shadingType == HENYEY_GREENSTEIN ) {
        pdf = HenyeyGreensteinPhaseFunction(-toCameraDir, toLightDirection, material.z);
    } else if ( shadingType == SCHLICK ) {
        pdf = SchlickPhaseFunction(-toCameraDir, toLightDirection, material.z);
    } else if ( shadingType == BLINN_PHONG ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = toCameraDir;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDirection);
        pdf = BlinnPdf(wo, wi, material.y);
    } else if ( shadingType == WARD ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = toCameraDir;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDirection);
        pdf = WardPdf(wo, wi, gradient, material.y, material.y);
    } else if ( shadingType == COOK_TORRANCE ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = toCameraDir;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDirection);
        pdf = CookTorrancePdf(wo, wi, gradient, material.y);
    } else if ( shadingType == ABC_MICROFACET ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = toCameraDir;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDirection);
        // TODO: Implement ABC pdf
        pdf = CookTorrancePdf(wo, wi, gradient, material.w);
    } else if ( shadingType == ASHIKHMIN ) {
        float3 gradient = normalize(gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float3 wo = toCameraDir;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDirection);
        pdf = AshikhminPdf(wo, wi, material.y);
    } else if ( shadingType == MIX ) {
        float3 gradient = (gradientCentralDiff(volumeTex, volumeParams, as_float4(pos)));
        float gradientMagnitude = length(gradient);
        if(material.w < gradientMagnitude) {
            pdf = uniformSpherePdf();
        } else {
            gradient = normalize(gradient);
            float3 wo = toCameraDir;
            float3 Nu = normalize(cross(gradient, wo));
            float3 Nv = normalize(cross(gradient, Nu));
            wo = worldToShading(Nu, Nv, gradient, wo);
            float3 wi = worldToShading(Nu, Nv, gradient, toLightDirection);
            pdf = AshikhminPdf(wo, wi, material.y);
        }
    } else {
        pdf = uniformSpherePdf();
    }
    return pdf;
}

float3 applyShading(const float3 toCameraDir, const float3 toLightDir, const float3 materialDiffuse, const float3 materialSpecular, const float4 material, float3 gradient, const ShadingType shadingType) {
    float3 f;
    //float3 Nu = cross(gradient, toCameraDir);
    //if ( dot(Nu, Nu) < 1.e-3f ) {
    //    Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    //    //if ( dot(Nu, Nu) < 1.e-3f ) {
    //    //    Nu = normalize(cross( gradient, (float3)( 0.0f, 1.0f, 0.0f ) ));
    //    //}
    //    //if ( dot(Nu, Nu) < 1.e-3f ) {
    //    //    Nu = normalize(cross( gradient, (float3)( 0.0f, 0.0f, 1.0f ) ));
    //    //}
    //}
    //Nu = normalize(Nu);
    //float3 Nv = normalize(cross(gradient, Nu));
    //float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    //float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
#ifdef PERFORMANCE
#if PHASE_FUNCTION == PHASE_FUNCTION_HENYEY_GREENSTEIN
    f = materialDiffuse*HenyeyGreensteinPhaseFunction(toCameraDir, toLightDir, material.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_SCHLICK
    f = materialDiffuse*SchlickPhaseFunction(toCameraDir, toLightDir, material.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_BLINN_PHONG
    float3 Nu = cross(gradient, toCameraDir);
    if ( dot(Nu, Nu) < 1.e-3f ) {
        Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    }
    Nu = fast_normalize(Nu);
    float3 Nv = fast_normalize(cross(gradient, Nu));
    float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
    f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*BlinnBRDF(wo, wi, material.y)*fabs(wi.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_WARD
    float3 Nu = cross(gradient, toCameraDir);
    if ( dot(Nu, Nu) < 1.e-3f ) {
        Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    }
    Nu = fast_normalize(Nu);
    float3 Nv = fast_normalize(cross(gradient, Nu));
    float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
    f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*WardBRDF(wo, wi, material.y, material.y)*fabs(wi.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_COOK_TORRANCE
    float3 Nu = cross(gradient, toCameraDir);
    if ( dot(Nu, Nu) < 1.e-3f ) {
        Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    }
    Nu = fast_normalize(Nu);
    float3 Nv = fast_normalize(cross(gradient, Nu));
    float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
    f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*CookTorranceBRDF(wo, wi, material.x, material.y)*fabs(wi.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_ABC_MICROFACET
    float3 Nu = cross(gradient, toCameraDir);
    if ( dot(Nu, Nu) < 1.e-3f ) {
        Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    }
    Nu = fast_normalize(Nu);
    float3 Nv = fast_normalize(cross(gradient, Nu));
    float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
    f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*ABCMicrofacetBRDF(wo, wi, material.x, material.y, material.z)*fabs(wi.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_ASHIKHMIN
    float3 Nu = cross(gradient, toCameraDir);
    if ( dot(Nu, Nu) < 1.e-3f ) {
        Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    }
    Nu = fast_normalize(Nu);
    float3 Nv = fast_normalize(cross(gradient, Nu));
    float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
    f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*AshikimBRDF(wo, wi, gradient, material.x, material.y)*fabs(wi.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_MIX
    float3 Nu = cross(gradient, toCameraDir);
    if ( dot(Nu, Nu) < 1.e-3f ) {
        Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    }
    Nu = fast_normalize(Nu);
    float3 Nv = fast_normalize(cross(gradient, Nu));
    float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
    f = mix(materialDiffuse*isotropicPhaseFunction(), materialDiffuse*lambertianBRDF()+materialSpecular*AshikimBRDF(wo, wi, gradient, material.x, material.y), material.w)*fabs(wi.z);
#else
    f = materialDiffuse*isotropicPhaseFunction();
#endif
#else
    if ( shadingType == HENYEY_GREENSTEIN ) {
        f = materialDiffuse*HenyeyGreensteinPhaseFunction(toCameraDir, -toLightDir, material.z);
    } else if ( shadingType == SCHLICK ) {
        f = materialDiffuse*SchlickPhaseFunction(toCameraDir, -toLightDir, material.z);
    } else if ( shadingType == BLINN_PHONG ) {
        float3 Nu = cross(gradient, toCameraDir);
        if ( dot(Nu, Nu) < 1.e-3f ) {
            Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
        }
        Nu = fast_normalize(Nu);
        float3 Nv = fast_normalize(cross(gradient, Nu));
        float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
        f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*BlinnBRDF(wo, wi, material.x, material.y)*fabs(wi.z);
    } else if ( shadingType == WARD ) {
        float3 Nu = cross(gradient, toCameraDir);
        if ( dot(Nu, Nu) < 1.e-3f ) {
            Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
        }
        Nu = fast_normalize(Nu);
        float3 Nv = fast_normalize(cross(gradient, Nu));
        float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
        f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*WardBRDF(wo, wi, material.x, material.y, material.y)*fabs(wi.z);
    } else if ( shadingType == COOK_TORRANCE ) {
        float3 Nu = cross(gradient, toCameraDir);
        if ( dot(Nu, Nu) < 1.e-3f ) {
            Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
        }
        Nu = fast_normalize(Nu);
        float3 Nv = fast_normalize(cross(gradient, Nu));
        float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
        f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*CookTorranceBRDF(wo, wi, material.x, material.y)*fabs(wi.z);
    } else if ( shadingType == ABC_MICROFACET ) {
        float3 Nu = cross(gradient, toCameraDir);
        if ( dot(Nu, Nu) < 1.e-3f ) {
            Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
        }
        Nu = fast_normalize(Nu);
        float3 Nv = fast_normalize(cross(gradient, Nu));
        float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
        f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*ABCMicrofacetBRDF(wo, wi, material.x, material.y, material.z)*fabs(wi.z);
    } else if ( shadingType == ASHIKHMIN ) {
        float3 Nu = cross(gradient, toCameraDir);
        if ( dot(Nu, Nu) < 1.e-3f ) {
            Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
        }
        Nu = fast_normalize(Nu);
        float3 Nv = fast_normalize(cross(gradient, Nu));
        float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
        f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*AshikimBRDF(wo, wi, gradient, material.x, material.y)*fabs(wi.z);
    } else if ( shadingType == MIX ) {
        float3 Nu = cross(gradient, toCameraDir);
        if ( dot(Nu, Nu) < 1.e-3f ) {
            Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
        }
        Nu = fast_normalize(Nu);
        float3 Nv = fast_normalize(cross(gradient, Nu));
        float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
        f = mix(materialDiffuse*isotropicPhaseFunction(), materialDiffuse*isotropicPhaseFunction()+materialSpecular*AshikimBRDF(wo, wi, gradient, material.x, material.y)*fabs(wi.z), material.w);
    } else {
        f = materialDiffuse*isotropicPhaseFunction();
    }
#endif
    // No need to use pdf since we are not generating samples based on phase function/BRDF
    return f;
}

#endif

/*********************************************************************************
 *
 * Inviwo - Interactive Visualization Workshop
 *
 * Copyright (c) 2013-2015 Inviwo Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *********************************************************************************/

#ifndef SHADING_CL
#define SHADING_CL

#include "shading/phasefunctions.cl"
#include "shading/blinnphongbrdf.cl"
#include "shading/cooktorrancebrdf.cl"
#include "shading/ashikiminbrdf.cl"
#include "shading/wardbrdf.cl"
#include "shading/abcbrdf.cl"
#include "datastructures/lightsource.cl"

#include "gradients.cl"
#include "samplers.cl"
// Take a look at this pdf, it explains many of the models used:
// http://www.cs.ucl.ac.uk/staff/j.kautz/GameCourse/04_PointLights.pdf

// Helper functions to calculate the shading
float3 shadeDiffuseCalculation(LightParameters light_, float3 materialDiffuseColor, float3 normal,
                             float3 toLightDir) {
    return materialDiffuseColor * light_.diffuseColor * max(dot(normal, toLightDir), 0.0f);
}

float3 shadeSpecularBlinnPhongCalculation(LightParameters light_, float3 materialSpecularColor,
                                        float3 normal, float3 toLightDir, float3 toCameraDir) {
    float3 halfway = toCameraDir + toLightDir;

    // check for special case where the light source is exactly opposite
    // to the view direction, i.e. the length of the halfway vector is zero
    if (dot(halfway, halfway) < 1.e-6f) {  // check for squared length
        return (float3)(0.0f);
    } else {
        halfway = normalize(halfway);
        return materialSpecularColor * light_.specularColor *
               native_powr(max(dot(normal, halfway), 0.f), light_.specularExponent);
    }
}

float3 shadeSpecularPhongCalculation(LightParameters light_, float3 materialSpecularColor, float3 normal,
                                   float3 toLightDir, float3 toCameraDir) {
    // Compute reflection (not that glsl uses incident direction)
    // Equivalent to: 2.0*dot(toLightDir, normal)*normal - toLightDir;
    float3 r = reflect(-toLightDir, normal);

    return materialSpecularColor * light_.specularColor *
           native_powr(max(dot(r, toCameraDir), 0.f), light_.specularExponent * 0.25f);
}

// Functions to apply different shading modes.
// All positions and directions should be in world space!
float3 shadeAmbient(LightParameters light_, float3 materialAmbientColor) {
    return materialAmbientColor * light_.ambientColor;
}

float3 shadeDiffuse(LightParameters light_, float3 materialDiffuseColor, float3 position, float3 normal) {
    return shadeDiffuseCalculation(light_, materialDiffuseColor, normal,
                                   normalize(light_.position - position));
}

float3 shadeSpecular(LightParameters light_, float3 materialSpecularColor, float3 position, float3 normal,
                   float3 toCameraDir) {
    return shadeSpecularPhongCalculation(light_, materialSpecularColor, normal,
                                         normalize(light_.position - position), toCameraDir);
}

float3 shadeBlinnPhong(LightParameters light_, float3 materialAmbientColor, float3 materialDiffuseColor,
                     float3 materialSpecularColor, float3 position, float3 normal, float3 toCameraDir) {
    float3 toLightDir = normalize(light_.position - position);
    float3 resAmb = shadeAmbient(light_, materialAmbientColor);
    float3 resDiff = shadeDiffuseCalculation(light_, materialDiffuseColor, normal, toLightDir);
    float3 resSpec = shadeSpecularBlinnPhongCalculation(light_, materialSpecularColor, normal,
                                                      toLightDir, toCameraDir);
    return resAmb + resDiff + resSpec;
}

float3 shadePhong(LightParameters light_, float3 materialAmbientColor, float3 materialDiffuseColor,
                float3 materialSpecularColor, float3 position, float3 normal, float3 toCameraDir) {
    float3 toLightDir = normalize(light_.position - position);
    float3 resAmb = shadeAmbient(light_, materialAmbientColor);
    float3 resDiff = shadeDiffuseCalculation(light_, materialDiffuseColor, normal, toLightDir);
    float3 resSpec = shadeSpecularPhongCalculation(light_, materialSpecularColor, normal, toLightDir,
                                                 toCameraDir);
    return resAmb + resDiff + resSpec;
}

/////// Physically based shading //////////
#define PHASE_FUNCTION_HENYEY_GREENSTEIN 0
#define PHASE_FUNCTION_SCHLICK 1
#define PHASE_FUNCTION_BLINN_PHONG 2
#define PHASE_FUNCTION_WARD 3
#define PHASE_FUNCTION_COOK_TORRANCE 4
#define PHASE_FUNCTION_ABC_MICROFACET 5
#define PHASE_FUNCTION_ASHIKHMIN 6
#define PHASE_FUNCTION_MIX 7
#define PHASE_FUNCTION_ISOTROPIC 8

typedef enum ShadingType {
    HENYEY_GREENSTEIN=0,
    SCHLICK,
    BLINN_PHONG,
    WARD,
    COOK_TORRANCE,
    ABC_MICROFACET,
    ASHIKHMIN,
    MIX,
    ISOTROPIC
} ShadingType;

#define BLINN_EXPONENT 10.0f
#define GRADIENT_MAGNITUDE_THRESHOLD (MAX_GRAD_MAG*0.5f)
    // Material properties:
    // x = fresnel factor f_0 = ((1-n)/(1+n))^2,
    // y = exponent (blinn)
    // z = phase function scattering
    // w = Mix between phase function and BRDF (MIX shadingtype)
#define EXPONENT_SCALE 100.f

// Maps parameter x from 0-1. Used for exposure in Blinn-Phong and other parameters.
inline float vrayParameterMapping(float x) {
    return 1.f/native_powr(1.f - x, 3.5f) - 1.f;
}
inline float WardParameterMapping(float x) {
    return fmax(x,1e-4f);
}
// Convert between the roughness m in Beckmann distribution and shininess exponent in Blinn-Phong distribution
float shininessToRoughness(float exponent) {
    return native_sqrt( 2.f/(exponent+2.f) );
}
float roughnessToShininess(float m) {
    return 2.f/(fmax(1e-4f, m*m))-2.f;
}
float2 BCParameterMapping(float2 bc) {
    return bc*(float2)(13000.f,1.f);
}
float HenyehGreensteinToSchlick(float g) {
    // Should work for intermediate values of k in Schlick
    return 1.55f*g-0.55f*pown(g, 3);
    //return 1.5f*g-(1.f-1.5f)*g*g*g;
}

inline float4 scaleShadingParameter(float4 material, const ShadingType shadingType) {
    float4 scaledMaterial = material;
#ifdef PERFORMANCE
#if PHASE_FUNCTION == PHASE_FUNCTION_HENYEY_GREENSTEIN
    scaledMaterial.z = -1.f+2.f*material.z;
#elif PHASE_FUNCTION == PHASE_FUNCTION_SCHLICK
    scaledMaterial.z = HenyehGreensteinToSchlick(-1.f+2.f*material.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_BLINN_PHONG
    scaledMaterial.y = roughnessToShininess(material.y);
#elif PHASE_FUNCTION == PHASE_FUNCTION_WARD
    scaledMaterial.y = WardParameterMapping(material.y);
#elif PHASE_FUNCTION == PHASE_FUNCTION_COOK_TORRANCE
    scaledMaterial.y = WardParameterMapping(material.y);
#elif PHASE_FUNCTION == PHASE_FUNCTION_ABC_MICROFACET
    scaledMaterial.yz = BCParameterMapping(material.yz);
#elif PHASE_FUNCTION == PHASE_FUNCTION_ASHIKHMIN
    scaledMaterial.y = roughnessToShininess(material.y);
#elif PHASE_FUNCTION == PHASE_FUNCTION_MIX
    scaledMaterial.y = roughnessToShininess(material.y);
#else
    scaledMaterial.y = material.y;
#endif
#else
    if ( shadingType == HENYEY_GREENSTEIN ) {
        scaledMaterial.z = -1.f+2.f*material.z;
    }else if ( shadingType == SCHLICK ) {
        scaledMaterial.z = HenyehGreensteinToSchlick(-1.f+2.f*material.z);
    }else if ( shadingType == BLINN_PHONG ) {
        scaledMaterial.y = roughnessToShininess(material.y);
    } else if ( shadingType == WARD ) {
        scaledMaterial.y = WardParameterMapping(material.y);
    } else if ( shadingType == COOK_TORRANCE ) {
        //scaledMaterial.y = roughnessToShininess(material.y);
        scaledMaterial.y = WardParameterMapping(material.y);
    } else if ( shadingType == ABC_MICROFACET ) {
        scaledMaterial.yz = BCParameterMapping(material.yz);
    } else if ( shadingType == ASHIKHMIN ) {
        scaledMaterial.y = roughnessToShininess(material.y);
    } else if ( shadingType == MIX ) {
        scaledMaterial.y = roughnessToShininess(material.y);
    } else {
        scaledMaterial.y = material.y;
    }
#endif
    // Scale index of refraction (IOR) f0 = ((n1-n2)/(n1+n2))^2, where n1 = 1
    // n2=20 gives f0=0.8185
    scaledMaterial.x *= 0.81859410430839002267573696145125f;
    //scaledMaterial.x *= 4.f;
    return scaledMaterial;
}

void sampleShadingFunction(read_only image3d_t volumeTex, float volumeSample, const float3 voxelSpacing, float4 material,
                 float3 sample,
                 float3* direction, float2 rnd, const ShadingType shadingType) {

    // Material properties:
    // x = fresnel factor f_0 = ((1-n)/(1+n))^2,
    // y = exponent (blinn)
    // z = phase function scattering

    if ( shadingType == HENYEY_GREENSTEIN ) {
        *direction = HenyeyGreensteinSample(-*direction, material.z, rnd);
    } else if ( shadingType == SCHLICK ) {
         *direction = SchlickSample(-*direction, material.z, rnd);
    } else if ( shadingType == BLINN_PHONG ) {
        float3 gradient = normalize(calcGradient(as_float4(sample), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = -*direction;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        *direction = shadingToWorld(Nu, Nv, gradient, BlinnSample(wo, gradient, material.y, rnd));
    } else if ( shadingType == WARD ) {
        float3 gradient = normalize(calcGradient(as_float4(sample), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = -*direction;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        *direction = shadingToWorld(Nu, Nv, gradient, WardSample(wo, gradient, material.y,material.y, rnd));
    } else if ( shadingType == COOK_TORRANCE ) {
        float3 gradient = normalize(calcGradient(as_float4(sample), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = -*direction;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        *direction = shadingToWorld(Nu, Nv, gradient, CookTorranceSample(wo, material.y, rnd));
    } else if ( shadingType == ABC_MICROFACET ) {
        float3 gradient = normalize(calcGradient(as_float4(sample), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = -*direction;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        *direction = shadingToWorld(Nu, Nv, gradient, ABCMicrofacetSample(wo, material.z, material.w, rnd));
    } else if ( shadingType == ASHIKHMIN ) {
        float3 gradient = normalize(calcGradient(as_float4(sample), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = -*direction;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        *direction = shadingToWorld(Nu, Nv, gradient, AshikhminSample(wo, gradient, material.y, rnd));
    } else if ( shadingType == MIX ) {
        float3 gradient = (calcGradient(as_float4(sample), volumeTex, as_float4(voxelSpacing)).xyz);
        float gradientMagnitude = length(gradient);
        if(material.w < gradientMagnitude) {
            *direction = uniformSampleSphere(rnd);
        } else {
            gradient = normalize(gradient);
            float3 wo = -*direction;
            float3 Nu = normalize(cross(gradient, wo));
            float3 Nv = normalize(cross(gradient, Nu));
            wo = worldToShading(Nu, Nv, gradient, wo);
            *direction = shadingToWorld(Nu, Nv, gradient, AshikhminSample(wo, gradient, material.y, rnd));
        }
    } else { // Isotropic phase function
        *direction = uniformSampleSphere(rnd);
    }

}

// directionInOut should contain toCameraDirection upon input and will contain toLightDirection upon output.
void sampleShadingFunctionPdf(read_only image3d_t volumeTex, float volumeSample, const float3 voxelSpacing, float4 material, float3 pos,
                 float3* directionInOut, float *pdf, float2 rnd, const ShadingType shadingType) {
    // directionInOut : A direction towards light source or camera upon input. Will be overwritten to new sample direction
    if ( shadingType == HENYEY_GREENSTEIN ) {
        float3 newDir = HenyeyGreensteinSample(-*directionInOut, material.z, rnd);
        *pdf = HenyeyGreensteinPhaseFunction(-*directionInOut, newDir, material.z);
        *directionInOut = newDir;
    } else if ( shadingType == SCHLICK ) {
         float3 newDir = SchlickSample(-*directionInOut, material.z, rnd);
          *pdf = SchlickPhaseFunction(-*directionInOut, newDir, material.z);
          *directionInOut = newDir;
    } else if ( shadingType == BLINN_PHONG ) {
        float3 gradient = normalize(calcGradient(as_float4(pos), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = -*directionInOut;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = BlinnSample(wo, gradient, material.y, rnd);
        *pdf = BlinnPdf(wo, wi, material.y);
        *directionInOut = shadingToWorld(Nu, Nv, gradient, wi);
    } else if ( shadingType == WARD ) {
        float3 gradient = normalize(calcGradient(as_float4(pos), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = -*directionInOut;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = WardSample(wo, gradient, material.y,material.y, rnd);
        *pdf = WardPdf(wo, wi, gradient, material.y, material.y);
        *directionInOut = shadingToWorld(Nu, Nv, gradient, wi);
    } else if ( shadingType == COOK_TORRANCE ) {
        float3 gradient = normalize(calcGradient(as_float4(pos), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = -*directionInOut;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = CookTorranceSample(wo, material.y, rnd);
        *pdf = CookTorrancePdf(wo, wi, gradient, material.y);
        *directionInOut = shadingToWorld(Nu, Nv, gradient, wi);
    } else if ( shadingType == ABC_MICROFACET ) {
        float3 gradient = normalize(calcGradient(as_float4(pos), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = -*directionInOut;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = ABCMicrofacetSample(wo, material.z, material.w, rnd);
        // TODO: Implement ABC pdf
        *pdf = CookTorrancePdf(wo, wi, gradient, material.w);
        *directionInOut = shadingToWorld(Nu, Nv, gradient, wi);
    } else if ( shadingType == ASHIKHMIN ) {
        float3 gradient = normalize(calcGradient(as_float4(pos), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = -*directionInOut;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = AshikhminSample(wo, gradient, material.y, rnd);
        *pdf = AshikhminPdf(wo, wi, material.y);
        *directionInOut = shadingToWorld(Nu, Nv, gradient, wi);
    } else if ( shadingType == MIX ) {
        float3 gradient = (calcGradient(as_float4(pos), volumeTex, as_float4(voxelSpacing)).xyz);
        float gradientMagnitude = length(gradient);
        if(material.w < gradientMagnitude) {
            *directionInOut = uniformSampleSphere(rnd);
            *pdf = uniformSpherePdf();
        } else {
            gradient = normalize(gradient);
            float3 wo = -*directionInOut;
            float3 Nu = normalize(cross(gradient, wo));
            float3 Nv = normalize(cross(gradient, Nu));
            wo = worldToShading(Nu, Nv, gradient, wo);
            float3 wi = AshikhminSample(wo, gradient, material.y, rnd);
            *pdf = AshikhminPdf(wo, wi, material.y);
            *directionInOut = shadingToWorld(Nu, Nv, gradient, wi);
        }
    } else {
        *directionInOut = normalize(uniformSampleSphere(rnd));
        *pdf = uniformSpherePdf();
    }

}
float shadingFunctionPdf(read_only image3d_t volumeTex, float volumeSample, const float3 voxelSpacing, float4 material,
                 float3 pos, float3 toCameraDir,
                 float3 toLightDirection, const ShadingType shadingType) {

    float pdf;
    if ( shadingType == HENYEY_GREENSTEIN ) {
        pdf = HenyeyGreensteinPhaseFunction(-toCameraDir, toLightDirection, material.z);
    } else if ( shadingType == SCHLICK ) {
        pdf = SchlickPhaseFunction(-toCameraDir, toLightDirection, material.z);
    } else if ( shadingType == BLINN_PHONG ) {
        float3 gradient = normalize(calcGradient(as_float4(pos), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = toCameraDir;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDirection);
        pdf = BlinnPdf(wo, wi, material.y);
    } else if ( shadingType == WARD ) {
        float3 gradient = normalize(calcGradient(as_float4(pos), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = toCameraDir;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDirection);
        pdf = WardPdf(wo, wi, gradient, material.y, material.y);
    } else if ( shadingType == COOK_TORRANCE ) {
        float3 gradient = normalize(calcGradient(as_float4(pos), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = toCameraDir;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDirection);
        pdf = CookTorrancePdf(wo, wi, gradient, material.y);
    } else if ( shadingType == ABC_MICROFACET ) {
        float3 gradient = normalize(calcGradient(as_float4(pos), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = toCameraDir;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDirection);
        // TODO: Implement ABC pdf
        pdf = CookTorrancePdf(wo, wi, gradient, material.w);
    } else if ( shadingType == ASHIKHMIN ) {
        float3 gradient = normalize(calcGradient(as_float4(pos), volumeTex, as_float4(voxelSpacing)).xyz);
        float3 wo = toCameraDir;
        float3 Nu = normalize(cross(gradient, wo));
        float3 Nv = normalize(cross(gradient, Nu));
        wo = worldToShading(Nu, Nv, gradient, wo);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDirection);
        pdf = AshikhminPdf(wo, wi, material.y);
    } else if ( shadingType == MIX ) {
        float3 gradient = (calcGradient(as_float4(pos), volumeTex, as_float4(voxelSpacing)).xyz);
        float gradientMagnitude = length(gradient);
        if(material.w < gradientMagnitude) {
            pdf = uniformSpherePdf();
        } else {
            gradient = normalize(gradient);
            float3 wo = toCameraDir;
            float3 Nu = normalize(cross(gradient, wo));
            float3 Nv = normalize(cross(gradient, Nu));
            wo = worldToShading(Nu, Nv, gradient, wo);
            float3 wi = worldToShading(Nu, Nv, gradient, toLightDirection);
            pdf = AshikhminPdf(wo, wi, material.y);
        }
    } else {
        pdf = uniformSpherePdf();
    }
    return pdf;
}

float3 applyShading(const float3 toCameraDir, const float3 toLightDir, const float3 materialDiffuse, const float3 materialSpecular, const float4 material, float3 gradient, const ShadingType shadingType) {
    float3 f;
    //float3 Nu = cross(gradient, toCameraDir);
    //if ( dot(Nu, Nu) < 1.e-3f ) {
    //    Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    //    //if ( dot(Nu, Nu) < 1.e-3f ) {
    //    //    Nu = normalize(cross( gradient, (float3)( 0.0f, 1.0f, 0.0f ) ));
    //    //}
    //    //if ( dot(Nu, Nu) < 1.e-3f ) {
    //    //    Nu = normalize(cross( gradient, (float3)( 0.0f, 0.0f, 1.0f ) ));
    //    //}
    //}
    //Nu = normalize(Nu);
    //float3 Nv = normalize(cross(gradient, Nu));
    //float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    //float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
#ifdef PERFORMANCE
#if PHASE_FUNCTION == PHASE_FUNCTION_HENYEY_GREENSTEIN
    f = materialDiffuse*HenyeyGreensteinPhaseFunction(toCameraDir, toLightDir, material.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_SCHLICK
    f = materialDiffuse*SchlickPhaseFunction(toCameraDir, toLightDir, material.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_BLINN_PHONG
    float3 Nu = cross(gradient, toCameraDir);
    if ( dot(Nu, Nu) < 1.e-3f ) {
        Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    }
    Nu = fast_normalize(Nu);
    float3 Nv = fast_normalize(cross(gradient, Nu));
    float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
    f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*BlinnBRDF(wo, wi, material.y)*fabs(wi.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_WARD
    float3 Nu = cross(gradient, toCameraDir);
    if ( dot(Nu, Nu) < 1.e-3f ) {
        Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    }
    Nu = fast_normalize(Nu);
    float3 Nv = fast_normalize(cross(gradient, Nu));
    float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
    f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*WardBRDF(wo, wi, material.y, material.y)*fabs(wi.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_COOK_TORRANCE
    float3 Nu = cross(gradient, toCameraDir);
    if ( dot(Nu, Nu) < 1.e-3f ) {
        Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    }
    Nu = fast_normalize(Nu);
    float3 Nv = fast_normalize(cross(gradient, Nu));
    float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
    f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*CookTorranceBRDF(wo, wi, material.x, material.y)*fabs(wi.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_ABC_MICROFACET
    float3 Nu = cross(gradient, toCameraDir);
    if ( dot(Nu, Nu) < 1.e-3f ) {
        Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    }
    Nu = fast_normalize(Nu);
    float3 Nv = fast_normalize(cross(gradient, Nu));
    float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
    f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*ABCMicrofacetBRDF(wo, wi, material.x, material.y, material.z)*fabs(wi.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_ASHIKHMIN
    float3 Nu = cross(gradient, toCameraDir);
    if ( dot(Nu, Nu) < 1.e-3f ) {
        Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    }
    Nu = fast_normalize(Nu);
    float3 Nv = fast_normalize(cross(gradient, Nu));
    float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
    f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*AshikimBRDF(wo, wi, gradient, material.x, material.y)*fabs(wi.z);
#elif PHASE_FUNCTION == PHASE_FUNCTION_MIX
    float3 Nu = cross(gradient, toCameraDir);
    if ( dot(Nu, Nu) < 1.e-3f ) {
        Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
    }
    Nu = fast_normalize(Nu);
    float3 Nv = fast_normalize(cross(gradient, Nu));
    float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
    float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
    f = mix(materialDiffuse*isotropicPhaseFunction(), materialDiffuse*lambertianBRDF()+materialSpecular*AshikimBRDF(wo, wi, gradient, material.x, material.y), material.w)*fabs(wi.z);
#else
    f = materialDiffuse*isotropicPhaseFunction();
#endif
#else
    if ( shadingType == HENYEY_GREENSTEIN ) {
        f = materialDiffuse*HenyeyGreensteinPhaseFunction(toCameraDir, -toLightDir, material.z);
    } else if ( shadingType == SCHLICK ) {
        f = materialDiffuse*SchlickPhaseFunction(toCameraDir, -toLightDir, material.z);
    } else if ( shadingType == BLINN_PHONG ) {
        float3 Nu = cross(gradient, toCameraDir);
        if ( dot(Nu, Nu) < 1.e-3f ) {
            Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
        }
        Nu = fast_normalize(Nu);
        float3 Nv = fast_normalize(cross(gradient, Nu));
        float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
        f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*BlinnBRDF(wo, wi, material.x, material.y)*fabs(wi.z);
    } else if ( shadingType == WARD ) {
        float3 Nu = cross(gradient, toCameraDir);
        if ( dot(Nu, Nu) < 1.e-3f ) {
            Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
        }
        Nu = fast_normalize(Nu);
        float3 Nv = fast_normalize(cross(gradient, Nu));
        float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
        f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*WardBRDF(wo, wi, material.x, material.y, material.y)*fabs(wi.z);
    } else if ( shadingType == COOK_TORRANCE ) {
        float3 Nu = cross(gradient, toCameraDir);
        if ( dot(Nu, Nu) < 1.e-3f ) {
            Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
        }
        Nu = fast_normalize(Nu);
        float3 Nv = fast_normalize(cross(gradient, Nu));
        float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
        f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*CookTorranceBRDF(wo, wi, material.x, material.y)*fabs(wi.z);
    } else if ( shadingType == ABC_MICROFACET ) {
        float3 Nu = cross(gradient, toCameraDir);
        if ( dot(Nu, Nu) < 1.e-3f ) {
            Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
        }
        Nu = fast_normalize(Nu);
        float3 Nv = fast_normalize(cross(gradient, Nu));
        float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
        f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*ABCMicrofacetBRDF(wo, wi, material.x, material.y, material.z)*fabs(wi.z);
    } else if ( shadingType == ASHIKHMIN ) {
        float3 Nu = cross(gradient, toCameraDir);
        if ( dot(Nu, Nu) < 1.e-3f ) {
            Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
        }
        Nu = fast_normalize(Nu);
        float3 Nv = fast_normalize(cross(gradient, Nu));
        float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
        f = materialDiffuse*isotropicPhaseFunction()+materialSpecular*AshikimBRDF(wo, wi, gradient, material.x, material.y)*fabs(wi.z);
    } else if ( shadingType == MIX ) {
        float3 Nu = cross(gradient, toCameraDir);
        if ( dot(Nu, Nu) < 1.e-3f ) {
            Nu = cross( gradient, (float3)( 1.0f, 0.0f, 0.0f ) );
        }
        Nu = fast_normalize(Nu);
        float3 Nv = fast_normalize(cross(gradient, Nu));
        float3 wo = worldToShading(Nu, Nv, gradient, toCameraDir);
        float3 wi = worldToShading(Nu, Nv, gradient, toLightDir);
        f = mix(materialDiffuse*isotropicPhaseFunction(), materialDiffuse*isotropicPhaseFunction()+materialSpecular*AshikimBRDF(wo, wi, gradient, material.x, material.y)*fabs(wi.z), material.w);
    } else {
        f = materialDiffuse*isotropicPhaseFunction();
    }
#endif
    // No need to use pdf since we are not generating samples based on phase function/BRDF
    return f;
}

#endif


fix error for clang-format

fix error for clang-format

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#pragma OPENCL EXTENSION cl_khr_fp16 : enable

#ifndef MIOPEN_TYPE
#define MIOPEN_TYPE float
#endif

__kernel void CopyTensor1d(const global MIOPEN_TYPE* __restrict src,
                           const int srcOffset,
                           const int srcStride0,
                           const int srcLen0,
                           global MIOPEN_TYPE* __restrict dst,
                           const int dstOffset,
                           const int dstStride0)
{
    const uint tidx = get_global_id(0);

    for(uint did0 = tidx; did0 < srcLen0; did0 += GLOBAL_WORK_SIZE_X)
    {
        const uint sindex = srcStride0 * did0;
        const uint dindex = dstStride0 * did0;

        dst[dindex + dstOffset] = src[sindex + srcOffset];
    }
}

__kernel void CopyTensor2d(const global MIOPEN_TYPE* __restrict src,
                           const int srcOffset,
                           const int srcStride0,
                           const int srcStride1,
                           const int srcLen0,
                           const int srcLen1,
                           global MIOPEN_TYPE* __restrict dst,
                           const int dstOffset,
                           const int dstStride0,
                           const int dstStride1)
{
    const uint tidx = get_global_id(0);
    const uint tidy = get_global_id(1);

    for(uint did0 = tidy; did0 < srcLen0; did0 += GLOBAL_WORK_SIZE_Y)
    {
        for(uint did1 = tidx; did1 < srcLen1; did1 += GLOBAL_WORK_SIZE_X)
        {
            const uint sindex = srcStride0 * did0 + srcStride1 * did1;
            const uint dindex = dstStride0 * did0 + dstStride1 * did1;

            dst[dindex + dstOffset] = src[sindex + srcOffset];
        }
    }
}

__kernel void CopyTensor3d(const global MIOPEN_TYPE* __restrict src,
                           const int srcOffset,
                           const int srcStride0,
                           const int srcStride1,
                           const int srcStride2,
                           const int srcLen0,
                           const int srcLen1,
                           const int srcLen2,
                           global MIOPEN_TYPE* __restrict dst,
                           const int dstOffset,
                           const int dstStride0,
                           const int dstStride1,
                           const int dstStride2)
{
    const uint tidx = get_global_id(0);
    const uint tidy = get_global_id(1);
    const uint tidz = get_global_id(2);

    for(uint did0 = tidz; did0 < srcLen0; did0 += GLOBAL_WORK_SIZE_Z)
    {
        for(uint did1 = tidy; did1 < srcLen1; did1 += GLOBAL_WORK_SIZE_Y)
        {
            for(uint did2 = tidx; did2 < srcLen2; did2 += GLOBAL_WORK_SIZE_X)
            {
                const uint sindex = srcStride0 * did0 + srcStride1 * did1 + srcStride2 * did2;
                const uint dindex = dstStride0 * did0 + dstStride1 * did1 + dstStride2 * did2;

                dst[dindex + dstOffset] = src[sindex + srcOffset];
            }
        }
    }
}

__kernel void CopyTensor4d(const global MIOPEN_TYPE* __restrict src,
                           const int srcOffset,
                           const int srcStride0,
                           const int srcStride1,
                           const int srcStride2,
                           const int srcStride3,
                           const int srcLen0,
                           const int srcLen1,
                           const int srcLen2,
                           const int srcLen3,
                           global MIOPEN_TYPE* __restrict dst,
                           const int dstOffset,
                           const int dstStride0,
                           const int dstStride1,
                           const int dstStride2,
                           const int dstStride3)
{
    const uint tidx = get_global_id(0);
    const uint tidy = get_global_id(1);
    const uint tidz = get_global_id(2);

    for(uint did0 = 0; did0 < srcLen0; did0++)
    {
        for(uint did1 = tidz; did1 < srcLen1; did1 += GLOBAL_WORK_SIZE_Z)
        {
            for(uint did2 = tidy; did2 < srcLen2; did2 += GLOBAL_WORK_SIZE_Y)
            {
                for(uint did3 = tidx; did3 < srcLen3; did3 += GLOBAL_WORK_SIZE_X)
                {
                    const uint sindex = srcStride0 * did0 + srcStride1 * did1 + srcStride2 * did2 +
                                        srcStride3 * did3;
                    const uint dindex = dstStride0 * did0 + dstStride1 * did1 + dstStride2 * did2 +
                                        dstStride3 * did3;

                    dst[dindex + dstOffset] = src[sindex + srcOffset];
                }
            }
        }
    }
}

__kernel void CopyTensor5d(const global MIOPEN_TYPE* __restrict src,
                           const int srcOffset,
                           const int srcStride0,
                           const int srcStride1,
                           const int srcStride2,
                           const int srcStride3,
                           const int srcStride4,
                           const int srcLen0,
                           const int srcLen1,
                           const int srcLen2,
                           const int srcLen3,
                           const int srcLen4,
                           global MIOPEN_TYPE* __restrict dst,
                           const int dstOffset,
                           const int dstStride0,
                           const int dstStride1,
                           const int dstStride2,
                           const int dstStride3,
                           const int dstStride4)
{
    const uint tidx = get_global_id(0);
    const uint tidy = get_global_id(1);
    const uint tidz = get_global_id(2);

    for(uint did0 = 0; did0 < srcLen0; did0++)
    {
        for(uint did1 = 0; did1 < srcLen1; did1++)
        {
            for(uint did2 = tidz; did2 < srcLen2; did2 += GLOBAL_WORK_SIZE_Z)
            {
                for(uint did3 = tidy; did3 < srcLen3; did3 += GLOBAL_WORK_SIZE_Y)
                {
                    for(uint did4 = tidx; did4 < srcLen4; did4 += GLOBAL_WORK_SIZE_X)
                    {
                        const uint sindex = srcStride0 * did0 + srcStride1 * did1 +
                                            srcStride2 * did2 + srcStride3 * did3 +
                                            srcStride4 * did4;
                        const uint dindex = dstStride0 * did0 + dstStride1 * did1 +
                                            dstStride2 * did2 + dstStride3 * did3 +
                                            srcStride4 * did4;

                        dst[dindex + dstOffset] = src[sindex + srcOffset];
                    }
                }
            }
        }
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

#pragma OPENCL EXTENSION cl_khr_fp16 : enable

#ifndef MIOPEN_TYPE
#define MIOPEN_TYPE float
#endif

__kernel void CopyTensor1d( const global MIOPEN_TYPE* __restrict src,
                            const int srcOffset,
                            const int srcStride0,
                            const int srcLen0,
                            global MIOPEN_TYPE* __restrict dst,
                            const int dstOffset,
                            const int dstStride0)
{
    const uint tidx = get_global_id(0);

    for(uint did0 = tidx; did0 < srcLen0; did0 += GLOBAL_WORK_SIZE_X)
    {
        const uint sindex = srcStride0 * did0;
        const uint dindex = dstStride0 * did0;

        dst[dindex + dstOffset] = src[sindex + srcOffset];
    }
}

__kernel void CopyTensor2d( const global MIOPEN_TYPE* __restrict src,
                            const int srcOffset,
                            const int srcStride0,
                            const int srcStride1,
                            const int srcLen0,
                            const int srcLen1,
                            global MIOPEN_TYPE* __restrict dst,
                            const int dstOffset,
                            const int dstStride0,
                            const int dstStride1)
{
    const uint tidx = get_global_id(0);
    const uint tidy = get_global_id(1);

    for(uint did0 = tidy; did0 < srcLen0; did0 += GLOBAL_WORK_SIZE_Y )
    {
        for(uint did1 = tidx; did1 < srcLen1; did1 += GLOBAL_WORK_SIZE_X )
        {
            const uint sindex = srcStride0 * did0 + srcStride1 * did1;
            const uint dindex = dstStride0 * did0 + dstStride1 * did1;

            dst[dindex + dstOffset] = src[sindex + srcOffset];
        }
    }
}

__kernel void CopyTensor3d( const global MIOPEN_TYPE* __restrict src,
                            const int srcOffset,
                            const int srcStride0,
                            const int srcStride1,
                            const int srcStride2,
                            const int srcLen0,
                            const int srcLen1,
                            const int srcLen2,
                            global MIOPEN_TYPE* __restrict dst,
                            const int dstOffset,
                            const int dstStride0,
                            const int dstStride1,
                            const int dstStride2)
{
    const uint tidx = get_global_id(0);
    const uint tidy = get_global_id(1);
    const uint tidz = get_global_id(2);

    for(uint did0 = tidz; did0 < srcLen0; did0 += GLOBAL_WORK_SIZE_Z)
    {
        for(uint did1 = tidy; did1 < srcLen1; did1 += GLOBAL_WORK_SIZE_Y)
        {
            for(uint did2 = tidx; did2 < srcLen2; did2 += GLOBAL_WORK_SIZE_X)
            {
                const uint sindex = srcStride0 * did0 + srcStride1 * did1 + srcStride2 * did2;
                const uint dindex = dstStride0 * did0 + dstStride1 * did1 + dstStride2 * did2;

                dst[dindex + dstOffset] = src[sindex + srcOffset];
            }
        }
    }
}

__kernel void CopyTensor4d( const global MIOPEN_TYPE* __restrict src,
                            const int srcOffset,
                            const int srcStride0,
                            const int srcStride1,
                            const int srcStride2,
                            const int srcStride3,
                            const int srcLen0,
                            const int srcLen1,
                            const int srcLen2,
                            const int srcLen3,
                            global MIOPEN_TYPE* __restrict dst,
                            const int dstOffset,
                            const int dstStride0,
                            const int dstStride1,
                            const int dstStride2,
                            const int dstStride3)
{
    const uint tidx = get_global_id(0);
    const uint tidy = get_global_id(1);
    const uint tidz = get_global_id(2);

    for(uint did0 = 0; did0 < srcLen0; did0++)
    {
        for(uint did1 = tidz; did1 < srcLen1; did1 += GLOBAL_WORK_SIZE_Z)
        {
            for(uint did2 = tidy; did2 < srcLen2; did2 += GLOBAL_WORK_SIZE_Y)
            {
                for(uint did3 = tidx; did3 < srcLen3; did3 += GLOBAL_WORK_SIZE_X)
                {
                    const uint sindex = srcStride0 * did0 + srcStride1 * did1 + srcStride2 * did2 + srcStride3 * did3;
                    const uint dindex = dstStride0 * did0 + dstStride1 * did1 + dstStride2 * did2 + dstStride3 * did3;

                    dst[dindex + dstOffset] = src[sindex + srcOffset];
                }
            }
        }
    }
}

__kernel void CopyTensor5d( const global MIOPEN_TYPE* __restrict src,
                            const int srcOffset,
                            const int srcStride0,
                            const int srcStride1,
                            const int srcStride2,
                            const int srcStride3,
                            const int srcStride4,
                            const int srcLen0,
                            const int srcLen1,
                            const int srcLen2,
                            const int srcLen3,
                            const int srcLen4,
                            global MIOPEN_TYPE* __restrict dst,
                            const int dstOffset,
                            const int dstStride0,
                            const int dstStride1,
                            const int dstStride2,
                            const int dstStride3,
                            const int dstStride4)
{
    const uint tidx = get_global_id(0);
    const uint tidy = get_global_id(1);
    const uint tidz = get_global_id(2);

    for(uint did0 = 0; did0 < srcLen0; did0++)
    {
        for(uint did1 = 0; did1 < srcLen1; did1++)
        {
            for(uint did2 = tidz; did2 < srcLen2; did2 += GLOBAL_WORK_SIZE_Z)
            {
                for(uint did3 = tidy; did3 < srcLen3; did3 += GLOBAL_WORK_SIZE_Y)
                {
                    for(uint did4 = tidx; did4 < srcLen4; did4 += GLOBAL_WORK_SIZE_X)
                    {
                        const uint sindex = srcStride0 * did0 + srcStride1 * did1 + srcStride2 * did2 + srcStride3 * did3 + srcStride4 * did4;
                        const uint dindex = dstStride0 * did0 + dstStride1 * did1 + dstStride2 * did2 + dstStride3 * did3 + srcStride4 * did4;

                        dst[dindex + dstOffset] = src[sindex + srcOffset];
                    }
                }
            }
        }
    }
}


fix erroneous definition of MLO_N_LCL_IN_MAPS_ONCE

The definition of MLO_N_LCL_IN_MAPS_ONCE was erroneously moved outside #if 0

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

// Trying to use float ATOMIC_ADD to increase total waves
// For example,  7x7 : 49
// Global 0:  H * W * N * (K/ 16 outplane per threads ) * (( Inputlanes/ 128 )   or 1)
// N * H * W =
//    [groupId / ((K/ 16 outplane per threads ) * (( Inputlanes/ 128 )   or 1))] * 64 + localId
// For example, H7*W7*N*C256*K64
// FIRST 4 waves
// Hit L2 for read
// try to Hit L2 for Atomic_ADD
// Assembly Shader can utlize LDS to Reductiion
// However Shader Compiler of OpenCL will compile any localId[1] to FLAT_BUFFER_LOAD not constant
// load

// WAVE 0  == N0_7x7 + N1_(7x2+1),  output  0-15  from K=64,  Inputplanes from 0-127
// WAVE 1  == N0_7x7 + N1_(7x2+1),  output  16-31 from K=64,  Inputplanes from 0-127
// WAVE 2  == N0_7x7 + N1_(7x2+1),  output  31-47 from K=64,  Inputplanes from 0-127
// WAVE 3  == N0_7x7 + N1_(7x2+1),  output  48-63 from K=64,  Inputplanes from 0-127
// 2nd 4 waves
// Hit L2 for read
// try to Hit L2 for Atomic_ADD

// WAVE 4  == N0_7x7 + N1_(7x2+1),  output  0-15  from K=64,  Inputplanes from 128-255
// WAVE 5  == N0_7x7 + N1_(7x2+1),  output  16-31 from K=64,  Inputplanes from 128-255
// WAVE 6  == N0_7x7 + N1_(7x2+1),  output  31-47 from K=64,  Inputplanes from 128-255
// WAVE 7  == N0_7x7 + N1_(7x2+1),  output  48-63 from K=64,  Inputplanes from 128-255

// example
#if 0 // ndef MLopen_RUNNING
#define BATCHSIZE 32
#define MLO_N_LCL_IN_MAPS_ONCE 8

#define H 28
#define W 28
#define C 192
#define K 64

#define MLO_IN_HEIGHT H
#define MLO_IN_WIDTH W
#define MLO_N_INPUTS C

//128 or MLO_N_INPUTS
#define MLO_N_LCL_IN_MAPS 192

#define MLO_N_OUTPUTS K

#define H_out 28
#define W_out 28
#define MLO_N_LCL_OUT_MAPS 16

#define MLO_N_IN_GROUPS ((MLO_N_INPUTS + MLO_N_LCL_IN_MAPS - 1) / MLO_N_LCL_IN_MAPS)
#define MLO_CLOOP0 (MLO_N_LCL_IN_MAPS / MLO_N_LCL_IN_MAPS_ONCE)
#define MLO_CLOOP2 \
    ((MLO_N_INPUTS - MLO_N_LCL_IN_MAPS * (MLO_N_IN_GROUPS - 1)) / MLO_N_LCL_IN_MAPS_ONCE)
#define MLO_CHEAT_SHADER_COMPILER 1

#endif

#define MLO_IN_CHANNEL_STRIDE (H * W)
#define MLO_IN_BATCH_STRIDE (H * W * C)

#define MLO_WEI_BSTRIDE (1 * 1 * C * K)
#define MLO_WEI_CHANNEL_STRIDE (1 * 1 * C)

#define MLO_OUT_BATCH_STRIDE (H_out * W_out * K)
#define MLO_OUT_CHANNEL_STRIDE (H_out * W_out)

#define FIXED_WORKGROUP_SIZE 64

#define MLO_N_OUT_GROUPS (MLO_N_OUTPUTS / MLO_N_LCL_OUT_MAPS)

#define MLO_GRP_SZ0 64
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1

#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#define _UNION_FLOAT_T half2
#define INIT(A) ((half2)(A[0], A[1]))
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#define _UNION_FLOAT_T float
#define INIT(A) (A[0])
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

#define MLO_CONV_BIAS 0
#define UNUSED __attribute((__unused__))

typedef union
{
    unsigned int intVal;
    _UNION_FLOAT_T floatVal;
} starVal;

inline void AtomicAdd(volatile __global _FLOAT* source, const _FLOAT operand)
{
    starVal newVal, prevVal;

    prevVal.floatVal = INIT(source);
    while(true)
    {
#if MIOPEN_USE_FP16 == 1
        newVal.floatVal = (_FLOAT2)(prevVal.floatVal.x + operand, source[1]);
#endif
#if MIOPEN_USE_FP32 == 1
        newVal.floatVal = prevVal.floatVal + operand;
#endif
        newVal.intVal =
            atomic_cmpxchg((volatile __global unsigned int*)source, prevVal.intVal, newVal.intVal);

        // equal to pass
        if(newVal.intVal == prevVal.intVal)
            break;

        prevVal.intVal = newVal.intVal;
    }
}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{

    uint grp_id0       = get_group_id(0);
    uint out_grp_block = grp_id0 % MLO_N_OUT_GROUPS;
    uint in_grp_block  = (uint)(grp_id0 / MLO_N_OUT_GROUPS) % MLO_N_IN_GROUPS;
    uint grp_id0_faked = (uint)(grp_id0 / MLO_N_OUT_GROUPS) / MLO_N_IN_GROUPS;

    uint local_id0 = get_local_id(0);
#if MLO_CHEAT_SHADER_COMPILER == 1
    uint grp_id2 = get_group_id(2);
#endif

    uint pos      = (grp_id0_faked * FIXED_WORKGROUP_SIZE + local_id0) % MLO_IN_CHANNEL_STRIDE;
    uint batch_id = (grp_id0_faked * FIXED_WORKGROUP_SIZE + local_id0) / MLO_IN_CHANNEL_STRIDE;

    if(batch_id >= BATCHSIZE)
        return;

    uint out_id = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off = batch_id * MLO_IN_BATCH_STRIDE +
                      in_grp_block * MLO_N_LCL_IN_MAPS * MLO_IN_CHANNEL_STRIDE + pos;

    uint wei_off = out_id * MLO_WEI_CHANNEL_STRIDE + in_grp_block * MLO_N_LCL_IN_MAPS;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS];
    _FLOAT weights[MLO_N_LCL_IN_MAPS_ONCE];
    _FLOAT dat[MLO_N_LCL_IN_MAPS_ONCE];
    _FLOAT dat2[MLO_N_LCL_IN_MAPS_ONCE];

//

// ATOMIC is needed if INPUTS in many waves
#if(MLO_N_LCL_IN_MAPS != MLO_N_INPUTS)

    if(in_grp_block == 0)
    {
        uint gbl_out_off = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE + pos;
        __global _FLOAT* q = out_ptr + gbl_out_off;

        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            *q = (_FLOAT)0;
            q += MLO_OUT_CHANNEL_STRIDE;
        }
    }
#endif

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        accum[o] = (_FLOAT)0;
    }

#if MLO_N_INPUTS == ((MLO_N_INPUTS / MLO_N_LCL_IN_MAPS) * MLO_N_LCL_IN_MAPS)
    // if(1)
    int loops = MLO_CLOOP0;

#if MLO_CHEAT_SHADER_COMPILER == 1
    // cheat shader compiler to disable loop unroll.  it will have better SQC performance
    if(grp_id2 == 0x1F)
    {
        loops = 377; // strange not to unroll loop
    }
#endif
#else
    int loops = MLO_CLOOP0;

    if(in_grp_block == (MLO_N_IN_GROUPS - 1))
    {
        loops = MLO_CLOOP2;
    }

#if MLO_CHEAT_SHADER_COMPILER == 1
    // cheat shader compiler to disable loop unroll.  it will have better SQC performance
    if(grp_id2 == 0x1F)
    {
        loops = 377; // strange not to unroll loop
    }
#endif

#endif
    {
        __global const _FLOAT* p = in_ptr + gbl_in_off;
        __constant _FLOAT* w     = wei_ptr + wei_off;

        // read data
        for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
        {

            dat[j] = *p;
            p += MLO_IN_CHANNEL_STRIDE;
        }

        for(uint ci = 0; ci < (loops - 2); ci += 2)
        {
            // read data
            for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
            {
                dat2[j] = *p;
                p += MLO_IN_CHANNEL_STRIDE;
            }

            // convolve
            __constant _FLOAT* w1 = w;
            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;

            // convolve
            w1 = w;
            for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
            {
                dat[j] = *p;
                p += MLO_IN_CHANNEL_STRIDE;
            }

            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat2[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;
        }

        //
        // last 2 iterations
        { // read data
            for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
            {
                dat2[j] = *p;
                p += MLO_IN_CHANNEL_STRIDE;
            }

            // convolve
            __constant _FLOAT* w1 = w;
            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;

            // convolve
            w1 = w;

            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat2[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;
        }
    }

    uint gbl_out_off   = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE + pos;
    __global _FLOAT* q = out_ptr + gbl_out_off;

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
#if(MLO_N_LCL_IN_MAPS == MLO_N_INPUTS)
        *q = accum[o];
        q += MLO_OUT_CHANNEL_STRIDE;
#else
        AtomicAdd(q, accum[o]);
        q += MLO_OUT_CHANNEL_STRIDE;
#endif
    }
}

/*******************************************************************************
 *
 * MIT License
 *
 * Copyright (c) 2017 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 *******************************************************************************/

// Trying to use float ATOMIC_ADD to increase total waves
// For example,  7x7 : 49
// Global 0:  H * W * N * (K/ 16 outplane per threads ) * (( Inputlanes/ 128 )   or 1)
// N * H * W =
//    [groupId / ((K/ 16 outplane per threads ) * (( Inputlanes/ 128 )   or 1))] * 64 + localId
// For example, H7*W7*N*C256*K64
// FIRST 4 waves
// Hit L2 for read
// try to Hit L2 for Atomic_ADD
// Assembly Shader can utlize LDS to Reductiion
// However Shader Compiler of OpenCL will compile any localId[1] to FLAT_BUFFER_LOAD not constant
// load

// WAVE 0  == N0_7x7 + N1_(7x2+1),  output  0-15  from K=64,  Inputplanes from 0-127
// WAVE 1  == N0_7x7 + N1_(7x2+1),  output  16-31 from K=64,  Inputplanes from 0-127
// WAVE 2  == N0_7x7 + N1_(7x2+1),  output  31-47 from K=64,  Inputplanes from 0-127
// WAVE 3  == N0_7x7 + N1_(7x2+1),  output  48-63 from K=64,  Inputplanes from 0-127
// 2nd 4 waves
// Hit L2 for read
// try to Hit L2 for Atomic_ADD

// WAVE 4  == N0_7x7 + N1_(7x2+1),  output  0-15  from K=64,  Inputplanes from 128-255
// WAVE 5  == N0_7x7 + N1_(7x2+1),  output  16-31 from K=64,  Inputplanes from 128-255
// WAVE 6  == N0_7x7 + N1_(7x2+1),  output  31-47 from K=64,  Inputplanes from 128-255
// WAVE 7  == N0_7x7 + N1_(7x2+1),  output  48-63 from K=64,  Inputplanes from 128-255

// example
#if 0 // ndef MLopen_RUNNING
#define BATCHSIZE 32
#define H 28
#define W 28
#define C 192
#define K 64

#define MLO_IN_HEIGHT H
#define MLO_IN_WIDTH W
#define MLO_N_INPUTS C

//128 or MLO_N_INPUTS
#define MLO_N_LCL_IN_MAPS 192

#define MLO_N_OUTPUTS K

#define H_out 28
#define W_out 28
#define MLO_N_LCL_OUT_MAPS 16

#define MLO_N_IN_GROUPS ((MLO_N_INPUTS + MLO_N_LCL_IN_MAPS - 1) / MLO_N_LCL_IN_MAPS)
#define MLO_CLOOP0 (MLO_N_LCL_IN_MAPS / MLO_N_LCL_IN_MAPS_ONCE)
#define MLO_CLOOP2 \
    ((MLO_N_INPUTS - MLO_N_LCL_IN_MAPS * (MLO_N_IN_GROUPS - 1)) / MLO_N_LCL_IN_MAPS_ONCE)
#define MLO_CHEAT_SHADER_COMPILER 1

#endif

#define MLO_IN_CHANNEL_STRIDE (H * W)
#define MLO_IN_BATCH_STRIDE (H * W * C)

#define MLO_WEI_BSTRIDE (1 * 1 * C * K)
#define MLO_WEI_CHANNEL_STRIDE (1 * 1 * C)

#define MLO_N_LCL_IN_MAPS_ONCE 8

#define MLO_OUT_BATCH_STRIDE (H_out * W_out * K)
#define MLO_OUT_CHANNEL_STRIDE (H_out * W_out)

#define FIXED_WORKGROUP_SIZE 64

#define MLO_N_OUT_GROUPS (MLO_N_OUTPUTS / MLO_N_LCL_OUT_MAPS)

#define MLO_GRP_SZ0 64
#define MLO_GRP_SZ1 1
#define MLO_GRP_SZ2 1

#define PPCAT_NX(A, B) A##B
#define PPCAT(A, B) PPCAT_NX(A, B)
#define TWO 2
#define FOUR 4
#define EIGHT 8

#if MIOPEN_USE_FP16 == 1
#pragma OPENCL EXTENSION cl_khr_fp16 : enable
#define _FLOAT half
#define _UNION_FLOAT_T half2
#define INIT(A) ((half2)(A[0], A[1]))
#ifndef HALF_MAX
#define MAX_VAL 65504 /* max value */
#else
#define MAX_VAL HALF_MAX
#endif
#endif
#if MIOPEN_USE_FP32 == 1
#define _FLOAT float
#define _UNION_FLOAT_T float
#define INIT(A) (A[0])
#ifndef FLT_MAX
#define MAX_VAL 3.402823466e+38F /* max value */
#else
#define MAX_VAL FLT_MAX
#endif
#endif

#define _FLOAT2 PPCAT(_FLOAT, TWO)
#define _FLOAT4 PPCAT(_FLOAT, FOUR)
#define _FLOAT8 PPCAT(_FLOAT, EIGHT)

#define MLO_CONV_BIAS 0
#define UNUSED __attribute((__unused__))

typedef union
{
    unsigned int intVal;
    _UNION_FLOAT_T floatVal;
} starVal;

inline void AtomicAdd(volatile __global _FLOAT* source, const _FLOAT operand)
{
    starVal newVal, prevVal;

    prevVal.floatVal = INIT(source);
    while(true)
    {
#if MIOPEN_USE_FP16 == 1
        newVal.floatVal = (_FLOAT2)(prevVal.floatVal.x + operand, source[1]);
#endif
#if MIOPEN_USE_FP32 == 1
        newVal.floatVal = prevVal.floatVal + operand;
#endif
        newVal.intVal =
            atomic_cmpxchg((volatile __global unsigned int*)source, prevVal.intVal, newVal.intVal);

        // equal to pass
        if(newVal.intVal == prevVal.intVal)
            break;

        prevVal.intVal = newVal.intVal;
    }
}

__attribute__((reqd_work_group_size(MLO_GRP_SZ0, MLO_GRP_SZ1, MLO_GRP_SZ2))) __kernel void
MIOpenConv1x1(const __global _FLOAT* __restrict in_ptr,
              __constant _FLOAT* __restrict wei_ptr,
#if MLO_CONV_BIAS
              const __global _FLOAT* __restrict bias,
#endif
              __global _FLOAT* __restrict out_ptr,
              UNUSED _FLOAT dummy_val // nothing
              )
{

    uint grp_id0       = get_group_id(0);
    uint out_grp_block = grp_id0 % MLO_N_OUT_GROUPS;
    uint in_grp_block  = (uint)(grp_id0 / MLO_N_OUT_GROUPS) % MLO_N_IN_GROUPS;
    uint grp_id0_faked = (uint)(grp_id0 / MLO_N_OUT_GROUPS) / MLO_N_IN_GROUPS;

    uint local_id0 = get_local_id(0);
#if MLO_CHEAT_SHADER_COMPILER == 1
    uint grp_id2 = get_group_id(2);
#endif

    uint pos      = (grp_id0_faked * FIXED_WORKGROUP_SIZE + local_id0) % MLO_IN_CHANNEL_STRIDE;
    uint batch_id = (grp_id0_faked * FIXED_WORKGROUP_SIZE + local_id0) / MLO_IN_CHANNEL_STRIDE;

    if(batch_id >= BATCHSIZE)
        return;

    uint out_id = out_grp_block * MLO_N_LCL_OUT_MAPS;

    uint gbl_in_off = batch_id * MLO_IN_BATCH_STRIDE +
                      in_grp_block * MLO_N_LCL_IN_MAPS * MLO_IN_CHANNEL_STRIDE + pos;

    uint wei_off = out_id * MLO_WEI_CHANNEL_STRIDE + in_grp_block * MLO_N_LCL_IN_MAPS;

    _FLOAT accum[MLO_N_LCL_OUT_MAPS];
    _FLOAT weights[MLO_N_LCL_IN_MAPS_ONCE];
    _FLOAT dat[MLO_N_LCL_IN_MAPS_ONCE];
    _FLOAT dat2[MLO_N_LCL_IN_MAPS_ONCE];

//

// ATOMIC is needed if INPUTS in many waves
#if(MLO_N_LCL_IN_MAPS != MLO_N_INPUTS)

    if(in_grp_block == 0)
    {
        uint gbl_out_off = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE + pos;
        __global _FLOAT* q = out_ptr + gbl_out_off;

        for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
        {
            *q = (_FLOAT)0;
            q += MLO_OUT_CHANNEL_STRIDE;
        }
    }
#endif

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
        accum[o] = (_FLOAT)0;
    }

#if MLO_N_INPUTS == ((MLO_N_INPUTS / MLO_N_LCL_IN_MAPS) * MLO_N_LCL_IN_MAPS)
    // if(1)
    int loops = MLO_CLOOP0;

#if MLO_CHEAT_SHADER_COMPILER == 1
    // cheat shader compiler to disable loop unroll.  it will have better SQC performance
    if(grp_id2 == 0x1F)
    {
        loops = 377; // strange not to unroll loop
    }
#endif
#else
    int loops = MLO_CLOOP0;

    if(in_grp_block == (MLO_N_IN_GROUPS - 1))
    {
        loops = MLO_CLOOP2;
    }

#if MLO_CHEAT_SHADER_COMPILER == 1
    // cheat shader compiler to disable loop unroll.  it will have better SQC performance
    if(grp_id2 == 0x1F)
    {
        loops = 377; // strange not to unroll loop
    }
#endif

#endif
    {
        __global const _FLOAT* p = in_ptr + gbl_in_off;
        __constant _FLOAT* w     = wei_ptr + wei_off;

        // read data
        for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
        {

            dat[j] = *p;
            p += MLO_IN_CHANNEL_STRIDE;
        }

        for(uint ci = 0; ci < (loops - 2); ci += 2)
        {
            // read data
            for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
            {
                dat2[j] = *p;
                p += MLO_IN_CHANNEL_STRIDE;
            }

            // convolve
            __constant _FLOAT* w1 = w;
            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;

            // convolve
            w1 = w;
            for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
            {
                dat[j] = *p;
                p += MLO_IN_CHANNEL_STRIDE;
            }

            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat2[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;
        }

        //
        // last 2 iterations
        { // read data
            for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
            {
                dat2[j] = *p;
                p += MLO_IN_CHANNEL_STRIDE;
            }

            // convolve
            __constant _FLOAT* w1 = w;
            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;

            // convolve
            w1 = w;

            for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
            {

                __constant _FLOAT* w2 = w1;

                for(uint j = 0; j < MLO_N_LCL_IN_MAPS_ONCE; ++j)
                {
                    weights[j] = *w2;
                    w2++;
                }
                w1 += MLO_WEI_CHANNEL_STRIDE;

                for(uint c = 0; c < MLO_N_LCL_IN_MAPS_ONCE; ++c)
                {
                    accum[o] += dat2[c] * weights[c];
                }
            }

            // move weights offset
            w += MLO_N_LCL_IN_MAPS_ONCE;
        }
    }

    uint gbl_out_off   = batch_id * MLO_OUT_BATCH_STRIDE + out_id * MLO_OUT_CHANNEL_STRIDE + pos;
    __global _FLOAT* q = out_ptr + gbl_out_off;

    for(uint o = 0; o < MLO_N_LCL_OUT_MAPS; ++o)
    {
#if(MLO_N_LCL_IN_MAPS == MLO_N_INPUTS)
        *q = accum[o];
        q += MLO_OUT_CHANNEL_STRIDE;
#else
        AtomicAdd(q, accum[o]);
        q += MLO_OUT_CHANNEL_STRIDE;
#endif
    }
}