<commit_before>/** \copyright
 * Copyright (c) 2013, Stuart W Baker
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are  permitted provided that the following conditions are met:
 *
 *  - Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 *  - Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * \file StlMap.hxx
 * This file provides a feature limited abstraction of std::map
 *
 * @author Stuart W. Baker
 * @date 2 December 2013
 */

#ifndef _UTILS_STLMAP_HXX_
#define _UTILS_STLMAP_HXX_

#include <map>

#include "utils/Allocator.hxx"
#include "utils/macros.h"

/** Though at the surface, this may seem like an unnecessary abstraction of
 * std::map, it has the purpose of limiting the implementation to features
 * found only in common to @ref SysMap and @ref LinearMap implementations.  In
 * this way, one can use @ref Map and based on compile time settings choose to
 * use any one of @ref StlMap, @ref SysMap, or @ref LinearMap from the same
 * source usage of @ref Map.
 */
template <typename Key, typename Value> class StlMap
{
public:
    /** Default Constructor which with no mapping entry limit.
     */
    StlMap()
        : mappingAllocator(NULL),
          mapping(new Mapping())
    {
    }

    /** Constructor that limits the number of mappings to a static pool.
     * @param entries number of nodes to statically create and track
     */
    StlMap(size_t entries)
        : mappingAllocator(new MappingAllocator(std::less<Key>(), Allocator<std::pair<const Key, Value>>(entries))),
          mapping(NULL)
    {
    }

    /** Destructor.
     */
    ~StlMap()
    {
        if (mappingAllocator)
        {
            delete mappingAllocator;
        }
        if (mapping)
        {
            delete mapping;
        }
    }

    /** This translation is done for consistency with @ref SysMap and @ref LinearMap */
    typedef pair<Key, Value> Pair;

    /** Short hand for the iterator type of a given instance */
    typedef typename std::map<Key, Value>::iterator Iterator;

    /** Removes all elements in the map. */
    void clear() {
        if (mapping) {
            mapping->clear();
        } else {
            mappingAllocator->clear();
        }
    }

    /** Remove an element from the tree.
     * @param key key for the element to remove
     * @return number of elements removed
     */
    size_t erase(Key key)
    {
        return mapping ? mapping->erase(key) : mappingAllocator->erase(key);
    }

    /** Remove a node from the tree.
     * @param it iterator index for the element to remove
     */
    void erase(Iterator it)
    {
        mapping ? mapping->erase(it) : mappingAllocator->erase(it);
    }

    /** Find the index associated with the key and create it if does not exist.
     * @param key key to lookup
     * @return value of the key by reference
     */
    Value& operator[](const Key &key)
    {
        return mapping ? (*mapping)[key] : (*mappingAllocator)[key];
    }

    /** Number of elements currently in the map.
     * @return number of elements in the map
     */
    size_t size()
    {
        return mapping ? mapping->size() : mappingAllocator->size();
    }

    /** Maximum theoretical number of elements in the map.
     * @return maximum theoretical number of elements in the map
     */
    size_t max_size()
    {
        return mapping ? mapping->max_size() : mappingAllocator->max_size();
    }

    /** Find an element matching the given key.
     * @param key key to search for
     * @return iterator index pointing to key, else iterator end() if not found
     */
    Iterator find( const Key &key )
    {
        return mapping ? mapping->find(key) : mappingAllocator->find(key);
    }

    /** Get an iterator index pointing one past the last element in mapping.
     * @return iterator index pointing to one past the last element in mapping
     */
    Iterator end()
    {
        return mapping ? mapping->end() : mappingAllocator->end();
    }

    /** Get an iterator index pointing one past the last element in mapping.
     * @return iterator index pointing to one past the last element in mapping
     */
    Iterator begin()
    {
        return mapping ? mapping->begin() : mappingAllocator->begin();
    }

private:
    /** short hand for the custom allocator std::map type */
    typedef std::map<Key, Value, std::less<Key>, Allocator<std::pair<const Key, Value>>> MappingAllocator;

    /** short hand for the default allocator std::map type */
    typedef std::map<Key, Value> Mapping;

    /** pointer to an std::map instance with a custom allocator */
    MappingAllocator *mappingAllocator;

    /** pointer to an std::map instance with a default allocator */
    Mapping *mapping;

    DISALLOW_COPY_AND_ASSIGN(StlMap);
};

#endif /* _UTILS_STLMAP_HXX_ */
<commit_msg>Makes StlMap work with entries==0 constructor.<commit_after>/** \copyright
 * Copyright (c) 2013, Stuart W Baker
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are  permitted provided that the following conditions are met:
 *
 *  - Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 *  - Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * \file StlMap.hxx
 * This file provides a feature limited abstraction of std::map
 *
 * @author Stuart W. Baker
 * @date 2 December 2013
 */

#ifndef _UTILS_STLMAP_HXX_
#define _UTILS_STLMAP_HXX_

#include <map>

#include "utils/Allocator.hxx"
#include "utils/macros.h"

/** Though at the surface, this may seem like an unnecessary abstraction of
 * std::map, it has the purpose of limiting the implementation to features
 * found only in common to @ref SysMap and @ref LinearMap implementations.  In
 * this way, one can use @ref Map and based on compile time settings choose to
 * use any one of @ref StlMap, @ref SysMap, or @ref LinearMap from the same
 * source usage of @ref Map.
 */
template <typename Key, typename Value> class StlMap
{
public:
    /** Default Constructor which with no mapping entry limit.
     */
    StlMap()
        : mappingAllocator(NULL),
          mapping(new Mapping())
    {
    }

    /** Constructor that limits the number of mappings to a static pool.
     * @param entries number of nodes to statically create and track
     */
    StlMap(size_t entries)
        : mappingAllocator(entries > 0 ? new MappingAllocator(std::less<Key>(), Allocator<std::pair<const Key, Value>>(entries)) : NULL),
          mapping(entries > 0 ? NULL : new Mapping())
    {
    }

    /** Destructor.
     */
    ~StlMap()
    {
        if (mappingAllocator)
        {
            delete mappingAllocator;
        }
        if (mapping)
        {
            delete mapping;
        }
    }

    /** This translation is done for consistency with @ref SysMap and @ref LinearMap */
    typedef pair<Key, Value> Pair;

    /** Short hand for the iterator type of a given instance */
    typedef typename std::map<Key, Value>::iterator Iterator;

    /** Removes all elements in the map. */
    void clear() {
        if (mapping) {
            mapping->clear();
        } else {
            mappingAllocator->clear();
        }
    }

    /** Remove an element from the tree.
     * @param key key for the element to remove
     * @return number of elements removed
     */
    size_t erase(Key key)
    {
        return mapping ? mapping->erase(key) : mappingAllocator->erase(key);
    }

    /** Remove a node from the tree.
     * @param it iterator index for the element to remove
     */
    void erase(Iterator it)
    {
        mapping ? mapping->erase(it) : mappingAllocator->erase(it);
    }

    /** Find the index associated with the key and create it if does not exist.
     * @param key key to lookup
     * @return value of the key by reference
     */
    Value& operator[](const Key &key)
    {
        return mapping ? (*mapping)[key] : (*mappingAllocator)[key];
    }

    /** Number of elements currently in the map.
     * @return number of elements in the map
     */
    size_t size()
    {
        return mapping ? mapping->size() : mappingAllocator->size();
    }

    /** Maximum theoretical number of elements in the map.
     * @return maximum theoretical number of elements in the map
     */
    size_t max_size()
    {
        return mapping ? mapping->max_size() : mappingAllocator->max_size();
    }

    /** Find an element matching the given key.
     * @param key key to search for
     * @return iterator index pointing to key, else iterator end() if not found
     */
    Iterator find( const Key &key )
    {
        return mapping ? mapping->find(key) : mappingAllocator->find(key);
    }

    /** Get an iterator index pointing one past the last element in mapping.
     * @return iterator index pointing to one past the last element in mapping
     */
    Iterator end()
    {
        return mapping ? mapping->end() : mappingAllocator->end();
    }

    /** Get an iterator index pointing one past the last element in mapping.
     * @return iterator index pointing to one past the last element in mapping
     */
    Iterator begin()
    {
        return mapping ? mapping->begin() : mappingAllocator->begin();
    }

private:
    /** short hand for the custom allocator std::map type */
    typedef std::map<Key, Value, std::less<Key>, Allocator<std::pair<const Key, Value>>> MappingAllocator;

    /** short hand for the default allocator std::map type */
    typedef std::map<Key, Value> Mapping;

    /** pointer to an std::map instance with a custom allocator */
    MappingAllocator *mappingAllocator;

    /** pointer to an std::map instance with a default allocator */
    Mapping *mapping;

    DISALLOW_COPY_AND_ASSIGN(StlMap);
};

#endif /* _UTILS_STLMAP_HXX_ */
<|endoftext|>


<commit_before>// This file is distributed under the MIT license.
// See the LICENSE file for details.

#pragma once

#ifndef VSNRAY_WHITTED_INL
#define VSNRAY_WHITTED_INL 1

#include <array>
#include <cstddef>

#include <visionaray/get_surface.h>
#include <visionaray/result_record.h>
#include <visionaray/traverse.h>

namespace visionaray
{
namespace whitted
{

namespace detail
{

//-------------------------------------------------------------------------------------------------
// TODO: consolidate this with "real" brdf sampling
// TODO: user should be able to customize this behavior
//

template <typename Vec3, typename Scalar>
struct bounce_result
{
    Vec3 reflected_dir;
    Vec3 refracted_dir;

    Scalar kr;
    Scalar kt;
};

// reflection
template <typename V, typename S>
VSNRAY_FUNC
inline auto make_bounce_result(V const& reflected_dir, S kr)
    -> bounce_result<V, S>
{
    return {
        reflected_dir,
        V(),
        kr,
        0.0f
        };
}

// reflection and refraction
template <typename V, typename S>
VSNRAY_FUNC
inline auto make_bounce_result(
        V const& reflected_dir,
        V const& refracted_dir,
        S kr,
        S kt
        )
    -> bounce_result<V, S>
{
    return {
        reflected_dir,
        refracted_dir,
        kr,
        kt
        };
}

//-------------------------------------------------------------------------------------------------
// specular_bounce() overloads for some materials
//

// fall-through, e.g. for plastic, assigns a dflt. reflectivity and no refraction
template <typename V, typename M>
VSNRAY_FUNC
inline auto specular_bounce(
        M const&    mat,
        V const&    view_dir,
        V const&    normal
        )
    -> decltype( make_bounce_result(V(), typename V::value_type()) )
{
    VSNRAY_UNUSED(mat);

    return make_bounce_result(
        reflect(view_dir, normal),
        typename V::value_type(0.1)
        );
}

// matte, no specular reflectivity, returns an arbitrary direction
template <typename V, typename S>
VSNRAY_FUNC
inline auto specular_bounce(
        matte<S> const& mat,
        V const&        view_dir,
        V const&        normal
        )
    -> decltype( make_bounce_result(V(), typename V::value_type()) )
{
    VSNRAY_UNUSED(mat);
    VSNRAY_UNUSED(view_dir);
    VSNRAY_UNUSED(normal);

    return make_bounce_result(
        V(),
        typename V::value_type(0.0)
        );
}

// mirror material, here we know kr
template <typename V, typename S>
VSNRAY_FUNC
inline auto specular_bounce(
        mirror<S> const&    mat,
        V const&            view_dir,
        V const&            normal
        )
    -> decltype( make_bounce_result(V(), typename V::value_type()) )
{
    return make_bounce_result(
        reflect(view_dir, normal),
        mat.get_kr()
        );
}

//-------------------------------------------------------------------------------------------------
// some special treatment for generic materials
//

template <typename V>
struct visitor
{
    using return_type = decltype( make_bounce_result(V(), typename V::value_type()) );

    VSNRAY_FUNC visitor(V const& vd, V const& n)
        : view_dir(vd)
        , normal(n)
    {
    }

    template <typename X>
    VSNRAY_FUNC
    return_type operator()(X const& ref) const
    {
        return specular_bounce(ref, view_dir, normal);
    }

    V const&    view_dir;
    V const&    normal;
};

template <typename V, typename ...Ts>
VSNRAY_FUNC
inline auto specular_bounce(
        generic_material<Ts...> const&  mat,
        V const&                        view_dir,
        V const&                        normal
        )
    -> decltype( make_bounce_result(V(), typename V::value_type()) )
{
    return apply_visitor(visitor<V>(view_dir, normal), mat);
}

template <
    size_t N,
    typename ...Ts,
    typename T,
    typename = typename std::enable_if<simd::is_simd_vector<T>::value>::type
    >
inline auto specular_bounce(
        simd::generic_material<N, Ts...> const& mat,
        vector<3, T> const&                     view_dir,
        vector<3, T> const&                     normal
        )
    -> bounce_result<vector<3, T>, T>
{
    using float_array = simd::aligned_array_t<T>;

    auto ms  = unpack(mat);
    auto vds = unpack(view_dir);
    auto ns  = unpack(normal);

    std::array<vector<3, float>, N> refl_dir;
    std::array<vector<3, float>, N> refr_dir;
    float_array                     kr;
    float_array                     kt;

    for (size_t i = 0; i < N; ++i)
    {
        auto res = specular_bounce(ms[i], vds[i], ns[i]);
        refl_dir[i] = res.reflected_dir;
        refr_dir[i] = res.refracted_dir;
        kr[i]       = res.kr;
        kt[i]       = res.kt;
    }

    return make_bounce_result(
        simd::pack(refl_dir),
        simd::pack(refr_dir),
        T(kr),
        T(kt)
        );
}

} // detail

//-------------------------------------------------------------------------------------------------
// Whitted kernel
//

template <typename Params>
struct kernel
{

    Params params;

    template <typename Intersector, typename R>
    VSNRAY_FUNC result_record<typename R::scalar_type> operator()(Intersector& isect, R ray) const
    {

        using S = typename R::scalar_type;
        using V = typename result_record<S>::vec_type;
        using C = spectrum<S>;

        result_record<S> result;

        auto hit_rec = closest_hit(ray, params.prims.begin, params.prims.end, isect);

        if (any(hit_rec.hit))
        {
            result.hit = hit_rec.hit;
            result.isect_pos = ray.ori + ray.dir * hit_rec.t;
        }
        else
        {
            result.hit = false;
            result.color = params.bg_color;
            return result;
        }

        C color(0.0);

        size_t depth = 0;
        auto no_hit_color = C(from_rgba(params.bg_color));
        auto throughput = S(1.0);
        while (any(hit_rec.hit) && any(throughput > S(params.epsilon)) && depth++ < params.num_bounces)
        {
            hit_rec.isect_pos = ray.ori + ray.dir * hit_rec.t;

            auto surf = get_surface(hit_rec, params);
            auto ambient = surf.material.ambient() * C(from_rgba(params.ambient_color));
            auto shaded_clr = select( hit_rec.hit, ambient, C(from_rgba(params.bg_color)) );
            auto view_dir = -ray.dir;

            auto n = surf.shading_normal;

#if 1 // two-sided
            n = faceforward( n, view_dir, surf.geometric_normal );
#endif

            for (auto it = params.lights.begin; it != params.lights.end; ++it)
            {
                auto light_dir = normalize( V(it->position()) - hit_rec.isect_pos );
                R shadow_ray
                (
                    hit_rec.isect_pos + light_dir * S(params.epsilon),
                    light_dir
                );

                // only cast a shadow if occluder between light source and hit pos
                auto shadow_rec  = any_hit(
                        shadow_ray,
                        params.prims.begin,
                        params.prims.end,
                        length(hit_rec.isect_pos - V(it->position())),
                        isect
                        );

                auto active_rays = hit_rec.hit & !shadow_rec.hit;

                auto sr         = make_shade_record<Params, S>();
                sr.active       = active_rays;
                sr.isect_pos    = hit_rec.isect_pos;
                sr.normal       = n;
                sr.view_dir     = view_dir;
                sr.light_dir    = light_dir;
                sr.light        = *it;
                auto clr        = surf.shade(sr);

                shaded_clr += select( active_rays, clr, C(0.0) );
            }

            color += select( hit_rec.hit, shaded_clr, no_hit_color ) * throughput;

            auto bounce = detail::specular_bounce(surf.material, view_dir, surf.shading_normal);

            if (any(bounce.kr > S(0.0)))
            {
                auto dir = bounce.reflected_dir;
                ray = R(
                    hit_rec.isect_pos + dir * S(params.epsilon),
                    dir
                    );
                hit_rec = closest_hit(ray, params.prims.begin, params.prims.end, isect);
            }
            throughput *= bounce.kr;
            no_hit_color = C(0.0);
        }

        result.color = select( result.hit, to_rgba(color), params.bg_color );

        return result;

    }

    template <typename R>
    VSNRAY_FUNC result_record<typename R::scalar_type> operator()(R ray) const
    {
        default_intersector ignore;
        return (*this)(ignore, ray);
    }
};

} // whitted
} // visionaray

#endif // VSNRAY_WHITTED_INL
<commit_msg>Simplify<commit_after>// This file is distributed under the MIT license.
// See the LICENSE file for details.

#pragma once

#ifndef VSNRAY_WHITTED_INL
#define VSNRAY_WHITTED_INL 1

#include <array>
#include <cstddef>

#include <visionaray/get_surface.h>
#include <visionaray/result_record.h>
#include <visionaray/traverse.h>

namespace visionaray
{
namespace whitted
{

namespace detail
{

//-------------------------------------------------------------------------------------------------
// TODO: consolidate this with "real" brdf sampling
// TODO: user should be able to customize this behavior
//

template <typename Vec3, typename Scalar>
struct bounce_result
{
    Vec3 reflected_dir;
    Vec3 refracted_dir;

    Scalar kr;
    Scalar kt;
};

// reflection
template <typename V, typename S>
VSNRAY_FUNC
inline auto make_bounce_result(V const& reflected_dir, S kr)
    -> bounce_result<V, S>
{
    return {
        reflected_dir,
        V(),
        kr,
        0.0f
        };
}

// reflection and refraction
template <typename V, typename S>
VSNRAY_FUNC
inline auto make_bounce_result(
        V const& reflected_dir,
        V const& refracted_dir,
        S kr,
        S kt
        )
    -> bounce_result<V, S>
{
    return {
        reflected_dir,
        refracted_dir,
        kr,
        kt
        };
}

//-------------------------------------------------------------------------------------------------
// specular_bounce() overloads for some materials
//

// fall-through, e.g. for plastic, assigns a dflt. reflectivity and no refraction
template <typename V, typename M>
VSNRAY_FUNC
inline auto specular_bounce(
        M const&    mat,
        V const&    view_dir,
        V const&    normal
        )
    -> decltype( make_bounce_result(V(), typename V::value_type()) )
{
    VSNRAY_UNUSED(mat);

    return make_bounce_result(
        reflect(view_dir, normal),
        typename V::value_type(0.1)
        );
}

// matte, no specular reflectivity, returns an arbitrary direction
template <typename V, typename S>
VSNRAY_FUNC
inline auto specular_bounce(
        matte<S> const& mat,
        V const&        view_dir,
        V const&        normal
        )
    -> decltype( make_bounce_result(V(), typename V::value_type()) )
{
    VSNRAY_UNUSED(mat);
    VSNRAY_UNUSED(view_dir);
    VSNRAY_UNUSED(normal);

    return make_bounce_result(
        V(),
        typename V::value_type(0.0)
        );
}

// mirror material, here we know kr
template <typename V, typename S>
VSNRAY_FUNC
inline auto specular_bounce(
        mirror<S> const&    mat,
        V const&            view_dir,
        V const&            normal
        )
    -> decltype( make_bounce_result(V(), typename V::value_type()) )
{
    return make_bounce_result(
        reflect(view_dir, normal),
        mat.get_kr()
        );
}

//-------------------------------------------------------------------------------------------------
// some special treatment for generic materials
//

template <typename V>
struct visitor
{
    using return_type = decltype( make_bounce_result(V(), typename V::value_type()) );

    VSNRAY_FUNC visitor(V const& vd, V const& n)
        : view_dir(vd)
        , normal(n)
    {
    }

    template <typename X>
    VSNRAY_FUNC
    return_type operator()(X const& ref) const
    {
        return specular_bounce(ref, view_dir, normal);
    }

    V const&    view_dir;
    V const&    normal;
};

template <typename V, typename ...Ts>
VSNRAY_FUNC
inline auto specular_bounce(
        generic_material<Ts...> const&  mat,
        V const&                        view_dir,
        V const&                        normal
        )
    -> decltype( make_bounce_result(V(), typename V::value_type()) )
{
    return apply_visitor(visitor<V>(view_dir, normal), mat);
}

template <
    size_t N,
    typename ...Ts,
    typename T,
    typename = typename std::enable_if<simd::is_simd_vector<T>::value>::type
    >
inline auto specular_bounce(
        simd::generic_material<N, Ts...> const& mat,
        vector<3, T> const&                     view_dir,
        vector<3, T> const&                     normal
        )
    -> bounce_result<vector<3, T>, T>
{
    using float_array = simd::aligned_array_t<T>;

    auto ms  = unpack(mat);
    auto vds = unpack(view_dir);
    auto ns  = unpack(normal);

    std::array<vector<3, float>, N> refl_dir;
    std::array<vector<3, float>, N> refr_dir;
    float_array                     kr;
    float_array                     kt;

    for (size_t i = 0; i < N; ++i)
    {
        auto res = specular_bounce(ms[i], vds[i], ns[i]);
        refl_dir[i] = res.reflected_dir;
        refr_dir[i] = res.refracted_dir;
        kr[i]       = res.kr;
        kt[i]       = res.kt;
    }

    return make_bounce_result(
        simd::pack(refl_dir),
        simd::pack(refr_dir),
        T(kr),
        T(kt)
        );
}

} // detail

//-------------------------------------------------------------------------------------------------
// Whitted kernel
//

template <typename Params>
struct kernel
{

    Params params;

    template <typename Intersector, typename R>
    VSNRAY_FUNC result_record<typename R::scalar_type> operator()(Intersector& isect, R ray) const
    {

        using S = typename R::scalar_type;
        using V = typename result_record<S>::vec_type;
        using C = spectrum<S>;

        result_record<S> result;

        auto hit_rec = closest_hit(ray, params.prims.begin, params.prims.end, isect);

        if (any(hit_rec.hit))
        {
            result.hit = hit_rec.hit;
            result.isect_pos = ray.ori + ray.dir * hit_rec.t;
        }
        else
        {
            result.hit = false;
            result.color = params.bg_color;
            return result;
        }

        C color(0.0);

        size_t depth = 0;
        C no_hit_color(from_rgba(params.bg_color));
        S throughput(1.0);
        while (any(hit_rec.hit) && any(throughput > S(params.epsilon)) && depth++ < params.num_bounces)
        {
            hit_rec.isect_pos = ray.ori + ray.dir * hit_rec.t;

            auto surf = get_surface(hit_rec, params);
            auto ambient = surf.material.ambient() * C(from_rgba(params.ambient_color));
            auto shaded_clr = select( hit_rec.hit, ambient, C(from_rgba(params.bg_color)) );
            auto view_dir = -ray.dir;

            auto n = surf.shading_normal;

#if 1 // two-sided
            n = faceforward( n, view_dir, surf.geometric_normal );
#endif

            for (auto it = params.lights.begin; it != params.lights.end; ++it)
            {
                auto light_dir = normalize( V(it->position()) - hit_rec.isect_pos );
                R shadow_ray
                (
                    hit_rec.isect_pos + light_dir * S(params.epsilon),
                    light_dir
                );

                // only cast a shadow if occluder between light source and hit pos
                auto shadow_rec  = any_hit(
                        shadow_ray,
                        params.prims.begin,
                        params.prims.end,
                        length(hit_rec.isect_pos - V(it->position())),
                        isect
                        );

                auto active_rays = hit_rec.hit & !shadow_rec.hit;

                auto sr         = make_shade_record<Params, S>();
                sr.active       = active_rays;
                sr.isect_pos    = hit_rec.isect_pos;
                sr.normal       = n;
                sr.view_dir     = view_dir;
                sr.light_dir    = light_dir;
                sr.light        = *it;
                auto clr        = surf.shade(sr);

                shaded_clr += select( active_rays, clr, C(0.0) );
            }

            color += select( hit_rec.hit, shaded_clr, no_hit_color ) * throughput;

            auto bounce = detail::specular_bounce(surf.material, view_dir, surf.shading_normal);

            if (any(bounce.kr > S(0.0)))
            {
                auto dir = bounce.reflected_dir;
                ray = R(
                    hit_rec.isect_pos + dir * S(params.epsilon),
                    dir
                    );
                hit_rec = closest_hit(ray, params.prims.begin, params.prims.end, isect);
            }
            throughput *= bounce.kr;
            no_hit_color = C(0.0);
        }

        result.color = select( result.hit, to_rgba(color), params.bg_color );

        return result;

    }

    template <typename R>
    VSNRAY_FUNC result_record<typename R::scalar_type> operator()(R ray) const
    {
        default_intersector ignore;
        return (*this)(ignore, ray);
    }
};

} // whitted
} // visionaray

#endif // VSNRAY_WHITTED_INL
<|endoftext|>


<commit_before>#include <gtest/gtest.h>
#include "utMadRace.h"

int main( int argc , char **argv )
{
    testing :: InitGoogleTest( &argc , argv ) ;
    return RUN_ALL_TESTS( ) ;
}
<commit_msg>Delete mainMadRace.cpp<commit_after><|endoftext|>


<commit_before>#ifndef CHANNEL_HELPER_H_
#define CHANNEL_HELPER_H_

/*
 * Channel parameter defaults to NO_CHANNEL.
 */
#ifndef CHANNEL
#define CHANNEL NO_CHANNEL
#endif

/*
 * HOST parameter.
 * Used for socket channels.
 */
#ifndef HOST
    #define HOST            {0}
    #define UNDEFINED_HOST  1
#else
    #define UNDEFINED_HOST  0
#endif

/*
 * PORT parameter.
 * Used for socket channels.
 */
#ifndef PORT
    #define PORT            0
    #define UNDEFINED_PORT  1
#else
    #define UNDEFINED_PORT  0
#endif

struct Channel
{
    int rx;
    int tx;
    bool dupable_to_stdout = true;

    CONSTRUCTOR Channel();
    METHOD int recv(void *buf, size_t count);
    METHOD int send(void *buf, size_t count);
};

enum channel_mode
{
    NO_CHANNEL,
    TCP_CONNECT,
    TCP_LISTEN,
    TCP6_CONNECT,
    TCP6_LISTEN,
    SCTP_CONNECT,
    SCTP_LISTEN,
    SCTP6_CONNECT,
    SCTP6_LISTEN,
    USE_STDOUT,
    USE_STDERR,
};

#include "socket.cc"
#include "fs.cc"

CONSTRUCTOR
Channel::Channel()
{
    const ip_addr_t host = { .ip6 = HOST };
    const ip_port_t port = PORT;

    switch ( CHANNEL )
    {
        case TCP_CONNECT:
            this->rx = this->tx = tcp_connect(host, port);
            break;

        case TCP6_CONNECT:
            this->rx = this->tx = tcp6_connect(host, port);
            break;

        case TCP_LISTEN:
            this->rx = this->tx = tcp_listen(host, port);
            break;

        case TCP6_LISTEN:
            this->rx = this->tx = tcp6_listen(host, port);
            break;

        case SCTP_CONNECT:
            this->rx = this->tx = sctp_connect(host, port);
            break;

        case SCTP6_CONNECT:
            this->rx = this->tx = sctp6_connect(host, port);
            break;

        case SCTP_LISTEN:
            this->rx = this->tx = sctp_listen(host, port);
            break;

        case SCTP6_LISTEN:
            this->rx = this->tx = sctp6_listen(host, port);
            break;

        case USE_STDOUT:
            this->dupable_to_stdout = false;
            this->rx = 0;
            this->tx = 1;
            break;

        case USE_STDERR:
            this->rx = 0;
            this->tx = 2;
            break;

        case NO_CHANNEL:
        default:
            this->dupable_to_stdout = false;
            this->rx = this->tx = -1;
            break;
    }
}

METHOD
int Channel::recv(void *buf, size_t count)
{
    size_t bytes_left = count;
    ssize_t nr_read;

    while ( bytes_left > 0 )
    {
        nr_read = Syscall::read(this->rx, (char *) buf + count - bytes_left, count);
        if ( nr_read < 0 )
            return -1;

        bytes_left -= nr_read;
    }

    return 0;
}

METHOD
int Channel::send(void *buf, size_t count)
{
    return Syscall::write(this->tx, buf, count);
}

#endif
<commit_msg>pico/channel: WIP<commit_after>#ifndef CHANNEL_HELPER_H_
#define CHANNEL_HELPER_H_

/*
 * Channel parameter defaults to NO_CHANNEL.
 */
#ifndef CHANNEL
#define CHANNEL NO_CHANNEL
#endif

/*
 * HOST parameter.
 * Used for socket channels.
 */
#ifndef HOST
    #define HOST            {0}
    #define UNDEFINED_HOST  1
#else
    #define UNDEFINED_HOST  0
#endif

/*
 * PORT parameter.
 * Used for socket channels.
 */
#ifndef PORT
    #define PORT            0
    #define UNDEFINED_PORT  1
#else
    #define UNDEFINED_PORT  0
#endif

namespace Pico {

    enum channel_mode
    {
        NO_CHANNEL,
        TCP_CONNECT,
        TCP_LISTEN,
        TCP6_CONNECT,
        TCP6_LISTEN,
        SCTP_CONNECT,
        SCTP_LISTEN,
        SCTP6_CONNECT,
        SCTP6_LISTEN,
        USE_STDOUT,
        USE_STDERR,
    };

    template <enum channel_mode>
    struct ChannelMode;

    #define DEFINE_CHANNEL_MODE(mode, type, dupable)            \
        template<>                                              \
        struct ChannelMode<mode>                                \
        {                                                       \
            typedef type stream_type;                           \
            static constexpr bool dupable_to_stdio = dupable;   \
        };                                                      \

    DEFINE_CHANNEL_MODE(NO_CHANNEL,     int,                    false);
    DEFINE_CHANNEL_MODE(USE_STDOUT,     Stream,                 false);
    DEFINE_CHANNEL_MODE(USE_STDERR,     Stream,                 false);
    DEFINE_CHANNEL_MODE(TCP_CONNECT,    Network::TcpSocket,     true);
    DEFINE_CHANNEL_MODE(TCP6_CONNECT,   Network::Tcp6Socket,    true);
    DEFINE_CHANNEL_MODE(TCP_LISTEN,     Network::TcpSocket,     true);
    DEFINE_CHANNEL_MODE(TCP6_LISTEN,    Network::Tcp6Socket,    true);
    DEFINE_CHANNEL_MODE(SCTP_CONNECT,   Network::SctpSocket,    true);
    DEFINE_CHANNEL_MODE(SCTP6_CONNECT,  Network::Sctp6Socket,   true);
    DEFINE_CHANNEL_MODE(SCTP_LISTEN,    Network::SctpSocket,    true);
    DEFINE_CHANNEL_MODE(SCTP6_LISTEN,   Network::Sctp6Socket,   true);

    /* XXX: WIP
    template <enum channel_mode M>
    struct Channel
    {
        typename ChannelMode<M>::stream_type rx;
        typename ChannelMode<M>::stream_type tx;
        static constexpr bool dupable_to_stdio = ChannelMode<M>::dupable_to_stdio;
        CONSTRUCTOR Channelng() = default;

        METHOD int recv(void *buf, size_t count) {
            return rx.read(buf, count);
        }
        METHOD int send(void *buf, size_t count) {
            return rx.write(buf, count);
        }
    };*/
}

struct Channel
{
    int rx;
    int tx;
    bool dupable_to_stdout = true;

    CONSTRUCTOR Channel();
    METHOD int recv(void *buf, size_t count);
    METHOD int send(void *buf, size_t count);
};

#include "socket.cc"
#include "fs.cc"

CONSTRUCTOR
Channel::Channel()
{
    const ip_addr_t host = { .ip6 = HOST };
    const ip_port_t port = PORT;

    switch ( CHANNEL )
    {
        case TCP_CONNECT:
            this->rx = this->tx = tcp_connect(host, port);
            break;

        case TCP6_CONNECT:
            this->rx = this->tx = tcp6_connect(host, port);
            break;

        case TCP_LISTEN:
            this->rx = this->tx = tcp_listen(host, port);
            break;

        case TCP6_LISTEN:
            this->rx = this->tx = tcp6_listen(host, port);
            break;

        case SCTP_CONNECT:
            this->rx = this->tx = sctp_connect(host, port);
            break;

        case SCTP6_CONNECT:
            this->rx = this->tx = sctp6_connect(host, port);
            break;

        case SCTP_LISTEN:
            this->rx = this->tx = sctp_listen(host, port);
            break;

        case SCTP6_LISTEN:
            this->rx = this->tx = sctp6_listen(host, port);
            break;

        case USE_STDOUT:
            this->dupable_to_stdout = false;
            this->rx = 0;
            this->tx = 1;
            break;

        case USE_STDERR:
            this->rx = 0;
            this->tx = 2;
            break;

        case NO_CHANNEL:
        default:
            this->dupable_to_stdout = false;
            this->rx = this->tx = -1;
            break;
    }
}

METHOD
int Channel::recv(void *buf, size_t count)
{
    size_t bytes_left = count;
    ssize_t nr_read;

    while ( bytes_left > 0 )
    {
        nr_read = Syscall::read(this->rx, (char *) buf + count - bytes_left, count);
        if ( nr_read < 0 )
            return -1;

        bytes_left -= nr_read;
    }

    return 0;
}

METHOD
int Channel::send(void *buf, size_t count)
{
    return Syscall::write(this->tx, buf, count);
}

#endif
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: previewadapter.hxx,v $
 *
 *  $Revision: 1.6 $
 *
 *  last change: $Author: obo $ $Date: 2006-10-12 10:54:28 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef _PREVIEWADAPTER_HXX_
#define _PREVIEWADAPTER_HXX_

#ifndef _SAL_TYPES_H_
#include <sal/types.h>
#endif

#ifndef _COM_SUN_STAR_UNO_SEQUENCE_HXX_
#include <com/sun/star/uno/Sequence.hxx>
#endif

#ifndef _COM_SUN_STAR_UNO_RUNTIMEEXCEPTION_HPP_
#include <com/sun/star/uno/RuntimeException.hpp>
#endif

#ifndef _COM_SUN_STAR_LANG_ILLEGALARGUMENTEXCEPTION_HPP_
#include <com/sun/star/lang/IllegalArgumentException.hpp>
#endif

#ifndef _COM_SUN_STAR_UNO_ANY_HXX_
#include <com/sun/star/uno/Any.hxx>
#endif

#if defined _MSC_VER
#pragma warning(push, 1)
#endif
#include <windows.h>
#if defined _MSC_VER
#pragma warning(pop)
#endif
#include <memory>

// forward declaration
class CPreviewAdapterImpl;

//---------------------------------------------
// A kind of a facade for the preview class.
// We want to hide the fact that the preview
// window may only become visible if there is
// a valid parent window (means, the FilePicker)
// is in execution mode. So unless someone sets
// the preview active with a valid parent
// window the preview may not be visible
//---------------------------------------------

class CPreviewAdapter
{
public:

    // ctor
    CPreviewAdapter(HINSTANCE instance);

    ~CPreviewAdapter();

    ::com::sun::star::uno::Sequence<sal_Int16> SAL_CALL getSupportedImageFormats();

    sal_Int32 SAL_CALL getTargetColorDepth();

    sal_Int32 SAL_CALL getAvailableWidth();

    sal_Int32 SAL_CALL getAvailableHeight();

    void SAL_CALL setImage(sal_Int16 aImageFormat, const ::com::sun::star::uno::Any& aImage)
        throw (::com::sun::star::lang::IllegalArgumentException, ::com::sun::star::uno::RuntimeException);

    sal_Bool SAL_CALL setShowState(sal_Bool bShowState);

    sal_Bool SAL_CALL getShowState();

    void SAL_CALL setParent(HWND parent);

    //--------------------------------------
    // notification from parent
    //--------------------------------------

    void SAL_CALL notifyParentShow(bool bShow);

    void SAL_CALL notifyParentSizeChanged();

    void SAL_CALL notifyParentWindowPosChanged();

private:
    // hide implementation details using the
    // bridge pattern
    std::auto_ptr<CPreviewAdapterImpl> m_pImpl;

// prevent copy and assignment
private:
    CPreviewAdapter(const CPreviewAdapter&);
    CPreviewAdapter& operator=(const CPreviewAdapter&);
};

#endif
<commit_msg>INTEGRATION: CWS changefileheader (1.6.154); FILE MERGED 2008/04/01 15:17:05 thb 1.6.154.3: #i85898# Stripping all external header guards 2008/04/01 12:30:51 thb 1.6.154.2: #i85898# Stripping all external header guards 2008/03/31 13:13:19 rt 1.6.154.1: #i87441# Change license header to LPGL v3.<commit_after>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2008 by Sun Microsystems, Inc.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * $RCSfile: previewadapter.hxx,v $
 * $Revision: 1.7 $
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

#ifndef _PREVIEWADAPTER_HXX_
#define _PREVIEWADAPTER_HXX_

#include <sal/types.h>
#include <com/sun/star/uno/Sequence.hxx>
#include <com/sun/star/uno/RuntimeException.hpp>
#include <com/sun/star/lang/IllegalArgumentException.hpp>
#include <com/sun/star/uno/Any.hxx>

#if defined _MSC_VER
#pragma warning(push, 1)
#endif
#include <windows.h>
#if defined _MSC_VER
#pragma warning(pop)
#endif
#include <memory>

// forward declaration
class CPreviewAdapterImpl;

//---------------------------------------------
// A kind of a facade for the preview class.
// We want to hide the fact that the preview
// window may only become visible if there is
// a valid parent window (means, the FilePicker)
// is in execution mode. So unless someone sets
// the preview active with a valid parent
// window the preview may not be visible
//---------------------------------------------

class CPreviewAdapter
{
public:

    // ctor
    CPreviewAdapter(HINSTANCE instance);

    ~CPreviewAdapter();

    ::com::sun::star::uno::Sequence<sal_Int16> SAL_CALL getSupportedImageFormats();

    sal_Int32 SAL_CALL getTargetColorDepth();

    sal_Int32 SAL_CALL getAvailableWidth();

    sal_Int32 SAL_CALL getAvailableHeight();

    void SAL_CALL setImage(sal_Int16 aImageFormat, const ::com::sun::star::uno::Any& aImage)
        throw (::com::sun::star::lang::IllegalArgumentException, ::com::sun::star::uno::RuntimeException);

    sal_Bool SAL_CALL setShowState(sal_Bool bShowState);

    sal_Bool SAL_CALL getShowState();

    void SAL_CALL setParent(HWND parent);

    //--------------------------------------
    // notification from parent
    //--------------------------------------

    void SAL_CALL notifyParentShow(bool bShow);

    void SAL_CALL notifyParentSizeChanged();

    void SAL_CALL notifyParentWindowPosChanged();

private:
    // hide implementation details using the
    // bridge pattern
    std::auto_ptr<CPreviewAdapterImpl> m_pImpl;

// prevent copy and assignment
private:
    CPreviewAdapter(const CPreviewAdapter&);
    CPreviewAdapter& operator=(const CPreviewAdapter&);
};

#endif
<|endoftext|>


<commit_before>#ifndef UVW_RESOURCE_INCLUDE_H
#define UVW_RESOURCE_INCLUDE_H

#include <memory>
#include <utility>
#include "emitter.h"
#include "underlying_type.hpp"

namespace uvw {

/**
 * @brief Common class for almost all the resources available in `uvw`.
 *
 * This is the base class for handles and requests.
 */
template<typename T, typename U>
class Resource: public UnderlyingType<T, U>, public Emitter<T>, public std::enable_shared_from_this<T> {
protected:
    using ConstructorAccess = typename UnderlyingType<T, U>::ConstructorAccess;

    auto parent() const noexcept {
        return this->loop().loop.get();
    }

    void leak() noexcept {
        sPtr = this->shared_from_this();
    }

    void reset() noexcept {
        sPtr.reset();
    }

    bool self() const noexcept {
        return static_cast<bool>(sPtr);
    }

public:
    explicit Resource(ConstructorAccess ca, std::shared_ptr<Loop> ref)
        : UnderlyingType<T, U>{ca, std::move(ref)},
          Emitter<T>{},
          std::enable_shared_from_this<T>{}
    {
        this->get()->data = this;
    }

    /**
     * @brief Gets user-defined data. `uvw` won't use this field in any case.
     * @return User-defined data if any, an invalid pointer otherwise.
     */
    template<typename R = void>
    std::shared_ptr<R> data() const {
        return std::static_pointer_cast<R>(userData);
    }

    /**
     * @brief Sets arbitrary data. `uvw` won't use this field in any case.
     * @param uData User-defined arbitrary data.
     */
    void data(std::shared_ptr<void> uData) {
        userData = std::move(uData);
    }

private:
    std::shared_ptr<void> userData{nullptr};
    std::shared_ptr<void> sPtr{nullptr};
};

}

#endif // UVW_RESOURCE_INCLUDE_H
<commit_msg>resolve an issue with ICC (close #208)<commit_after>#ifndef UVW_RESOURCE_INCLUDE_H
#define UVW_RESOURCE_INCLUDE_H

#include <memory>
#include <utility>
#include "emitter.h"
#include "underlying_type.hpp"

namespace uvw {

/**
 * @brief Common class for almost all the resources available in `uvw`.
 *
 * This is the base class for handles and requests.
 */
template<typename T, typename U>
class Resource: public UnderlyingType<T, U>, public Emitter<T>, public std::enable_shared_from_this<T> {
protected:
    using ConstructorAccess = typename UnderlyingType<T, U>::ConstructorAccess;

    auto parent() const noexcept {
        return this->loop().loop.get();
    }

    void leak() noexcept {
        sPtr = this->shared_from_this();
    }

    void reset() noexcept {
        sPtr.reset();
    }

    bool self() const noexcept {
        return static_cast<bool>(sPtr);
    }

public:
    explicit Resource(ConstructorAccess ca, std::shared_ptr<Loop> ref)
        : UnderlyingType<T, U>{ca, std::move(ref)}
    {
        this->get()->data = this;
    }

    /**
     * @brief Gets user-defined data. `uvw` won't use this field in any case.
     * @return User-defined data if any, an invalid pointer otherwise.
     */
    template<typename R = void>
    std::shared_ptr<R> data() const {
        return std::static_pointer_cast<R>(userData);
    }

    /**
     * @brief Sets arbitrary data. `uvw` won't use this field in any case.
     * @param uData User-defined arbitrary data.
     */
    void data(std::shared_ptr<void> uData) {
        userData = std::move(uData);
    }

private:
    std::shared_ptr<void> userData{nullptr};
    std::shared_ptr<void> sPtr{nullptr};
};

}

#endif // UVW_RESOURCE_INCLUDE_H
<|endoftext|>


<commit_before>#pragma once

/*
MIT License
Copyright (c) 2019 Arlen Keshabyan (arlen.albert@gmail.com)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

#include <condition_variable>
#include <deque>
#include <functional>
#include <future>
#include <memory>
#include <mutex>
#include <queue>
#include <stdexcept>
#include <thread>

namespace nstd
{

class thread_pool_lite
{
public:
    thread_pool_lite(long num_threads = std::max(std::thread::hardware_concurrency(), 2u) - 1u)
    {
        if (num_threads < 1) num_threads = 1;

        while (--num_threads >= 0)
        {
            _worker_threads.emplace_back([this]
            {
                while (true)
                {
                    std::function<void()> new_task;

                    {
                        std::unique_lock<std::mutex> lock(_queue_mutex);

                        _condition.wait(lock, [this]{ return _cancelled || !std::empty(_tasks); });

                        if(_cancelled && std::empty(_tasks)) return;

                        new_task = std::move(_tasks.front());

                        _tasks.pop();
                    }

                    new_task();
                }
            });
        }
    }

    template<class Functor, class... Args>
    auto enqueue(Functor&& functor, Args&&... args)
    {
        using return_type = std::result_of_t<Functor(Args...)>;

        auto new_task { std::make_shared<std::packaged_task<return_type()>>(std::bind(std::forward<Functor>(functor), std::forward<Args>(args)...)) };

        std::future<return_type> result = new_task->get_future();

        {
            std::scoped_lock lock { _queue_mutex };

            if(_cancelled) throw std::runtime_error("Enqueueing on cancelled thread_pool is not allowed!");

            _tasks.emplace([new_task = std::move(new_task)](){ (*new_task)(); });
        }

        _condition.notify_one();

        return result;
    }

    auto size() const { return std::size(_worker_threads); }

    ~thread_pool_lite()
    {
        {
            std::scoped_lock lock { _queue_mutex };
            _cancelled = true;
        }

        _condition.notify_all();

        for(auto &worker: _worker_threads) worker.join();
    }

private:
    std::deque<std::thread> _worker_threads;
    std::queue<std::function<void()>> _tasks;

    std::mutex _queue_mutex {};
    std::condition_variable _condition {};
    bool _cancelled { false };
};

template <typename T>
class thread_safe_queue
{
public:
    ~thread_safe_queue()
    {
        invalidate();
    }

    bool try_pop(T& out)
    {
        std::scoped_lock lock { _mutex };

        if(_queue.empty() || !_valid) return false;

        out = std::move(_queue.front());

        _queue.pop();

        return true;
    }

    bool wait_pop(T& out)
    {
        std::unique_lock<std::mutex> lock{_mutex};

        _condition.wait(lock, [this]() { return !_queue.empty() || !_valid; });

        if(!_valid) return false;

        out = std::move(_queue.front());

        _queue.pop();

        return true;
    }

    void push(T value)
    {
        std::scoped_lock lock { _mutex };

        _queue.push(std::move(value));
        _condition.notify_one();
    }

    bool empty() const
    {
        std::scoped_lock lock { _mutex };

        return _queue.empty();
    }

    void clear()
    {
        std::scoped_lock lock { _mutex };

        decltype(_queue) empty;

        std::swap(_queue, empty);

        _condition.notify_all();
    }

    void invalidate()
    {
        std::scoped_lock lock { _mutex };

        _valid = false;

        _condition.notify_all();
    }

    bool is_valid() const
    {
        std::scoped_lock lock{ _mutex };

        return _valid;
    }

private:
    std::atomic_bool _valid { true };
    mutable std::mutex _mutex;
    std::queue<T> _queue;
    std::condition_variable _condition;
};

class thread_pool
{
private:
    class thread_task_base
    {
    public:
        thread_task_base() = default;
        virtual ~thread_task_base() = default;
        thread_task_base(const thread_task_base& rhs) = delete;
        thread_task_base& operator=(const thread_task_base& rhs) = delete;
        thread_task_base(thread_task_base&& other) = default;
        thread_task_base& operator=(thread_task_base&& other) = default;
        virtual void execute() = 0;
    };

    template <typename Functor>
    class thread_task: public thread_task_base
    {
    public:
        thread_task(Functor&& functor) :_functor { std::move(functor) } { }
        ~thread_task() override = default;
        thread_task(const thread_task& rhs) = delete;
        thread_task& operator=(const thread_task& rhs) = delete;
        thread_task(thread_task&& other) = default;
        thread_task& operator=(thread_task&& other) = default;
        void execute() override { _functor(); }

    private:
        Functor _functor;
    };

public:
    template <typename T>
    class task_future
    {
    public:
        task_future(std::future<T>&& future) : _future{ std::move(future) } { }
        task_future(const task_future& rhs) = delete;
        task_future& operator=(const task_future& rhs) = delete;
        task_future(task_future&& other) = default;
        task_future& operator=(task_future&& other) = default;
        ~task_future() { if(_future.valid()) _future.get(); }
        auto get() { return _future.get(); }
        void wait() { _future.wait(); }

    private:
        std::future<T> _future;
    };

public:
    explicit thread_pool(long num_threads = std::max(std::thread::hardware_concurrency(), 2u) - 1u) : _cancelled { false }, _task_queue{}, _worker_threads{}
    {
        if (num_threads < 1) num_threads = 1;

        try
        {
            while (--num_threads >= 0) _worker_threads.emplace_back(&thread_pool::worker, this);
        }
        catch(...)
        {
            destroy();
            throw;
        }
    }

    thread_pool(const thread_pool& rhs) = delete;
    thread_pool& operator=(const thread_pool& rhs) = delete;
    ~thread_pool() { destroy(); }

    template <typename Functor, typename... Args>
    auto enqueue(Functor&& functor, Args&&... args)
    {
        auto bound_task = std::bind(std::forward<Functor>(functor), std::forward<Args>(args)...);
        using result_type = std::result_of_t<decltype(bound_task)()>;
        using packaged_task = std::packaged_task<result_type()>;
        using task_type = thread_task<packaged_task>;

        packaged_task task{ std::move(bound_task) };
        task_future<result_type> result { task.get_future() };

        _task_queue.push(std::make_unique<task_type>(std::move(task)));

        return result;
    }

    auto size() const { return std::size(_worker_threads); }

private:
    void worker()
    {
        while(!_cancelled)
        {
            std::unique_ptr<thread_task_base> task { nullptr };

            if(_task_queue.wait_pop(task)) task->execute();
        }
    }

    void destroy()
    {
        _cancelled = true;

        _task_queue.invalidate();

        for(auto& thread : _worker_threads) if(thread.joinable()) thread.join();
    }

private:
    std::atomic_bool _cancelled;
    thread_safe_queue<std::unique_ptr<thread_task_base>> _task_queue;
    std::deque<std::thread> _worker_threads;
};

namespace global_thread_pool
{
    inline thread_pool& get_thread_pool()
    {
        static thread_pool static_thread_pool;

        return static_thread_pool;
    }

    template <typename Functor, typename... Args>
    inline auto enqueue(Functor&& functor, Args&&... args)
    {
        return get_thread_pool().enqueue(std::forward<Functor>(functor), std::forward<Args>(args)...);
    }
}

}
<commit_msg>small refinements<commit_after>#pragma once

/*
MIT License
Copyright (c) 2019 Arlen Keshabyan (arlen.albert@gmail.com)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

#include <condition_variable>
#include <deque>
#include <functional>
#include <future>
#include <memory>
#include <mutex>
#include <queue>
#include <stdexcept>
#include <thread>

namespace nstd
{

class thread_pool_lite
{
public:
    thread_pool_lite(long num_threads = std::max(std::thread::hardware_concurrency(), 2u) - 1u)
    {
        if (num_threads < 1) num_threads = 1;

        while (--num_threads >= 0)
        {
            _worker_threads.emplace_back([this]
            {
                while (true)
                {
                    std::function<void()> new_task;

                    {
                        std::unique_lock<std::mutex> lock(_queue_mutex);

                        _condition.wait(lock, [this]{ return _cancelled || !std::empty(_tasks); });

                        if(_cancelled && std::empty(_tasks)) return;

                        new_task = std::move(_tasks.front());

                        _tasks.pop();
                    }

                    new_task();
                }
            });
        }
    }

    template<class Functor, class... Args>
    auto enqueue(Functor&& functor, Args&&... args)
    {
        using return_type = std::result_of_t<Functor(Args...)>;

        auto new_task { std::make_shared<std::packaged_task<return_type()>>(std::bind(std::forward<Functor>(functor), std::forward<Args>(args)...)) };

        std::future<return_type> result = new_task->get_future();

        {
            std::scoped_lock lock { _queue_mutex };

            if(_cancelled) throw std::runtime_error("Enqueueing on cancelled thread_pool is not allowed!");

            _tasks.emplace([new_task = std::move(new_task)](){ (*new_task)(); });
        }

        _condition.notify_one();

        return result;
    }

    auto size() const { return std::size(_worker_threads); }

    ~thread_pool_lite()
    {
        {
            std::scoped_lock lock { _queue_mutex };
            _cancelled = true;
        }

        _condition.notify_all();

        for(auto &worker: _worker_threads) worker.join();
    }

private:
    std::deque<std::thread> _worker_threads {};
    std::queue<std::function<void()>> _tasks {};

    std::mutex _queue_mutex {};
    std::condition_variable _condition {};
    bool _cancelled { false };
};

template <typename T>
class thread_safe_queue
{
public:
    ~thread_safe_queue()
    {
        invalidate();
    }

    bool try_pop(T& out)
    {
        std::scoped_lock lock { _mutex };

        if(_queue.empty() || !_valid) return false;

        out = std::move(_queue.front());

        _queue.pop();

        return true;
    }

    bool wait_pop(T& out)
    {
        std::unique_lock<std::mutex> lock{_mutex};

        _condition.wait(lock, [this]() { return !_queue.empty() || !_valid; });

        if(!_valid) return false;

        out = std::move(_queue.front());

        _queue.pop();

        return true;
    }

    void push(T value)
    {
        std::scoped_lock lock { _mutex };

        _queue.push(std::move(value));
        _condition.notify_one();
    }

    bool empty() const
    {
        std::scoped_lock lock { _mutex };

        return _queue.empty();
    }

    void clear()
    {
        std::scoped_lock lock { _mutex };

        decltype(_queue) empty;

        std::swap(_queue, empty);

        _condition.notify_all();
    }

    void invalidate()
    {
        std::scoped_lock lock { _mutex };

        _valid = false;

        _condition.notify_all();
    }

    bool is_valid() const
    {
        std::scoped_lock lock{ _mutex };

        return _valid;
    }

private:
    std::atomic_bool _valid { true };
    mutable std::mutex _mutex {};
    std::queue<T> _queue {};
    std::condition_variable _condition {};
};

class thread_pool
{
private:
    class thread_task_base
    {
    public:
        thread_task_base() = default;
        virtual ~thread_task_base() = default;
        thread_task_base(const thread_task_base& rhs) = delete;
        thread_task_base& operator=(const thread_task_base& rhs) = delete;
        thread_task_base(thread_task_base&& other) = default;
        thread_task_base& operator=(thread_task_base&& other) = default;
        virtual void execute() = 0;
    };

    template <typename Functor>
    class thread_task: public thread_task_base
    {
    public:
        thread_task(Functor&& functor) :_functor { std::move(functor) } { }
        ~thread_task() override = default;
        thread_task(const thread_task& rhs) = delete;
        thread_task& operator=(const thread_task& rhs) = delete;
        thread_task(thread_task&& other) = default;
        thread_task& operator=(thread_task&& other) = default;
        void execute() override { _functor(); }

    private:
        Functor _functor;
    };

public:
    explicit thread_pool(long num_threads = std::max(std::thread::hardware_concurrency(), 2u) - 1u)
    {
        if (num_threads < 1) num_threads = 1;

        try
        {
            while (--num_threads >= 0) _worker_threads.emplace_back(&thread_pool::worker, this);
        }
        catch(...)
        {
            destroy();
            throw;
        }
    }

    thread_pool(const thread_pool& rhs) = delete;
    thread_pool& operator=(const thread_pool& rhs) = delete;
    ~thread_pool() { destroy(); }

    template <typename Functor, typename... Args>
    auto enqueue(Functor&& functor, Args&&... args)
    {
        auto bound_task = std::bind(std::forward<Functor>(functor), std::forward<Args>(args)...);
        using result_type = std::result_of_t<decltype(bound_task)()>;
        using packaged_task = std::packaged_task<result_type()>;
        using task_type = thread_task<packaged_task>;

        packaged_task task{ std::move(bound_task) };
        std::future<result_type> result { task.get_future() };

        _task_queue.push(std::make_unique<task_type>(std::move(task)));

        return result;
    }

    auto size() const { return std::size(_worker_threads); }

private:
    void worker()
    {
        while(!_cancelled)
        {
            std::unique_ptr<thread_task_base> task { nullptr };

            if(_task_queue.wait_pop(task)) task->execute();
        }
    }

    void destroy()
    {
        _cancelled = true;

        _task_queue.invalidate();

        for(auto& thread : _worker_threads) if(thread.joinable()) thread.join();
    }

private:
    std::atomic_bool _cancelled { false };
    thread_safe_queue<std::unique_ptr<thread_task_base>> _task_queue {};
    std::deque<std::thread> _worker_threads {};
};

namespace global_thread_pool
{
    inline thread_pool& get_thread_pool()
    {
        static thread_pool static_thread_pool;

        return static_thread_pool;
    }

    template <typename Functor, typename... Args>
    inline auto enqueue(Functor&& functor, Args&&... args)
    {
        return get_thread_pool().enqueue(std::forward<Functor>(functor), std::forward<Args>(args)...);
    }
}

}
<|endoftext|>


<commit_before>#include "vast/ingestor.h"

#include "vast/segment.h"
#include "vast/source/file.h"
#include "vast/io/serialization.h"

#ifdef VAST_HAVE_BROCCOLI
#include "vast/source/broccoli.h"
#endif

namespace vast {

using namespace cppa;

ingestor_actor::ingestor_actor(path dir,
                               actor_ptr receiver,
                               size_t max_events_per_chunk,
                               size_t max_segment_size,
                               uint64_t batch_size)
  : dir_{std::move(dir)},
    receiver_{receiver},
    max_events_per_chunk_{max_events_per_chunk},
    max_segment_size_{max_segment_size},
    batch_size_{batch_size}
{
}

void ingestor_actor::act()
{
  trap_exit(true);

  // FIXME: figure out why detaching the segmentizer yields a two-fold
  // performance increase in the ingestion rate.
  sink_ = spawn<segmentizer, monitored>(
      self, max_events_per_chunk_, max_segment_size_);

  auto segment_dir = dir_ / "ingest" / "segments";
  traverse(
      segment_dir,
      [&](path const& p) -> bool
      {
        VAST_LOG_ACTOR_INFO("found orphaned segment: " << p.basename());
        orphaned_.insert(p.basename());
        return true;
      });

  become(
      on(atom("terminate"), arg_match) >> [=](uint32_t reason)
      {
        if (segments_.empty())
        {
          quit(reason);
        }
        else
        {
          VAST_LOG_ACTOR_INFO("writes un-acked segments to stable storage");

          if (! exists(segment_dir) && ! mkdir(segment_dir))
          {
            VAST_LOG_ACTOR_ERROR("failed to create directory " << segment_dir);
          }
          else
          {
            for (auto& p : segments_)
            {
              auto const path = segment_dir / to<string>(p.first);
              VAST_LOG_ACTOR_INFO("saves " << path);
              if (! io::archive(path, p.second))
                VAST_LOG_ACTOR_ERROR("failed to save " << path);
            }
          }

          quit(exit::error);
        }
      },
      on(atom("shutdown"), arg_match) >> [=](uint32_t reason)
      {
        if (segments_.empty())
        {
          quit(reason);
        }
        else
        {
          delayed_send(self, std::chrono::seconds(10),
                       atom("terminate"), reason);

          VAST_LOG_ACTOR_INFO(
              "waits 10 seconds for " << segments_.size() << " segment ACKs");
        }
      },
      on(atom("EXIT"), arg_match) >> [=](uint32_t reason)
      {
        if (! source_)
          send(self, atom("shutdown"), reason);
        else
          // Tell the source to exit, it will in turn propagate the exit
          // message to the sink.
          send_exit(source_, exit::stop);
      },
      on(atom("DOWN"), arg_match) >> [=](uint32_t /* reason */)
      {
        // Once we have received DOWN from the sink, the ingestor has nothing
        // else left todo and can shutdown.
        delayed_send(self, std::chrono::seconds(5), atom("shutdown"), exit::done);
      },
      on(atom("submit")) >> [=]
      {
        for (auto& base : orphaned_)
        {
          auto p = segment_dir / base;
          segment s;
          if (! io::unarchive(p, s))
          {
            VAST_LOG_ACTOR_ERROR("failed to load orphaned segment " << base);
            continue;
          }

          send(self, std::move(s));
        }
      },
#ifdef VAST_HAVE_BROCCOLI
      on(atom("ingest"), atom("broccoli"), arg_match) >>
        [=](std::string const& host, unsigned port,
            std::vector<std::string> const& events)
      {
        // TODO.
      },
#endif
      on(atom("ingest"), "bro2", arg_match) >> [=](std::string const& file)
      {
        source_ = spawn<source::bro2, detached>(sink_, file);
        source_->link_to(sink_);
        send(source_, atom("batch size"), batch_size_);
        send(source_, atom("run"));
      },
      on(atom("ingest"), val<std::string>, arg_match) >> [=](std::string const&)
      {
        VAST_LOG_ACTOR_ERROR("got invalid ingestion file type");
      },
      on_arg_match >> [=](segment& s)
      {
        VAST_LOG_ACTOR_DEBUG(
            "relays segment " << s.id() << " to " << VAST_ACTOR_ID(receiver_));

        auto cs = cow<segment>{std::move(s)};
        segments_[cs->id()] = cs;
        receiver_ << cs;
      },
      on(atom("ack"), arg_match) >> [=](uuid const& id)
      {
        VAST_LOG_ACTOR_DEBUG("got ack for segment " << id);
        auto i = segments_.find(id);
        assert(i != segments_.end());
        segments_.erase(i);

        auto j = orphaned_.find(path{to<string>(id)});
        if (j != orphaned_.end())
        {
          VAST_LOG_ACTOR_INFO("deletes accepted segment " << id);
          rm(segment_dir / *j);
          orphaned_.erase(j);
        }
      },
      on(atom("nack"), arg_match) >> [=](uuid const& id)
      {
        VAST_LOG_ACTOR_ERROR("got nack for segment " << id);
        send(self, atom("shutdown"), exit::error);
      });
}

char const* ingestor_actor::description() const
{
  return "ingestor";
}

} // namespace vast
<commit_msg>Fix ingestor shutdown procedure.<commit_after>#include "vast/ingestor.h"

#include "vast/segment.h"
#include "vast/source/file.h"
#include "vast/io/serialization.h"

#ifdef VAST_HAVE_BROCCOLI
#include "vast/source/broccoli.h"
#endif

namespace vast {

using namespace cppa;

ingestor_actor::ingestor_actor(path dir,
                               actor_ptr receiver,
                               size_t max_events_per_chunk,
                               size_t max_segment_size,
                               uint64_t batch_size)
  : dir_{std::move(dir)},
    receiver_{receiver},
    max_events_per_chunk_{max_events_per_chunk},
    max_segment_size_{max_segment_size},
    batch_size_{batch_size}
{
}

void ingestor_actor::act()
{
  trap_exit(true);

  // FIXME: figure out why detaching the segmentizer yields a two-fold
  // performance increase in the ingestion rate.
  sink_ = spawn<segmentizer, monitored>(
      self, max_events_per_chunk_, max_segment_size_);

  auto segment_dir = dir_ / "ingest" / "segments";
  traverse(
      segment_dir,
      [&](path const& p) -> bool
      {
        VAST_LOG_ACTOR_INFO("found orphaned segment: " << p.basename());
        orphaned_.insert(p.basename());
        return true;
      });

  become(
      on(atom("terminate"), arg_match) >> [=](uint32_t reason)
      {
        if (segments_.empty())
        {
          quit(reason);
        }
        else
        {
          VAST_LOG_ACTOR_INFO("writes un-acked segments to stable storage");

          if (! exists(segment_dir) && ! mkdir(segment_dir))
          {
            VAST_LOG_ACTOR_ERROR("failed to create directory " << segment_dir);
          }
          else
          {
            for (auto& p : segments_)
            {
              auto const path = segment_dir / to<string>(p.first);
              VAST_LOG_ACTOR_INFO("saves " << path);
              if (! io::archive(path, p.second))
                VAST_LOG_ACTOR_ERROR("failed to save " << path);
            }
          }

          quit(exit::error);
        }
      },
      on(atom("shutdown"), arg_match) >> [=](uint32_t reason)
      {
        if (segments_.empty())
        {
          quit(reason);
        }
        else
        {
          delayed_send(self, std::chrono::seconds(10),
                       atom("terminate"), reason);

          VAST_LOG_ACTOR_INFO(
              "waits 10 seconds for " << segments_.size() << " segment ACKs");
        }
      },
      on(atom("EXIT"), arg_match) >> [=](uint32_t reason)
      {
        if (! source_)
        {
          send(self, atom("shutdown"), reason);
          send_exit(sink_, reason);
        }
        else
        {
          // Tell the source to exit, it will in turn propagate the exit
          // message to the sink.
          send_exit(source_, exit::stop);
        }
      },
      on(atom("DOWN"), arg_match) >> [=](uint32_t /* reason */)
      {
        // Once we have received DOWN from the sink, the ingestor has nothing
        // else left todo and can shutdown.
        delayed_send(self, std::chrono::seconds(5), atom("shutdown"), exit::done);
      },
      on(atom("submit")) >> [=]
      {
        for (auto& base : orphaned_)
        {
          auto p = segment_dir / base;
          segment s;
          if (! io::unarchive(p, s))
          {
            VAST_LOG_ACTOR_ERROR("failed to load orphaned segment " << base);
            continue;
          }

          send(self, std::move(s));
        }
      },
#ifdef VAST_HAVE_BROCCOLI
      on(atom("ingest"), atom("broccoli"), arg_match) >>
        [=](std::string const& host, unsigned port,
            std::vector<std::string> const& events)
      {
        // TODO.
      },
#endif
      on(atom("ingest"), "bro2", arg_match) >> [=](std::string const& file)
      {
        source_ = spawn<source::bro2, detached>(sink_, file);
        source_->link_to(sink_);
        send(source_, atom("batch size"), batch_size_);
        send(source_, atom("run"));
      },
      on(atom("ingest"), val<std::string>, arg_match) >> [=](std::string const&)
      {
        VAST_LOG_ACTOR_ERROR("got invalid ingestion file type");
      },
      on_arg_match >> [=](segment& s)
      {
        VAST_LOG_ACTOR_DEBUG(
            "relays segment " << s.id() << " to " << VAST_ACTOR_ID(receiver_));

        auto cs = cow<segment>{std::move(s)};
        segments_[cs->id()] = cs;
        receiver_ << cs;
      },
      on(atom("ack"), arg_match) >> [=](uuid const& id)
      {
        VAST_LOG_ACTOR_DEBUG("got ack for segment " << id);
        auto i = segments_.find(id);
        assert(i != segments_.end());
        segments_.erase(i);

        auto j = orphaned_.find(path{to<string>(id)});
        if (j != orphaned_.end())
        {
          VAST_LOG_ACTOR_INFO("deletes accepted segment " << id);
          rm(segment_dir / *j);
          orphaned_.erase(j);
        }
      },
      on(atom("nack"), arg_match) >> [=](uuid const& id)
      {
        VAST_LOG_ACTOR_ERROR("got nack for segment " << id);
        send(self, atom("shutdown"), exit::error);
      });
}

char const* ingestor_actor::description() const
{
  return "ingestor";
}

} // namespace vast
<|endoftext|>


<commit_before>/*
 * Copyright 2014 Milian Wolff <mail@milianw.de>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Library General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

#include <cstdio>
#include <cassert>
#include <cstring>
#include <cstdlib>

#include <unordered_map>
#include <atomic>
#include <string>
#include <vector>

#include <boost/functional/hash.hpp>

#include <dlfcn.h>
#include <unistd.h>

#define UNW_LOCAL_ONLY
#include <libunwind.h>

using namespace std;

namespace {

using malloc_t = void* (*) (size_t);
using free_t = void (*) (void*);
using realloc_t = void* (*) (void*, size_t);
using calloc_t = void* (*) (size_t, size_t);
using posix_memalign_t = int (*) (void **, size_t, size_t);
using valloc_t = void* (*) (size_t);
using aligned_alloc_t = void* (*) (size_t, size_t);

malloc_t real_malloc = nullptr;
free_t real_free = nullptr;
realloc_t real_realloc = nullptr;
calloc_t real_calloc = nullptr;
posix_memalign_t real_posix_memalign = nullptr;
valloc_t real_valloc = nullptr;
aligned_alloc_t real_aligned_alloc = nullptr;

struct IPCacheEntry
{
    unsigned int id;
    bool skip;
    bool stop;
};

atomic<unsigned int> next_cache_id;
atomic<unsigned int> next_thread_id;

// must be kept separately from ThreadData to ensure it stays valid
// even until after ThreadData is destroyed
thread_local bool in_handler = false;

string env(const char* variable)
{
    const char* value = getenv(variable);
    return value ? string(value) : string();
}

struct Tree
{
    static const unsigned int MAX_DEPTH = 64;
    unsigned int data[MAX_DEPTH];
    unsigned int depth;

    bool operator==(const Tree& o) const
    {
        return depth == o.depth && !memcmp(data, o.data, sizeof(unsigned int) * depth);
    }
};

struct TreeHasher
{
    size_t operator()(const Tree& tree) const
    {
        size_t seed = 0;
        hash<unsigned int> nodeHash;
        for (unsigned int i = 0; i < tree.depth; ++i) {
            boost::hash_combine(seed, nodeHash(tree.data[i]));
        }
        return seed;
    }
};

//TODO: merge per-thread output into single file
struct ThreadData
{
    ThreadData()
        : thread_id(next_thread_id++)
        , out(nullptr)
    {
        bool wasInHandler = in_handler;
        in_handler = true;
        ipCache.reserve(16384);
        treeCache.reserve(16384);
        string outputFileName = env("DUMP_MALLOC_TRACE_OUTPUT") + to_string(getpid()) + '.' + to_string(thread_id);
        out = fopen(outputFileName.c_str(), "wa");
        if (!out) {
            fprintf(stderr, "Failed to open output file: %s\n", outputFileName.c_str());
            exit(1);
        }
        in_handler = wasInHandler;
    }

    ~ThreadData()
    {
        in_handler = true;
        fclose(out);
    }

    unordered_map<unw_word_t, IPCacheEntry> ipCache;
    unordered_map<Tree, unsigned int, TreeHasher> treeCache;
    unsigned int thread_id;
    FILE* out;
};

thread_local ThreadData threadData;

unsigned int print_caller()
{
    unw_context_t uc;
    unw_getcontext (&uc);

    unw_cursor_t cursor;
    unw_init_local (&cursor, &uc);

    // skip handleMalloc & malloc
    for (int i = 0; i < 2; ++i) {
        if (unw_step(&cursor) <= 0) {
            return 0;
        }
    }

    auto& ipCache = threadData.ipCache;

    Tree tree;
    tree.depth = 0;
    while (unw_step(&cursor) > 0 && tree.depth < Tree::MAX_DEPTH) {
        unw_word_t ip;
        unw_get_reg(&cursor, UNW_REG_IP, &ip);

        // check cache for known ip's
        auto it = ipCache.find(ip);
        if (it == ipCache.end()) {
            // not cached yet, get data
            const size_t BUF_SIZE = 256;
            char name[BUF_SIZE];
            name[0] = '\0';
            unw_word_t offset;
            unw_get_proc_name(&cursor, name, BUF_SIZE, &offset);
            // skip operator new (_Znwm) and operator new[] (_Znam)
            const bool skip = name[0] == '_' && name[1] == 'Z' && name[2] == 'n'
                        && name[4] == 'm' && name[5] == '\0'
                        && (name[3] == 'w' || name[3] == 'a');
            const bool stop = !skip && (!strcmp(name, "main")
                                    || !strcmp(name, "_GLOBAL__sub_I_main")
                                    || !strcmp(name, "_Z41__static_initialization_and_destruction_0ii"));
            const unsigned int id = next_cache_id++;

            // and store it in the cache
            it = ipCache.insert(it, make_pair(ip, IPCacheEntry{id, skip, stop}));

            if (!skip) {
                fprintf(threadData.out, "%u=%lx@%s+0x%lx\n", id, ip, name, offset);
            }
        }

        const auto& frame = it->second;
        if (!frame.skip) {
            tree.data[tree.depth++] = frame.id;
        }
        if (frame.stop) {
            break;
        }
    }

    auto& treeCache = threadData.treeCache;
    auto it = treeCache.find(tree);
    if (it == treeCache.end()) {
        it = treeCache.insert(it, make_pair(tree, next_cache_id++));

        fprintf(threadData.out, "%u=", it->second);
        for (unsigned int i = 0; i < tree.depth; ++i) {
            fprintf(threadData.out, "%u;", tree.data[i]);
        }
        fputs("\n", threadData.out);
    }

    return it->second;
}

template<typename T>
T findReal(const char* name)
{
    auto ret = dlsym(RTLD_NEXT, name);
    if (!ret) {
        fprintf(stderr, "could not find original function %s\n", name);
        exit(1);
    }
    return reinterpret_cast<T>(ret);
}

/**
 * Dummy implementation, since the call to dlsym from findReal triggers a call to calloc.
 *
 * This is only called at startup and will eventually be replaced by the "proper" calloc implementation.
 */
void* dummy_calloc(size_t num, size_t size)
{
    const size_t MAX_SIZE = 1024;
    static char* buf[MAX_SIZE];
    static size_t offset = 0;
    if (!offset) {
        memset(buf, 0, MAX_SIZE);
    }
    size_t oldOffset = offset;
    offset += num * size;
    if (offset >= MAX_SIZE) {
        fprintf(stderr, "failed to initialize, dummy calloc buf size exhausted: %lu requested, %lu available\n", offset, MAX_SIZE);
        exit(1);
    }
    return buf + oldOffset;
}

void init()
{
    if (in_handler) {
        fprintf(stderr, "initialization recursion detected\n");
        exit(1);
    }

    in_handler = true;

    real_calloc = &dummy_calloc;
    real_calloc = findReal<calloc_t>("calloc");
    real_malloc = findReal<malloc_t>("malloc");
    real_free = findReal<free_t>("free");
    real_realloc = findReal<realloc_t>("realloc");
    real_posix_memalign = findReal<posix_memalign_t>("posix_memalign");
    real_valloc = findReal<valloc_t>("valloc");
    real_aligned_alloc = findReal<aligned_alloc_t>("aligned_alloc");

    in_handler = false;
}

void handleMalloc(void* ptr, size_t size)
{
    const unsigned int treeId = print_caller();
    fprintf(threadData.out, "+%lu:%p %u\n", size, ptr, treeId);
}

void handleFree(void* ptr)
{
    fprintf(threadData.out, "-%p\n", ptr);
}

}

extern "C" {

/// TODO: memalign, pvalloc, ...?

void* malloc(size_t size)
{
    if (!real_malloc) {
        init();
    }

    void* ret = real_malloc(size);

    if (!in_handler) {
        in_handler = true;
        handleMalloc(ret, size);
        in_handler = false;
    }

    return ret;
}

void free(void* ptr)
{
    if (!real_free) {
        init();
    }

    real_free(ptr);

    if (!in_handler) {
        in_handler = true;
        handleFree(ptr);
        in_handler = false;
    }
}

void* realloc(void* ptr, size_t size)
{
    if (!real_realloc) {
        init();
    }

    void* ret = real_realloc(ptr, size);

    if (!in_handler) {
        in_handler = true;
        handleFree(ptr);
        handleMalloc(ret, size);
        in_handler = false;
    }

    return ret;
}

void* calloc(size_t num, size_t size)
{
    if (!real_calloc) {
        init();
    }

    void* ret = real_calloc(num, size);

    if (!in_handler) {
        in_handler = true;
        handleMalloc(ret, num*size);
        in_handler = false;
    }

    return ret;
}

int posix_memalign(void **memptr, size_t alignment, size_t size)
{
    if (!real_posix_memalign) {
        init();
    }

    int ret = real_posix_memalign(memptr, alignment, size);

    if (!in_handler) {
        in_handler = true;
        handleMalloc(*memptr, size);
        in_handler = false;
    }

    return ret;
}

void* aligned_alloc(size_t alignment, size_t size)
{
    if (!real_aligned_alloc) {
        init();
    }

    void* ret = real_aligned_alloc(alignment, size);

    if (!in_handler) {
        in_handler = true;
        handleMalloc(ret, size);
        in_handler = false;
    }

    return ret;
}

void* valloc(size_t size)
{
    if (!real_valloc) {
        init();
    }

    void* ret = real_valloc(size);

    if (!in_handler) {
        in_handler = true;
        handleMalloc(ret, size);
        in_handler = false;
    }

    return ret;
}

}
<commit_msg>Don't create single-entry trees output.<commit_after>/*
 * Copyright 2014 Milian Wolff <mail@milianw.de>
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Library General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

#include <cstdio>
#include <cassert>
#include <cstring>
#include <cstdlib>

#include <unordered_map>
#include <atomic>
#include <string>
#include <vector>

#include <boost/functional/hash.hpp>

#include <dlfcn.h>
#include <unistd.h>

#define UNW_LOCAL_ONLY
#include <libunwind.h>

using namespace std;

namespace {

using malloc_t = void* (*) (size_t);
using free_t = void (*) (void*);
using realloc_t = void* (*) (void*, size_t);
using calloc_t = void* (*) (size_t, size_t);
using posix_memalign_t = int (*) (void **, size_t, size_t);
using valloc_t = void* (*) (size_t);
using aligned_alloc_t = void* (*) (size_t, size_t);

malloc_t real_malloc = nullptr;
free_t real_free = nullptr;
realloc_t real_realloc = nullptr;
calloc_t real_calloc = nullptr;
posix_memalign_t real_posix_memalign = nullptr;
valloc_t real_valloc = nullptr;
aligned_alloc_t real_aligned_alloc = nullptr;

struct IPCacheEntry
{
    unsigned int id;
    bool skip;
    bool stop;
};

atomic<unsigned int> next_cache_id;
atomic<unsigned int> next_thread_id;

// must be kept separately from ThreadData to ensure it stays valid
// even until after ThreadData is destroyed
thread_local bool in_handler = false;

string env(const char* variable)
{
    const char* value = getenv(variable);
    return value ? string(value) : string();
}

struct Tree
{
    static const unsigned int MAX_DEPTH = 64;
    unsigned int data[MAX_DEPTH];
    unsigned int depth;

    bool operator==(const Tree& o) const
    {
        return depth == o.depth && !memcmp(data, o.data, sizeof(unsigned int) * depth);
    }
};

struct TreeHasher
{
    size_t operator()(const Tree& tree) const
    {
        size_t seed = 0;
        hash<unsigned int> nodeHash;
        for (unsigned int i = 0; i < tree.depth; ++i) {
            boost::hash_combine(seed, nodeHash(tree.data[i]));
        }
        return seed;
    }
};

//TODO: merge per-thread output into single file
struct ThreadData
{
    ThreadData()
        : thread_id(next_thread_id++)
        , out(nullptr)
    {
        bool wasInHandler = in_handler;
        in_handler = true;
        ipCache.reserve(16384);
        treeCache.reserve(16384);
        string outputFileName = env("DUMP_MALLOC_TRACE_OUTPUT") + to_string(getpid()) + '.' + to_string(thread_id);
        out = fopen(outputFileName.c_str(), "wa");
        if (!out) {
            fprintf(stderr, "Failed to open output file: %s\n", outputFileName.c_str());
            exit(1);
        }
        in_handler = wasInHandler;
    }

    ~ThreadData()
    {
        in_handler = true;
        fclose(out);
    }

    unordered_map<unw_word_t, IPCacheEntry> ipCache;
    unordered_map<Tree, unsigned int, TreeHasher> treeCache;
    unsigned int thread_id;
    FILE* out;
};

thread_local ThreadData threadData;

unsigned int print_caller()
{
    unw_context_t uc;
    unw_getcontext (&uc);

    unw_cursor_t cursor;
    unw_init_local (&cursor, &uc);

    // skip handleMalloc & malloc
    for (int i = 0; i < 2; ++i) {
        if (unw_step(&cursor) <= 0) {
            return 0;
        }
    }

    auto& ipCache = threadData.ipCache;

    Tree tree;
    tree.depth = 0;
    while (unw_step(&cursor) > 0 && tree.depth < Tree::MAX_DEPTH) {
        unw_word_t ip;
        unw_get_reg(&cursor, UNW_REG_IP, &ip);

        // check cache for known ip's
        auto it = ipCache.find(ip);
        if (it == ipCache.end()) {
            // not cached yet, get data
            const size_t BUF_SIZE = 256;
            char name[BUF_SIZE];
            name[0] = '\0';
            unw_word_t offset;
            unw_get_proc_name(&cursor, name, BUF_SIZE, &offset);
            // skip operator new (_Znwm) and operator new[] (_Znam)
            const bool skip = name[0] == '_' && name[1] == 'Z' && name[2] == 'n'
                        && name[4] == 'm' && name[5] == '\0'
                        && (name[3] == 'w' || name[3] == 'a');
            const bool stop = !skip && (!strcmp(name, "main")
                                    || !strcmp(name, "_GLOBAL__sub_I_main")
                                    || !strcmp(name, "_Z41__static_initialization_and_destruction_0ii"));
            const unsigned int id = next_cache_id++;

            // and store it in the cache
            it = ipCache.insert(it, make_pair(ip, IPCacheEntry{id, skip, stop}));

            if (!skip) {
                fprintf(threadData.out, "%u=%lx@%s+0x%lx\n", id, ip, name, offset);
            }
        }

        const auto& frame = it->second;
        if (!frame.skip) {
            tree.data[tree.depth++] = frame.id;
        }
        if (frame.stop) {
            break;
        }
    }

    if (tree.depth == 1) {
        return tree.data[0];
    }

    auto& treeCache = threadData.treeCache;
    auto it = treeCache.find(tree);
    if (it == treeCache.end()) {
        it = treeCache.insert(it, make_pair(tree, next_cache_id++));

        fprintf(threadData.out, "%u=", it->second);
        for (unsigned int i = 0; i < tree.depth; ++i) {
            fprintf(threadData.out, "%u;", tree.data[i]);
        }
        fputs("\n", threadData.out);
    }

    return it->second;
}

template<typename T>
T findReal(const char* name)
{
    auto ret = dlsym(RTLD_NEXT, name);
    if (!ret) {
        fprintf(stderr, "could not find original function %s\n", name);
        exit(1);
    }
    return reinterpret_cast<T>(ret);
}

/**
 * Dummy implementation, since the call to dlsym from findReal triggers a call to calloc.
 *
 * This is only called at startup and will eventually be replaced by the "proper" calloc implementation.
 */
void* dummy_calloc(size_t num, size_t size)
{
    const size_t MAX_SIZE = 1024;
    static char* buf[MAX_SIZE];
    static size_t offset = 0;
    if (!offset) {
        memset(buf, 0, MAX_SIZE);
    }
    size_t oldOffset = offset;
    offset += num * size;
    if (offset >= MAX_SIZE) {
        fprintf(stderr, "failed to initialize, dummy calloc buf size exhausted: %lu requested, %lu available\n", offset, MAX_SIZE);
        exit(1);
    }
    return buf + oldOffset;
}

void init()
{
    if (in_handler) {
        fprintf(stderr, "initialization recursion detected\n");
        exit(1);
    }

    in_handler = true;

    real_calloc = &dummy_calloc;
    real_calloc = findReal<calloc_t>("calloc");
    real_malloc = findReal<malloc_t>("malloc");
    real_free = findReal<free_t>("free");
    real_realloc = findReal<realloc_t>("realloc");
    real_posix_memalign = findReal<posix_memalign_t>("posix_memalign");
    real_valloc = findReal<valloc_t>("valloc");
    real_aligned_alloc = findReal<aligned_alloc_t>("aligned_alloc");

    in_handler = false;
}

void handleMalloc(void* ptr, size_t size)
{
    const unsigned int treeId = print_caller();
    fprintf(threadData.out, "+%lu:%p %u\n", size, ptr, treeId);
}

void handleFree(void* ptr)
{
    fprintf(threadData.out, "-%p\n", ptr);
}

}

extern "C" {

/// TODO: memalign, pvalloc, ...?

void* malloc(size_t size)
{
    if (!real_malloc) {
        init();
    }

    void* ret = real_malloc(size);

    if (!in_handler) {
        in_handler = true;
        handleMalloc(ret, size);
        in_handler = false;
    }

    return ret;
}

void free(void* ptr)
{
    if (!real_free) {
        init();
    }

    real_free(ptr);

    if (!in_handler) {
        in_handler = true;
        handleFree(ptr);
        in_handler = false;
    }
}

void* realloc(void* ptr, size_t size)
{
    if (!real_realloc) {
        init();
    }

    void* ret = real_realloc(ptr, size);

    if (!in_handler) {
        in_handler = true;
        handleFree(ptr);
        handleMalloc(ret, size);
        in_handler = false;
    }

    return ret;
}

void* calloc(size_t num, size_t size)
{
    if (!real_calloc) {
        init();
    }

    void* ret = real_calloc(num, size);

    if (!in_handler) {
        in_handler = true;
        handleMalloc(ret, num*size);
        in_handler = false;
    }

    return ret;
}

int posix_memalign(void **memptr, size_t alignment, size_t size)
{
    if (!real_posix_memalign) {
        init();
    }

    int ret = real_posix_memalign(memptr, alignment, size);

    if (!in_handler) {
        in_handler = true;
        handleMalloc(*memptr, size);
        in_handler = false;
    }

    return ret;
}

void* aligned_alloc(size_t alignment, size_t size)
{
    if (!real_aligned_alloc) {
        init();
    }

    void* ret = real_aligned_alloc(alignment, size);

    if (!in_handler) {
        in_handler = true;
        handleMalloc(ret, size);
        in_handler = false;
    }

    return ret;
}

void* valloc(size_t size)
{
    if (!real_valloc) {
        init();
    }

    void* ret = real_valloc(size);

    if (!in_handler) {
        in_handler = true;
        handleMalloc(ret, size);
        in_handler = false;
    }

    return ret;
}

}
<|endoftext|>