<commit_before>
#include <gloperate/painter/ContextFormat.h>

#include <cassert>
#include <sstream>
#include <map>

#include <globjects/base/baselogging.h>

using namespace globjects;

namespace gloperate
{

ContextFormat::ContextFormat()
: m_version(glbinding::Version(4, 5))
, m_profile(Profile::None)
, m_debugContext(false)
, m_forwardCompatibility(false)
, m_redBufferSize(8)
, m_greenBufferSize(8)
, m_blueBufferSize(8)
, m_alphaBufferSize(8)
, m_depthBufferSize(24)
, m_stencilBufferSize(0)
, m_stereo(false)
, m_swapBehavior(SwapBehavior::DoubleBuffering)
, m_samples(0)
{
}

ContextFormat::~ContextFormat()
{
}

void ContextFormat::setVersion(
    const unsigned int majorVersion
,   const unsigned int minorVersion)
{
    setVersion(glbinding::Version(majorVersion, minorVersion));
}

void ContextFormat::setVersion(const glbinding::Version & version)
{
    m_version = version;
}

glbinding::Version ContextFormat::validateVersion(const glbinding::Version &requestedVersion
,   const glbinding::Version &_maximumVersion)
{
    glbinding::Version maximumVersion = _maximumVersion;
    if (maximumVersion.isNull())
    {
#ifdef __APPLE__
        maximumVersion = glbinding::Version(3, 2);
#else
        maximumVersion = glbinding::Version(3, 0);
#endif
    }

    if (requestedVersion.isNull() || requestedVersion > maximumVersion)
        return maximumVersion;

    if (!requestedVersion.isValid())
    {
        glbinding::Version nearest = requestedVersion.nearest();
        return nearest > maximumVersion ? maximumVersion : nearest;
    }
    return requestedVersion;
}

int ContextFormat::majorVersion() const
{
    return m_version.m_major;
}

int ContextFormat::minorVersion() const
{
    return m_version.m_minor;
}

const glbinding::Version & ContextFormat::version() const
{
    return m_version;
}

ContextFormat::Profile ContextFormat::profile() const
{
    return m_profile;
}

void ContextFormat::setProfile(const ContextFormat::Profile profile)
{
    m_profile = profile;
}

bool ContextFormat::debugContext() const
{
    return m_debugContext;
}

void ContextFormat::setDebugContext(const bool on)
{
    m_debugContext = on;
}

bool ContextFormat::forwardCompatible() const
{
    return m_forwardCompatibility;
}

void ContextFormat::setForwardCompatible(const bool on)
{
    m_forwardCompatibility = on;
}

int ContextFormat::redBufferSize() const
{
    return m_redBufferSize;
}

void ContextFormat::setRedBufferSize(const int size)
{
    m_redBufferSize = size;
}

int ContextFormat::greenBufferSize() const
{
    return m_greenBufferSize;
}

void ContextFormat::setGreenBufferSize(const int size)
{
    m_greenBufferSize = size;
}

int  ContextFormat::blueBufferSize() const
{
    return m_blueBufferSize;
}

void ContextFormat::setBlueBufferSize(const int size)
{
    m_blueBufferSize = size;
}

int ContextFormat::alphaBufferSize() const
{
    return m_alphaBufferSize;
}

void ContextFormat::setAlphaBufferSize(const int size)
{
    m_alphaBufferSize = size;
}

int ContextFormat::depthBufferSize() const
{
    return m_depthBufferSize;
}

void ContextFormat::setDepthBufferSize(const int size)
{
    m_depthBufferSize = size;
}

int ContextFormat::stencilBufferSize() const
{
    return m_stencilBufferSize;
}

void ContextFormat::setStencilBufferSize(const int size)
{
    m_stencilBufferSize = size;
}

ContextFormat::SwapBehavior ContextFormat::swapBehavior() const
{
    return m_swapBehavior;
}

void ContextFormat::setSwapBehavior(const ContextFormat::SwapBehavior behavior)
{
    m_swapBehavior = behavior;
}

bool ContextFormat::stereo() const
{
    return m_stereo;
}

void ContextFormat::setStereo(const bool enable)
{
    m_stereo = enable;
}

int ContextFormat::samples() const
{
    return m_samples;
}

void ContextFormat::setSamples(const int samples)
{
    m_samples = samples;
}

const std::string & ContextFormat::profileString(const Profile profile)
{
    static const std::map<Profile, std::string> profileIdentifier = {
        { Profile::Core,          "Core" }
    ,   { Profile::Compatibility, "Compatibility" }
    ,   { Profile::None,          "None" } };

    return profileIdentifier.at(profile);
}

const std::string & ContextFormat::swapBehaviorString(const SwapBehavior swapBehavior)
{
    static const std::map<SwapBehavior, std::string> swapbIdentifier = {
        { SwapBehavior::Default,         "Default" }
    ,   { SwapBehavior::DoubleBuffering, "DoubleBuffering" }
    ,   { SwapBehavior::SingleBuffering, "SingleBuffering" }
    ,   { SwapBehavior::TripleBuffering, "TripleBuffering" } };

    return swapbIdentifier.at(swapBehavior);
}

bool ContextFormat::verify(const ContextFormat & requested, const ContextFormat & created)
{
    return
        verifyVersionAndProfile(requested, created) &&
        verifyPixelFormat(requested, created);
}

bool ContextFormat::verify(const ContextFormat & requested) const
{
    return verify(requested, *this);
}

bool ContextFormat::verifyVersionAndProfile(const ContextFormat & requested, const ContextFormat & created)
{
    const bool sameProfiles(requested.profile() == created.profile());

    if (!sameProfiles)
    {
        warning() << "Profile mismatch for the current context: "
            << profileString(requested.profile()) << " requested, "
            << profileString(created.profile())   << " created.";
    }

    if (requested.version() != created.version())
    {
        warning() << "Version mismatch for the current context: "
            << requested.version().toString() << " requested, "
            << created.version().toString()   << " created.";

        if (requested.profile() == Profile::Core)
            return false;
    }
    return sameProfiles;
}

inline void ContextFormat::verifyBufferSize(
    const unsigned int sizeRequested
,   const unsigned int sizeInitialized
,   const std::string & warning
,   std::vector<std::string> & issues)
{
    if (sizeRequested == sizeInitialized)
        return;

    std::stringstream ss;
    ss << warning << " size mismatch: " << sizeRequested << " requested, " << sizeInitialized << " created.";

    issues.push_back(ss.str());
}

bool ContextFormat::verifyPixelFormat(
    const ContextFormat & requested
,   const ContextFormat & created)
{
    std::vector<std::string> issues;

    const bool sameSwapBehaviors(requested.swapBehavior() == created.swapBehavior());

    if (!sameSwapBehaviors)
    {
        warning() << "Swap behavior mismatch for the current context: "
            << swapBehaviorString(requested.swapBehavior()) << " requested, "
            << swapBehaviorString(created.swapBehavior())   << " created.";
    }

    if (requested.depthBufferSize())
    {
        if (!created.depthBufferSize())
            issues.push_back("- Depth Buffer requested, but none created.");
        else
            verifyBufferSize(requested.depthBufferSize(), created.depthBufferSize()
                , "- Depth Buffer", issues);
    }

    verifyBufferSize(requested.redBufferSize(),   created.redBufferSize()
        , "- Red Buffer", issues);
    verifyBufferSize(requested.greenBufferSize(), created.greenBufferSize()
        , "- Green Buffer", issues);
    verifyBufferSize(requested.blueBufferSize(),  created.blueBufferSize()
        , "- Blue Buffer", issues);
    verifyBufferSize(requested.alphaBufferSize(), created.alphaBufferSize()
        , "- Alpha Buffer", issues);

    if (requested.stencilBufferSize())
    {
        if (!created.stencilBufferSize())
            issues.push_back("- Stencil Buffer requested, but none created.");
        else
            verifyBufferSize(requested.stencilBufferSize(), created.stencilBufferSize()
                , "- Stencil Buffer", issues);
    }

    if (requested.stereo() && !created.stereo())
        issues.push_back("- Stereo Buffering requested, but not initialized.");

    if (requested.samples())
    {
        if (!created.samples())
            issues.push_back("- Sample Buffers requested, but none initialized.");
        else
            verifyBufferSize(requested.samples(), created.samples(), "- Samples ", issues);
    }

    if (issues.empty())
        return true;

    warning() << "Pixelformat mismatch for the current context:";
    for(const std::string & issue : issues)
        warning() << issue;

    return false;
}

} // namespace gloperate
<commit_msg>fix access to protected glbinding::Version member<commit_after>
#include <gloperate/painter/ContextFormat.h>

#include <cassert>
#include <sstream>
#include <map>

#include <globjects/base/baselogging.h>

using namespace globjects;

namespace gloperate
{

ContextFormat::ContextFormat()
: m_version(glbinding::Version(4, 5))
, m_profile(Profile::None)
, m_debugContext(false)
, m_forwardCompatibility(false)
, m_redBufferSize(8)
, m_greenBufferSize(8)
, m_blueBufferSize(8)
, m_alphaBufferSize(8)
, m_depthBufferSize(24)
, m_stencilBufferSize(0)
, m_stereo(false)
, m_swapBehavior(SwapBehavior::DoubleBuffering)
, m_samples(0)
{
}

ContextFormat::~ContextFormat()
{
}

void ContextFormat::setVersion(
    const unsigned int majorVersion
,   const unsigned int minorVersion)
{
    setVersion(glbinding::Version(majorVersion, minorVersion));
}

void ContextFormat::setVersion(const glbinding::Version & version)
{
    m_version = version;
}

glbinding::Version ContextFormat::validateVersion(const glbinding::Version &requestedVersion
,   const glbinding::Version &_maximumVersion)
{
    glbinding::Version maximumVersion = _maximumVersion;
    if (maximumVersion.isNull())
    {
#ifdef __APPLE__
        maximumVersion = glbinding::Version(3, 2);
#else
        maximumVersion = glbinding::Version(3, 0);
#endif
    }

    if (requestedVersion.isNull() || requestedVersion > maximumVersion)
        return maximumVersion;

    if (!requestedVersion.isValid())
    {
        glbinding::Version nearest = requestedVersion.nearest();
        return nearest > maximumVersion ? maximumVersion : nearest;
    }
    return requestedVersion;
}

int ContextFormat::majorVersion() const
{
    return m_version.majorVersion();
}

int ContextFormat::minorVersion() const
{
    return m_version.minorVersion();
}

const glbinding::Version & ContextFormat::version() const
{
    return m_version;
}

ContextFormat::Profile ContextFormat::profile() const
{
    return m_profile;
}

void ContextFormat::setProfile(const ContextFormat::Profile profile)
{
    m_profile = profile;
}

bool ContextFormat::debugContext() const
{
    return m_debugContext;
}

void ContextFormat::setDebugContext(const bool on)
{
    m_debugContext = on;
}

bool ContextFormat::forwardCompatible() const
{
    return m_forwardCompatibility;
}

void ContextFormat::setForwardCompatible(const bool on)
{
    m_forwardCompatibility = on;
}

int ContextFormat::redBufferSize() const
{
    return m_redBufferSize;
}

void ContextFormat::setRedBufferSize(const int size)
{
    m_redBufferSize = size;
}

int ContextFormat::greenBufferSize() const
{
    return m_greenBufferSize;
}

void ContextFormat::setGreenBufferSize(const int size)
{
    m_greenBufferSize = size;
}

int  ContextFormat::blueBufferSize() const
{
    return m_blueBufferSize;
}

void ContextFormat::setBlueBufferSize(const int size)
{
    m_blueBufferSize = size;
}

int ContextFormat::alphaBufferSize() const
{
    return m_alphaBufferSize;
}

void ContextFormat::setAlphaBufferSize(const int size)
{
    m_alphaBufferSize = size;
}

int ContextFormat::depthBufferSize() const
{
    return m_depthBufferSize;
}

void ContextFormat::setDepthBufferSize(const int size)
{
    m_depthBufferSize = size;
}

int ContextFormat::stencilBufferSize() const
{
    return m_stencilBufferSize;
}

void ContextFormat::setStencilBufferSize(const int size)
{
    m_stencilBufferSize = size;
}

ContextFormat::SwapBehavior ContextFormat::swapBehavior() const
{
    return m_swapBehavior;
}

void ContextFormat::setSwapBehavior(const ContextFormat::SwapBehavior behavior)
{
    m_swapBehavior = behavior;
}

bool ContextFormat::stereo() const
{
    return m_stereo;
}

void ContextFormat::setStereo(const bool enable)
{
    m_stereo = enable;
}

int ContextFormat::samples() const
{
    return m_samples;
}

void ContextFormat::setSamples(const int samples)
{
    m_samples = samples;
}

const std::string & ContextFormat::profileString(const Profile profile)
{
    static const std::map<Profile, std::string> profileIdentifier = {
        { Profile::Core,          "Core" }
    ,   { Profile::Compatibility, "Compatibility" }
    ,   { Profile::None,          "None" } };

    return profileIdentifier.at(profile);
}

const std::string & ContextFormat::swapBehaviorString(const SwapBehavior swapBehavior)
{
    static const std::map<SwapBehavior, std::string> swapbIdentifier = {
        { SwapBehavior::Default,         "Default" }
    ,   { SwapBehavior::DoubleBuffering, "DoubleBuffering" }
    ,   { SwapBehavior::SingleBuffering, "SingleBuffering" }
    ,   { SwapBehavior::TripleBuffering, "TripleBuffering" } };

    return swapbIdentifier.at(swapBehavior);
}

bool ContextFormat::verify(const ContextFormat & requested, const ContextFormat & created)
{
    return
        verifyVersionAndProfile(requested, created) &&
        verifyPixelFormat(requested, created);
}

bool ContextFormat::verify(const ContextFormat & requested) const
{
    return verify(requested, *this);
}

bool ContextFormat::verifyVersionAndProfile(const ContextFormat & requested, const ContextFormat & created)
{
    const bool sameProfiles(requested.profile() == created.profile());

    if (!sameProfiles)
    {
        warning() << "Profile mismatch for the current context: "
            << profileString(requested.profile()) << " requested, "
            << profileString(created.profile())   << " created.";
    }

    if (requested.version() != created.version())
    {
        warning() << "Version mismatch for the current context: "
            << requested.version().toString() << " requested, "
            << created.version().toString()   << " created.";

        if (requested.profile() == Profile::Core)
            return false;
    }
    return sameProfiles;
}

inline void ContextFormat::verifyBufferSize(
    const unsigned int sizeRequested
,   const unsigned int sizeInitialized
,   const std::string & warning
,   std::vector<std::string> & issues)
{
    if (sizeRequested == sizeInitialized)
        return;

    std::stringstream ss;
    ss << warning << " size mismatch: " << sizeRequested << " requested, " << sizeInitialized << " created.";

    issues.push_back(ss.str());
}

bool ContextFormat::verifyPixelFormat(
    const ContextFormat & requested
,   const ContextFormat & created)
{
    std::vector<std::string> issues;

    const bool sameSwapBehaviors(requested.swapBehavior() == created.swapBehavior());

    if (!sameSwapBehaviors)
    {
        warning() << "Swap behavior mismatch for the current context: "
            << swapBehaviorString(requested.swapBehavior()) << " requested, "
            << swapBehaviorString(created.swapBehavior())   << " created.";
    }

    if (requested.depthBufferSize())
    {
        if (!created.depthBufferSize())
            issues.push_back("- Depth Buffer requested, but none created.");
        else
            verifyBufferSize(requested.depthBufferSize(), created.depthBufferSize()
                , "- Depth Buffer", issues);
    }

    verifyBufferSize(requested.redBufferSize(),   created.redBufferSize()
        , "- Red Buffer", issues);
    verifyBufferSize(requested.greenBufferSize(), created.greenBufferSize()
        , "- Green Buffer", issues);
    verifyBufferSize(requested.blueBufferSize(),  created.blueBufferSize()
        , "- Blue Buffer", issues);
    verifyBufferSize(requested.alphaBufferSize(), created.alphaBufferSize()
        , "- Alpha Buffer", issues);

    if (requested.stencilBufferSize())
    {
        if (!created.stencilBufferSize())
            issues.push_back("- Stencil Buffer requested, but none created.");
        else
            verifyBufferSize(requested.stencilBufferSize(), created.stencilBufferSize()
                , "- Stencil Buffer", issues);
    }

    if (requested.stereo() && !created.stereo())
        issues.push_back("- Stereo Buffering requested, but not initialized.");

    if (requested.samples())
    {
        if (!created.samples())
            issues.push_back("- Sample Buffers requested, but none initialized.");
        else
            verifyBufferSize(requested.samples(), created.samples(), "- Samples ", issues);
    }

    if (issues.empty())
        return true;

    warning() << "Pixelformat mismatch for the current context:";
    for(const std::string & issue : issues)
        warning() << issue;

    return false;
}

} // namespace gloperate
<|endoftext|>


<commit_before>//=======================================================================
// Copyright (c) 2014-2017 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

namespace etl {

/*!
 * \brief Indicates if an 1D evaluation should run in paralle
 * \param n The size of the evaluation
 * \param threshold The parallel threshold
 * \return true if the evaluation should be done in paralle, false otherwise
 */
inline bool select_parallel(std::size_t n, std::size_t threshold = parallel_threshold) {
    return threads > 1 && ((parallel_support && local_context().parallel)|| (is_parallel && n >= threshold && !local_context().serial));
}

/*!
 * \brief Indicates if an 2D evaluation should run in paralle
 * \param n1 The first dimension of the evaluation
 * \param t1 The first parallel threshold
 * \param n2 The second dimension of the evaluation
 * \param t2 The second parallel threshold
 * \return true if the evaluation should be done in paralle, false otherwise
 */
inline bool select_parallel_2d(std::size_t n1, std::size_t t1, std::size_t n2, std::size_t t2) {
    return threads > 1 && ((parallel_support && local_context().parallel) || (is_parallel && n1 >= t1 && n2 >= t2 && !local_context().serial));
}

/*!
 * \brief Dispatch the elements of a range to a functor in a parallel manner
 * \param pool The pool to use
 * \param p Boolean tag to indicate if parallel dispatching must be done
 * \param functor The functor to execute
 * \param T The number of threads to use
 * \param first The beginning of the range
 * \param last The end of the range
 */
template <typename Functor>
inline void dispatch_1d(cpp::default_thread_pool<>& pool, bool p, Functor&& functor, size_t T, std::size_t first, std::size_t last) {
    if (p) {
        const auto n     = last - first;
        const auto batch = n / T;

        for (std::size_t t = 0; t < T - 1; ++t) {
            pool.do_task(functor, first + t * batch, first + (t + 1) * batch);
        }

        functor(first + (T - 1) * batch, last);

        pool.wait();
    } else {
        functor(first, last);
    }
}

/*!
 * \brief Dispatch the elements of a range to a functor in a parallel manner
 * \param pool The pool to use
 * \param p Boolean tag to indicate if parallel dispatching must be done
 * \param functor The functor to execute
 * \param first The beginning of the range
 * \param last The end of the range
 */
template <typename Functor>
inline void dispatch_1d(cpp::default_thread_pool<>& pool, bool p, Functor&& functor, std::size_t first, std::size_t last) {
    if (p) {
        dispatch_1d(pool, p, std::forward<Functor>(functor), pool.size() + 1, first, last);
    } else {
        functor(first, last);
    }
}

/*!
 * \brief Dispatch the elements of a range to a functor in a parallel manner
 * \param p Boolean tag to indicate if parallel dispatching must be done
 * \param functor The functor to execute
 * \param first The beginning of the range
 * \param last The end of the range
 */
template <typename Functor>
inline void dispatch_1d(bool p, Functor&& functor, std::size_t first, std::size_t last) {
    if (p) {
        thread_local cpp::default_thread_pool<> pool(threads - 1);
        dispatch_1d(pool, p, std::forward<Functor>(functor), threads, first, last);
    } else {
        functor(first, last);
    }
}

/*!
 * \brief Dispatch the elements of a range to a functor in a parallel manner
 * \param p Boolean tag to indicate if parallel dispatching must be done
 * \param functor The functor to execute
 * \param first The beginning of the range
 * \param last The end of the range
 */
template <typename Functor>
inline void dispatch_1d_any(bool p, Functor&& functor, std::size_t first, std::size_t last) {
    if (p) {
        auto n = last - first;
        size_t T = std::min(n, threads);
        thread_local cpp::default_thread_pool<> pool(threads - 1);
        dispatch_1d(pool, p, std::forward<Functor>(functor), T, first, last);
    } else {
        functor(first, last);
    }
}

/*!
 * \brief Dispatch the elements of a range to a functor in a parallel manner and use an accumulator functor to accumulate the results
 * \param p Boolean tag to indicate if parallel dispatching must be done
 * \param functor The functor to execute
 * \param acc_functor The functor to accumulate results
 * \param first The beginning of the range
 * \param last The end of the range
 */
template <typename T, typename Functor, typename AccFunctor>
inline void dispatch_1d_acc(bool p, Functor&& functor, AccFunctor&& acc_functor, std::size_t first, std::size_t last) {
    if (p) {
        std::vector<T> futures(threads - 1);
        thread_local cpp::default_thread_pool<> pool(threads - 1);

        auto n     = last - first;
        auto batch = n / threads;

        auto sub_functor = [&futures, &functor](std::size_t t, std::size_t first, std::size_t last) {
            futures[t]   = functor(first, last);
        };

        for (std::size_t t = 0; t < threads - 1; ++t) {
            pool.do_task(sub_functor, t, first + t * batch, first + (t + 1) * batch);
        }

        acc_functor(functor(first + (threads - 1) * batch, last));

        pool.wait();

        for (auto fut : futures) {
            acc_functor(fut);
        }
    } else {
        acc_functor(functor(first, last));
    }
}

} //end of namespace etl
<commit_msg>New dispatch function<commit_after>//=======================================================================
// Copyright (c) 2014-2017 Baptiste Wicht
// Distributed under the terms of the MIT License.
// (See accompanying file LICENSE or copy at
//  http://opensource.org/licenses/MIT)
//=======================================================================

#pragma once

namespace etl {

/*!
 * \brief Indicates if an 1D evaluation should run in paralle
 * \param n The size of the evaluation
 * \param threshold The parallel threshold
 * \return true if the evaluation should be done in paralle, false otherwise
 */
inline bool select_parallel(std::size_t n, std::size_t threshold = parallel_threshold) {
    return threads > 1 && ((parallel_support && local_context().parallel)|| (is_parallel && n >= threshold && !local_context().serial));
}

/*!
 * \brief Indicates if an 2D evaluation should run in paralle
 * \param n1 The first dimension of the evaluation
 * \param t1 The first parallel threshold
 * \param n2 The second dimension of the evaluation
 * \param t2 The second parallel threshold
 * \return true if the evaluation should be done in paralle, false otherwise
 */
inline bool select_parallel_2d(std::size_t n1, std::size_t t1, std::size_t n2, std::size_t t2) {
    return threads > 1 && ((parallel_support && local_context().parallel) || (is_parallel && n1 >= t1 && n2 >= t2 && !local_context().serial));
}

/*!
 * \brief Dispatch the elements of a range to a functor in a parallel manner
 * \param pool The pool to use
 * \param p Boolean tag to indicate if parallel dispatching must be done
 * \param functor The functor to execute
 * \param T The number of threads to use
 * \param first The beginning of the range
 * \param last The end of the range
 */
template <typename Functor>
inline void dispatch_1d(cpp::default_thread_pool<>& pool, bool p, Functor&& functor, size_t T, std::size_t first, std::size_t last) {
    if (p) {
        const auto n     = last - first;
        const auto batch = n / T;

        for (std::size_t t = 0; t < T - 1; ++t) {
            pool.do_task(functor, first + t * batch, first + (t + 1) * batch);
        }

        functor(first + (T - 1) * batch, last);

        pool.wait();
    } else {
        functor(first, last);
    }
}

/*!
 * \brief Dispatch the elements of a range to a functor in a parallel manner
 * \param pool The pool to use
 * \param p Boolean tag to indicate if parallel dispatching must be done
 * \param functor The functor to execute
 * \param first The beginning of the range
 * \param last The end of the range
 */
template <typename Functor>
inline void dispatch_1d(cpp::default_thread_pool<>& pool, bool p, Functor&& functor, std::size_t first, std::size_t last) {
    if (p) {
        dispatch_1d(pool, p, std::forward<Functor>(functor), pool.size() + 1, first, last);
    } else {
        functor(first, last);
    }
}

/*!
 * \brief Dispatch the elements of a range to a functor in a parallel manner
 * \param p Boolean tag to indicate if parallel dispatching must be done
 * \param functor The functor to execute
 * \param first The beginning of the range
 * \param last The end of the range
 */
template <typename Functor>
inline void dispatch_1d(bool p, Functor&& functor, std::size_t first, std::size_t last) {
    if (p) {
        thread_local cpp::default_thread_pool<> pool(threads - 1);
        dispatch_1d(pool, p, std::forward<Functor>(functor), threads, first, last);
    } else {
        functor(first, last);
    }
}

/*!
 * \brief Dispatch the elements of a range to a functor in a parallel manner
 * \param p Boolean tag to indicate if parallel dispatching must be done
 * \param functor The functor to execute
 * \param first The beginning of the range
 * \param last The end of the range
 */
template <typename Functor>
inline void dispatch_1d_any(bool p, Functor&& functor, std::size_t first, std::size_t last) {
    if (p) {
        auto n = last - first;
        size_t T = std::min(n, threads);
        thread_local cpp::default_thread_pool<> pool(threads - 1);
        dispatch_1d(pool, p, std::forward<Functor>(functor), T, first, last);
    } else {
        functor(first, last);
    }
}

/*!
 * \brief Dispatch the elements of a range to a functor in a parallel manner.
 *
 * The dispatching will be done in batch. That is to say that the
 * functor will be called with a range of data.
 *
 * This will only be dispatched in parallel if etl is running in
 * parallel mode and if the range is bigger than the treshold.
 *
 * \param functor The functor to execute
 * \param first The beginning of the range
 * \param last The end of the range. Must be bigger or equal to first.
 * \param threshold The threshold for parallelization
 */
template <typename Functor>
inline void smart_dispatch_1d_any(Functor&& functor, size_t first, size_t last, size_t threshold) {
    if (etl::is_parallel) {
        auto n = last - first;

        if (select_parallel(n, threshold)) {
            size_t T = std::min(n, threads);
            thread_local cpp::default_thread_pool<> pool(threads - 1);
            dispatch_1d(pool, true, std::forward<Functor>(functor), T, first, last);
        } else {
            functor(first, last);
        }
    } else {
        functor(first, last);
    }
}

/*!
 * \brief Dispatch the elements of a range to a functor in a parallel manner and use an accumulator functor to accumulate the results
 * \param p Boolean tag to indicate if parallel dispatching must be done
 * \param functor The functor to execute
 * \param acc_functor The functor to accumulate results
 * \param first The beginning of the range
 * \param last The end of the range
 */
template <typename T, typename Functor, typename AccFunctor>
inline void dispatch_1d_acc(bool p, Functor&& functor, AccFunctor&& acc_functor, std::size_t first, std::size_t last) {
    if (p) {
        std::vector<T> futures(threads - 1);
        thread_local cpp::default_thread_pool<> pool(threads - 1);

        auto n     = last - first;
        auto batch = n / threads;

        auto sub_functor = [&futures, &functor](std::size_t t, std::size_t first, std::size_t last) {
            futures[t]   = functor(first, last);
        };

        for (std::size_t t = 0; t < threads - 1; ++t) {
            pool.do_task(sub_functor, t, first + t * batch, first + (t + 1) * batch);
        }

        acc_functor(functor(first + (threads - 1) * batch, last));

        pool.wait();

        for (auto fut : futures) {
            acc_functor(fut);
        }
    } else {
        acc_functor(functor(first, last));
    }
}

} //end of namespace etl
<|endoftext|>


<commit_before>
#include "ApplicationContext.hpp"

#include <boost/algorithm/string.hpp>
#include <cstdio>

#include "Werk/Config/ReloadConfigCommand.hpp"
#include "Werk/OS/Signals.hpp"
#include "Werk/Profiling/WriteProfilesAction.hpp"

#include "QuitCommand.hpp"

namespace werk
{

ApplicationContext::ApplicationContext(const std::string &configPath)
{
	//And let's pull ourselves up by our bootstraps...

	/********** Pre-Initialization **********/

	//Setup handlers for certain signals
	setupSegfaultHandler();

	/********** Stdout Log **********/

	_stdoutLog = new AsyncLog("StdoutLog", &_backgroundThread.backgroundClock());
	_logManager.add(_stdoutLog);
	_backgroundThread.addTask(_stdoutLog);

	/********** Config **********/

	_config = new Config("Config", _stdoutLog);

	//Synchronously execute the reload since the values are needed for the next step
	_config->addConfigSource(new IniConfigSource(configPath));
	_config->reloadConfig();
	_config->execute();

	//Register other config files
	const char *configPathsStr = _config->getString("Application.ConfigPaths");
	if (nullptr != configPathsStr) {
		//Add all the new configs
		std::vector<std::string> configPaths;
		boost::split(configPaths, configPathsStr, boost::is_any_of(","));
		for (auto &path : configPaths) {
			_config->addConfigSource(new IniConfigSource(path));
		}
	}

	//Synchronously reload with everything
	_config->addConfigSource(new IniConfigSource(configPath));
	_config->reloadConfig();
	_config->execute();

	//Finally, with all files loaded, start reloading in the background
	_backgroundThread.addTask(_config);
	_stdoutLog->logRaw(LogLevel::SUCCESS, "<Config> Initialized.");

	/********** Main Log **********/

	const char *logPath = _config->getString("Application.LogPath");
	FILE *file = nullptr == logPath ? stdout : std::fopen(logPath, "a");
	if (nullptr == file) {
		_stdoutLog->logRaw(LogLevel::ERROR, "Could not open log file, redirecting to stderr.\n");
		file = stderr;
	}

	_log = new AsyncLog("Log", &_backgroundThread.backgroundClock(), file);
	_logManager.add(_log);
	_backgroundThread.addTask(_log);
	_config->setLog(_log);
	_log->logRaw(LogLevel::SUCCESS, "<Log> Initialized.");

	/********** Command Manager **********/

	_commandManager = new CommandManager(_log);
	_commandManager->add("reload", new ReloadConfigCommand(*_config));
	Command *quitCommand = new QuitCommand(this);
	_commandManager->add("quit", quitCommand);
	_log->logRaw(LogLevel::SUCCESS, "<CommandManager> Initialized.");

	const char *profilesPath = _config->getString("Application.ProfilesPath");
	if (nullptr != profilesPath) {
		_log->log(LogLevel::INFO, "Writing profiles to %s on shutdown.", profilesPath);
		_shutdownActions.push_back(new WriteProfilesAction("WriteProfiles", _log, _profileManager, profilesPath));
	}

	/********** Finish Initialization **********/

	//Setup remaining signals
	//SIGHUP -> reload config
	setupSignalHandler(SIGHUP, _config->getReloadConfigAction());

	//Load and run startup commands
	//TODO: consider defering this until later, once the user has setup everything they need?
	const char *startupCommandsStr = _config->getString("Application.StartupCommands");
	if (nullptr != startupCommandsStr) {
		//Split and run each command
		boost::split(_startupCommands, startupCommandsStr, boost::is_any_of(";"));
		for (auto &command : _startupCommands) {
			_commandManager->execute(command);
		}
	}

	_log->logRaw(LogLevel::SUCCESS, "<ApplicationContext> Initialized.");
}

ApplicationContext::~ApplicationContext()
{
	//Shut down, if not already done
	if (!isShutdown()) {
		shutdown();
	}
}

bool ApplicationContext::isShutdown()
{
	return _backgroundThread.stopped();
}

void ApplicationContext::shutdown()
{
	//Don't shut down twice
	if (isShutdown()) {
		fprintf(stderr, "ApplicationContext::shutdown - Already shut down.\n");
		return;
	}

	//Load and run shutdown commands
	const char *shutdownCommandsStr = _config->getString("Application.ShutdownCommands");
	if (nullptr != shutdownCommandsStr) {
		//Split on ; and run each command
		boost::split(_shutdownCommands, shutdownCommandsStr, boost::is_any_of(";"));
		for (auto &command : _startupCommands) {
			_commandManager->execute(command);
		}
	}

	//Run shutdown actions
	_log->logRaw(LogLevel::INFO, "Running shutdown actions...");
	for (Action *action : _shutdownActions) {
		action->execute();
	}
	_log->logRaw(LogLevel::SUCCESS, "Shutdown actions complete.");

	_log->logRaw(LogLevel::INFO, "Shutting down...");
	_backgroundThread.stop();
}

}
<commit_msg>Add missing header<commit_after>
#include "ApplicationContext.hpp"

#include <boost/algorithm/string.hpp>
#include <cstdio>
#include <signal.h>

#include "Werk/Config/ReloadConfigCommand.hpp"
#include "Werk/OS/Signals.hpp"
#include "Werk/Profiling/WriteProfilesAction.hpp"

#include "QuitCommand.hpp"

namespace werk
{

ApplicationContext::ApplicationContext(const std::string &configPath)
{
	//And let's pull ourselves up by our bootstraps...

	/********** Pre-Initialization **********/

	//Setup handlers for certain signals
	setupSegfaultHandler();

	/********** Stdout Log **********/

	_stdoutLog = new AsyncLog("StdoutLog", &_backgroundThread.backgroundClock());
	_logManager.add(_stdoutLog);
	_backgroundThread.addTask(_stdoutLog);

	/********** Config **********/

	_config = new Config("Config", _stdoutLog);

	//Synchronously execute the reload since the values are needed for the next step
	_config->addConfigSource(new IniConfigSource(configPath));
	_config->reloadConfig();
	_config->execute();

	//Register other config files
	const char *configPathsStr = _config->getString("Application.ConfigPaths");
	if (nullptr != configPathsStr) {
		//Add all the new configs
		std::vector<std::string> configPaths;
		boost::split(configPaths, configPathsStr, boost::is_any_of(","));
		for (auto &path : configPaths) {
			_config->addConfigSource(new IniConfigSource(path));
		}
	}

	//Synchronously reload with everything
	_config->addConfigSource(new IniConfigSource(configPath));
	_config->reloadConfig();
	_config->execute();

	//Finally, with all files loaded, start reloading in the background
	_backgroundThread.addTask(_config);
	_stdoutLog->logRaw(LogLevel::SUCCESS, "<Config> Initialized.");

	/********** Main Log **********/

	const char *logPath = _config->getString("Application.LogPath");
	FILE *file = nullptr == logPath ? stdout : std::fopen(logPath, "a");
	if (nullptr == file) {
		_stdoutLog->logRaw(LogLevel::ERROR, "Could not open log file, redirecting to stderr.\n");
		file = stderr;
	}

	_log = new AsyncLog("Log", &_backgroundThread.backgroundClock(), file);
	_logManager.add(_log);
	_backgroundThread.addTask(_log);
	_config->setLog(_log);
	_log->logRaw(LogLevel::SUCCESS, "<Log> Initialized.");

	/********** Command Manager **********/

	_commandManager = new CommandManager(_log);
	_commandManager->add("reload", new ReloadConfigCommand(*_config));
	Command *quitCommand = new QuitCommand(this);
	_commandManager->add("quit", quitCommand);
	_log->logRaw(LogLevel::SUCCESS, "<CommandManager> Initialized.");

	const char *profilesPath = _config->getString("Application.ProfilesPath");
	if (nullptr != profilesPath) {
		_log->log(LogLevel::INFO, "Writing profiles to %s on shutdown.", profilesPath);
		_shutdownActions.push_back(new WriteProfilesAction("WriteProfiles", _log, _profileManager, profilesPath));
	}

	/********** Finish Initialization **********/

	//Setup remaining signals
	//SIGHUP -> reload config
	setupSignalHandler(SIGHUP, _config->getReloadConfigAction());

	//Load and run startup commands
	//TODO: consider defering this until later, once the user has setup everything they need?
	const char *startupCommandsStr = _config->getString("Application.StartupCommands");
	if (nullptr != startupCommandsStr) {
		//Split and run each command
		boost::split(_startupCommands, startupCommandsStr, boost::is_any_of(";"));
		for (auto &command : _startupCommands) {
			_commandManager->execute(command);
		}
	}

	_log->logRaw(LogLevel::SUCCESS, "<ApplicationContext> Initialized.");
}

ApplicationContext::~ApplicationContext()
{
	//Shut down, if not already done
	if (!isShutdown()) {
		shutdown();
	}
}

bool ApplicationContext::isShutdown()
{
	return _backgroundThread.stopped();
}

void ApplicationContext::shutdown()
{
	//Don't shut down twice
	if (isShutdown()) {
		fprintf(stderr, "ApplicationContext::shutdown - Already shut down.\n");
		return;
	}

	//Load and run shutdown commands
	const char *shutdownCommandsStr = _config->getString("Application.ShutdownCommands");
	if (nullptr != shutdownCommandsStr) {
		//Split on ; and run each command
		boost::split(_shutdownCommands, shutdownCommandsStr, boost::is_any_of(";"));
		for (auto &command : _startupCommands) {
			_commandManager->execute(command);
		}
	}

	//Run shutdown actions
	_log->logRaw(LogLevel::INFO, "Running shutdown actions...");
	for (Action *action : _shutdownActions) {
		action->execute();
	}
	_log->logRaw(LogLevel::SUCCESS, "Shutdown actions complete.");

	_log->logRaw(LogLevel::INFO, "Shutting down...");
	_backgroundThread.stop();
}

}
<|endoftext|>


<commit_before>#include "data/shortlist.h"
#include "microsoft/shortlist/utils/ParameterTree.h"
#include "marian.h"

#if BLAS_FOUND
#include "3rd_party/faiss/IndexLSH.h"
#endif

namespace marian {
namespace data {

// cast current void pointer to T pointer and move forward by num elements
template <typename T>
const T* get(const void*& current, size_t num = 1) {
  const T* ptr = (const T*)current;
  current = (const T*)current + num;
  return ptr;
}

//////////////////////////////////////////////////////////////////////////////////////
Shortlist::Shortlist(const std::vector<WordIndex>& indices)
  : indices_(indices)
  , done_(false) {}

Shortlist::~Shortlist() {}

WordIndex Shortlist::reverseMap(int , int , int idx) const { return indices_[idx]; }

WordIndex Shortlist::tryForwardMap(int , int , WordIndex wIdx) const {
  auto first = std::lower_bound(indices_.begin(), indices_.end(), wIdx);
  if(first != indices_.end() && *first == wIdx)         // check if element not less than wIdx has been found and if equal to wIdx
    return (int)std::distance(indices_.begin(), first); // return coordinate if found
  else
    return npos;                                        // return npos if not found, @TODO: replace with std::optional once we switch to C++17?
}

void Shortlist::filter(Expr input, Expr weights, bool isLegacyUntransposedW, Expr b, Expr lemmaEt) {
  if (done_) {
    return;
  }

  auto forward = [this](Expr out, const std::vector<Expr>& ) {
    out->val()->set(indices_);
  };

  int k = (int) indices_.size();
  Shape kShape({k});
  indicesExpr_ = lambda({input, weights}, kShape, Type::uint32, forward);

  //std::cerr << "indicesExpr_=" << indicesExpr_->shape() << std::endl;
  createCachedTensors(weights, isLegacyUntransposedW, b, lemmaEt, k);
  done_ = true;
}

Expr Shortlist::getIndicesExpr() const {
  int k = indicesExpr_->shape()[0];
  Expr out = reshape(indicesExpr_, {1, 1, k});
  return out;
}

void Shortlist::createCachedTensors(Expr weights,
                          bool isLegacyUntransposedW,
                          Expr b,
                          Expr lemmaEt,
                          int k) {
  ABORT_IF(isLegacyUntransposedW, "Legacy untranspose W not yet tested");
  cachedShortWt_ = index_select(weights, isLegacyUntransposedW ? -1 : 0, indicesExpr_);
  cachedShortWt_ = reshape(cachedShortWt_, {1, 1, cachedShortWt_->shape()[0], cachedShortWt_->shape()[1]});

  if (b) {
    ABORT("Bias not yet tested");
    cachedShortb_ = index_select(b, -1, indicesExpr_);
    cachedShortb_ = reshape(cachedShortb_, {1, k, 1, cachedShortb_->shape()[1]}); // not tested
  }

  cachedShortLemmaEt_ = index_select(lemmaEt, -1, indicesExpr_);
  cachedShortLemmaEt_ = reshape(cachedShortLemmaEt_, {1, 1, cachedShortLemmaEt_->shape()[0], k});
}

///////////////////////////////////////////////////////////////////////////////////
Ptr<faiss::IndexLSH> LSHShortlist::index_;

LSHShortlist::LSHShortlist(int k, int nbits)
: Shortlist(std::vector<WordIndex>())
, k_(k), nbits_(nbits) {
  //std::cerr << "LSHShortlist" << std::endl;
  /*
  for (int i = 0; i < k_; ++i) {
    indices_.push_back(i);
  }
  */
}

//#define BLAS_FOUND 1

WordIndex LSHShortlist::reverseMap(int beamIdx, int batchIdx, int idx) const {
  //int currBeamSize = indicesExpr_->shape()[0];
  int currBatchSize = indicesExpr_->shape()[1];
  idx = (k_ * currBatchSize * beamIdx) + (k_ * batchIdx) + idx;
  assert(idx < indices_.size());
  return indices_[idx];
}

WordIndex LSHShortlist::tryForwardMap(int , int , WordIndex wIdx) const {
  auto first = std::lower_bound(indices_.begin(), indices_.end(), wIdx);
  bool found = first != indices_.end();
  if(found && *first == wIdx)         // check if element not less than wIdx has been found and if equal to wIdx
    return (int)std::distance(indices_.begin(), first); // return coordinate if found
  else
    return npos;                                        // return npos if not found, @TODO: replace with std::optional once we switch to C++17?
}

Expr LSHShortlist::getIndicesExpr() const {
  return indicesExpr_;
}

#define BLAS_FOUND 1

void LSHShortlist::filter(Expr input, Expr weights, bool isLegacyUntransposedW, Expr b, Expr lemmaEt) {
#if BLAS_FOUND
  ABORT_IF(input->graph()->getDeviceId().type == DeviceType::gpu,
           "LSH index (--output-approx-knn) currently not implemented for GPU");

  int currBeamSize = input->shape()[0];
  int batchSize = input->shape()[2];
  int numHypos = currBeamSize * batchSize;

  auto forward = [this, numHypos](Expr out, const std::vector<Expr>& inputs) {
    auto query  = inputs[0];
    auto values = inputs[1];
    int dim = values->shape()[-1];

    if(!index_) {
      //std::cerr << "build lsh index" << std::endl;
      LOG(info, "Building LSH index for vector dim {} and with hash size {} bits", dim, nbits_);
      index_.reset(new faiss::IndexLSH(dim, nbits_,
                                       /*rotate=*/dim != nbits_,
                                       /*train_thesholds*/false));
      int vRows = 32121; //47960; //values->shape().elements() / dim;
      index_->train(vRows, values->val()->data<float>());
      index_->add(  vRows, values->val()->data<float>());
    }

    int qRows = query->shape().elements() / dim;
    std::vector<float> distances(qRows * k_);
    std::vector<faiss::Index::idx_t> ids(qRows * k_);

    index_->search(qRows, query->val()->data<float>(), k_,
                   distances.data(), ids.data());

    indices_.clear();
    for(auto iter = ids.begin(); iter != ids.end(); ++iter) {
      faiss::Index::idx_t id = *iter;
      indices_.push_back((WordIndex)id);
    }

    for (size_t hypoIdx = 0; hypoIdx < numHypos; ++hypoIdx) {
      size_t startIdx = k_ * hypoIdx;
      size_t endIdx = startIdx + k_;
      std::sort(indices_.begin() + startIdx, indices_.begin() + endIdx);
    }
    out->val()->set(indices_);
  };

  Shape kShape({currBeamSize, batchSize, k_});
  indicesExpr_ = lambda({input, weights}, kShape, Type::uint32, forward);

  createCachedTensors(weights, isLegacyUntransposedW, b, lemmaEt, k_);

#else
  input; weights; isLegacyUntransposedW; b; lemmaEt;
  ABORT("LSH output layer requires a CPU BLAS library");
#endif
}

void LSHShortlist::createCachedTensors(Expr weights,
                          bool isLegacyUntransposedW,
                          Expr b,
                          Expr lemmaEt,
                          int k) {
  int currBeamSize = indicesExpr_->shape()[0];
  int batchSize = indicesExpr_->shape()[1];
  ABORT_IF(isLegacyUntransposedW, "Legacy untranspose W not yet tested");

  Expr indicesExprFlatten = reshape(indicesExpr_, {indicesExpr_->shape().elements()});

  cachedShortWt_ = index_select(weights, isLegacyUntransposedW ? -1 : 0, indicesExprFlatten);
  cachedShortWt_ = reshape(cachedShortWt_, {currBeamSize, batchSize, k, cachedShortWt_->shape()[1]});

  if (b) {
    ABORT("Bias not yet tested");
    cachedShortb_ = index_select(b, -1, indicesExprFlatten);
    cachedShortb_ = reshape(cachedShortb_, {currBeamSize, k, batchSize, cachedShortb_->shape()[1]}); // not tested
  }

  int dim = lemmaEt->shape()[0];
  cachedShortLemmaEt_ = index_select(lemmaEt, -1, indicesExprFlatten);
  cachedShortLemmaEt_ = reshape(cachedShortLemmaEt_, {dim, currBeamSize, batchSize, k});
  cachedShortLemmaEt_ = transpose(cachedShortLemmaEt_, {1, 2, 0, 3});
}

LSHShortlistGenerator::LSHShortlistGenerator(int k, int nbits)
  : k_(k), nbits_(nbits) {
  //std::cerr << "LSHShortlistGenerator" << std::endl;
}

Ptr<Shortlist> LSHShortlistGenerator::generate(Ptr<data::CorpusBatch> batch) const {
  return New<LSHShortlist>(k_, nbits_);
}

//////////////////////////////////////////////////////////////////////////////////////
QuicksandShortlistGenerator::QuicksandShortlistGenerator(Ptr<Options> options,
                                                         Ptr<const Vocab> srcVocab,
                                                         Ptr<const Vocab> trgVocab,
                                                         size_t srcIdx,
                                                         size_t /*trgIdx*/,
                                                         bool /*shared*/)
    : options_(options),
      srcVocab_(srcVocab),
      trgVocab_(trgVocab),
      srcIdx_(srcIdx) {
  std::vector<std::string> vals = options_->get<std::vector<std::string>>("shortlist");

  ABORT_IF(vals.empty(), "No path to filter path given");
  std::string fname = vals[0];

  auto firstNum   = vals.size() > 1 ? std::stoi(vals[1]) : 0;
  auto bestNum    = vals.size() > 2 ? std::stoi(vals[2]) : 0;
  float threshold = vals.size() > 3 ? std::stof(vals[3]) : 0;

  if(firstNum != 0 || bestNum != 0 || threshold != 0) {
    LOG(warn, "You have provided additional parameters for the Quicksand shortlist, but they are ignored.");
  }

  mmap_ = mio::mmap_source(fname); // memory-map the binary file once
  const void* current = mmap_.data(); // pointer iterator over binary file

  // compare magic number in binary file to make sure we are reading the right thing
  const int32_t MAGIC_NUMBER = 1234567890;
  int32_t header_magic_number = *get<int32_t>(current);
  ABORT_IF(header_magic_number != MAGIC_NUMBER, "Trying to mmap Quicksand shortlist but encountered wrong magic number");

  auto config = ::quicksand::ParameterTree::FromBinaryReader(current);
  use16bit_ = config->GetBoolReq("use_16_bit");

  LOG(info, "[data] Mapping Quicksand shortlist from {}", fname);

  idSize_ = sizeof(int32_t);
  if (use16bit_) {
    idSize_ = sizeof(uint16_t);
  }

  // mmap the binary shortlist pieces
  numDefaultIds_        = *get<int32_t>(current);
  defaultIds_           =  get<int32_t>(current, numDefaultIds_);
  numSourceIds_         = *get<int32_t>(current);
  sourceLengths_        =  get<int32_t>(current, numSourceIds_);
  sourceOffsets_        =  get<int32_t>(current, numSourceIds_);
  numShortlistIds_      = *get<int32_t>(current);
  sourceToShortlistIds_ =  get<uint8_t>(current, idSize_ * numShortlistIds_);

  // display parameters
  LOG(info,
      "[data] Quicksand shortlist has {} source ids, {} default ids and {} shortlist ids",
      numSourceIds_,
      numDefaultIds_,
      numShortlistIds_);
}

Ptr<Shortlist> QuicksandShortlistGenerator::generate(Ptr<data::CorpusBatch> batch) const {
  auto srcBatch = (*batch)[srcIdx_];
  auto maxShortlistSize = trgVocab_->size();

  std::unordered_set<int32_t> indexSet;
  for(int32_t i = 0; i < numDefaultIds_ && i < maxShortlistSize; ++i) {
    int32_t id = defaultIds_[i];
    indexSet.insert(id);
  }

  // State
  std::vector<std::pair<const uint8_t*, int32_t>> curShortlists(maxShortlistSize);
  auto curShortlistIt = curShortlists.begin();

  // Because we might fill up our shortlist before reaching max_shortlist_size, we fill the shortlist in order of rank.
  // E.g., first rank of word 0, first rank of word 1, ... second rank of word 0, ...
  int32_t maxLength = 0;
  for (Word word : srcBatch->data()) {
    int32_t sourceId = (int32_t)word.toWordIndex();
    srcVocab_->transcodeToShortlistInPlace((WordIndex*)&sourceId, 1);

    if (sourceId < numSourceIds_) { // if it's a valid source id
      const uint8_t* curShortlistIds = sourceToShortlistIds_ + idSize_ * sourceOffsets_[sourceId]; // start position for mapping
      int32_t length = sourceLengths_[sourceId]; // how many mappings are there
      curShortlistIt->first  = curShortlistIds;
      curShortlistIt->second = length;
      curShortlistIt++;

      if (length > maxLength)
        maxLength = length;
    }
  }

  // collect the actual shortlist mappings
  for (int32_t i = 0; i < maxLength && indexSet.size() < maxShortlistSize; i++) {
    for (int32_t j = 0; j < curShortlists.size() && indexSet.size() < maxShortlistSize; j++) {
      int32_t length = curShortlists[j].second;
      if (i < length) {
        const uint8_t* source_shortlist_ids_bytes = curShortlists[j].first;
        int32_t id = 0;
        if (use16bit_) {
          const uint16_t* source_shortlist_ids = reinterpret_cast<const uint16_t*>(source_shortlist_ids_bytes);
          id = (int32_t)source_shortlist_ids[i];
        }
        else {
          const int32_t* source_shortlist_ids = reinterpret_cast<const int32_t*>(source_shortlist_ids_bytes);
          id = source_shortlist_ids[i];
        }
        indexSet.insert(id);
      }
    }
  }

  // turn into vector and sort (selected indices)
  std::vector<WordIndex> indices;
  indices.reserve(indexSet.size());
  for(auto i : indexSet)
    indices.push_back((WordIndex)i);

  std::sort(indices.begin(), indices.end());
  return New<Shortlist>(indices);
}

Ptr<ShortlistGenerator> createShortlistGenerator(Ptr<Options> options,
                                                 Ptr<const Vocab> srcVocab,
                                                 Ptr<const Vocab> trgVocab,
                                                 const std::vector<int> &lshOpts,
                                                 size_t srcIdx,
                                                 size_t trgIdx,
                                                 bool shared) {
  if (lshOpts.size() == 2) {
    return New<LSHShortlistGenerator>(lshOpts[0], lshOpts[1]);
  }
  else {
    std::vector<std::string> vals = options->get<std::vector<std::string>>("shortlist");
    ABORT_IF(vals.empty(), "No path to shortlist given");
    std::string fname = vals[0];
    if(filesystem::Path(fname).extension().string() == ".bin") {
      return New<QuicksandShortlistGenerator>(options, srcVocab, trgVocab, srcIdx, trgIdx, shared);
    } else {
      return New<LexicalShortlistGenerator>(options, srcVocab, trgVocab, srcIdx, trgIdx, shared);
    }
  }
}

}  // namespace data
}  // namespace marian
<commit_msg>don't define BLAS_FOUND<commit_after>#include "data/shortlist.h"
#include "microsoft/shortlist/utils/ParameterTree.h"
#include "marian.h"

#if BLAS_FOUND
#include "3rd_party/faiss/IndexLSH.h"
#endif

namespace marian {
namespace data {

// cast current void pointer to T pointer and move forward by num elements
template <typename T>
const T* get(const void*& current, size_t num = 1) {
  const T* ptr = (const T*)current;
  current = (const T*)current + num;
  return ptr;
}

//////////////////////////////////////////////////////////////////////////////////////
Shortlist::Shortlist(const std::vector<WordIndex>& indices)
  : indices_(indices)
  , done_(false) {}

Shortlist::~Shortlist() {}

WordIndex Shortlist::reverseMap(int , int , int idx) const { return indices_[idx]; }

WordIndex Shortlist::tryForwardMap(int , int , WordIndex wIdx) const {
  auto first = std::lower_bound(indices_.begin(), indices_.end(), wIdx);
  if(first != indices_.end() && *first == wIdx)         // check if element not less than wIdx has been found and if equal to wIdx
    return (int)std::distance(indices_.begin(), first); // return coordinate if found
  else
    return npos;                                        // return npos if not found, @TODO: replace with std::optional once we switch to C++17?
}

void Shortlist::filter(Expr input, Expr weights, bool isLegacyUntransposedW, Expr b, Expr lemmaEt) {
  if (done_) {
    return;
  }

  auto forward = [this](Expr out, const std::vector<Expr>& ) {
    out->val()->set(indices_);
  };

  int k = (int) indices_.size();
  Shape kShape({k});
  indicesExpr_ = lambda({input, weights}, kShape, Type::uint32, forward);

  //std::cerr << "indicesExpr_=" << indicesExpr_->shape() << std::endl;
  createCachedTensors(weights, isLegacyUntransposedW, b, lemmaEt, k);
  done_ = true;
}

Expr Shortlist::getIndicesExpr() const {
  int k = indicesExpr_->shape()[0];
  Expr out = reshape(indicesExpr_, {1, 1, k});
  return out;
}

void Shortlist::createCachedTensors(Expr weights,
                          bool isLegacyUntransposedW,
                          Expr b,
                          Expr lemmaEt,
                          int k) {
  ABORT_IF(isLegacyUntransposedW, "Legacy untranspose W not yet tested");
  cachedShortWt_ = index_select(weights, isLegacyUntransposedW ? -1 : 0, indicesExpr_);
  cachedShortWt_ = reshape(cachedShortWt_, {1, 1, cachedShortWt_->shape()[0], cachedShortWt_->shape()[1]});

  if (b) {
    ABORT("Bias not yet tested");
    cachedShortb_ = index_select(b, -1, indicesExpr_);
    cachedShortb_ = reshape(cachedShortb_, {1, k, 1, cachedShortb_->shape()[1]}); // not tested
  }

  cachedShortLemmaEt_ = index_select(lemmaEt, -1, indicesExpr_);
  cachedShortLemmaEt_ = reshape(cachedShortLemmaEt_, {1, 1, cachedShortLemmaEt_->shape()[0], k});
}

///////////////////////////////////////////////////////////////////////////////////
Ptr<faiss::IndexLSH> LSHShortlist::index_;

LSHShortlist::LSHShortlist(int k, int nbits)
: Shortlist(std::vector<WordIndex>())
, k_(k), nbits_(nbits) {
  //std::cerr << "LSHShortlist" << std::endl;
  /*
  for (int i = 0; i < k_; ++i) {
    indices_.push_back(i);
  }
  */
}

//#define BLAS_FOUND 1

WordIndex LSHShortlist::reverseMap(int beamIdx, int batchIdx, int idx) const {
  //int currBeamSize = indicesExpr_->shape()[0];
  int currBatchSize = indicesExpr_->shape()[1];
  idx = (k_ * currBatchSize * beamIdx) + (k_ * batchIdx) + idx;
  assert(idx < indices_.size());
  return indices_[idx];
}

WordIndex LSHShortlist::tryForwardMap(int , int , WordIndex wIdx) const {
  auto first = std::lower_bound(indices_.begin(), indices_.end(), wIdx);
  bool found = first != indices_.end();
  if(found && *first == wIdx)         // check if element not less than wIdx has been found and if equal to wIdx
    return (int)std::distance(indices_.begin(), first); // return coordinate if found
  else
    return npos;                                        // return npos if not found, @TODO: replace with std::optional once we switch to C++17?
}

Expr LSHShortlist::getIndicesExpr() const {
  return indicesExpr_;
}

void LSHShortlist::filter(Expr input, Expr weights, bool isLegacyUntransposedW, Expr b, Expr lemmaEt) {
#if BLAS_FOUND
  ABORT_IF(input->graph()->getDeviceId().type == DeviceType::gpu,
           "LSH index (--output-approx-knn) currently not implemented for GPU");

  int currBeamSize = input->shape()[0];
  int batchSize = input->shape()[2];
  int numHypos = currBeamSize * batchSize;

  auto forward = [this, numHypos](Expr out, const std::vector<Expr>& inputs) {
    auto query  = inputs[0];
    auto values = inputs[1];
    int dim = values->shape()[-1];

    if(!index_) {
      //std::cerr << "build lsh index" << std::endl;
      LOG(info, "Building LSH index for vector dim {} and with hash size {} bits", dim, nbits_);
      index_.reset(new faiss::IndexLSH(dim, nbits_,
                                       /*rotate=*/dim != nbits_,
                                       /*train_thesholds*/false));
      int vRows = 32121; //47960; //values->shape().elements() / dim;
      index_->train(vRows, values->val()->data<float>());
      index_->add(  vRows, values->val()->data<float>());
    }

    int qRows = query->shape().elements() / dim;
    std::vector<float> distances(qRows * k_);
    std::vector<faiss::Index::idx_t> ids(qRows * k_);

    index_->search(qRows, query->val()->data<float>(), k_,
                   distances.data(), ids.data());

    indices_.clear();
    for(auto iter = ids.begin(); iter != ids.end(); ++iter) {
      faiss::Index::idx_t id = *iter;
      indices_.push_back((WordIndex)id);
    }

    for (size_t hypoIdx = 0; hypoIdx < numHypos; ++hypoIdx) {
      size_t startIdx = k_ * hypoIdx;
      size_t endIdx = startIdx + k_;
      std::sort(indices_.begin() + startIdx, indices_.begin() + endIdx);
    }
    out->val()->set(indices_);
  };

  Shape kShape({currBeamSize, batchSize, k_});
  indicesExpr_ = lambda({input, weights}, kShape, Type::uint32, forward);

  createCachedTensors(weights, isLegacyUntransposedW, b, lemmaEt, k_);

#else
  input; weights; isLegacyUntransposedW; b; lemmaEt;
  ABORT("LSH output layer requires a CPU BLAS library");
#endif
}

void LSHShortlist::createCachedTensors(Expr weights,
                          bool isLegacyUntransposedW,
                          Expr b,
                          Expr lemmaEt,
                          int k) {
  int currBeamSize = indicesExpr_->shape()[0];
  int batchSize = indicesExpr_->shape()[1];
  ABORT_IF(isLegacyUntransposedW, "Legacy untranspose W not yet tested");

  Expr indicesExprFlatten = reshape(indicesExpr_, {indicesExpr_->shape().elements()});

  cachedShortWt_ = index_select(weights, isLegacyUntransposedW ? -1 : 0, indicesExprFlatten);
  cachedShortWt_ = reshape(cachedShortWt_, {currBeamSize, batchSize, k, cachedShortWt_->shape()[1]});

  if (b) {
    ABORT("Bias not yet tested");
    cachedShortb_ = index_select(b, -1, indicesExprFlatten);
    cachedShortb_ = reshape(cachedShortb_, {currBeamSize, k, batchSize, cachedShortb_->shape()[1]}); // not tested
  }

  int dim = lemmaEt->shape()[0];
  cachedShortLemmaEt_ = index_select(lemmaEt, -1, indicesExprFlatten);
  cachedShortLemmaEt_ = reshape(cachedShortLemmaEt_, {dim, currBeamSize, batchSize, k});
  cachedShortLemmaEt_ = transpose(cachedShortLemmaEt_, {1, 2, 0, 3});
}

LSHShortlistGenerator::LSHShortlistGenerator(int k, int nbits)
  : k_(k), nbits_(nbits) {
  //std::cerr << "LSHShortlistGenerator" << std::endl;
}

Ptr<Shortlist> LSHShortlistGenerator::generate(Ptr<data::CorpusBatch> batch) const {
  return New<LSHShortlist>(k_, nbits_);
}

//////////////////////////////////////////////////////////////////////////////////////
QuicksandShortlistGenerator::QuicksandShortlistGenerator(Ptr<Options> options,
                                                         Ptr<const Vocab> srcVocab,
                                                         Ptr<const Vocab> trgVocab,
                                                         size_t srcIdx,
                                                         size_t /*trgIdx*/,
                                                         bool /*shared*/)
    : options_(options),
      srcVocab_(srcVocab),
      trgVocab_(trgVocab),
      srcIdx_(srcIdx) {
  std::vector<std::string> vals = options_->get<std::vector<std::string>>("shortlist");

  ABORT_IF(vals.empty(), "No path to filter path given");
  std::string fname = vals[0];

  auto firstNum   = vals.size() > 1 ? std::stoi(vals[1]) : 0;
  auto bestNum    = vals.size() > 2 ? std::stoi(vals[2]) : 0;
  float threshold = vals.size() > 3 ? std::stof(vals[3]) : 0;

  if(firstNum != 0 || bestNum != 0 || threshold != 0) {
    LOG(warn, "You have provided additional parameters for the Quicksand shortlist, but they are ignored.");
  }

  mmap_ = mio::mmap_source(fname); // memory-map the binary file once
  const void* current = mmap_.data(); // pointer iterator over binary file

  // compare magic number in binary file to make sure we are reading the right thing
  const int32_t MAGIC_NUMBER = 1234567890;
  int32_t header_magic_number = *get<int32_t>(current);
  ABORT_IF(header_magic_number != MAGIC_NUMBER, "Trying to mmap Quicksand shortlist but encountered wrong magic number");

  auto config = ::quicksand::ParameterTree::FromBinaryReader(current);
  use16bit_ = config->GetBoolReq("use_16_bit");

  LOG(info, "[data] Mapping Quicksand shortlist from {}", fname);

  idSize_ = sizeof(int32_t);
  if (use16bit_) {
    idSize_ = sizeof(uint16_t);
  }

  // mmap the binary shortlist pieces
  numDefaultIds_        = *get<int32_t>(current);
  defaultIds_           =  get<int32_t>(current, numDefaultIds_);
  numSourceIds_         = *get<int32_t>(current);
  sourceLengths_        =  get<int32_t>(current, numSourceIds_);
  sourceOffsets_        =  get<int32_t>(current, numSourceIds_);
  numShortlistIds_      = *get<int32_t>(current);
  sourceToShortlistIds_ =  get<uint8_t>(current, idSize_ * numShortlistIds_);

  // display parameters
  LOG(info,
      "[data] Quicksand shortlist has {} source ids, {} default ids and {} shortlist ids",
      numSourceIds_,
      numDefaultIds_,
      numShortlistIds_);
}

Ptr<Shortlist> QuicksandShortlistGenerator::generate(Ptr<data::CorpusBatch> batch) const {
  auto srcBatch = (*batch)[srcIdx_];
  auto maxShortlistSize = trgVocab_->size();

  std::unordered_set<int32_t> indexSet;
  for(int32_t i = 0; i < numDefaultIds_ && i < maxShortlistSize; ++i) {
    int32_t id = defaultIds_[i];
    indexSet.insert(id);
  }

  // State
  std::vector<std::pair<const uint8_t*, int32_t>> curShortlists(maxShortlistSize);
  auto curShortlistIt = curShortlists.begin();

  // Because we might fill up our shortlist before reaching max_shortlist_size, we fill the shortlist in order of rank.
  // E.g., first rank of word 0, first rank of word 1, ... second rank of word 0, ...
  int32_t maxLength = 0;
  for (Word word : srcBatch->data()) {
    int32_t sourceId = (int32_t)word.toWordIndex();
    srcVocab_->transcodeToShortlistInPlace((WordIndex*)&sourceId, 1);

    if (sourceId < numSourceIds_) { // if it's a valid source id
      const uint8_t* curShortlistIds = sourceToShortlistIds_ + idSize_ * sourceOffsets_[sourceId]; // start position for mapping
      int32_t length = sourceLengths_[sourceId]; // how many mappings are there
      curShortlistIt->first  = curShortlistIds;
      curShortlistIt->second = length;
      curShortlistIt++;

      if (length > maxLength)
        maxLength = length;
    }
  }

  // collect the actual shortlist mappings
  for (int32_t i = 0; i < maxLength && indexSet.size() < maxShortlistSize; i++) {
    for (int32_t j = 0; j < curShortlists.size() && indexSet.size() < maxShortlistSize; j++) {
      int32_t length = curShortlists[j].second;
      if (i < length) {
        const uint8_t* source_shortlist_ids_bytes = curShortlists[j].first;
        int32_t id = 0;
        if (use16bit_) {
          const uint16_t* source_shortlist_ids = reinterpret_cast<const uint16_t*>(source_shortlist_ids_bytes);
          id = (int32_t)source_shortlist_ids[i];
        }
        else {
          const int32_t* source_shortlist_ids = reinterpret_cast<const int32_t*>(source_shortlist_ids_bytes);
          id = source_shortlist_ids[i];
        }
        indexSet.insert(id);
      }
    }
  }

  // turn into vector and sort (selected indices)
  std::vector<WordIndex> indices;
  indices.reserve(indexSet.size());
  for(auto i : indexSet)
    indices.push_back((WordIndex)i);

  std::sort(indices.begin(), indices.end());
  return New<Shortlist>(indices);
}

Ptr<ShortlistGenerator> createShortlistGenerator(Ptr<Options> options,
                                                 Ptr<const Vocab> srcVocab,
                                                 Ptr<const Vocab> trgVocab,
                                                 const std::vector<int> &lshOpts,
                                                 size_t srcIdx,
                                                 size_t trgIdx,
                                                 bool shared) {
  if (lshOpts.size() == 2) {
    return New<LSHShortlistGenerator>(lshOpts[0], lshOpts[1]);
  }
  else {
    std::vector<std::string> vals = options->get<std::vector<std::string>>("shortlist");
    ABORT_IF(vals.empty(), "No path to shortlist given");
    std::string fname = vals[0];
    if(filesystem::Path(fname).extension().string() == ".bin") {
      return New<QuicksandShortlistGenerator>(options, srcVocab, trgVocab, srcIdx, trgIdx, shared);
    } else {
      return New<LexicalShortlistGenerator>(options, srcVocab, trgVocab, srcIdx, trgIdx, shared);
    }
  }
}

}  // namespace data
}  // namespace marian
<|endoftext|>


<commit_before>#include "LogServerWorkThread.h"

#include <glog/logging.h>

namespace sf1r
{

LogServerWorkThread::LogServerWorkThread()
    : workThread_(&LogServerWorkThread::run, this)
    , drumRequestQueue_(LogServerCfg::get()->getRpcRequestQueueSize())
{
}

LogServerWorkThread::~LogServerWorkThread()
{
}

void LogServerWorkThread::stop()
{
    workThread_.interrupt();
    workThread_.join();
}

void LogServerWorkThread::putUuidRequestData(const UUID2DocidList& uuid2DocidList)
{
    DrumRequestData drumReqData;

    drumReqData.uuid2DocidList.reset(new UUID2DocidList(uuid2DocidList));
    drumRequestQueue_.push(drumReqData);
}

void LogServerWorkThread::putSyncRequestData(const SynchronizeData& syncReqData)
{
    DrumRequestData drumReqData;

    drumReqData.syncReqData.reset(new SynchronizeData(syncReqData));
    drumRequestQueue_.push(drumReqData);
}

void LogServerWorkThread::run()
{
    try
    {
        DrumRequestData drumReqData;
        while (true)
        {
            // process requests
            drumRequestQueue_.pop(drumReqData);
            process(drumReqData);

            // terminate execution if interrupted
            boost::this_thread::interruption_point();
        }
    }
    catch (const std::exception& e)
    {
        // nothing
    }
}

void LogServerWorkThread::process(const DrumRequestData& drumReqData)
{
    if (drumReqData.uuid2DocidList)
    {
        process(*drumReqData.uuid2DocidList);
    }

    if (drumReqData.syncReqData)
    {
        process(*drumReqData.syncReqData);
    }
}

void LogServerWorkThread::process(const UUID2DocidList& uuid2DocidList)
{
    boost::lock_guard<boost::mutex> lock(LogServerStorage::get()->uuidDrumMutex());
    LogServerStorage::get()->uuidDrum()->Update(uuid2DocidList.uuid_, uuid2DocidList.docidList_);
}

void LogServerWorkThread::process(const SynchronizeData& syncReqData)
{
    {
        boost::lock_guard<boost::mutex> lock(LogServerStorage::get()->uuidDrumMutex());
        LOG(INFO) << "synchronizing drum for uuids (locked)";
        LogServerStorage::get()->uuidDrum()->Synchronize();
        LOG(INFO) << "finished synchronizing drum for uuids";
    }

    {
        boost::lock_guard<boost::mutex> lock(LogServerStorage::get()->docidDrumMutex());
        LOG(INFO) << "synchronizing drum for docids (locked)";
        LogServerStorage::get()->docidDrum()->Synchronize();
        LOG(INFO) << "finished synchronizing drum for docids";
    }
}

}
<commit_msg>release memory for request queue of LogServer<commit_after>#include "LogServerWorkThread.h"

#include <glog/logging.h>

namespace sf1r
{

LogServerWorkThread::LogServerWorkThread()
    : workThread_(&LogServerWorkThread::run, this)
    , drumRequestQueue_(LogServerCfg::get()->getRpcRequestQueueSize())
{
}

LogServerWorkThread::~LogServerWorkThread()
{
}

void LogServerWorkThread::stop()
{
    workThread_.interrupt();
    workThread_.join();
}

void LogServerWorkThread::putUuidRequestData(const UUID2DocidList& uuid2DocidList)
{
    DrumRequestData drumReqData;

    drumReqData.uuid2DocidList.reset(new UUID2DocidList(uuid2DocidList));
    drumRequestQueue_.push(drumReqData);
}

void LogServerWorkThread::putSyncRequestData(const SynchronizeData& syncReqData)
{
    DrumRequestData drumReqData;

    drumReqData.syncReqData.reset(new SynchronizeData(syncReqData));
    drumRequestQueue_.push(drumReqData);
}

void LogServerWorkThread::run()
{
    try
    {
        DrumRequestData drumReqData;
        while (true)
        {
            drumRequestQueue_.pop(drumReqData);
            process(drumReqData);

            if (drumRequestQueue_.empty())
            {
                drumRequestQueue_.clear();
            }

            // terminate execution if interrupted
            boost::this_thread::interruption_point();
        }
    }
    catch (const std::exception& e)
    {
        // nothing
    }
}

void LogServerWorkThread::process(const DrumRequestData& drumReqData)
{
    if (drumReqData.uuid2DocidList)
    {
        process(*drumReqData.uuid2DocidList);
    }

    if (drumReqData.syncReqData)
    {
        process(*drumReqData.syncReqData);
    }
}

void LogServerWorkThread::process(const UUID2DocidList& uuid2DocidList)
{
    boost::lock_guard<boost::mutex> lock(LogServerStorage::get()->uuidDrumMutex());
    LogServerStorage::get()->uuidDrum()->Update(uuid2DocidList.uuid_, uuid2DocidList.docidList_);
}

void LogServerWorkThread::process(const SynchronizeData& syncReqData)
{
    {
        boost::lock_guard<boost::mutex> lock(LogServerStorage::get()->uuidDrumMutex());
        LOG(INFO) << "synchronizing drum for uuids (locked)";
        LogServerStorage::get()->uuidDrum()->Synchronize();
        LOG(INFO) << "finished synchronizing drum for uuids";
    }

    {
        boost::lock_guard<boost::mutex> lock(LogServerStorage::get()->docidDrumMutex());
        LOG(INFO) << "synchronizing drum for docids (locked)";
        LogServerStorage::get()->docidDrum()->Synchronize();
        LOG(INFO) << "finished synchronizing drum for docids";
    }
}

}
<|endoftext|>


<commit_before>// Copyright 2017 Global Phasing Ltd.

// Writing cif::Document or its parts to std::ostream.

#ifndef GEMMI_TO_CIF_HPP_
#define GEMMI_TO_CIF_HPP_

#include <ostream>
#include "cifdoc.hpp"

namespace gemmi {
namespace cif {

enum class Style {
  Simple,
  NoBlankLines,
  PreferPairs,  // write single-row loops as pairs
  Pdbx,         // PreferPairs + put '#' (empty comments) between categories
  Indent35,     // start values in pairs from 35th column
  Aligned,      // columns in tables are left-aligned
};

// CIF files are read in binary mode. It makes difference only for text fields.
// If the text field with \r\n would be written as is in text mode on Windows
// \r would get duplicated. As a workaround, here we convert \r\n to \n.
// Hopefully \r that gets removed here is never meaningful.
inline void write_text_field(std::ostream& os, const std::string& value) {
  for (size_t pos = 0, end = 0; end != std::string::npos; pos = end + 1) {
    end = value.find("\r\n", pos);
    size_t len = (end == std::string::npos ? value.size() : end) - pos;
    os.write(value.c_str() + pos, len);
  }
}

inline void write_out_pair(std::ostream& os, const std::string& name,
                           const std::string& value, Style style) {
  os << name;
  if (is_text_field(value)) {
    os.put('\n');
    write_text_field(os, value);
  } else {
    if (name.size() + value.size() > 120)
      os.put('\n');
    else if ((style == Style::Indent35 || style == Style::Aligned) && name.size() < 34)
      os.write("                                  ", 34 - name.size());
    else
      os.put(' ');
    os << value;
  }
  os.put('\n');
}

inline void write_out_loop(std::ostream& os, const Loop& loop, Style style) {
  constexpr size_t max_padding = 30;  // if increased, adjust os.write() below
  if (loop.values.empty())
    return;
  if ((style == Style::PreferPairs || style == Style::Pdbx) &&
      loop.length() == 1) {
    for (size_t i = 0; i != loop.tags.size(); ++i)
      write_out_pair(os, loop.tags[i], loop.values[i], style);
    return;
  }
  // tags
  os << "loop_";
  for (const std::string& tag : loop.tags)
    os << '\n' << tag;
  // values
  size_t ncol = loop.tags.size();

  std::vector<size_t> col_width;
  if (style == Style::Aligned) {
    col_width.resize(ncol, 1);
    size_t col = 0;
    for (const std::string& val : loop.values) {
      col_width[col] = std::max(col_width[col], val.size());
      if (++col == ncol)
        col = 0;
    }
    for (size_t& w : col_width)
      w = std::min(w, max_padding);
  }

  size_t col = 0;
  for (const std::string& val : loop.values) {
    bool text_field = is_text_field(val);
    os.put(col == 0 || text_field ? '\n' : ' ');
    if (text_field)
      write_text_field(os, val);
    else
      os << val;
    if (col != ncol - 1) {
      if (!col_width.empty() && val.size() < col_width[col])
        os.write("                                  ", col_width[col] - val.size());
      ++col;
    } else {
      col = 0;
    }
  }
  os.put('\n');
}

inline void write_out_item(std::ostream& os, const Item& item, Style style) {
  switch (item.type) {
    case ItemType::Pair:
      write_out_pair(os, item.pair[0], item.pair[1], style);
      break;
    case ItemType::Loop:
      write_out_loop(os, item.loop, style);
      break;
    case ItemType::Frame:
      os << "save_" << item.frame.name << '\n';
      for (const Item& inner_item : item.frame.items)
        write_out_item(os, inner_item, style);
      os << "save_\n";
      break;
    case ItemType::Comment:
      os << item.pair[1] << '\n';
      break;
    case ItemType::Erased:
      break;
  }
}

inline bool should_be_separated_(const Item& a, const Item& b) {
  if (a.type == ItemType::Comment || b.type == ItemType::Comment)
    return false;
  if (a.type != ItemType::Pair || b.type != ItemType::Pair)
    return true;
  // check if we have mmcif-like tags from different categories
  auto adot = a.pair[0].find('.');
  if (adot == std::string::npos)
    return false;
  auto bdot = b.pair[0].find('.');
  return adot != bdot || a.pair[0].compare(0, adot, b.pair[0], 0, adot) != 0;
}

inline void write_cif_block_to_stream(std::ostream& os, const Block& block,
                                      Style style=Style::Simple) {
  os << "data_" << block.name << '\n';
  if (style == Style::Pdbx)
    os << "#\n";
  const Item* prev = nullptr;
  for (const Item& item : block.items)
    if (item.type != ItemType::Erased) {
      if (prev && style != Style::NoBlankLines &&
          should_be_separated_(*prev, item))
        os << (style == Style::Pdbx ? "#\n" : "\n");
      write_out_item(os, item, style);
      prev = &item;
    }
  if (style == Style::Pdbx)
    os << "#\n";
}

inline void write_cif_to_stream(std::ostream& os, const Document& doc,
                                Style style=Style::Simple) {
  bool first = true;
  for (const Block& block : doc.blocks) {
    if (!first)
      os.put('\n'); // extra blank line for readability
    write_cif_block_to_stream(os, block, style);
    first = false;
  }
}

} // namespace cif
} // namespace gemmi

#endif
<commit_msg>to_cif: when calculating alignment ignore text fields<commit_after>// Copyright 2017 Global Phasing Ltd.

// Writing cif::Document or its parts to std::ostream.

#ifndef GEMMI_TO_CIF_HPP_
#define GEMMI_TO_CIF_HPP_

#include <ostream>
#include "cifdoc.hpp"

namespace gemmi {
namespace cif {

enum class Style {
  Simple,
  NoBlankLines,
  PreferPairs,  // write single-row loops as pairs
  Pdbx,         // PreferPairs + put '#' (empty comments) between categories
  Indent35,     // start values in pairs from 35th column
  Aligned,      // columns in tables are left-aligned
};

// CIF files are read in binary mode. It makes difference only for text fields.
// If the text field with \r\n would be written as is in text mode on Windows
// \r would get duplicated. As a workaround, here we convert \r\n to \n.
// Hopefully \r that gets removed here is never meaningful.
inline void write_text_field(std::ostream& os, const std::string& value) {
  for (size_t pos = 0, end = 0; end != std::string::npos; pos = end + 1) {
    end = value.find("\r\n", pos);
    size_t len = (end == std::string::npos ? value.size() : end) - pos;
    os.write(value.c_str() + pos, len);
  }
}

inline void write_out_pair(std::ostream& os, const std::string& name,
                           const std::string& value, Style style) {
  os << name;
  if (is_text_field(value)) {
    os.put('\n');
    write_text_field(os, value);
  } else {
    if (name.size() + value.size() > 120)
      os.put('\n');
    else if ((style == Style::Indent35 || style == Style::Aligned) && name.size() < 34)
      os.write("                                  ", 34 - name.size());
    else
      os.put(' ');
    os << value;
  }
  os.put('\n');
}

inline void write_out_loop(std::ostream& os, const Loop& loop, Style style) {
  constexpr size_t max_padding = 30;  // if increased, adjust os.write() below
  if (loop.values.empty())
    return;
  if ((style == Style::PreferPairs || style == Style::Pdbx) &&
      loop.length() == 1) {
    for (size_t i = 0; i != loop.tags.size(); ++i)
      write_out_pair(os, loop.tags[i], loop.values[i], style);
    return;
  }
  // tags
  os << "loop_";
  for (const std::string& tag : loop.tags)
    os << '\n' << tag;
  // values
  size_t ncol = loop.tags.size();

  std::vector<size_t> col_width;
  if (style == Style::Aligned) {
    col_width.resize(ncol, 1);
    size_t col = 0;
    for (const std::string& val : loop.values) {
      if (!is_text_field(val))
        col_width[col] = std::max(col_width[col], val.size());
      if (++col == ncol)
        col = 0;
    }
    for (size_t& w : col_width)
      w = std::min(w, max_padding);
  }

  size_t col = 0;
  for (const std::string& val : loop.values) {
    bool text_field = is_text_field(val);
    os.put(col == 0 || text_field ? '\n' : ' ');
    if (text_field)
      write_text_field(os, val);
    else
      os << val;
    if (col != ncol - 1) {
      if (!col_width.empty() && val.size() < col_width[col])
        os.write("                                  ", col_width[col] - val.size());
      ++col;
    } else {
      col = 0;
    }
  }
  os.put('\n');
}

inline void write_out_item(std::ostream& os, const Item& item, Style style) {
  switch (item.type) {
    case ItemType::Pair:
      write_out_pair(os, item.pair[0], item.pair[1], style);
      break;
    case ItemType::Loop:
      write_out_loop(os, item.loop, style);
      break;
    case ItemType::Frame:
      os << "save_" << item.frame.name << '\n';
      for (const Item& inner_item : item.frame.items)
        write_out_item(os, inner_item, style);
      os << "save_\n";
      break;
    case ItemType::Comment:
      os << item.pair[1] << '\n';
      break;
    case ItemType::Erased:
      break;
  }
}

inline bool should_be_separated_(const Item& a, const Item& b) {
  if (a.type == ItemType::Comment || b.type == ItemType::Comment)
    return false;
  if (a.type != ItemType::Pair || b.type != ItemType::Pair)
    return true;
  // check if we have mmcif-like tags from different categories
  auto adot = a.pair[0].find('.');
  if (adot == std::string::npos)
    return false;
  auto bdot = b.pair[0].find('.');
  return adot != bdot || a.pair[0].compare(0, adot, b.pair[0], 0, adot) != 0;
}

inline void write_cif_block_to_stream(std::ostream& os, const Block& block,
                                      Style style=Style::Simple) {
  os << "data_" << block.name << '\n';
  if (style == Style::Pdbx)
    os << "#\n";
  const Item* prev = nullptr;
  for (const Item& item : block.items)
    if (item.type != ItemType::Erased) {
      if (prev && style != Style::NoBlankLines &&
          should_be_separated_(*prev, item))
        os << (style == Style::Pdbx ? "#\n" : "\n");
      write_out_item(os, item, style);
      prev = &item;
    }
  if (style == Style::Pdbx)
    os << "#\n";
}

inline void write_cif_to_stream(std::ostream& os, const Document& doc,
                                Style style=Style::Simple) {
  bool first = true;
  for (const Block& block : doc.blocks) {
    if (!first)
      os.put('\n'); // extra blank line for readability
    write_cif_block_to_stream(os, block, style);
    first = false;
  }
}

} // namespace cif
} // namespace gemmi

#endif
<|endoftext|>