<commit_before>#ifndef regex_impl_hh_INCLUDED
#define regex_impl_hh_INCLUDED

#include "unicode.hh"
#include "utf8.hh"
#include "utf8_iterator.hh"
#include "vector.hh"

namespace Kakoune
{

struct CompiledRegex
{
    enum Op : char
    {
        Match,
        Literal,
        LiteralIgnoreCase,
        AnyChar,
        Matcher,
        Jump,
        Split_PrioritizeParent,
        Split_PrioritizeChild,
        Save,
        LineStart,
        LineEnd,
        WordBoundary,
        NotWordBoundary,
        SubjectBegin,
        SubjectEnd,
        LookAhead,
        LookBehind,
        NegativeLookAhead,
        NegativeLookBehind,
    };

    using Offset = unsigned;
    static constexpr Offset search_prefix_size = 3 + 2 * sizeof(Offset);

    explicit operator bool() const { return not bytecode.empty(); }

    Vector<char> bytecode;
    Vector<std::function<bool (Codepoint)>> matchers;
    size_t save_count;
};

CompiledRegex compile_regex(StringView re);

template<typename Iterator>
struct ThreadedRegexVM
{
    ThreadedRegexVM(const CompiledRegex& program)
      : m_program{program} { kak_assert(m_program); }

    struct Thread
    {
        const char* inst;
        Vector<Iterator> saves = {};
    };

    enum class StepResult { Consumed, Matched, Failed };
    StepResult step(size_t thread_index)
    {
        const auto prog_start = m_program.bytecode.data();
        const auto prog_end = prog_start + m_program.bytecode.size();
        while (true)
        {
            auto& thread = m_threads[thread_index];
            const Codepoint cp = m_pos == m_end ? 0 : *m_pos;
            const CompiledRegex::Op op = (CompiledRegex::Op)*thread.inst++;
            switch (op)
            {
                case CompiledRegex::Literal:
                    if (utf8::read_codepoint(thread.inst, prog_end) == cp)
                        return StepResult::Consumed;
                    return StepResult::Failed;
                case CompiledRegex::LiteralIgnoreCase:
                    if (utf8::read_codepoint(thread.inst, prog_end) == to_lower(cp))
                        return StepResult::Consumed;
                    return StepResult::Failed;
                case CompiledRegex::AnyChar:
                    return StepResult::Consumed;
                case CompiledRegex::Jump:
                {
                    auto inst = prog_start + *reinterpret_cast<const CompiledRegex::Offset*>(thread.inst);
                    // if instruction is already going to be executed by another thread, drop this thread
                    if (std::find_if(m_threads.begin(), m_threads.end(),
                                     [inst](const Thread& t) { return t.inst == inst; }) != m_threads.end())
                        return StepResult::Failed;
                    thread.inst = inst;
                    break;
                }
                case CompiledRegex::Split_PrioritizeParent:
                {
                    add_thread(thread_index+1, *reinterpret_cast<const CompiledRegex::Offset*>(thread.inst), thread.saves);
                    // thread is invalidated now, as we mutated the m_thread vector
                    m_threads[thread_index].inst += sizeof(CompiledRegex::Offset);
                    break;
                }
                case CompiledRegex::Split_PrioritizeChild:
                {
                    add_thread(thread_index+1, thread.inst + sizeof(CompiledRegex::Offset) - prog_start, thread.saves);
                    // thread is invalidated now, as we mutated the m_thread vector
                    m_threads[thread_index].inst = prog_start + *reinterpret_cast<const CompiledRegex::Offset*>(m_threads[thread_index].inst);
                    break;
                }
                case CompiledRegex::Save:
                {
                    const char index = *thread.inst++;
                    thread.saves[index] = m_pos.base();
                    break;
                }
                case CompiledRegex::Matcher:
                {
                    const int matcher_id = *thread.inst++;
                    return m_program.matchers[matcher_id](*m_pos) ?
                        StepResult::Consumed : StepResult::Failed;
                }
                case CompiledRegex::LineStart:
                    if (not is_line_start())
                        return StepResult::Failed;
                    break;
                case CompiledRegex::LineEnd:
                    if (not is_line_end())
                        return StepResult::Failed;
                    break;
                case CompiledRegex::WordBoundary:
                    if (not is_word_boundary())
                        return StepResult::Failed;
                    break;
                case CompiledRegex::NotWordBoundary:
                    if (is_word_boundary())
                        return StepResult::Failed;
                    break;
                case CompiledRegex::SubjectBegin:
                    if (m_pos != m_begin)
                        return StepResult::Failed;
                    break;
                case CompiledRegex::SubjectEnd:
                    if (m_pos != m_end)
                        return StepResult::Failed;
                    break;
                case CompiledRegex::LookAhead:
                case CompiledRegex::NegativeLookAhead:
                {
                    int count = *thread.inst++;
                    for (auto it = m_pos; count and it != m_end; ++it, --count)
                        if (*it != utf8::read(thread.inst))
                            break;
                    if ((op == CompiledRegex::LookAhead and count != 0) or
                        (op == CompiledRegex::NegativeLookAhead and count == 0))
                        return StepResult::Failed;
                    thread.inst = utf8::advance(thread.inst, prog_end, CharCount{count - 1});
                    break;
                }
                case CompiledRegex::LookBehind:
                case CompiledRegex::NegativeLookBehind:
                {
                    int count = *thread.inst++;
                    for (auto it = m_pos-1; count and it >= m_begin; --it, --count)
                        if (*it != utf8::read(thread.inst))
                            break;
                    if ((op == CompiledRegex::LookBehind and count != 0) or
                        (op == CompiledRegex::NegativeLookBehind and count == 0))
                        return StepResult::Failed;
                    thread.inst = utf8::advance(thread.inst, prog_end, CharCount{count - 1});
                    break;
                }
                case CompiledRegex::Match:
                    thread.inst = nullptr;
                    return StepResult::Matched;
            }
        }
        return StepResult::Failed;
    }

    bool exec(Iterator begin, Iterator end, bool match = true, bool longest = false)
    {
        bool found_match = false;
        m_threads.clear();
        add_thread(0, match ? CompiledRegex::search_prefix_size : 0,
                   Vector<Iterator>(m_program.save_count, Iterator{}));

        m_begin = begin;
        m_end = end;

        for (m_pos = Utf8It{m_begin, m_begin, m_end}; m_pos != m_end; ++m_pos)
        {
            for (int i = 0; i < m_threads.size(); ++i)
            {
                const auto res = step(i);
                if (res == StepResult::Matched)
                {
                    if (match)
                        continue; // We are not at end, this is not a full match

                    m_captures = std::move(m_threads[i].saves);
                    found_match = true;
                    m_threads.resize(i); // remove this and lower priority threads
                    if (not longest)
                        return true;
                }
                else if (res == StepResult::Failed)
                    m_threads[i].inst = nullptr;
            }
            m_threads.erase(std::remove_if(m_threads.begin(), m_threads.end(),
                                           [](const Thread& t) { return t.inst == nullptr; }), m_threads.end());
            if (m_threads.empty())
                return found_match;
        }

        // Step remaining threads to see if they match without consuming anything else
        for (int i = 0; i < m_threads.size(); ++i)
        {
            if (step(i) == StepResult::Matched)
            {
                m_captures = std::move(m_threads[i].saves);
                found_match = true;
                m_threads.resize(i); // remove this and lower priority threads
                if (not longest)
                    return true;
            }
        }
        return found_match;
    }

    void add_thread(int index, CompiledRegex::Offset pos, Vector<Iterator> saves)
    {
        const char* inst = m_program.bytecode.data() + pos;
        if (std::find_if(m_threads.begin(), m_threads.end(),
                         [inst](const Thread& t) { return t.inst == inst; }) == m_threads.end())
            m_threads.insert(m_threads.begin() + index, {inst, std::move(saves)});
    }

    bool is_line_start() const
    {
        return m_pos == m_begin or *(m_pos-1) == '\n';
    }

    bool is_line_end() const
    {
        return m_pos == m_end or *m_pos == '\n';
    }

    bool is_word_boundary() const
    {
        return m_pos == m_begin or m_pos == m_end or
               is_word(*(m_pos-1)) != is_word(*m_pos);
    }

    const CompiledRegex& m_program;
    Vector<Thread> m_threads;

    using Utf8It = utf8::iterator<Iterator>;

    Iterator m_begin;
    Iterator m_end;
    Utf8It m_pos;

    Vector<Iterator> m_captures;
};

template<typename It>
bool regex_match(It begin, It end, const CompiledRegex& re)
{
    ThreadedRegexVM<It> vm{re};
    return vm.exec(begin, end, true, false);
}

template<typename It>
bool regex_match(It begin, It end, Vector<It>& captures, const CompiledRegex& re)
{
    ThreadedRegexVM<It> vm{re};
    if (vm.exec(begin, end, true, true))
    {
        captures = std::move(vm.m_captures);
        return true;
    }
    return false;
}

template<typename It>
bool regex_search(It begin, It end, const CompiledRegex& re)
{
    ThreadedRegexVM<It> vm{re};
    return vm.exec(begin, end, false, false);
}

template<typename It>
bool regex_search(It begin, It end, Vector<It>& captures, const CompiledRegex& re)
{
    ThreadedRegexVM<It> vm{re};
    if (vm.exec(begin, end, false, true))
    {
        captures = std::move(vm.m_captures);
        return true;
    }
    return false;
}

}

#endif // regex_impl_hh_INCLUDED
<commit_msg>Regex: Ensure we dont have a thread explosion in ThreadedRegexVM<commit_after>#ifndef regex_impl_hh_INCLUDED
#define regex_impl_hh_INCLUDED

#include "unicode.hh"
#include "utf8.hh"
#include "utf8_iterator.hh"
#include "vector.hh"

namespace Kakoune
{

struct CompiledRegex
{
    enum Op : char
    {
        Match,
        Literal,
        LiteralIgnoreCase,
        AnyChar,
        Matcher,
        Jump,
        Split_PrioritizeParent,
        Split_PrioritizeChild,
        Save,
        LineStart,
        LineEnd,
        WordBoundary,
        NotWordBoundary,
        SubjectBegin,
        SubjectEnd,
        LookAhead,
        LookBehind,
        NegativeLookAhead,
        NegativeLookBehind,
    };

    using Offset = unsigned;
    static constexpr Offset search_prefix_size = 3 + 2 * sizeof(Offset);

    explicit operator bool() const { return not bytecode.empty(); }

    Vector<char> bytecode;
    Vector<std::function<bool (Codepoint)>> matchers;
    size_t save_count;
};

CompiledRegex compile_regex(StringView re);

template<typename Iterator>
struct ThreadedRegexVM
{
    ThreadedRegexVM(const CompiledRegex& program)
      : m_program{program} { kak_assert(m_program); }

    struct Thread
    {
        const char* inst;
        Vector<Iterator> saves = {};
    };

    enum class StepResult { Consumed, Matched, Failed };
    StepResult step(size_t thread_index)
    {
        const auto prog_start = m_program.bytecode.data();
        const auto prog_end = prog_start + m_program.bytecode.size();
        while (true)
        {
            auto& thread = m_threads[thread_index];
            const Codepoint cp = m_pos == m_end ? 0 : *m_pos;
            const CompiledRegex::Op op = (CompiledRegex::Op)*thread.inst++;
            switch (op)
            {
                case CompiledRegex::Literal:
                    if (utf8::read_codepoint(thread.inst, prog_end) == cp)
                        return StepResult::Consumed;
                    return StepResult::Failed;
                case CompiledRegex::LiteralIgnoreCase:
                    if (utf8::read_codepoint(thread.inst, prog_end) == to_lower(cp))
                        return StepResult::Consumed;
                    return StepResult::Failed;
                case CompiledRegex::AnyChar:
                    return StepResult::Consumed;
                case CompiledRegex::Jump:
                {
                    auto inst = prog_start + *reinterpret_cast<const CompiledRegex::Offset*>(thread.inst);
                    // if instruction is already going to be executed by another thread, drop this thread
                    if (std::find_if(m_threads.begin(), m_threads.end(),
                                     [inst](const Thread& t) { return t.inst == inst; }) != m_threads.end())
                        return StepResult::Failed;
                    thread.inst = inst;
                    break;
                }
                case CompiledRegex::Split_PrioritizeParent:
                {
                    add_thread(thread_index+1, *reinterpret_cast<const CompiledRegex::Offset*>(thread.inst), thread.saves);
                    // thread is invalidated now, as we mutated the m_thread vector
                    m_threads[thread_index].inst += sizeof(CompiledRegex::Offset);
                    break;
                }
                case CompiledRegex::Split_PrioritizeChild:
                {
                    add_thread(thread_index+1, thread.inst + sizeof(CompiledRegex::Offset) - prog_start, thread.saves);
                    // thread is invalidated now, as we mutated the m_thread vector
                    m_threads[thread_index].inst = prog_start + *reinterpret_cast<const CompiledRegex::Offset*>(m_threads[thread_index].inst);
                    break;
                }
                case CompiledRegex::Save:
                {
                    const char index = *thread.inst++;
                    thread.saves[index] = m_pos.base();
                    break;
                }
                case CompiledRegex::Matcher:
                {
                    const int matcher_id = *thread.inst++;
                    return m_program.matchers[matcher_id](*m_pos) ?
                        StepResult::Consumed : StepResult::Failed;
                }
                case CompiledRegex::LineStart:
                    if (not is_line_start())
                        return StepResult::Failed;
                    break;
                case CompiledRegex::LineEnd:
                    if (not is_line_end())
                        return StepResult::Failed;
                    break;
                case CompiledRegex::WordBoundary:
                    if (not is_word_boundary())
                        return StepResult::Failed;
                    break;
                case CompiledRegex::NotWordBoundary:
                    if (is_word_boundary())
                        return StepResult::Failed;
                    break;
                case CompiledRegex::SubjectBegin:
                    if (m_pos != m_begin)
                        return StepResult::Failed;
                    break;
                case CompiledRegex::SubjectEnd:
                    if (m_pos != m_end)
                        return StepResult::Failed;
                    break;
                case CompiledRegex::LookAhead:
                case CompiledRegex::NegativeLookAhead:
                {
                    int count = *thread.inst++;
                    for (auto it = m_pos; count and it != m_end; ++it, --count)
                        if (*it != utf8::read(thread.inst))
                            break;
                    if ((op == CompiledRegex::LookAhead and count != 0) or
                        (op == CompiledRegex::NegativeLookAhead and count == 0))
                        return StepResult::Failed;
                    thread.inst = utf8::advance(thread.inst, prog_end, CharCount{count - 1});
                    break;
                }
                case CompiledRegex::LookBehind:
                case CompiledRegex::NegativeLookBehind:
                {
                    int count = *thread.inst++;
                    for (auto it = m_pos-1; count and it >= m_begin; --it, --count)
                        if (*it != utf8::read(thread.inst))
                            break;
                    if ((op == CompiledRegex::LookBehind and count != 0) or
                        (op == CompiledRegex::NegativeLookBehind and count == 0))
                        return StepResult::Failed;
                    thread.inst = utf8::advance(thread.inst, prog_end, CharCount{count - 1});
                    break;
                }
                case CompiledRegex::Match:
                    return StepResult::Matched;
            }
        }
        return StepResult::Failed;
    }

    bool exec(Iterator begin, Iterator end, bool match = true, bool longest = false)
    {
        bool found_match = false;
        m_threads.clear();
        add_thread(0, match ? CompiledRegex::search_prefix_size : 0,
                   Vector<Iterator>(m_program.save_count, Iterator{}));

        m_begin = begin;
        m_end = end;

        for (m_pos = Utf8It{m_begin, m_begin, m_end}; m_pos != m_end; ++m_pos)
        {
            for (int i = 0; i < m_threads.size(); )
            {
                const auto res = step(i);
                if (res == StepResult::Matched)
                {
                    if (match)
                    {
                        m_threads.erase(m_threads.begin() + i);
                        continue; // We are not at end, this is not a full match
                    }

                    m_captures = std::move(m_threads[i].saves);
                    found_match = true;
                    m_threads.resize(i); // remove this and lower priority threads
                    if (not longest)
                        return true;
                }
                else if (res == StepResult::Failed)
                    m_threads.erase(m_threads.begin() + i);
                else
                {
                    auto it = m_threads.begin() + i;
                    if (std::find_if(m_threads.begin(), it, [inst = it->inst](auto& t)
                                     { return t.inst == inst; }) != it)
                        m_threads.erase(it);
                    else
                        ++i;
                }
            }
            // we should never have more than one thread on the same instruction
            kak_assert(m_threads.size() <= m_program.bytecode.size());
            if (m_threads.empty())
                return found_match;
        }

        // Step remaining threads to see if they match without consuming anything else
        for (int i = 0; i < m_threads.size(); ++i)
        {
            if (step(i) == StepResult::Matched)
            {
                m_captures = std::move(m_threads[i].saves);
                found_match = true;
                m_threads.resize(i); // remove this and lower priority threads
                if (not longest)
                    return true;
            }
        }
        return found_match;
    }

    void add_thread(int index, CompiledRegex::Offset pos, Vector<Iterator> saves)
    {
        const char* inst = m_program.bytecode.data() + pos;
        if (std::find_if(m_threads.begin(), m_threads.end(),
                         [inst](const Thread& t) { return t.inst == inst; }) == m_threads.end())
            m_threads.insert(m_threads.begin() + index, {inst, std::move(saves)});
        kak_assert(m_threads.size() < m_program.bytecode.size());
    }

    bool is_line_start() const
    {
        return m_pos == m_begin or *(m_pos-1) == '\n';
    }

    bool is_line_end() const
    {
        return m_pos == m_end or *m_pos == '\n';
    }

    bool is_word_boundary() const
    {
        return m_pos == m_begin or m_pos == m_end or
               is_word(*(m_pos-1)) != is_word(*m_pos);
    }

    const CompiledRegex& m_program;
    Vector<Thread> m_threads;

    using Utf8It = utf8::iterator<Iterator>;

    Iterator m_begin;
    Iterator m_end;
    Utf8It m_pos;

    Vector<Iterator> m_captures;
};

template<typename It>
bool regex_match(It begin, It end, const CompiledRegex& re)
{
    ThreadedRegexVM<It> vm{re};
    return vm.exec(begin, end, true, false);
}

template<typename It>
bool regex_match(It begin, It end, Vector<It>& captures, const CompiledRegex& re)
{
    ThreadedRegexVM<It> vm{re};
    if (vm.exec(begin, end, true, true))
    {
        captures = std::move(vm.m_captures);
        return true;
    }
    return false;
}

template<typename It>
bool regex_search(It begin, It end, const CompiledRegex& re)
{
    ThreadedRegexVM<It> vm{re};
    return vm.exec(begin, end, false, false);
}

template<typename It>
bool regex_search(It begin, It end, Vector<It>& captures, const CompiledRegex& re)
{
    ThreadedRegexVM<It> vm{re};
    if (vm.exec(begin, end, false, true))
    {
        captures = std::move(vm.m_captures);
        return true;
    }
    return false;
}

}

#endif // regex_impl_hh_INCLUDED
<|endoftext|>


<commit_before>//=======================================================================
// Copyright Baptiste Wicht 2013-2016.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)
//=======================================================================

/*
 * This file contains the OS specific layer
 */

#include "acpica.hpp"

#include "kalloc.hpp"
#include "console.hpp"
#include "scheduler.hpp"

extern "C" {

void* AcpiOsAllocate(ACPI_SIZE size){
    return kalloc::k_malloc(size);
}

void AcpiOsFree(void* p){
    kalloc::k_free(p);
}

void AcpiOsPrintf(const char* format, ...){
    va_list va;
    va_start(va, format);

    printf(format, va);

    va_end(va);
}

void AcpiOsVprintf(const char* format, va_list va){
    printf(format, va);
}

ACPI_THREAD_ID AcpiOsGetThreadId(void){
    return scheduler::get_pid();
}

ACPI_PHYSICAL_ADDRESS AcpiOsGetRootPointer(){
	ACPI_PHYSICAL_ADDRESS  root_pointer;
	root_pointer = 0;
	AcpiFindRootPointer(&root_pointer);
	return root_pointer;
}

} //end of extern "C"
<commit_msg>Get ACPI root pointer<commit_after>//=======================================================================
// Copyright Baptiste Wicht 2013-2016.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
//  http://www.boost.org/LICENSE_1_0.txt)
//=======================================================================

/*
 * This file contains the OS specific layer
 */

#include "acpica.hpp"

#include "kalloc.hpp"
#include "console.hpp"
#include "scheduler.hpp"

extern "C" {

void* AcpiOsAllocate(ACPI_SIZE size){
    return kalloc::k_malloc(size);
}

void AcpiOsFree(void* p){
    kalloc::k_free(p);
}

void AcpiOsPrintf(const char* format, ...){
    va_list va;
    va_start(va, format);

    printf(format, va);

    va_end(va);
}

void AcpiOsVprintf(const char* format, va_list va){
    printf(format, va);
}

ACPI_THREAD_ID AcpiOsGetThreadId(void){
    return scheduler::get_pid();
}

ACPI_PHYSICAL_ADDRESS AcpiOsGetRootPointer(){
    ACPI_PHYSICAL_ADDRESS  root_pointer;
    root_pointer = 0;
    AcpiFindRootPointer(&root_pointer);
    return root_pointer;
}

} //end of extern "C"
<|endoftext|>


<commit_before>/***********************************************
 * Author: Jun Jiang - jiangjun4@sina.com
 * Create: 2017-09-05 16:04
 * Last modified : 2017-09-05 16:05
 * Filename	  : Layer.cpp
 * Description   : RNN network Layer
 **********************************************/
#include "rnn/Layer.h"

namespace abcdl{
namespace rnn{

void Layer::farward(const abcdl::algebra::Mat& train_seq_data,
					const abcdl::algebra::Mat& weight,
					const abcdl::algebra::Mat& pre_weight,
					const abcdl::algebra::Mat& act_weight,
					abcdl::algebra::Mat& state,
					abcdl::algebra::Mat& activation){

	size_t seq_rows = train_seq_data.rows();
	size_t seq_cols = train_seq_data.cols();

	state.reset(0, seq_rows, _hidden_dim);
	activation.reset(0, seq_rows, seq_cols);

	abcdl::algebra::Mat s_t;
	for(size_t t = 0; t != seq_rows; t++){
		//s[t] = tanh(U*x[t] + W*s[t-1])
		//o[t] = softmax(V* s[t])
		if(t > 0){
			s_t = (helper.dot(weight, train_seq_data.get_row(t).Ts()) + helper.dot(pre_weight, state.get_row(t - 1).Ts())).tanh().Ts();
		}else{
			s_t = helper.dot(weight, train_seq_data.get_row(t).Ts()).tanh().Ts();
		}
		state.set_row(t, s_t);

		activation.set_row(t, helper.dot(act_weight, s_t).softmax().Ts())
	}
}

void Layer::backward(const abcdl::algebra::Mat& train_seq_data,
					 const abcdl::algebra::Mat& train_seq_label,
					 const abcdl::algebra::Mat& weight,
					 const abcdl::algebra::Mat& pre_weight,
					 const abcdl::algebra::Mat& act_weight,
					 abcdl::algebra::Mat& derivate_weight,
					 abcdl::algebra::Mat& derivate_pre_weight,
					 abcdl::algebra::Mat& derivate_act_weight){

    auto now = []{return std::chrono::system_clock::now();};
    auto start_time = now();

	farward(train_seq_data, weight, pre_weight, act_weight, state, derivate_output);
	derivate_output.subtract(train_seq_label);

	size_t seq_size = train_seq_data.rows();

	for(sizt_t t = seq_size - 1; t >= 0 ; t--){
		auto derivate_output_tt = derivate_output.get_row(t).Ts();
        //update derivate_act_weight
		dervate_act_weight += derivate_output_tt.outer(state.get_row(t).Ts());

        //calc derivate_t
		auto derivate_t = helper.dot(act_weight.Ts(), derivate_output_tt) * state.get_row(t).Ts().sigmoid_derivative();

		//back_propagation steps
		for(size_t step = 0; step < _bptt_truncate && (int)step <= t; step++){
			int bptt_step = t - step;

            if(debug){
    			printf("Backpropagation step t=%d bptt step=%d\n", t, bptt_step);
            }

			abcdl::algebra::Mat derivate_state_t;
			if(bptt_step > 0){
				state.get_row(&derivate_state_t, bptt_step -1);
			}else{
				derivate_state_t.reset(0, state.get_cols(), 1);
			}

            //update derivate_pre_weight
			derivate_pre_weight += helper.outer(derivate_t, derivate_state_t.Ts());

            //update derivate_weight
			train_seq_data.get_row_data(bptt_step, train_data_t);
            derivate_weight.clone(derivate_weight_t);

			derivate_weight_t.dot(train_data_t.transpose());
			derivate_weight_t.add(derivate_t);

            size_t idx = train_data_t.argmax(0, 1);
            derivate_weight.get_col_data(idx, derivate_weight_t_c);
            derivate_weight_t_c.add(derivate_weight_t);
            derivate_weight.set_col_data(idx, derivate_weight_t_c);

			//update delta
			if(bptt_step > 0){
				derivate_state_t.multiply(derivate_state_t);
				derivate_state_t.multiply(-1);
				derivate_state_t.add(1);

				pre_weight.clone(derivate_pre_weight_t);
				derivate_pre_weight_t.transpose().dot(derivate_t);
				derivate_pre_weight_t.multiply(derivate_state_t);
				derivate_pre_weight_t.clone(derivate_t);
			}
		}
	}

	delete derivate_output;
	delete derivate_output_t;
    delete derivate_weight_t;
    delete derivate_weight_t_c;
	delete derivate_t;
	delete derivate_pre_weight_t;
	delete derivate_state_t;
	delete train_data_t;

    auto end_time = now();
    printf("thread[%lu][%ld-%ld][%ld].\n", std::this_thread::get_id(), start_time, end_time, std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count());
}

}//namespace rnn
}//namespace abcdl
<commit_msg>initialize<commit_after>/***********************************************
 * Author: Jun Jiang - jiangjun4@sina.com
 * Create: 2017-09-05 16:04
 * Last modified : 2017-09-06 14:42
 * Filename	  : Layer.cpp
 * Description   : RNN network Layer
 **********************************************/
#include "rnn/Layer.h"

namespace abcdl{
namespace rnn{

void Layer::farward(const abcdl::algebra::Mat& train_seq_data,
					const abcdl::algebra::Mat& weight,
					const abcdl::algebra::Mat& pre_weight,
					const abcdl::algebra::Mat& act_weight,
					abcdl::algebra::Mat& state,
					abcdl::algebra::Mat& activation){

	size_t seq_rows = train_seq_data.rows();
	size_t seq_cols = train_seq_data.cols();

	state.reset(0, seq_rows, _hidden_dim);
	activation.reset(0, seq_rows, seq_cols);

	abcdl::algebra::Mat s_t;
	for(size_t t = 0; t != seq_rows; t++){
		//s[t] = tanh(U*x[t] + W*s[t-1])
		//o[t] = softmax(V* s[t])
		if(t > 0){
			s_t = (helper.dot(weight, train_seq_data.get_row(t).Ts()) + helper.dot(pre_weight, state.get_row(t - 1).Ts())).tanh().Ts();
		}else{
			s_t = helper.dot(weight, train_seq_data.get_row(t).Ts()).tanh().Ts();
		}
		state.set_row(t, s_t);

		activation.set_row(t, helper.dot(act_weight, s_t).softmax().Ts())
	}
}

void Layer::backward(const abcdl::algebra::Mat& train_seq_data,
					 const abcdl::algebra::Mat& train_seq_label,
					 const abcdl::algebra::Mat& weight,
					 const abcdl::algebra::Mat& pre_weight,
					 const abcdl::algebra::Mat& act_weight,
					 abcdl::algebra::Mat& derivate_weight,
					 abcdl::algebra::Mat& derivate_pre_weight,
					 abcdl::algebra::Mat& derivate_act_weight){

    auto now = []{return std::chrono::system_clock::now();};
    auto start_time = now();

	farward(train_seq_data, weight, pre_weight, act_weight, state, derivate_output);
	derivate_output.subtract(train_seq_label);

	size_t seq_size = train_seq_data.rows();

	for(sizt_t t = seq_size - 1; t >= 0 ; t--){
		auto derivate_output_tt = derivate_output.get_row(t).Ts();
        //update derivate_act_weight
		dervate_act_weight += derivate_output_tt.outer(state.get_row(t).Ts());

        //calc derivate_t
		auto derivate_t = helper.dot(act_weight.Ts(), derivate_output_tt) * state.get_row(t).Ts().sigmoid_derivative();

		//back_propagation steps
		for(size_t step = 0; step < _bptt_truncate && (int)step <= t; step++){
			int bptt_step = t - step;

            if(debug){
    			printf("Backpropagation step t=%d bptt step=%d\n", t, bptt_step);
            }

			abcdl::algebra::Mat derivate_state_t;
			if(bptt_step > 0){
				state.get_row(&derivate_state_t, bptt_step -1);
			}else{
				derivate_state_t.reset(0, state.get_cols(), 1);
			}

            //update derivate_pre_weight
			derivate_pre_weight += helper.outer(derivate_t, derivate_state_t.Ts());

            //update derivate_weight
            auto derivate_weight_t = helper.dot(derivate_weight, train_seq_data.get_row(bptt_step).Ts()) + derivate_t;

            size_t idx = train_data_t.argmax(0, 1);
            derivate_weight.get_col_data(idx, derivate_weight_t_c);
            derivate_weight_t_c.add(derivate_weight_t);
            derivate_weight.set_col_data(idx, derivate_weight_t_c);

			//update delta
			if(bptt_step > 0){
				derivate_state_t.multiply(derivate_state_t);
				derivate_state_t.multiply(-1);
				derivate_state_t.add(1);

				pre_weight.clone(derivate_pre_weight_t);
				derivate_pre_weight_t.transpose().dot(derivate_t);
				derivate_pre_weight_t.multiply(derivate_state_t);
				derivate_pre_weight_t.clone(derivate_t);
			}
		}
	}

	delete derivate_output;
	delete derivate_output_t;
    delete derivate_weight_t;
    delete derivate_weight_t_c;
	delete derivate_t;
	delete derivate_pre_weight_t;
	delete derivate_state_t;
	delete train_data_t;

    auto end_time = now();
    printf("thread[%lu][%ld-%ld][%ld].\n", std::this_thread::get_id(), start_time, end_time, std::chrono::duration_cast<std::chrono::milliseconds>(end_time - start_time).count());
}

}//namespace rnn
}//namespace abcdl
<|endoftext|>


<commit_before>/*  -*- mode: c++; coding: utf-8; c-file-style: "stroustrup"; -*-

    Contributors: Asier Aguirre

    All rights reserved. Use of this source code is governed by a
    BSD-style license that can be found in the LICENSE.txt file.
*/

#include "widget_layout.h"
#include "input.h"
#include "nanovg.h"
#include "context.h"
#include "type_widget.h"
#include <cmath>

using namespace std;
using namespace webui;

namespace {

    TypeWidget widgetLayoutHorType = {
        Identifier::LayoutHor, sizeof(WidgetLayout), { }
    };

    TypeWidget widgetLayoutVerType = {
        Identifier::LayoutVer, sizeof(WidgetLayout), {
            { Identifier::margin,         PROP(WidgetLayout, margin,     Float,        4, 0, 0) },
        }
    };

}

namespace webui {

    WidgetLayout::WidgetLayout(Widget* parent, int coord):
        Widget(parent), margin(0), scrolling(false), coord(coord), positionTarget(1e10f), dragDrop(nullptr) {
        typeWidget = coord ? &widgetLayoutVerType : &widgetLayoutHorType;
    }

    TypeWidget& WidgetLayout::getTypeHor() {
        return widgetLayoutHorType;
    }

    TypeWidget& WidgetLayout::getTypeVer() {
        return widgetLayoutVerType;
    }

    void WidgetLayout::render(int alphaMult) {
        alphaMult = renderBase(alphaMult);
        if (alphaMult) {
            // render only visible widgets taking into account they are sorted (use renderVisibilityBox intersection)
            Box4f boxVisibilitySave(Context::renderVisibilityBox);
            Context::renderVisibilityBox.intersect(box);

            float maxCoord(Context::renderVisibilityBox.pos[coord] + Context::renderVisibilityBox.size[coord]);
            for (auto* child: children) {
                if (child->box.pos[coord] >= maxCoord) break;
                child->render(alphaMult);
            }
            if (dragDrop) {
                alphaMult = Context::render.multAlpha(alphaMult, alpha);
                dragDrop->translate(Input::cursor - Input::cursorLeftPress);
                dragDrop->render(alphaMult >> 1);
                dragDrop->translate(Input::cursorLeftPress - Input::cursor);
            }

            Context::renderVisibilityBox = boxVisibilitySave;
        }
    }

    bool WidgetLayout::input() {
        // scroll content
        if (scrollable && Input::keyboardAction && (Input::action == GLFW_PRESS || Input::action == GLFW_REPEAT)) {
            /**/ if (Input::keyButton == GLFW_KEY_HOME)      { positionTarget  = margin; return true; }
            else if (Input::keyButton == GLFW_KEY_END)       { positionTarget  = -1e5f;  return true; }
            else if (Input::keyButton == GLFW_KEY_PAGE_UP)   { positionTarget += 150;    return true; }
            else if (Input::keyButton == GLFW_KEY_PAGE_DOWN) { positionTarget -= 150;    return true; }
            else if (Input::keyButton == GLFW_KEY_UP)        { positionTarget +=  50;    return true; }
            else if (Input::keyButton == GLFW_KEY_DOWN)      { positionTarget -=  50;    return true; }
        }
        if (scrollable && Context::cursor == Cursor::Hand && Input::mouseButtonAction &&
            Input::keyButton == GLFW_MOUSE_BUTTON_LEFT && Input::action == GLFW_PRESS) {
            // this widge takes care of the event and does not propagate upwards
            Input::mouseButtonAction = false;
            Input::mouseButtonWidget = this;
            Input::cursorLeftPress[coord] -= positionTarget;
            scrolling = true;
        } else if (scrolling) {
            if (Input::keyButton == GLFW_MOUSE_BUTTON_LEFT && Input::action == GLFW_RELEASE) {
                scrolling = false;
            } else {
                positionTarget = Input::cursor[coord] - Input::cursorLeftPress[coord];
                return true;
            }
        } else if (scrollable && Input::scroll[coord] != 0.0f) {
            positionTarget += Input::scroll[coord] * 50;
            return true;
        }

        // drag & drop
        if (Input::mouseButtonWidget && Input::cursorLeftPress.manhatan(Input::cursor) > 16) {
            if (draggable && !dragDrop) {
                // check if clicked in one of layout widgets
                for (size_t idx = 0; idx < children.size(); idx++)
                    if (children[idx] == Input::mouseButtonWidget) {
                        dragDrop = Input::mouseButtonWidget;
                        break;
                    }
            }
            if (Input::keyButton == GLFW_MOUSE_BUTTON_LEFT &&
                Input::action == GLFW_RELEASE && dragDrop) {
                dragDrop = Input::mouseButtonWidget = nullptr;
                return true;
            }
            return true;
        }
        return false;
    }

    bool WidgetLayout::layout(const Box4f& boxAvail) {
        bool stable(true);
        box = boxAvail;
        if (visible && box.size[coord] >= 0) {
            // margin (only once)
            if (positionTarget == 1e10f) position = positionTarget = margin;
            int coord2(coord ^ 1);
            // drag & drop
            float prevCoord(0);
            if (dragDrop) {
                // remove from list
                for (size_t idx = 0; idx < children.size(); idx++)
                    if (children[idx] == dragDrop)
                        children.erase(children.begin() + idx);
                // find position
                float midCoord(dragDrop->box.pos[coord] + (dragDrop->box.size[coord] * 0.5f) + Input::cursor[coord] - Input::cursorLeftPress[coord]);
                float last(numeric_limits<float>::min());
                size_t pos(children.size());
                for (size_t idx = 0; idx < children.size(); idx++) {
                    auto* child(children[idx]);
                    float mid(child->box.pos[coord] + (child->box.size[coord] * 0.5f));
                    if (midCoord >= last && midCoord < mid) {
                        pos = idx;
                        break;
                    }
                    last = mid;
                }
                // add it in correct position
                children.insert(children.begin() + pos, dragDrop);
                prevCoord = dragDrop->box.pos[coord];
            }

            // use linear arrangement util
            LinearArrangement<float>::Elem elems[children.size() + 1];
            LinearArrangement<float> la(elems, boxAvail.pos[coord] + position);
            for (auto child: children)
                if (child->isVisible())
                    la.add(child->box.size[coord], child->size[coord].get(boxAvail.size[coord]), child->size[coord].relative);
                else
                    la.add(child->box.size[coord], 0, true);
            stable = la.calculate(boxAvail.size[coord]);

            float lastCoord(la.get(0));
            for (size_t idx = 0; idx < children.size(); idx++) {
                auto* child(children[idx]);
                Box4f b;
                float newCoord = la.get(idx + 1);
                // calculate children box
                b.pos[coord]   = lastCoord;
                b.pos[coord2]  = box.pos[coord2];
                b.size[coord]  = newCoord - lastCoord;
                b.size[coord2] = child->size[coord2].get(box.size[coord2]);
                lastCoord      = newCoord;

                // recurse layout
                stable &= child->layout(b);
            }
            // adaptative size
            float requiredSize(elems[children.size()].posTarget - elems[0].posTarget);
            if (size[coord].adapt) {
                size[coord].size = requiredSize; // total space required
                scrollable = 0;
            } else {
                scrollable = box.size[coord] + 1.1f < requiredSize;
                // converge position
                float diff(0);
                if (positionTarget > margin) diff = margin - positionTarget;
                else if (positionTarget < margin && positionTarget < box.size[coord] - requiredSize - margin)
                    diff = box.size[coord] - requiredSize - margin - positionTarget;
                if (scrolling) {
                    // when scrolling with the mouse, don't separate content too much from limits
                    const float tension(0.7f);
                    float diffTension(diff > 0 ? -powf(diff, tension) : powf(-diff, tension));
                    position = positionTarget + diff + diffTension;
                } else {
                    // scrolling with mouse wheel or converging to position
                    positionTarget += diff;
                    stable &= ctx.getCloser(position, positionTarget);
                }
            }
            if (size[coord2].adapt) {
                if (children.empty())
                    size[coord2].size = 0;
                else
                    size[coord2].size = children[0]->box.size[coord2]; // TODO: take max of all children?
                if (size[coord2].size != box.size[coord2]) stable = false;
            }

            // drag & drop
            if (dragDrop)
                Input::cursorLeftPress[coord] += dragDrop->box.pos[coord] - prevCoord;
        }
        return animeAlpha() && stable;
    }

}
<commit_msg>Reverse wheel actions.<commit_after>/*  -*- mode: c++; coding: utf-8; c-file-style: "stroustrup"; -*-

    Contributors: Asier Aguirre

    All rights reserved. Use of this source code is governed by a
    BSD-style license that can be found in the LICENSE.txt file.
*/

#include "widget_layout.h"
#include "input.h"
#include "nanovg.h"
#include "context.h"
#include "type_widget.h"
#include <cmath>

using namespace std;
using namespace webui;

namespace {

    TypeWidget widgetLayoutHorType = {
        Identifier::LayoutHor, sizeof(WidgetLayout), { }
    };

    TypeWidget widgetLayoutVerType = {
        Identifier::LayoutVer, sizeof(WidgetLayout), {
            { Identifier::margin,         PROP(WidgetLayout, margin,     Float,        4, 0, 0) },
        }
    };

}

namespace webui {

    WidgetLayout::WidgetLayout(Widget* parent, int coord):
        Widget(parent), margin(0), scrolling(false), coord(coord), positionTarget(1e10f), dragDrop(nullptr) {
        typeWidget = coord ? &widgetLayoutVerType : &widgetLayoutHorType;
    }

    TypeWidget& WidgetLayout::getTypeHor() {
        return widgetLayoutHorType;
    }

    TypeWidget& WidgetLayout::getTypeVer() {
        return widgetLayoutVerType;
    }

    void WidgetLayout::render(int alphaMult) {
        alphaMult = renderBase(alphaMult);
        if (alphaMult) {
            // render only visible widgets taking into account they are sorted (use renderVisibilityBox intersection)
            Box4f boxVisibilitySave(Context::renderVisibilityBox);
            Context::renderVisibilityBox.intersect(box);

            float maxCoord(Context::renderVisibilityBox.pos[coord] + Context::renderVisibilityBox.size[coord]);
            for (auto* child: children) {
                if (child->box.pos[coord] >= maxCoord) break;
                child->render(alphaMult);
            }
            if (dragDrop) {
                alphaMult = Context::render.multAlpha(alphaMult, alpha);
                dragDrop->translate(Input::cursor - Input::cursorLeftPress);
                dragDrop->render(alphaMult >> 1);
                dragDrop->translate(Input::cursorLeftPress - Input::cursor);
            }

            Context::renderVisibilityBox = boxVisibilitySave;
        }
    }

    bool WidgetLayout::input() {
        // scroll content
        if (scrollable && Input::keyboardAction && (Input::action == GLFW_PRESS || Input::action == GLFW_REPEAT)) {
            /**/ if (Input::keyButton == GLFW_KEY_HOME)      { positionTarget  = margin; return true; }
            else if (Input::keyButton == GLFW_KEY_END)       { positionTarget  = -1e5f;  return true; }
            else if (Input::keyButton == GLFW_KEY_PAGE_UP)   { positionTarget += 150;    return true; }
            else if (Input::keyButton == GLFW_KEY_PAGE_DOWN) { positionTarget -= 150;    return true; }
            else if (Input::keyButton == GLFW_KEY_UP)        { positionTarget +=  50;    return true; }
            else if (Input::keyButton == GLFW_KEY_DOWN)      { positionTarget -=  50;    return true; }
        }
        if (scrollable && Context::cursor == Cursor::Hand && Input::mouseButtonAction &&
            Input::keyButton == GLFW_MOUSE_BUTTON_LEFT && Input::action == GLFW_PRESS) {
            // this widge takes care of the event and does not propagate upwards
            Input::mouseButtonAction = false;
            Input::mouseButtonWidget = this;
            Input::cursorLeftPress[coord] -= positionTarget;
            scrolling = true;
        } else if (scrolling) {
            if (Input::keyButton == GLFW_MOUSE_BUTTON_LEFT && Input::action == GLFW_RELEASE) {
                scrolling = false;
            } else {
                positionTarget = Input::cursor[coord] - Input::cursorLeftPress[coord];
                return true;
            }
        } else if (scrollable && Input::scroll[coord] != 0.0f) {
            positionTarget -= Input::scroll[coord] * 50;
            return true;
        }

        // drag & drop
        if (Input::mouseButtonWidget && Input::cursorLeftPress.manhatan(Input::cursor) > 16) {
            if (draggable && !dragDrop) {
                // check if clicked in one of layout widgets
                for (size_t idx = 0; idx < children.size(); idx++)
                    if (children[idx] == Input::mouseButtonWidget) {
                        dragDrop = Input::mouseButtonWidget;
                        break;
                    }
            }
            if (Input::keyButton == GLFW_MOUSE_BUTTON_LEFT &&
                Input::action == GLFW_RELEASE && dragDrop) {
                dragDrop = Input::mouseButtonWidget = nullptr;
                return true;
            }
            return true;
        }
        return false;
    }

    bool WidgetLayout::layout(const Box4f& boxAvail) {
        bool stable(true);
        box = boxAvail;
        if (visible && box.size[coord] >= 0) {
            // margin (only once)
            if (positionTarget == 1e10f) position = positionTarget = margin;
            int coord2(coord ^ 1);
            // drag & drop
            float prevCoord(0);
            if (dragDrop) {
                // remove from list
                for (size_t idx = 0; idx < children.size(); idx++)
                    if (children[idx] == dragDrop)
                        children.erase(children.begin() + idx);
                // find position
                float midCoord(dragDrop->box.pos[coord] + (dragDrop->box.size[coord] * 0.5f) + Input::cursor[coord] - Input::cursorLeftPress[coord]);
                float last(numeric_limits<float>::min());
                size_t pos(children.size());
                for (size_t idx = 0; idx < children.size(); idx++) {
                    auto* child(children[idx]);
                    float mid(child->box.pos[coord] + (child->box.size[coord] * 0.5f));
                    if (midCoord >= last && midCoord < mid) {
                        pos = idx;
                        break;
                    }
                    last = mid;
                }
                // add it in correct position
                children.insert(children.begin() + pos, dragDrop);
                prevCoord = dragDrop->box.pos[coord];
            }

            // use linear arrangement util
            LinearArrangement<float>::Elem elems[children.size() + 1];
            LinearArrangement<float> la(elems, boxAvail.pos[coord] + position);
            for (auto child: children)
                if (child->isVisible())
                    la.add(child->box.size[coord], child->size[coord].get(boxAvail.size[coord]), child->size[coord].relative);
                else
                    la.add(child->box.size[coord], 0, true);
            stable = la.calculate(boxAvail.size[coord]);

            float lastCoord(la.get(0));
            for (size_t idx = 0; idx < children.size(); idx++) {
                auto* child(children[idx]);
                Box4f b;
                float newCoord = la.get(idx + 1);
                // calculate children box
                b.pos[coord]   = lastCoord;
                b.pos[coord2]  = box.pos[coord2];
                b.size[coord]  = newCoord - lastCoord;
                b.size[coord2] = child->size[coord2].get(box.size[coord2]);
                lastCoord      = newCoord;

                // recurse layout
                stable &= child->layout(b);
            }
            // adaptative size
            float requiredSize(elems[children.size()].posTarget - elems[0].posTarget);
            if (size[coord].adapt) {
                size[coord].size = requiredSize; // total space required
                scrollable = 0;
            } else {
                scrollable = box.size[coord] + 1.1f < requiredSize;
                // converge position
                float diff(0);
                if (positionTarget > margin) diff = margin - positionTarget;
                else if (positionTarget < margin && positionTarget < box.size[coord] - requiredSize - margin)
                    diff = box.size[coord] - requiredSize - margin - positionTarget;
                if (scrolling) {
                    // when scrolling with the mouse, don't separate content too much from limits
                    const float tension(0.7f);
                    float diffTension(diff > 0 ? -powf(diff, tension) : powf(-diff, tension));
                    position = positionTarget + diff + diffTension;
                } else {
                    // scrolling with mouse wheel or converging to position
                    positionTarget += diff;
                    stable &= ctx.getCloser(position, positionTarget);
                }
            }
            if (size[coord2].adapt) {
                if (children.empty())
                    size[coord2].size = 0;
                else
                    size[coord2].size = children[0]->box.size[coord2]; // TODO: take max of all children?
                if (size[coord2].size != box.size[coord2]) stable = false;
            }

            // drag & drop
            if (dragDrop)
                Input::cursorLeftPress[coord] += dragDrop->box.pos[coord] - prevCoord;
        }
        return animeAlpha() && stable;
    }

}
<|endoftext|>


<commit_before>#include "node_impl.hpp"

#include <future>

namespace obelisk {

using namespace bc;
using std::placeholders::_1;
using std::placeholders::_2;
using std::placeholders::_3;
using std::placeholders::_4;

void log_to_file(std::ofstream& file, log_level level,
    const std::string& domain, const std::string& body)
{
    if (body.empty())
        return;
    file << level_repr(level);
    if (!domain.empty())
        file << " [" << domain << "]";
    file << ": " << body << std::endl;
}
void log_to_both(std::ostream& device, std::ofstream& file, log_level level,
    const std::string& domain, const std::string& body)
{
    if (body.empty())
        return;
    std::ostringstream output;
    output << level_repr(level);
    if (!domain.empty())
        output << " [" << domain << "]";
    output << ": " << body;
    device << output.str() << std::endl;
    file << output.str() << std::endl;
}

void output_file(std::ofstream& file, log_level level,
    const std::string& domain, const std::string& body)
{
    log_to_file(file, level, domain, body);
}
void output_both(std::ofstream& file, log_level level,
    const std::string& domain, const std::string& body)
{
    log_to_both(std::cout, file, level, domain, body);
}

void error_file(std::ofstream& file, log_level level,
    const std::string& domain, const std::string& body)
{
    log_to_file(file, level, domain, body);
}
void error_both(std::ofstream& file, log_level level,
    const std::string& domain, const std::string& body)
{
    log_to_both(std::cerr, file, level, domain, body);
}

node_impl::node_impl()
  : network_pool_(1), disk_pool_(6), mem_pool_(1),
    hosts_(network_pool_),
    handshake_(network_pool_),
    network_(network_pool_),
    protocol_(network_pool_, hosts_, handshake_, network_),
    chain_(disk_pool_),
    poller_(mem_pool_, chain_),
    txpool_(mem_pool_, chain_),
    indexer_(mem_pool_),
    session_(mem_pool_, {
        handshake_, protocol_, chain_, poller_, txpool_})
{
}

bool node_impl::start(config_map_type& config)
{
    auto file_mode = std::ofstream::out | std::ofstream::app;
    outfile_.open(config["output-file"], file_mode);
    errfile_.open(config["error-file"], file_mode);
    log_debug().set_output_function(
        std::bind(output_file, std::ref(outfile_), _1, _2, _3));
    log_info().set_output_function(
        std::bind(output_both, std::ref(outfile_), _1, _2, _3));
    log_warning().set_output_function(
        std::bind(error_file, std::ref(errfile_), _1, _2, _3));
    log_error().set_output_function(
        std::bind(error_both, std::ref(errfile_), _1, _2, _3));
    log_fatal().set_output_function(
        std::bind(error_both, std::ref(errfile_), _1, _2, _3));
    protocol_.subscribe_channel(
        std::bind(&node_impl::monitor_tx, this, _1, _2));
    // Start blockchain.
    std::promise<std::error_code> ec_chain;
    auto blockchain_started =
        [&](const std::error_code& ec)
        {
            ec_chain.set_value(ec);
        };
    chain_.start(config["blockchain-path"], blockchain_started);
    // Query the error_code and wait for startup completion.
    std::error_code ec = ec_chain.get_future().get();
    if (ec)
    {
        log_error() << "Couldn't start blockchain: " << ec.message();
        return false;
    }
    chain_.subscribe_reorganize(
        std::bind(&node_impl::reorganize, this, _1, _2, _3, _4));
    // Transaction pool
    txpool_.start();
    // Start session
    std::promise<std::error_code> ec_session;
    auto session_started =
        [&](const std::error_code& ec)
        {
            ec_session.set_value(ec);
        };
    session_.start(session_started);
    // Query the error_code and wait for startup completion.
    ec = ec_session.get_future().get();
    if (ec)
    {
        log_error() << "Unable to start session: " << ec.message();
        return false;
    }
    return true;
}

void session_stop(const std::error_code& ec)
{
    if (ec)
        log_error() << "Problem stopping session: " << ec.message();
}
bool node_impl::stop()
{
    session_.stop(session_stop);
    network_pool_.stop();
    disk_pool_.stop();
    mem_pool_.stop();
    network_pool_.join();
    disk_pool_.join();
    mem_pool_.join();
    chain_.stop();
    return true;
}

void node_impl::subscribe_blocks(block_notify_callback notify_block)
{
    notify_blocks_.push_back(notify_block);
}
void node_impl::subscribe_transactions(transaction_notify_callback notify_tx)
{
    notify_txs_.push_back(notify_tx);
}

blockchain& node_impl::blockchain()
{
    return chain_;
}
transaction_pool& node_impl::transaction_pool()
{
    return txpool_;
}
transaction_indexer& node_impl::transaction_indexer()
{
    return indexer_;
}
protocol& node_impl::protocol()
{
    return protocol_;
}
threadpool& node_impl::memory_related_threadpool()
{
    return mem_pool_;
}

void node_impl::monitor_tx(const std::error_code& ec, channel_ptr node)
{
    if (ec)
    {
        log_warning() << "Couldn't start connection: " << ec.message();
        return;
    }
    node->subscribe_transaction(
        std::bind(&node_impl::recv_transaction, this, _1, _2, node));
    protocol_.subscribe_channel(
        std::bind(&node_impl::monitor_tx, this, _1, _2));
}

void node_impl::recv_transaction(const std::error_code& ec,
    const transaction_type& tx, channel_ptr node)
{
    if (ec)
    {
        log_error() << "recv_transaction: " << ec.message();
        return;
    }
    auto handle_deindex = [](const std::error_code& ec)
        {
            if (ec)
                log_error() << "Deindex error: " << ec.message();
        };
    // Called when the transaction becomes confirmed in a block.
    auto handle_confirm = [this, tx, handle_deindex](
        const std::error_code& ec)
        {
            log_debug() << "Confirm transaction: " << ec.message()
                << " " << hash_transaction(tx);
            // Always try to deindex tx.
            // The error could be error::forced_removal from txpool.
            indexer_.deindex(tx, handle_deindex);
        };
    txpool_.store(tx, handle_confirm,
        std::bind(&node_impl::handle_mempool_store, this, _1, _2, tx, node));
    node->subscribe_transaction(
        std::bind(&node_impl::recv_transaction, this, _1, _2, node));
}

void node_impl::handle_mempool_store(
    const std::error_code& ec, const index_list& unconfirmed,
    const transaction_type& tx, channel_ptr node)
{
    if (ec)
    {
        log_warning()
            << "Error storing memory pool transaction "
            << hash_transaction(tx) << ": " << ec.message();
        return;
    }
    auto handle_index = [](const std::error_code& ec)
        {
            if (ec)
                log_error() << "Index error: " << ec.message();
        };
    indexer_.index(tx, handle_index);
    log_info() << "Accepted transaction: " << hash_transaction(tx);
    for (auto notify: notify_txs_)
        notify(tx);
}

void node_impl::reorganize(const std::error_code& ec,
    size_t fork_point,
    const bc::blockchain::block_list& new_blocks,
    const bc::blockchain::block_list& replaced_blocks)
{
    // Don't bother publishing blocks when in the initial blockchain download.
    if (fork_point > 235866)
        for (size_t i = 0; i < new_blocks.size(); ++i)
        {
            size_t height = fork_point + i + 1;
            const block_type& blk = *new_blocks[i];
            for (auto notify: notify_blocks_)
                notify(height, blk);
        }
    chain_.subscribe_reorganize(
        std::bind(&node_impl::reorganize, this, _1, _2, _3, _4));
}

} // namespace obelisk

<commit_msg>more usable style for lambdas.<commit_after>#include "node_impl.hpp"

#include <future>

namespace obelisk {

using namespace bc;
using std::placeholders::_1;
using std::placeholders::_2;
using std::placeholders::_3;
using std::placeholders::_4;

void log_to_file(std::ofstream& file, log_level level,
    const std::string& domain, const std::string& body)
{
    if (body.empty())
        return;
    file << level_repr(level);
    if (!domain.empty())
        file << " [" << domain << "]";
    file << ": " << body << std::endl;
}
void log_to_both(std::ostream& device, std::ofstream& file, log_level level,
    const std::string& domain, const std::string& body)
{
    if (body.empty())
        return;
    std::ostringstream output;
    output << level_repr(level);
    if (!domain.empty())
        output << " [" << domain << "]";
    output << ": " << body;
    device << output.str() << std::endl;
    file << output.str() << std::endl;
}

void output_file(std::ofstream& file, log_level level,
    const std::string& domain, const std::string& body)
{
    log_to_file(file, level, domain, body);
}
void output_both(std::ofstream& file, log_level level,
    const std::string& domain, const std::string& body)
{
    log_to_both(std::cout, file, level, domain, body);
}

void error_file(std::ofstream& file, log_level level,
    const std::string& domain, const std::string& body)
{
    log_to_file(file, level, domain, body);
}
void error_both(std::ofstream& file, log_level level,
    const std::string& domain, const std::string& body)
{
    log_to_both(std::cerr, file, level, domain, body);
}

node_impl::node_impl()
  : network_pool_(1), disk_pool_(6), mem_pool_(1),
    hosts_(network_pool_),
    handshake_(network_pool_),
    network_(network_pool_),
    protocol_(network_pool_, hosts_, handshake_, network_),
    chain_(disk_pool_),
    poller_(mem_pool_, chain_),
    txpool_(mem_pool_, chain_),
    indexer_(mem_pool_),
    session_(mem_pool_, {
        handshake_, protocol_, chain_, poller_, txpool_})
{
}

bool node_impl::start(config_map_type& config)
{
    auto file_mode = std::ofstream::out | std::ofstream::app;
    outfile_.open(config["output-file"], file_mode);
    errfile_.open(config["error-file"], file_mode);
    log_debug().set_output_function(
        std::bind(output_file, std::ref(outfile_), _1, _2, _3));
    log_info().set_output_function(
        std::bind(output_both, std::ref(outfile_), _1, _2, _3));
    log_warning().set_output_function(
        std::bind(error_file, std::ref(errfile_), _1, _2, _3));
    log_error().set_output_function(
        std::bind(error_both, std::ref(errfile_), _1, _2, _3));
    log_fatal().set_output_function(
        std::bind(error_both, std::ref(errfile_), _1, _2, _3));
    protocol_.subscribe_channel(
        std::bind(&node_impl::monitor_tx, this, _1, _2));
    // Start blockchain.
    std::promise<std::error_code> ec_chain;
    auto blockchain_started =
        [&](const std::error_code& ec)
    {
        ec_chain.set_value(ec);
    };
    chain_.start(config["blockchain-path"], blockchain_started);
    // Query the error_code and wait for startup completion.
    std::error_code ec = ec_chain.get_future().get();
    if (ec)
    {
        log_error() << "Couldn't start blockchain: " << ec.message();
        return false;
    }
    chain_.subscribe_reorganize(
        std::bind(&node_impl::reorganize, this, _1, _2, _3, _4));
    // Transaction pool
    txpool_.start();
    // Start session
    std::promise<std::error_code> ec_session;
    auto session_started =
        [&](const std::error_code& ec)
    {
        ec_session.set_value(ec);
    };
    session_.start(session_started);
    // Query the error_code and wait for startup completion.
    ec = ec_session.get_future().get();
    if (ec)
    {
        log_error() << "Unable to start session: " << ec.message();
        return false;
    }
    return true;
}

void session_stop(const std::error_code& ec)
{
    if (ec)
        log_error() << "Problem stopping session: " << ec.message();
}
bool node_impl::stop()
{
    session_.stop(session_stop);
    network_pool_.stop();
    disk_pool_.stop();
    mem_pool_.stop();
    network_pool_.join();
    disk_pool_.join();
    mem_pool_.join();
    chain_.stop();
    return true;
}

void node_impl::subscribe_blocks(block_notify_callback notify_block)
{
    notify_blocks_.push_back(notify_block);
}
void node_impl::subscribe_transactions(transaction_notify_callback notify_tx)
{
    notify_txs_.push_back(notify_tx);
}

blockchain& node_impl::blockchain()
{
    return chain_;
}
transaction_pool& node_impl::transaction_pool()
{
    return txpool_;
}
transaction_indexer& node_impl::transaction_indexer()
{
    return indexer_;
}
protocol& node_impl::protocol()
{
    return protocol_;
}
threadpool& node_impl::memory_related_threadpool()
{
    return mem_pool_;
}

void node_impl::monitor_tx(const std::error_code& ec, channel_ptr node)
{
    if (ec)
    {
        log_warning() << "Couldn't start connection: " << ec.message();
        return;
    }
    node->subscribe_transaction(
        std::bind(&node_impl::recv_transaction, this, _1, _2, node));
    protocol_.subscribe_channel(
        std::bind(&node_impl::monitor_tx, this, _1, _2));
}

void node_impl::recv_transaction(const std::error_code& ec,
    const transaction_type& tx, channel_ptr node)
{
    if (ec)
    {
        log_error() << "recv_transaction: " << ec.message();
        return;
    }
    auto handle_deindex = [](const std::error_code& ec)
    {
        if (ec)
            log_error() << "Deindex error: " << ec.message();
    };
    // Called when the transaction becomes confirmed in a block.
    auto handle_confirm = [this, tx, handle_deindex](
        const std::error_code& ec)
    {
        log_debug() << "Confirm transaction: " << ec.message()
            << " " << hash_transaction(tx);
        // Always try to deindex tx.
        // The error could be error::forced_removal from txpool.
        indexer_.deindex(tx, handle_deindex);
    };
    txpool_.store(tx, handle_confirm,
        std::bind(&node_impl::handle_mempool_store, this, _1, _2, tx, node));
    node->subscribe_transaction(
        std::bind(&node_impl::recv_transaction, this, _1, _2, node));
}

void node_impl::handle_mempool_store(
    const std::error_code& ec, const index_list& unconfirmed,
    const transaction_type& tx, channel_ptr node)
{
    if (ec)
    {
        log_warning()
            << "Error storing memory pool transaction "
            << hash_transaction(tx) << ": " << ec.message();
        return;
    }
    auto handle_index = [](const std::error_code& ec)
    {
        if (ec)
            log_error() << "Index error: " << ec.message();
    };
    indexer_.index(tx, handle_index);
    log_info() << "Accepted transaction: " << hash_transaction(tx);
    for (auto notify: notify_txs_)
        notify(tx);
}

void node_impl::reorganize(const std::error_code& ec,
    size_t fork_point,
    const bc::blockchain::block_list& new_blocks,
    const bc::blockchain::block_list& replaced_blocks)
{
    // Don't bother publishing blocks when in the initial blockchain download.
    if (fork_point > 235866)
        for (size_t i = 0; i < new_blocks.size(); ++i)
        {
            size_t height = fork_point + i + 1;
            const block_type& blk = *new_blocks[i];
            for (auto notify: notify_blocks_)
                notify(height, blk);
        }
    chain_.subscribe_reorganize(
        std::bind(&node_impl::reorganize, this, _1, _2, _3, _4));
}

} // namespace obelisk

<|endoftext|>


<commit_before>/**
 * Copyright (c) 2013 Eugene Lazin <4lazin@gmail.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

#include <thread>
#include <boost/heap/skew_heap.hpp>
#include "akumuli_def.h"
#include "sequencer.h"
#include "util.h"

#define PARAM_ID 1
#define TIMESTMP 0

using namespace std;

namespace Akumuli {

template<typename RunType>
bool top_element_less(const RunType& x, const RunType& y)
{
    return x.back() < y.back();
}

template<typename RunType>
bool top_element_more(const RunType& x, const RunType& y)
{
    return top_element_less(y, x);
}

TimeSeriesValue::TimeSeriesValue() {}

TimeSeriesValue::TimeSeriesValue(aku_TimeStamp ts, aku_ParamId id, aku_EntryOffset offset)
    : key_(ts, id)
    , value(offset)
{
}

bool operator < (TimeSeriesValue const& lhs, TimeSeriesValue const& rhs) {
    return lhs.key_ < rhs.key_;
}

// Sequencer

Sequencer::Sequencer(PageHeader const* page, aku_Duration window_size)
    : window_size_(window_size)
    , page_(page)
    , top_timestamp_()
    , checkpoint_(0u)
    , run_lock_flags_(RUN_LOCK_FLAGS_SIZE)
{
    key_.push_back(TimeSeriesValue());

    for(auto& flag: run_lock_flags_) {
        flag.clear();
    }
}

//! Checkpoint id = ⌊timestamp/window_size⌋
uint32_t Sequencer::get_checkpoint_(aku_TimeStamp ts) const {
    // TODO: use fast integer division (libdivision or else)
    return ts / window_size_;
}

//! Convert checkpoint id to timestamp
aku_TimeStamp Sequencer::get_timestamp_(uint32_t cp) const {
    return cp*window_size_;
}

// move sorted runs to ready_ collection
void Sequencer::make_checkpoint_(uint32_t new_checkpoint, Lock& lock) {
    if(!lock.try_lock()) {
        return;
    }
    lock_all_runs();  // stop all consurrent searches
    auto old_top = get_timestamp_(checkpoint_);
    checkpoint_ = new_checkpoint;
    if (!ready_.empty()) {
        throw runtime_error("sequencer invariant is broken");
    }
    vector<SortedRun> new_runs;
    for (auto& sorted_run: runs_) {
        auto it = lower_bound(sorted_run.begin(), sorted_run.end(), TimeSeriesValue(old_top, AKU_LIMITS_MAX_ID, 0));
        if (it == sorted_run.begin()) {
            // all timestamps are newer than old_top, do nothing
            new_runs.push_back(move(sorted_run));
            continue;
        } else if (it == sorted_run.end()) {
            // all timestamps are older than old_top, move them
            ready_.push_back(move(sorted_run));
        } else {
            // it is in between of the sorted run - split
            SortedRun run;
            copy(sorted_run.begin(), it, back_inserter(run));  // copy old
            ready_.push_back(move(run));
            run.clear();
            copy(it, sorted_run.end(), back_inserter(run));  // copy new
            new_runs.push_back(move(run));
        }
    }
    swap(runs_, new_runs);
    unlock_all_runs();
    atomic_thread_fence(memory_order_acq_rel);
    return;
}

/** Check timestamp and make checkpoint if timestamp is large enough.
  * @returns error code and flag that indicates whether or not new checkpoint is created
  */
int Sequencer::check_timestamp_(aku_TimeStamp ts, Lock& lock) {
    int error_code = AKU_SUCCESS;
    if (ts < top_timestamp_) {
        auto delta = top_timestamp_ - ts;
        if (delta > window_size_) {
            error_code = AKU_ELATE_WRITE;
        }
        return error_code;
    }
    auto point = get_checkpoint_(ts);
    if (point > checkpoint_) {
        // Create new checkpoint
        make_checkpoint_(point, lock);
        if (!lock.owns_lock()) {
            // Previous checkpoint not completed
            error_code = AKU_EBUSY;
        }
    }
    top_timestamp_ = ts;
    return error_code;
}

std::tuple<int, Sequencer::Lock> Sequencer::add(TimeSeriesValue const& value) {
    // FIXME: max_cache_size_ is not used
    int status;
    Lock lock(progress_flag_, defer_lock);
    status = check_timestamp_(get<0>(value.key_), lock);
    if (status != AKU_SUCCESS) {
        return make_tuple(status, move(lock));
    }
    key_.pop_back();
    key_.push_back(value);
    auto begin = runs_.begin();
    auto end = runs_.end();
    auto insert_it = lower_bound(begin, end, key_, top_element_more<SortedRun>);
    if (insert_it == runs_.end()) {
        SortedRun new_pile;
        new_pile.push_back(value);
        runs_.push_back(new_pile);
    } else {
        int run_ix = distance(begin, insert_it);
        lock_run(run_ix);
        insert_it->push_back(value);
        unlock_run(run_ix);
    }
    return make_tuple(AKU_SUCCESS, move(lock));
}

Sequencer::Lock Sequencer::close() {
    Lock lock(progress_flag_, defer_lock);
    if (!lock.try_lock()) {
        return move(lock);
    }
    if (!ready_.empty()) {
        throw runtime_error("sequencer invariant is broken");
    }
    lock_all_runs();
    for (auto& sorted_run: runs_) {
        ready_.push_back(move(sorted_run));
    }
    unlock_all_runs();
    runs_.clear();
    atomic_thread_fence(memory_order_acq_rel);
    return move(lock);
}

template<class TKey, int dir>
struct MergePred;

template<class TKey>
struct MergePred<TKey, AKU_CURSOR_DIR_FORWARD> {
    greater<TKey> greater_;
    bool operator () (TKey const& lhs, TKey const& rhs) const {
        return greater_(lhs, rhs);
    }
};

template<class TKey>
struct MergePred<TKey, AKU_CURSOR_DIR_BACKWARD> {
    less<TKey> less_;
    bool operator () (TKey const& lhs, TKey const& rhs) const {
        return less_(lhs, rhs);
    }
};

template<class TRun, int dir>
struct RunIter;

template<class TRun>
struct RunIter<TRun, AKU_CURSOR_DIR_FORWARD> {
    typedef typename TRun::const_iterator iterator;
    static iterator begin(TRun const& run) {
        return run.begin();
    }
    static iterator end(TRun const& run) {
        return run.end();
    }
};

template<class TRun>
struct RunIter<TRun, AKU_CURSOR_DIR_BACKWARD> {
    typedef typename TRun::const_reverse_iterator iterator;
    static iterator begin(TRun const& run) {
        return run.rbegin();
    }
    static iterator end(TRun const& run) {
        return run.rend();
    }
};

template <int dir, class TRun>
void kway_merge(vector<TRun> const& runs, Caller& caller, InternalCursor* out_iter, PageHeader const* page) {
    size_t n = runs.size();
    typedef typename RunIter<TRun, dir>::iterator iter_t;
    typedef typename TRun::value_type value_t;
    iter_t iter[n], ends[n];
    int cnt = 0;
    for(auto i = runs.begin(); i != runs.end(); i++) {
        iter[cnt] = RunIter<TRun, dir>::begin(*i);
        ends[cnt] = RunIter<TRun, dir>::end(*i);
        cnt++;
    }

    typedef tuple<value_t, int> HeapItem;
    typedef MergePred<HeapItem, dir> Comp;
    typedef boost::heap::skew_heap<HeapItem, boost::heap::compare<Comp>> Heap;
    Heap heap;

    for(auto index = 0u; index < n; index++) {
        if (iter[index] != ends[index]) {
            auto value = *iter[index];
            iter[index]++;
            heap.push(make_tuple(value, index));
        }
    }

    while(!heap.empty()) {
        auto item = heap.top();
        auto point = get<0>(item);
        int index = get<1>(item);
        out_iter->put(caller, point.value, page);
        heap.pop();
        if (iter[index] != ends[index]) {
            auto point = *iter[index];
            iter[index]++;
            heap.push(make_tuple(point, index));
        }
    }
}

void Sequencer::merge(Caller& caller, InternalCursor* cur, Lock&& lock) {
    bool owns_lock = lock.owns_lock();
    if (!owns_lock) {
        // Error! Merge called too early
        cur->set_error(caller, AKU_EBUSY);
        return;
    }

    if (ready_.size() == 0) {
        // Things go crazy
        cur->set_error(caller, AKU_ENO_DATA);
        return;
    }

    kway_merge<AKU_CURSOR_DIR_FORWARD>(ready_, caller, cur, page_);

    // Sequencer invariant - if progress_flag_ is unset - ready_ flag must be empty
    // we've got only one place to store ready to sync data, if such data is present
    // progress_flag_ must be set (it indicates that merge/sync procedure is in progress)
    // after that we must clear ready_ collection and free some space for new data, after
    // that progress_flag_ can be cleared.

    ready_.clear();
    atomic_thread_fence(memory_order_acq_rel);
    cur->complete(caller);
}

void Sequencer::lock_run(int ix) const {
    int busy_wait_counter = RUN_LOCK_BUSY_COUNT;
    int backoff_len = 0;
    while(true) {
        bool is_busy = run_lock_flags_[RUN_LOCK_FLAGS_MASK & ix].test_and_set();
        if (!is_busy) {
            return;
        } else {
            // Busy wait
            if (busy_wait_counter) {
                busy_wait_counter--;
            } else {
                std::this_thread::sleep_for(std::chrono::milliseconds(backoff_len));
                if (backoff_len < RUN_LOCK_MAX_BACKOFF) {
                    backoff_len++;
                }
            }
        }
    }
}

void Sequencer::unlock_run(int ix) const {
    run_lock_flags_[RUN_LOCK_FLAGS_MASK & ix].clear();
}

void Sequencer::lock_all_runs() const {
   for (int i = 0; i < RUN_LOCK_FLAGS_SIZE; i++) {
       lock_run(i);
   }
}

void Sequencer::unlock_all_runs() const {
   for (int i = 0; i < RUN_LOCK_FLAGS_SIZE; i++) {
       unlock_run(i);
   }
}

struct SearchPredicate {
    SearchQuery const& query;
    SearchPredicate(SearchQuery const& q) : query(q) {}

    bool operator () (TimeSeriesValue const& value) const {
        if (query.lowerbound < get<TIMESTMP>(value.key_) &&
            query.upperbound > get<TIMESTMP>(value.key_))
        {
            if (query.param_pred(get<PARAM_ID>(value.key_)) == SearchQuery::MATCH) {
                return true;
            }
        }
        return false;
    }
};

void Sequencer::filter(SortedRun const& run, SearchQuery const& q, std::vector<SortedRun>* results) const {
    // TODO: use more effective algorithm, based on binary search
    SortedRun result;
    copy_if(run.begin(), run.end(), std::back_inserter(result), SearchPredicate(q));
    results->push_back(move(result));
}

void Sequencer::search(Caller& caller, InternalCursor* cur, SearchQuery query) const {
    std::lock_guard<std::mutex> guard(progress_flag_);
    // we can get here only before checkpoint (or after merge was completed)
    // that means that ready_ is empty
    assert(ready_.empty());
    std::vector<SortedRun> filtered;
    int run_ix = 0;
    for (const auto& run: runs_) {
        lock_run(run_ix);
        filter(run, query, &filtered);
        unlock_run(run_ix);
        run_ix++;
    }
    if (query.direction == AKU_CURSOR_DIR_FORWARD) {
        kway_merge<AKU_CURSOR_DIR_FORWARD>(filtered, caller, cur, page_);
    } else {
        kway_merge<AKU_CURSOR_DIR_BACKWARD>(filtered, caller, cur, page_);
    }
}
}  // namespace Akumuli
<commit_msg>Replace iterators with ranges in merge procedure<commit_after>/**
 * Copyright (c) 2013 Eugene Lazin <4lazin@gmail.com>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

#include <thread>
#include <boost/heap/skew_heap.hpp>
#include "akumuli_def.h"
#include "sequencer.h"
#include "util.h"
#include <boost/range.hpp>
#include <boost/range/iterator_range.hpp>

#define PARAM_ID 1
#define TIMESTMP 0

using namespace std;

namespace Akumuli {

template<typename RunType>
bool top_element_less(const RunType& x, const RunType& y)
{
    return x.back() < y.back();
}

template<typename RunType>
bool top_element_more(const RunType& x, const RunType& y)
{
    return top_element_less(y, x);
}

TimeSeriesValue::TimeSeriesValue() {}

TimeSeriesValue::TimeSeriesValue(aku_TimeStamp ts, aku_ParamId id, aku_EntryOffset offset)
    : key_(ts, id)
    , value(offset)
{
}

bool operator < (TimeSeriesValue const& lhs, TimeSeriesValue const& rhs) {
    return lhs.key_ < rhs.key_;
}

// Sequencer

Sequencer::Sequencer(PageHeader const* page, aku_Duration window_size)
    : window_size_(window_size)
    , page_(page)
    , top_timestamp_()
    , checkpoint_(0u)
    , run_lock_flags_(RUN_LOCK_FLAGS_SIZE)
{
    key_.push_back(TimeSeriesValue());

    for(auto& flag: run_lock_flags_) {
        flag.clear();
    }
}

//! Checkpoint id = ⌊timestamp/window_size⌋
uint32_t Sequencer::get_checkpoint_(aku_TimeStamp ts) const {
    // TODO: use fast integer division (libdivision or else)
    return ts / window_size_;
}

//! Convert checkpoint id to timestamp
aku_TimeStamp Sequencer::get_timestamp_(uint32_t cp) const {
    return cp*window_size_;
}

// move sorted runs to ready_ collection
void Sequencer::make_checkpoint_(uint32_t new_checkpoint, Lock& lock) {
    if(!lock.try_lock()) {
        return;
    }
    lock_all_runs();  // stop all consurrent searches
    auto old_top = get_timestamp_(checkpoint_);
    checkpoint_ = new_checkpoint;
    if (!ready_.empty()) {
        throw runtime_error("sequencer invariant is broken");
    }
    vector<SortedRun> new_runs;
    for (auto& sorted_run: runs_) {
        auto it = lower_bound(sorted_run.begin(), sorted_run.end(), TimeSeriesValue(old_top, AKU_LIMITS_MAX_ID, 0));
        if (it == sorted_run.begin()) {
            // all timestamps are newer than old_top, do nothing
            new_runs.push_back(move(sorted_run));
            continue;
        } else if (it == sorted_run.end()) {
            // all timestamps are older than old_top, move them
            ready_.push_back(move(sorted_run));
        } else {
            // it is in between of the sorted run - split
            SortedRun run;
            copy(sorted_run.begin(), it, back_inserter(run));  // copy old
            ready_.push_back(move(run));
            run.clear();
            copy(it, sorted_run.end(), back_inserter(run));  // copy new
            new_runs.push_back(move(run));
        }
    }
    swap(runs_, new_runs);
    unlock_all_runs();
    atomic_thread_fence(memory_order_acq_rel);
    return;
}

/** Check timestamp and make checkpoint if timestamp is large enough.
  * @returns error code and flag that indicates whether or not new checkpoint is created
  */
int Sequencer::check_timestamp_(aku_TimeStamp ts, Lock& lock) {
    int error_code = AKU_SUCCESS;
    if (ts < top_timestamp_) {
        auto delta = top_timestamp_ - ts;
        if (delta > window_size_) {
            error_code = AKU_ELATE_WRITE;
        }
        return error_code;
    }
    auto point = get_checkpoint_(ts);
    if (point > checkpoint_) {
        // Create new checkpoint
        make_checkpoint_(point, lock);
        if (!lock.owns_lock()) {
            // Previous checkpoint not completed
            error_code = AKU_EBUSY;
        }
    }
    top_timestamp_ = ts;
    return error_code;
}

std::tuple<int, Sequencer::Lock> Sequencer::add(TimeSeriesValue const& value) {
    // FIXME: max_cache_size_ is not used
    int status;
    Lock lock(progress_flag_, defer_lock);
    status = check_timestamp_(get<0>(value.key_), lock);
    if (status != AKU_SUCCESS) {
        return make_tuple(status, move(lock));
    }
    key_.pop_back();
    key_.push_back(value);
    auto begin = runs_.begin();
    auto end = runs_.end();
    auto insert_it = lower_bound(begin, end, key_, top_element_more<SortedRun>);
    if (insert_it == runs_.end()) {
        SortedRun new_pile;
        new_pile.push_back(value);
        runs_.push_back(new_pile);
    } else {
        int run_ix = distance(begin, insert_it);
        lock_run(run_ix);
        insert_it->push_back(value);
        unlock_run(run_ix);
    }
    return make_tuple(AKU_SUCCESS, move(lock));
}

Sequencer::Lock Sequencer::close() {
    Lock lock(progress_flag_, defer_lock);
    if (!lock.try_lock()) {
        return move(lock);
    }
    if (!ready_.empty()) {
        throw runtime_error("sequencer invariant is broken");
    }
    lock_all_runs();
    for (auto& sorted_run: runs_) {
        ready_.push_back(move(sorted_run));
    }
    unlock_all_runs();
    runs_.clear();
    atomic_thread_fence(memory_order_acq_rel);
    return move(lock);
}

template<class TKey, int dir>
struct MergePred;

template<class TKey>
struct MergePred<TKey, AKU_CURSOR_DIR_FORWARD> {
    greater<TKey> greater_;
    bool operator () (TKey const& lhs, TKey const& rhs) const {
        return greater_(lhs, rhs);
    }
};

template<class TKey>
struct MergePred<TKey, AKU_CURSOR_DIR_BACKWARD> {
    less<TKey> less_;
    bool operator () (TKey const& lhs, TKey const& rhs) const {
        return less_(lhs, rhs);
    }
};

template<class TRun, int dir>
struct RunIter;

template<class TRun>
struct RunIter<TRun, AKU_CURSOR_DIR_FORWARD> {
    typedef boost::iterator_range<typename TRun::const_iterator> range_type;
    static range_type make_range(TRun const& run) {
        return boost::make_iterator_range(run);
    }
};

template<class TRun>
struct RunIter<TRun, AKU_CURSOR_DIR_BACKWARD> {
    typedef boost::iterator_range<typename TRun::const_reverse_iterator> range_type;
    static range_type make_range(TRun const& run) {
        return boost::make_iterator_range(run.rbegin(), run.rend());
    }
};

template <int dir, class TRun>
void kway_merge(vector<TRun> const& runs, Caller& caller, InternalCursor* out_iter, PageHeader const* page) {
    typedef RunIter<TRun, dir> RIter;
    typedef typename RIter::range_type range_t;
    typedef typename TRun::value_type KeyType;
    std::vector<range_t> ranges;
    for(auto i = runs.begin(); i != runs.end(); i++) {
        ranges.push_back(RIter::make_range(*i));
    }

    typedef tuple<KeyType, int> HeapItem;
    typedef MergePred<HeapItem, dir> Comp;
    typedef boost::heap::skew_heap<HeapItem, boost::heap::compare<Comp>> Heap;
    Heap heap;

    int index = 0;
    for(auto& range: ranges) {
        if (!range.empty()) {
            KeyType value = range.front();
            range.advance_begin(1);
            heap.push(make_tuple(value, index));
        }
        index++;
    }

    while(!heap.empty()) {
        HeapItem item = heap.top();
        KeyType point = get<0>(item);
        int index = get<1>(item);
        out_iter->put(caller, point.value, page);
        heap.pop();
        if (!ranges[index].empty()) {
            KeyType point = ranges[index].front();
            ranges[index].advance_begin(1);
            heap.push(make_tuple(point, index));
        }
    }
}

void Sequencer::merge(Caller& caller, InternalCursor* cur, Lock&& lock) {
    bool owns_lock = lock.owns_lock();
    if (!owns_lock) {
        // Error! Merge called too early
        cur->set_error(caller, AKU_EBUSY);
        return;
    }

    if (ready_.size() == 0) {
        // Things go crazy
        cur->set_error(caller, AKU_ENO_DATA);
        return;
    }

    kway_merge<AKU_CURSOR_DIR_FORWARD>(ready_, caller, cur, page_);

    // Sequencer invariant - if progress_flag_ is unset - ready_ flag must be empty
    // we've got only one place to store ready to sync data, if such data is present
    // progress_flag_ must be set (it indicates that merge/sync procedure is in progress)
    // after that we must clear ready_ collection and free some space for new data, after
    // that progress_flag_ can be cleared.

    ready_.clear();
    atomic_thread_fence(memory_order_acq_rel);
    cur->complete(caller);
}

void Sequencer::lock_run(int ix) const {
    int busy_wait_counter = RUN_LOCK_BUSY_COUNT;
    int backoff_len = 0;
    while(true) {
        bool is_busy = run_lock_flags_[RUN_LOCK_FLAGS_MASK & ix].test_and_set();
        if (!is_busy) {
            return;
        } else {
            // Busy wait
            if (busy_wait_counter) {
                busy_wait_counter--;
            } else {
                std::this_thread::sleep_for(std::chrono::milliseconds(backoff_len));
                if (backoff_len < RUN_LOCK_MAX_BACKOFF) {
                    backoff_len++;
                }
            }
        }
    }
}

void Sequencer::unlock_run(int ix) const {
    run_lock_flags_[RUN_LOCK_FLAGS_MASK & ix].clear();
}

void Sequencer::lock_all_runs() const {
   for (int i = 0; i < RUN_LOCK_FLAGS_SIZE; i++) {
       lock_run(i);
   }
}

void Sequencer::unlock_all_runs() const {
   for (int i = 0; i < RUN_LOCK_FLAGS_SIZE; i++) {
       unlock_run(i);
   }
}

struct SearchPredicate {
    SearchQuery const& query;
    SearchPredicate(SearchQuery const& q) : query(q) {}

    bool operator () (TimeSeriesValue const& value) const {
        if (query.lowerbound < get<TIMESTMP>(value.key_) &&
            query.upperbound > get<TIMESTMP>(value.key_))
        {
            if (query.param_pred(get<PARAM_ID>(value.key_)) == SearchQuery::MATCH) {
                return true;
            }
        }
        return false;
    }
};

void Sequencer::filter(SortedRun const& run, SearchQuery const& q, std::vector<SortedRun>* results) const {
    // TODO: use more effective algorithm, based on binary search
    SortedRun result;
    copy_if(run.begin(), run.end(), std::back_inserter(result), SearchPredicate(q));
    results->push_back(move(result));
}

void Sequencer::search(Caller& caller, InternalCursor* cur, SearchQuery query) const {
    std::lock_guard<std::mutex> guard(progress_flag_);
    // we can get here only before checkpoint (or after merge was completed)
    // that means that ready_ is empty
    assert(ready_.empty());
    std::vector<SortedRun> filtered;
    int run_ix = 0;
    for (const auto& run: runs_) {
        lock_run(run_ix);
        filter(run, query, &filtered);
        unlock_run(run_ix);
        run_ix++;
    }
    if (query.direction == AKU_CURSOR_DIR_FORWARD) {
        kway_merge<AKU_CURSOR_DIR_FORWARD>(filtered, caller, cur, page_);
    } else {
        kway_merge<AKU_CURSOR_DIR_BACKWARD>(filtered, caller, cur, page_);
    }
}
}  // namespace Akumuli
<|endoftext|>


<commit_before>#ifndef LISTHOOKEDPOINTING_HPP
#define LISTHOOKEDPOINTING_HPP

#include "base.hpp"
#include "CallBackWrapper.hpp"
#include "ListHookedDevice.hpp"

namespace org_pqrs_KeyRemap4MacBook {
  class ListHookedPointing : public ListHookedDevice {
  public:
    class Item : public ListHookedDevice::Item {
    public:
      Item(IOHIDevice* d);
      ~Item(void);

      IOHIPointing* get(void) const { return OSDynamicCast(IOHIPointing, ListHookedDevice::Item::get()); }

      RelativePointerEventCallback getOrig_relativePointerEventAction(void) const { return orig_relativePointerEventAction_; }
      ScrollWheelEventCallback getOrig_scrollWheelEventAction(void) const { return orig_scrollWheelEventAction_; }
      OSObject* getOrig_relativePointerEventTarget(void) const { return orig_relativePointerEventTarget_; }
      OSObject* getOrig_scrollWheelEventTarget(void) const { return orig_scrollWheelEventTarget_; }

      Buttons get_previousbuttons(void) const { return previousbuttons_; }
      void set_previousbuttons(Buttons newval) { previousbuttons_ = newval; }

      void apply(const Params_RelativePointerEventCallback& params);
      void apply(const Params_ScrollWheelEventCallback& params);

    private:
      RelativePointerEventCallback orig_relativePointerEventAction_;
      ScrollWheelEventCallback orig_scrollWheelEventAction_;
      OSObject* orig_relativePointerEventTarget_;
      OSObject* orig_scrollWheelEventTarget_;

      Buttons previousbuttons_;

      IOLock* replacerestore_lock_;

      bool refresh_callback(void);
      bool isReplaced(void) const { return orig_relativePointerEventAction_ != NULL; }

      /** return true if event action is replaced. */
      bool replaceEventAction(void);
      /** return true if event action is replaced. */
      bool restoreEventAction(void);
    };

    static ListHookedPointing& instance(void);
    ListHookedPointing::Item* get(void) { return static_cast<ListHookedPointing::Item*>(ListHookedDevice::get()); }
    ListHookedPointing::Item* get(const IOHIPointing* pointing) { return static_cast<ListHookedPointing::Item*>(ListHookedDevice::get(pointing)); }

    static void hook_RelativePointerEventCallback_queued(Params_RelativePointerEventCallback& params);
    static void hook_ScrollWheelEventCallback_queued(Params_ScrollWheelEventCallback& params);

    void apply(const Params_RelativePointerEventCallback& params);
    void apply(const Params_ScrollWheelEventCallback& params);
  };
}

#endif
<commit_msg>cleanup<commit_after>#ifndef LISTHOOKEDPOINTING_HPP
#define LISTHOOKEDPOINTING_HPP

#include "base.hpp"
#include "CallBackWrapper.hpp"
#include "ListHookedDevice.hpp"

namespace org_pqrs_KeyRemap4MacBook {
  class ListHookedPointing : public ListHookedDevice {
  public:
    class Item : public ListHookedDevice::Item {
    public:
      Item(IOHIDevice* d);
      ~Item(void);

      Buttons get_previousbuttons(void) const { return previousbuttons_; }
      void set_previousbuttons(Buttons newval) { previousbuttons_ = newval; }

      void apply(const Params_RelativePointerEventCallback& params);
      void apply(const Params_ScrollWheelEventCallback& params);

    private:
      RelativePointerEventCallback orig_relativePointerEventAction_;
      ScrollWheelEventCallback orig_scrollWheelEventAction_;
      OSObject* orig_relativePointerEventTarget_;
      OSObject* orig_scrollWheelEventTarget_;

      Buttons previousbuttons_;

      IOLock* replacerestore_lock_;

      bool refresh_callback(void);
      bool isReplaced(void) const { return orig_relativePointerEventAction_ != NULL; }

      /** return true if event action is replaced. */
      bool replaceEventAction(void);
      /** return true if event action is replaced. */
      bool restoreEventAction(void);
    };

    static ListHookedPointing& instance(void);
    ListHookedPointing::Item* get(void) { return static_cast<ListHookedPointing::Item*>(ListHookedDevice::get()); }
    ListHookedPointing::Item* get(const IOHIPointing* pointing) { return static_cast<ListHookedPointing::Item*>(ListHookedDevice::get(pointing)); }

    static void hook_RelativePointerEventCallback_queued(Params_RelativePointerEventCallback& params);
    static void hook_ScrollWheelEventCallback_queued(Params_ScrollWheelEventCallback& params);

    void apply(const Params_RelativePointerEventCallback& params);
    void apply(const Params_ScrollWheelEventCallback& params);
  };
}

#endif
<|endoftext|>