<commit_before>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2018.  All rights reserved
 */
#ifndef _Stroika_Foundation_Containers_Private_PatchingDataStructures_STLContainerWrapper_inl_
#define _Stroika_Foundation_Containers_Private_PatchingDataStructures_STLContainerWrapper_inl_ 1

#include <algorithm>

#include "../../../Debug/Assertions.h"

namespace Stroika {
    namespace Foundation {
        namespace Containers {
            namespace Private {
                namespace PatchingDataStructures {

                    /*
                     ********************************************************************************
                     ******* PatchingDataStructures::STLContainerWrapper<STL_CONTAINER_OF_T> ********
                     ********************************************************************************
                     */
                    template <typename STL_CONTAINER_OF_T>
                    template <typename... CONTAINER_EXTRA_ARGS>
                    inline STLContainerWrapper<STL_CONTAINER_OF_T>::STLContainerWrapper (CONTAINER_EXTRA_ARGS&&... stdContainerArgs)
                        : inherited (eCreateNewConstructorSelector, std::forward<CONTAINER_EXTRA_ARGS> (stdContainerArgs)...)
                    {
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline STLContainerWrapper<STL_CONTAINER_OF_T>::STLContainerWrapper (STLContainerWrapper<STL_CONTAINER_OF_T>* rhs, IteratorOwnerID newOwnerID)
                        : inherited (rhs, newOwnerID, (ForwardIterator*)nullptr)
                    {
                        RequireNotNull (rhs);
                        rhs->Invariant ();
                        Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    template <typename INSERT_VALUE_TYPE>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::insert_toVector_WithPatching (typename STL_CONTAINER_OF_T::iterator i, INSERT_VALUE_TYPE v)
                    {
                        Invariant ();
                        Memory::SmallStackBuffer<size_t> patchOffsets (0);
                        TwoPhaseIteratorPatcherAll2FromOffsetsPass1 (&patchOffsets, i);
                        this->insert (i, v);
                        TwoPhaseIteratorPatcherAll2FromOffsetsPass2 (patchOffsets);
                        Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::erase_WithPatching (typename STL_CONTAINER_OF_T::iterator i)
                    {
                        Invariant ();
                        Memory::SmallStackBuffer<ForwardIterator*> items2Patch (0);
                        TwoPhaseIteratorPatcherPass1 (i, &items2Patch);
                        auto newI = this->erase (i);
                        TwoPhaseIteratorPatcherPass2 (&items2Patch, newI);
                        Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::clear_WithPatching ()
                    {
                        Invariant ();
                        this->clear ();
                        this->template _ApplyToEachOwnedIterator<ForwardIterator> ([this](ForwardIterator* ai) {
                            ai->TwoPhaseIteratorPatcherPass2 (this->end ());
                        });
                        Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::TwoPhaseIteratorPatcherPass1 (typename STL_CONTAINER_OF_T::iterator oldI, Memory::SmallStackBuffer<ForwardIterator*>* items2Patch) const
                    {
                        this->template _ApplyToEachOwnedIterator<ForwardIterator> ([oldI, items2Patch](ForwardIterator* ai) {
                            ai->TwoPhaseIteratorPatcherPass1 (oldI, items2Patch);
                        });
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::TwoPhaseIteratorPatcherPass2 (const Memory::SmallStackBuffer<ForwardIterator*>* items2Patch, typename STL_CONTAINER_OF_T::iterator newI)
                    {
                        for (size_t i = 0; i < items2Patch->GetSize (); ++i) {
                            (*items2Patch)[i]->TwoPhaseIteratorPatcherPass2 (newI);
                        }
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::TwoPhaseIteratorPatcherAll2FromOffsetsPass1 (Memory::SmallStackBuffer<size_t>* patchOffsets) const
                    {
                        RequireNotNull (patchOffsets);
                        this->template _ApplyToEachOwnedIterator<ForwardIterator> ([this, patchOffsets](ForwardIterator* ai) {
                            size_t idx = ai->fStdIterator - this->begin ();
                            patchOffsets->push_back (idx);
                        });
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::TwoPhaseIteratorPatcherAll2FromOffsetsPass1 (Memory::SmallStackBuffer<size_t>* patchOffsets, typename STL_CONTAINER_OF_T::iterator incIfGreaterOrEqual) const
                    {
                        RequireNotNull (patchOffsets);
                        this->template _ApplyToEachOwnedIterator<ForwardIterator> ([&](ForwardIterator* ai) {
                            typename STL_CONTAINER_OF_T::iterator aIt = ai->fStdIterator;
                            size_t                                idx = ai->fStdIterator - this->begin ();
                            if (incIfGreaterOrEqual <= aIt) {
                                idx++;
                            }
                            patchOffsets->push_back (idx);
                        });
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::TwoPhaseIteratorPatcherAll2FromOffsetsPass2 (const Memory::SmallStackBuffer<size_t>& patchOffsets)
                    {
                        size_t i = 0;
                        this->template _ApplyToEachOwnedIterator<ForwardIterator> ([this, &patchOffsets, &i](ForwardIterator* ai) {
                            Assert (patchOffsets[i] < this->size ());
                            ai->fStdIterator = this->begin () + patchOffsets[i];
                            ++i;
                        });
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::Invariant () const
                    {
#if qDebug
                        _Invariant ();
#endif
                    }
#if qDebug
                    template <typename STL_CONTAINER_OF_T>
                    void STLContainerWrapper<STL_CONTAINER_OF_T>::_Invariant () const
                    {
                        this->template _ApplyToEachOwnedIterator<ForwardIterator> ([](ForwardIterator* ai) {
                            ai->Invariant ();
                        });
                    }
#endif

                    /*
                     ********************************************************************************
                     * PatchingDataStructures::STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator *
                     ********************************************************************************
                     */
                    template <typename STL_CONTAINER_OF_T>
                    inline STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::ForwardIterator (IteratorOwnerID ownerID, CONTAINER_TYPE* data)
                        : inherited_DataStructure (data)
                        , inherited_PatchHelper (const_cast<STLContainerWrapper<STL_CONTAINER_OF_T>*> (data), ownerID)
                    {
                        RequireNotNull (data);
                        this->Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::ForwardIterator (const ForwardIterator& from)
                        : inherited_DataStructure (from)
                        , inherited_PatchHelper (from)
                    {
                        this->Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::~ForwardIterator ()
                    {
                        this->Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::RemoveCurrent ()
                    {
                        AssertNotNull (this->fData);
                        static_cast<STLContainerWrapper<STL_CONTAINER_OF_T>*> (this->fData)->erase_WithPatching (this->fStdIterator);
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::TwoPhaseIteratorPatcherPass1 (typename STL_CONTAINER_OF_T::iterator oldI, Memory::SmallStackBuffer<ForwardIterator*>* items2Patch)
                    {
                        if (this->fStdIterator == oldI) {
                            items2Patch->push_back (this);
                        }
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::TwoPhaseIteratorPatcherPass2 (typename STL_CONTAINER_OF_T::iterator newI)
                    {
                        this->SetCurrentLink (newI);
                        this->fSuppressMore = true;
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::Invariant () const
                    {
#if qDebug
                        _Invariant ();
#endif
                    }
#if qDebug
                    template <typename STL_CONTAINER_OF_T>
                    void STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::_Invariant () const
                    {
                        AssertNotNull (this->fData);
                        // apparently pointless test is just to force triggering any check code in iterator classes for valid iterators
                        Assert (this->fStdIterator == this->fData->end () or this->fStdIterator != this->fData->end ());
                        // Find some way to check iterator traits to see if random access and do this test if possible
                        //Assert ((fCurrent >= fStart) and (fCurrent <= fEnd));   // ANSI C requires this is always TRUE
                    }
#endif
                }
            }
        }
    }
}
#endif /* _Stroika_Foundation_Containers_Private_PatchingDataStructures_STLContainerWrapper_inl_ */
<commit_msg>gcc compat<commit_after>/*
 * Copyright(c) Sophist Solutions, Inc. 1990-2018.  All rights reserved
 */
#ifndef _Stroika_Foundation_Containers_Private_PatchingDataStructures_STLContainerWrapper_inl_
#define _Stroika_Foundation_Containers_Private_PatchingDataStructures_STLContainerWrapper_inl_ 1

#include <algorithm>

#include "../../../Debug/Assertions.h"

namespace Stroika {
    namespace Foundation {
        namespace Containers {
            namespace Private {
                namespace PatchingDataStructures {

                    /*
                     ********************************************************************************
                     ******* PatchingDataStructures::STLContainerWrapper<STL_CONTAINER_OF_T> ********
                     ********************************************************************************
                     */
                    template <typename STL_CONTAINER_OF_T>
                    template <typename... CONTAINER_EXTRA_ARGS>
                    inline STLContainerWrapper<STL_CONTAINER_OF_T>::STLContainerWrapper (CONTAINER_EXTRA_ARGS&&... stdContainerArgs)
                        : inherited (inherited::eCreateNewConstructorSelector, std::forward<CONTAINER_EXTRA_ARGS> (stdContainerArgs)...)
                    {
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline STLContainerWrapper<STL_CONTAINER_OF_T>::STLContainerWrapper (STLContainerWrapper<STL_CONTAINER_OF_T>* rhs, IteratorOwnerID newOwnerID)
                        : inherited (rhs, newOwnerID, (ForwardIterator*)nullptr)
                    {
                        RequireNotNull (rhs);
                        rhs->Invariant ();
                        Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    template <typename INSERT_VALUE_TYPE>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::insert_toVector_WithPatching (typename STL_CONTAINER_OF_T::iterator i, INSERT_VALUE_TYPE v)
                    {
                        Invariant ();
                        Memory::SmallStackBuffer<size_t> patchOffsets (0);
                        TwoPhaseIteratorPatcherAll2FromOffsetsPass1 (&patchOffsets, i);
                        this->insert (i, v);
                        TwoPhaseIteratorPatcherAll2FromOffsetsPass2 (patchOffsets);
                        Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::erase_WithPatching (typename STL_CONTAINER_OF_T::iterator i)
                    {
                        Invariant ();
                        Memory::SmallStackBuffer<ForwardIterator*> items2Patch (0);
                        TwoPhaseIteratorPatcherPass1 (i, &items2Patch);
                        auto newI = this->erase (i);
                        TwoPhaseIteratorPatcherPass2 (&items2Patch, newI);
                        Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::clear_WithPatching ()
                    {
                        Invariant ();
                        this->clear ();
                        this->template _ApplyToEachOwnedIterator<ForwardIterator> ([this](ForwardIterator* ai) {
                            ai->TwoPhaseIteratorPatcherPass2 (this->end ());
                        });
                        Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::TwoPhaseIteratorPatcherPass1 (typename STL_CONTAINER_OF_T::iterator oldI, Memory::SmallStackBuffer<ForwardIterator*>* items2Patch) const
                    {
                        this->template _ApplyToEachOwnedIterator<ForwardIterator> ([oldI, items2Patch](ForwardIterator* ai) {
                            ai->TwoPhaseIteratorPatcherPass1 (oldI, items2Patch);
                        });
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::TwoPhaseIteratorPatcherPass2 (const Memory::SmallStackBuffer<ForwardIterator*>* items2Patch, typename STL_CONTAINER_OF_T::iterator newI)
                    {
                        for (size_t i = 0; i < items2Patch->GetSize (); ++i) {
                            (*items2Patch)[i]->TwoPhaseIteratorPatcherPass2 (newI);
                        }
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::TwoPhaseIteratorPatcherAll2FromOffsetsPass1 (Memory::SmallStackBuffer<size_t>* patchOffsets) const
                    {
                        RequireNotNull (patchOffsets);
                        this->template _ApplyToEachOwnedIterator<ForwardIterator> ([this, patchOffsets](ForwardIterator* ai) {
                            size_t idx = ai->fStdIterator - this->begin ();
                            patchOffsets->push_back (idx);
                        });
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::TwoPhaseIteratorPatcherAll2FromOffsetsPass1 (Memory::SmallStackBuffer<size_t>* patchOffsets, typename STL_CONTAINER_OF_T::iterator incIfGreaterOrEqual) const
                    {
                        RequireNotNull (patchOffsets);
                        this->template _ApplyToEachOwnedIterator<ForwardIterator> ([&](ForwardIterator* ai) {
                            typename STL_CONTAINER_OF_T::iterator aIt = ai->fStdIterator;
                            size_t                                idx = ai->fStdIterator - this->begin ();
                            if (incIfGreaterOrEqual <= aIt) {
                                idx++;
                            }
                            patchOffsets->push_back (idx);
                        });
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::TwoPhaseIteratorPatcherAll2FromOffsetsPass2 (const Memory::SmallStackBuffer<size_t>& patchOffsets)
                    {
                        size_t i = 0;
                        this->template _ApplyToEachOwnedIterator<ForwardIterator> ([this, &patchOffsets, &i](ForwardIterator* ai) {
                            Assert (patchOffsets[i] < this->size ());
                            ai->fStdIterator = this->begin () + patchOffsets[i];
                            ++i;
                        });
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::Invariant () const
                    {
#if qDebug
                        _Invariant ();
#endif
                    }
#if qDebug
                    template <typename STL_CONTAINER_OF_T>
                    void STLContainerWrapper<STL_CONTAINER_OF_T>::_Invariant () const
                    {
                        this->template _ApplyToEachOwnedIterator<ForwardIterator> ([](ForwardIterator* ai) {
                            ai->Invariant ();
                        });
                    }
#endif

                    /*
                     ********************************************************************************
                     * PatchingDataStructures::STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator *
                     ********************************************************************************
                     */
                    template <typename STL_CONTAINER_OF_T>
                    inline STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::ForwardIterator (IteratorOwnerID ownerID, CONTAINER_TYPE* data)
                        : inherited_DataStructure (data)
                        , inherited_PatchHelper (const_cast<STLContainerWrapper<STL_CONTAINER_OF_T>*> (data), ownerID)
                    {
                        RequireNotNull (data);
                        this->Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::ForwardIterator (const ForwardIterator& from)
                        : inherited_DataStructure (from)
                        , inherited_PatchHelper (from)
                    {
                        this->Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::~ForwardIterator ()
                    {
                        this->Invariant ();
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::RemoveCurrent ()
                    {
                        AssertNotNull (this->fData);
                        static_cast<STLContainerWrapper<STL_CONTAINER_OF_T>*> (this->fData)->erase_WithPatching (this->fStdIterator);
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::TwoPhaseIteratorPatcherPass1 (typename STL_CONTAINER_OF_T::iterator oldI, Memory::SmallStackBuffer<ForwardIterator*>* items2Patch)
                    {
                        if (this->fStdIterator == oldI) {
                            items2Patch->push_back (this);
                        }
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::TwoPhaseIteratorPatcherPass2 (typename STL_CONTAINER_OF_T::iterator newI)
                    {
                        this->SetCurrentLink (newI);
                        this->fSuppressMore = true;
                    }
                    template <typename STL_CONTAINER_OF_T>
                    inline void STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::Invariant () const
                    {
#if qDebug
                        _Invariant ();
#endif
                    }
#if qDebug
                    template <typename STL_CONTAINER_OF_T>
                    void STLContainerWrapper<STL_CONTAINER_OF_T>::ForwardIterator::_Invariant () const
                    {
                        AssertNotNull (this->fData);
                        // apparently pointless test is just to force triggering any check code in iterator classes for valid iterators
                        Assert (this->fStdIterator == this->fData->end () or this->fStdIterator != this->fData->end ());
                        // Find some way to check iterator traits to see if random access and do this test if possible
                        //Assert ((fCurrent >= fStart) and (fCurrent <= fEnd));   // ANSI C requires this is always TRUE
                    }
#endif
                }
            }
        }
    }
}
#endif /* _Stroika_Foundation_Containers_Private_PatchingDataStructures_STLContainerWrapper_inl_ */
<|endoftext|>


<commit_before>/*
 *  Copyright(C) 2021 hikyuu.org
 *
 *  Create on: 2021-02-16
 *     Author: fasiondog
 */

#include <csignal>
#include <unordered_set>
#include "../utilities/os.h"
#include "../utilities/IniParser.h"
#include "../global/schedule/scheduler.h"
#include "StrategyBase.h"

namespace hku {

std::atomic_bool StrategyBase::ms_keep_running = true;

void StrategyBase::sig_handler(int sig) {
    if (sig == SIGINT) {
        ms_keep_running = false;
    }
}

StrategyBase::StrategyBase() : StrategyBase("Strategy") {}

StrategyBase::StrategyBase(const string& name) {
    string home = getUserHome();
    HKU_ERROR_IF(home == "", "Failed get user home path!");
#if HKU_OS_WINOWS
    m_config_file = format("{}\\{}", home, ".hikyuu\\hikyuu.ini");
#else
    m_config_file = format("{}/{}", home, ".hikyuu/hikyuu.ini");
#endif
}

StrategyBase::StrategyBase(const string& name, const string& config_file)
: m_name(name), m_config_file(config_file) {}

StrategyBase::~StrategyBase() {
    HKU_INFO("[Strategy {}] Quit Strategy!", m_name);
}

void StrategyBase::run() {
    HKU_INFO("[Strategy {}] strategy is running! You can press Ctrl-C to terminte ...", m_name);

    // 注册 ctrl-c 终止信号
    std::signal(SIGINT, sig_handler);

    // 调用 strategy 自身的初始化方法
    init();

    // 加载上下文指定的证券数据
    IniParser config;
    try {
        config.read(m_config_file);

    } catch (std::exception& e) {
        HKU_FATAL("[Strategy {}] Failed read configure file (\"{}\")! {}", m_name, m_config_file,
                  e.what());
        HKU_INFO("[Strategy {}] Exit Strategy", m_name);
        exit(1);
    } catch (...) {
        HKU_FATAL("[Strategy {}] Failed read configure file (\"{}\")! Unknow error!", m_name,
                  m_config_file);
        HKU_INFO("[Strategy {}] Exit Strategy", m_name);
        exit(1);
    }

    Parameter baseParam, blockParam, kdataParam, preloadParam, hkuParam;

    hkuParam.set<string>("tmpdir", config.get("hikyuu", "tmpdir", "."));
    hkuParam.set<string>("datadir", config.get("hikyuu", "datadir", "."));

    if (!config.hasSection("baseinfo")) {
        HKU_FATAL("Missing configure of baseinfo!");
        exit(1);
    }

    IniParser::StringListPtr option = config.getOptionList("baseinfo");
    for (auto iter = option->begin(); iter != option->end(); ++iter) {
        string value = config.get("baseinfo", *iter);
        baseParam.set<string>(*iter, value);
    }

    IniParser::StringListPtr block_config = config.getOptionList("block");
    for (auto iter = block_config->begin(); iter != block_config->end(); ++iter) {
        string value = config.get("block", *iter);
        blockParam.set<string>(*iter, value);
    }

    option = config.getOptionList("kdata");
    for (auto iter = option->begin(); iter != option->end(); ++iter) {
        kdataParam.set<string>(*iter, config.get("kdata", *iter));
    }

    // 设置预加载参数，只加载指定的 ktype 至内存
    auto ktype_list = m_context.getKTypeList();
    if (ktype_list.empty()) {
        // 如果为空，则默认加载日线数据
        ktype_list.push_back(KQuery::DAY);
    }

    for (auto ktype : ktype_list) {
        to_lower(ktype);
        preloadParam.set<bool>(ktype, true);
        string key(format("{}_max", ktype));
        try {
            preloadParam.set<int>(key, config.getInt("preload", key));
        } catch (...) {
            preloadParam.set<int>(key, 4096);
        }
    }

    StockManager& sm = StockManager::instance();
    sm.init(baseParam, blockParam, kdataParam, preloadParam, hkuParam, m_context);

    const auto& stk_code_list = getStockCodeList();
    m_stock_list.reserve(stk_code_list.size());
    for (const auto& code : stk_code_list) {
        Stock stk = getStock(code);
        if (!stk.isNull()) {
            m_stock_list.push_back(stk);
        } else {
            HKU_WARN("[Strategy {}] Invalid code: {}, can't find the stock!", m_name, code);
        }
    }
    HKU_WARN_IF(m_stock_list.empty(), "[Strategy {}] stock list is empty!", m_name);

    if (m_stock_list.size() > 0) {
        Stock& ref_stk = m_stock_list[0];
        for (auto& ktype : ktype_list) {
            size_t count = ref_stk.getCount(ktype);
            if (count > 0) {
                KRecord k = ref_stk.getKRecord(count - 1, ktype);
                m_ref_last_time[ktype] = k.datetime;
            } else {
                m_ref_last_time[ktype] = Null<Datetime>();
            }
        }
    }

    // 启动行情接收代理
    auto& agent = *getGlobalSpotAgent();
    agent.addProcess([this](const SpotRecord& spot) { this->receivedSpot(spot); });
    agent.addPostProcess([this](Datetime revTime) { this->finishReceivedSpot(revTime); });
    startSpotAgent(false);

    _startEventLoop();
}

void StrategyBase::receivedSpot(const SpotRecord& spot) {
    Stock stk = getStock(format("{}{}", spot.market, spot.code));
    if (!stk.isNull()) {
        m_spot_map[stk] = spot;
    }
}

void StrategyBase::finishReceivedSpot(Datetime revTime) {
    HKU_IF_RETURN(m_stock_list.empty(), void());
    event([this]() { this->onTick(); });

    Stock& ref_stk = m_stock_list[0];
    const auto& ktype_list = getKTypeList();
    for (const auto& ktype : ktype_list) {
        size_t count = ref_stk.getCount(ktype);
        if (count > 0) {
            KRecord k = ref_stk.getKRecord(count - 1, ktype);
            if (k.datetime != m_ref_last_time[ktype]) {
                m_ref_last_time[ktype] = k.datetime;
                event([this, ktype]() { this->onBar(ktype); });
            }
        }
    }
}

/*
 * 在主线程中处理事件队列，避免 python GIL
 */
void StrategyBase::_startEventLoop() {
    while (ms_keep_running) {
        event_type task;
        m_event_queue.wait_and_pop(task);
        if (task.isNullTask()) {
            ms_keep_running = false;
        } else {
            task();
        }
    }
}

}  // namespace hku<commit_msg>fixed Strategy run<commit_after>/*
 *  Copyright(C) 2021 hikyuu.org
 *
 *  Create on: 2021-02-16
 *     Author: fasiondog
 */

#include <csignal>
#include <unordered_set>
#include "../utilities/os.h"
#include "../utilities/IniParser.h"
#include "../global/schedule/scheduler.h"
#include "StrategyBase.h"

namespace hku {

std::atomic_bool StrategyBase::ms_keep_running = true;

void StrategyBase::sig_handler(int sig) {
    if (sig == SIGINT) {
        ms_keep_running = false;
    }
}

StrategyBase::StrategyBase() : StrategyBase("Strategy") {}

StrategyBase::StrategyBase(const string& name) {
    string home = getUserHome();
    HKU_ERROR_IF(home == "", "Failed get user home path!");
#if HKU_OS_WINOWS
    m_config_file = format("{}\\{}", home, ".hikyuu\\hikyuu.ini");
#else
    m_config_file = format("{}/{}", home, ".hikyuu/hikyuu.ini");
#endif
}

StrategyBase::StrategyBase(const string& name, const string& config_file)
: m_name(name), m_config_file(config_file) {}

StrategyBase::~StrategyBase() {
    HKU_INFO("[Strategy {}] Quit Strategy!", m_name);
}

void StrategyBase::run() {
    HKU_INFO("[Strategy {}] strategy is running! You can press Ctrl-C to terminte ...", m_name);

    // 注册 ctrl-c 终止信号
    std::signal(SIGINT, sig_handler);

    // 调用 strategy 自身的初始化方法
    init();

    // 加载上下文指定的证券数据
    IniParser config;
    try {
        config.read(m_config_file);

    } catch (std::exception& e) {
        HKU_FATAL("[Strategy {}] Failed read configure file (\"{}\")! {}", m_name, m_config_file,
                  e.what());
        HKU_INFO("[Strategy {}] Exit Strategy", m_name);
        exit(1);
    } catch (...) {
        HKU_FATAL("[Strategy {}] Failed read configure file (\"{}\")! Unknow error!", m_name,
                  m_config_file);
        HKU_INFO("[Strategy {}] Exit Strategy", m_name);
        exit(1);
    }

    Parameter baseParam, blockParam, kdataParam, preloadParam, hkuParam;

    hkuParam.set<string>("tmpdir", config.get("hikyuu", "tmpdir", "."));
    hkuParam.set<string>("datadir", config.get("hikyuu", "datadir", "."));

    if (!config.hasSection("baseinfo")) {
        HKU_FATAL("Missing configure of baseinfo!");
        exit(1);
    }

    IniParser::StringListPtr option = config.getOptionList("baseinfo");
    for (auto iter = option->begin(); iter != option->end(); ++iter) {
        string value = config.get("baseinfo", *iter);
        baseParam.set<string>(*iter, value);
    }

    IniParser::StringListPtr block_config = config.getOptionList("block");
    for (auto iter = block_config->begin(); iter != block_config->end(); ++iter) {
        string value = config.get("block", *iter);
        blockParam.set<string>(*iter, value);
    }

    option = config.getOptionList("kdata");
    for (auto iter = option->begin(); iter != option->end(); ++iter) {
        kdataParam.set<string>(*iter, config.get("kdata", *iter));
    }

    // 设置预加载参数，只加载指定的 ktype 至内存
    auto ktype_list = m_context.getKTypeList();
    if (ktype_list.empty()) {
        // 如果为空，则默认加载日线数据
        ktype_list.push_back(KQuery::DAY);
    }

    for (auto ktype : ktype_list) {
        to_lower(ktype);
        preloadParam.set<bool>(ktype, true);
        string key(format("{}_max", ktype));
        try {
            preloadParam.set<int>(key, config.getInt("preload", key));
        } catch (...) {
            preloadParam.set<int>(key, 4096);
        }
    }

    StockManager& sm = StockManager::instance();
    sm.init(baseParam, blockParam, kdataParam, preloadParam, hkuParam, m_context);

    const auto& stk_code_list = getStockCodeList();
    m_stock_list.reserve(stk_code_list.size());
    for (const auto& code : stk_code_list) {
        Stock stk = getStock(code);
        if (!stk.isNull()) {
            m_stock_list.push_back(stk);
        } else {
            HKU_WARN("[Strategy {}] Invalid code: {}, can't find the stock!", m_name, code);
        }
    }
    HKU_WARN_IF(m_stock_list.empty(), "[Strategy {}] stock list is empty!", m_name);

    if (m_stock_list.size() > 0) {
        Stock& ref_stk = m_stock_list[0];
        for (auto& ktype : ktype_list) {
            // 由于异步初始化，此处不用通过先getCount再getKRecord的方式获取最后的KRecord
            KRecordList klist = ref_stk.getKRecordList(KQueryByIndex(0, Null<int64_t>(), ktype));
            size_t count = klist.size();
            if (count > 0) {
                m_ref_last_time[ktype] = klist[count - 1].datetime;
            } else {
                m_ref_last_time[ktype] = Null<Datetime>();
            }
        }
    }

    // 启动行情接收代理
    auto& agent = *getGlobalSpotAgent();
    agent.addProcess([this](const SpotRecord& spot) { this->receivedSpot(spot); });
    agent.addPostProcess([this](Datetime revTime) { this->finishReceivedSpot(revTime); });
    startSpotAgent(false);

    _startEventLoop();
}

void StrategyBase::receivedSpot(const SpotRecord& spot) {
    Stock stk = getStock(format("{}{}", spot.market, spot.code));
    if (!stk.isNull()) {
        m_spot_map[stk] = spot;
    }
}

void StrategyBase::finishReceivedSpot(Datetime revTime) {
    HKU_IF_RETURN(m_stock_list.empty(), void());
    event([this]() { this->onTick(); });

    Stock& ref_stk = m_stock_list[0];
    const auto& ktype_list = getKTypeList();
    for (const auto& ktype : ktype_list) {
        size_t count = ref_stk.getCount(ktype);
        if (count > 0) {
            KRecord k = ref_stk.getKRecord(count - 1, ktype);
            if (k.datetime != m_ref_last_time[ktype]) {
                m_ref_last_time[ktype] = k.datetime;
                event([this, ktype]() { this->onBar(ktype); });
            }
        }
    }
}

/*
 * 在主线程中处理事件队列，避免 python GIL
 */
void StrategyBase::_startEventLoop() {
    while (ms_keep_running) {
        event_type task;
        m_event_queue.wait_and_pop(task);
        if (task.isNullTask()) {
            ms_keep_running = false;
        } else {
            task();
        }
    }
}

}  // namespace hku<|endoftext|>


<commit_before>#include <nonius/nonius.h++>

#include <vector>

#include <libprecisegc/details/allocators/bitmap_pool_chunk.hpp>
#include <libprecisegc/details/allocators/freelist_pool_chunk.hpp>
#include <libprecisegc/details/allocators/managed_pool_chunk.hpp>

#include "deoptimize.hpp"

using namespace precisegc::details;
using namespace precisegc::details::allocators;

static const size_t OBJ_SIZE = 32;

template <typename Chunk>
void chunk_allocate_benchmark(nonius::chronometer meter)
{
    size_t chunk_size = std::min((size_t) 64, Chunk::CHUNK_MAXSIZE);
    size_t chunks_cnt = (meter.runs() + chunk_size - 1) / chunk_size;
    size_t bytes_cnt = OBJ_SIZE * chunk_size;

    std::vector<std::unique_ptr<byte[]>> mem(chunks_cnt);
    std::unique_ptr<byte[]> chunks_storage(new byte[chunks_cnt * sizeof(Chunk)]);
    Chunk* chunks = (Chunk*) chunks_storage.get();

    for (size_t i = 0; i < chunks_cnt; ++i) {
        mem[i].reset(new byte[bytes_cnt]);
        new (chunks + i) Chunk(mem[i].get(), bytes_cnt, OBJ_SIZE);
    }

    meter.measure([&chunks, chunk_size] (size_t i) {
        return chunks[i / chunk_size].allocate(OBJ_SIZE);
    });
}

template <typename Chunk>
void chunk_deallocate_benchmark(nonius::chronometer meter)
{
    std::vector<typename Chunk::pointer_type> ps(meter.runs());

    size_t chunk_size = std::min((size_t) 64, Chunk::CHUNK_MAXSIZE);
    size_t chunks_cnt = (meter.runs() + chunk_size - 1) / chunk_size;
    size_t bytes_cnt = OBJ_SIZE * chunk_size;

    std::vector<std::unique_ptr<byte[]>> mem(chunks_cnt);
    std::unique_ptr<byte[]> chunks_storage(new byte[chunks_cnt * sizeof(Chunk)]);
    Chunk* chunks = (Chunk*) chunks_storage.get();

    for (size_t i = 0; i < chunks_cnt; ++i) {
        mem[i].reset(new byte[bytes_cnt]);
        new (chunks + i) Chunk(mem[i].get(), bytes_cnt, OBJ_SIZE);
        for (size_t j = 0; j < chunk_size; ++j) {
            size_t idx = i * chunk_size + j;
            if (idx >= meter.runs()) {
                break;
            }
            ps[idx] = chunks[i].allocate(OBJ_SIZE);
        }
    }

    meter.measure([&ps, &chunks, chunk_size] (size_t i) {
        chunks[i / chunk_size].deallocate(ps[i], OBJ_SIZE);
    });
}

NONIUS_BENCHMARK("pool_chunks.bitmap_pool_chunk.allocate", [](nonius::chronometer meter)
{
    chunk_allocate_benchmark<bitmap_pool_chunk>(meter);
});

NONIUS_BENCHMARK("pool_chunks.bitmap_pool_chunk.deallocate", [](nonius::chronometer meter)
{
    chunk_deallocate_benchmark<bitmap_pool_chunk>(meter);
});

NONIUS_BENCHMARK("pool_chunks.freelist_pool_chunk.allocate", [](nonius::chronometer meter)
{
    chunk_allocate_benchmark<freelist_pool_chunk>(meter);
});

NONIUS_BENCHMARK("pool_chunks.freelist_pool_chunk.deallocate", [](nonius::chronometer meter)
{
    chunk_deallocate_benchmark<freelist_pool_chunk>(meter);
});

NONIUS_BENCHMARK("pool_chunks.managed_pool_chunk.allocate", [](nonius::chronometer meter)
{
    chunk_allocate_benchmark<managed_pool_chunk>(meter);
});

NONIUS_BENCHMARK("pool_chunks.managed_pool_chunk.deallocate", [](nonius::chronometer meter)
{
    chunk_deallocate_benchmark<managed_pool_chunk>(meter);
});<commit_msg>fix benchmark<commit_after>#include <nonius/nonius.h++>

#include <vector>

#include <libprecisegc/details/allocators/bitmap_pool_chunk.hpp>
#include <libprecisegc/details/allocators/freelist_pool_chunk.hpp>
#include <libprecisegc/details/allocators/managed_pool_chunk.hpp>

#include "deoptimize.hpp"

using namespace precisegc::details;
using namespace precisegc::details::allocators;

static const size_t OBJ_SIZE = 32;

template <typename Chunk>
void chunk_allocate_benchmark(nonius::chronometer meter)
{
    size_t chunk_size = std::min((size_t) 64, (size_t) Chunk::CHUNK_MAXSIZE);
    size_t chunks_cnt = (meter.runs() + chunk_size - 1) / chunk_size;
    size_t bytes_cnt = OBJ_SIZE * chunk_size;

    std::vector<std::unique_ptr<byte[]>> mem(chunks_cnt);
    std::unique_ptr<byte[]> chunks_storage(new byte[chunks_cnt * sizeof(Chunk)]);
    Chunk* chunks = (Chunk*) chunks_storage.get();

    for (size_t i = 0; i < chunks_cnt; ++i) {
        mem[i].reset(new byte[bytes_cnt]);
        new (chunks + i) Chunk(mem[i].get(), bytes_cnt, OBJ_SIZE);
    }

    meter.measure([&chunks, chunk_size] (size_t i) {
        return chunks[i / chunk_size].allocate(OBJ_SIZE);
    });
}

template <typename Chunk>
void chunk_deallocate_benchmark(nonius::chronometer meter)
{
    std::vector<typename Chunk::pointer_type> ps(meter.runs());

    size_t chunk_size = std::min((size_t) 64, (size_t) Chunk::CHUNK_MAXSIZE);
    size_t chunks_cnt = (meter.runs() + chunk_size - 1) / chunk_size;
    size_t bytes_cnt = OBJ_SIZE * chunk_size;

    std::vector<std::unique_ptr<byte[]>> mem(chunks_cnt);
    std::unique_ptr<byte[]> chunks_storage(new byte[chunks_cnt * sizeof(Chunk)]);
    Chunk* chunks = (Chunk*) chunks_storage.get();

    for (size_t i = 0; i < chunks_cnt; ++i) {
        mem[i].reset(new byte[bytes_cnt]);
        new (chunks + i) Chunk(mem[i].get(), bytes_cnt, OBJ_SIZE);
        for (size_t j = 0; j < chunk_size; ++j) {
            size_t idx = i * chunk_size + j;
            if (idx >= meter.runs()) {
                break;
            }
            ps[idx] = chunks[i].allocate(OBJ_SIZE);
        }
    }

    meter.measure([&ps, &chunks, chunk_size] (size_t i) {
        chunks[i / chunk_size].deallocate(ps[i], OBJ_SIZE);
    });
}

NONIUS_BENCHMARK("pool_chunks.bitmap_pool_chunk.allocate", [](nonius::chronometer meter)
{
    chunk_allocate_benchmark<bitmap_pool_chunk>(meter);
});

NONIUS_BENCHMARK("pool_chunks.bitmap_pool_chunk.deallocate", [](nonius::chronometer meter)
{
    chunk_deallocate_benchmark<bitmap_pool_chunk>(meter);
});

NONIUS_BENCHMARK("pool_chunks.freelist_pool_chunk.allocate", [](nonius::chronometer meter)
{
    chunk_allocate_benchmark<freelist_pool_chunk>(meter);
});

NONIUS_BENCHMARK("pool_chunks.freelist_pool_chunk.deallocate", [](nonius::chronometer meter)
{
    chunk_deallocate_benchmark<freelist_pool_chunk>(meter);
});

NONIUS_BENCHMARK("pool_chunks.managed_pool_chunk.allocate", [](nonius::chronometer meter)
{
    chunk_allocate_benchmark<managed_pool_chunk>(meter);
});

NONIUS_BENCHMARK("pool_chunks.managed_pool_chunk.deallocate", [](nonius::chronometer meter)
{
    chunk_deallocate_benchmark<managed_pool_chunk>(meter);
});<|endoftext|>


<commit_before>//-----------------------------------------------------------------------bl-
//--------------------------------------------------------------------------
//
// GRINS - General Reacting Incompressible Navier-Stokes
//
// Copyright (C) 2010-2012 The PECOS Development Team
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the Version 2 GNU General
// Public License as published by the Free Software Foundation.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this library; if not, write to the Free Software
// Foundation, Inc. 51 Franklin Street, Fifth Floor, Boston, MA
// 02110-1301 USA
//
//-----------------------------------------------------------------------el-
//
// $Id$
//
//--------------------------------------------------------------------------
//--------------------------------------------------------------------------

#include "qoi_base.h"

namespace GRINS
{
  QoIBase::QoIBase()
    : DifferentiableQoI()
  {
    return;
  }

  QoIBase::~QoIBase()
  {
    return;
  }

  void QoIBase::init_qoi( std::vector<Number>& sys_qoi )
  {
    sys_qoi.resize(1, 0.0);
    return;
  }

  void QoIBase::parallel_op( std::vector<Number>& sys_qoi, std::vector<Number>& local_qoi,
			     const QoISet& qoi_indices )
  {
    libMesh::DifferentiableQoI::parallel_op( sys_qoi, local_qoi, qoi_indices );
    _qoi_cache = sys_qoi;
    return;
  }

  void QoIBase::output_qoi( std::ostream& out ) const
  {
    if( !_qoi_cache.empty() )
      {
	out << "========================================================================" << std::endl;

	for(  unsigned int i = 0; i < _qoi_cache.size(); i++ )
	  {
	    out << "QoI #" << i << " = " << std::setprecision(10) << _qoi_cache[i] << std::endl;
	  }

	out << "========================================================================" << std::endl;
      }

    return;
  }

  Number QoIBase::get_qoi( unsigned int qoi_index ) const
  {
    return _qoi_cache[qoi_index];
  }

} // namespace GRINS
<commit_msg>[GRINS]: Output QoI with more digits and in scientific notation.<commit_after>//-----------------------------------------------------------------------bl-
//--------------------------------------------------------------------------
//
// GRINS - General Reacting Incompressible Navier-Stokes
//
// Copyright (C) 2010-2012 The PECOS Development Team
//
// This program is free software; you can redistribute it and/or
// modify it under the terms of the Version 2 GNU General
// Public License as published by the Free Software Foundation.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
// General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this library; if not, write to the Free Software
// Foundation, Inc. 51 Franklin Street, Fifth Floor, Boston, MA
// 02110-1301 USA
//
//-----------------------------------------------------------------------el-
//
// $Id$
//
//--------------------------------------------------------------------------
//--------------------------------------------------------------------------

#include "qoi_base.h"

namespace GRINS
{
  QoIBase::QoIBase()
    : DifferentiableQoI()
  {
    return;
  }

  QoIBase::~QoIBase()
  {
    return;
  }

  void QoIBase::init_qoi( std::vector<Number>& sys_qoi )
  {
    sys_qoi.resize(1, 0.0);
    return;
  }

  void QoIBase::parallel_op( std::vector<Number>& sys_qoi, std::vector<Number>& local_qoi,
			     const QoISet& qoi_indices )
  {
    libMesh::DifferentiableQoI::parallel_op( sys_qoi, local_qoi, qoi_indices );
    _qoi_cache = sys_qoi;
    return;
  }

  void QoIBase::output_qoi( std::ostream& out ) const
  {
    if( !_qoi_cache.empty() )
      {
	out << "========================================================================" << std::endl;

	for(  unsigned int i = 0; i < _qoi_cache.size(); i++ )
	  {
	    out << "QoI #" << i << " = "
		<< std::setprecision(16)
		<< std::scientific
		<< _qoi_cache[i] << std::endl;
	  }

	out << "========================================================================" << std::endl;
      }

    return;
  }

  Number QoIBase::get_qoi( unsigned int qoi_index ) const
  {
    return _qoi_cache[qoi_index];
  }

} // namespace GRINS
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "iree/tools/vm_util.h"

#include <ostream>

#include "absl/strings/numbers.h"
#include "absl/strings/string_view.h"
#include "absl/strings/strip.h"
#include "absl/types/span.h"
#include "iree/base/api_util.h"
#include "iree/base/buffer_string_util.h"
#include "iree/base/shape.h"
#include "iree/base/shaped_buffer.h"
#include "iree/base/shaped_buffer_string_util.h"
#include "iree/base/signature_mangle.h"
#include "iree/base/status.h"
#include "iree/modules/hal/hal_module.h"
#include "iree/vm/bytecode_module.h"
#include "iree/vm/module.h"
#include "iree/vm/variant_list.h"

namespace iree {

Status ValidateFunctionAbi(const iree_vm_function_t& function) {
  iree_string_view_t sig_fv =
      iree_vm_function_reflection_attr(&function, iree_make_cstring_view("fv"));
  if (absl::string_view{sig_fv.data, sig_fv.size} != "1") {
    return iree::UnimplementedErrorBuilder(IREE_LOC)
           << "Unsupported function ABI";
  }
  return OkStatus();
}

StatusOr<std::vector<RawSignatureParser::Description>> ParseInputSignature(
    iree_vm_function_t& function) {
  iree_string_view_t sig_f =
      iree_vm_function_reflection_attr(&function, iree_make_cstring_view("f"));
  RawSignatureParser sig_parser;
  std::vector<RawSignatureParser::Description> input_descs;
  sig_parser.VisitInputs(absl::string_view{sig_f.data, sig_f.size},
                         [&](const RawSignatureParser::Description& desc) {
                           input_descs.push_back(desc);
                         });
  if (sig_parser.GetError()) {
    return InvalidArgumentErrorBuilder(IREE_LOC)
           << "Parsing function signature '" << sig_f.data
           << "' failed getting input";
  }
  return input_descs;
}

StatusOr<std::vector<RawSignatureParser::Description>> ParseOutputSignature(
    const iree_vm_function_t& function) {
  iree_string_view_t sig_f =
      iree_vm_function_reflection_attr(&function, iree_make_cstring_view("f"));
  RawSignatureParser sig_parser;
  std::vector<RawSignatureParser::Description> output_descs;
  sig_parser.VisitResults(absl::string_view{sig_f.data, sig_f.size},
                          [&](const RawSignatureParser::Description& desc) {
                            output_descs.push_back(desc);
                          });
  if (sig_parser.GetError()) {
    return InvalidArgumentErrorBuilder(IREE_LOC)
           << "Parsing function signature '" << sig_f.data
           << "' failed getting results";
  }
  return output_descs;
}

StatusOr<iree_vm_variant_list_t*> ParseToVariantList(
    absl::Span<const RawSignatureParser::Description> descs,
    iree_hal_allocator_t* allocator,
    absl::Span<const std::string> input_strings) {
  if (input_strings.size() != descs.size()) {
    return FailedPreconditionErrorBuilder(IREE_LOC)
           << "Signature mismatch; expected " << descs.size()
           << " buffer strings but received " << input_strings.size();
  }
  iree_vm_variant_list_t* variant_list = nullptr;
  RETURN_IF_ERROR(FromApiStatus(
      iree_vm_variant_list_alloc(input_strings.size(), IREE_ALLOCATOR_SYSTEM,
                                 &variant_list),
      IREE_LOC));
  for (int i = 0; i < input_strings.size(); ++i) {
    auto input_string = input_strings[i];
    auto desc = descs[i];
    std::string desc_str;
    desc.ToString(desc_str);
    switch (desc.type) {
      case RawSignatureParser::Type::kScalar: {
        if (desc.scalar.type != AbiConstants::ScalarType::kSint32) {
          return UnimplementedErrorBuilder(IREE_LOC)
                 << "Unsupported signature scalar type: " << desc_str;
        }
        absl::string_view input_view = absl::StripAsciiWhitespace(input_string);
        input_view = absl::StripPrefix(input_view, "\"");
        input_view = absl::StripSuffix(input_view, "\"");
        if (!absl::ConsumePrefix(&input_view, "i32=")) {
          return InvalidArgumentErrorBuilder(IREE_LOC)
                 << "Parsing '" << input_string
                 << "'. Has i32 descriptor but does not start with 'i32='";
        }
        int32_t val;
        if (!absl::SimpleAtoi(input_view, &val)) {
          return InvalidArgumentErrorBuilder(IREE_LOC)
                 << "Converting '" << input_view << "' to i32 when parsing '"
                 << input_string << "'";
        }
        iree_vm_variant_list_append_value(variant_list,
                                          IREE_VM_VALUE_MAKE_I32(val));
        break;
      }
      case RawSignatureParser::Type::kBuffer: {
        ASSIGN_OR_RETURN(auto shaped_buffer,
                         ParseShapedBufferFromString(input_string),
                         _ << "Parsing value '" << input_string << "'");
        iree_hal_buffer_t* buf = nullptr;
        // TODO(benvanik): combined function for linear to optimal upload.
        iree_device_size_t allocation_size =
            shaped_buffer.shape().element_count() *
            shaped_buffer.element_size();
        RETURN_IF_ERROR(FromApiStatus(
            iree_hal_allocator_allocate_buffer(
                allocator,
                static_cast<iree_hal_memory_type_t>(
                    IREE_HAL_MEMORY_TYPE_HOST_LOCAL |
                    IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE),
                static_cast<iree_hal_buffer_usage_t>(
                    IREE_HAL_BUFFER_USAGE_ALL | IREE_HAL_BUFFER_USAGE_CONSTANT),
                allocation_size, &buf),
            IREE_LOC))
            << "Allocating buffer";
        RETURN_IF_ERROR(FromApiStatus(
            iree_hal_buffer_write_data(buf, 0, shaped_buffer.contents().data(),
                                       shaped_buffer.contents().size()),
            IREE_LOC))
            << "Populating buffer contents ";
        auto buf_ref = iree_hal_buffer_move_ref(buf);
        RETURN_IF_ERROR(FromApiStatus(
            iree_vm_variant_list_append_ref_move(variant_list, &buf_ref),
            IREE_LOC));
        break;
      }
      default:
        return UnimplementedErrorBuilder(IREE_LOC)
               << "Unsupported signature type: " << desc_str;
    }
  }
  return variant_list;
}

Status PrintVariantList(absl::Span<const RawSignatureParser::Description> descs,
                        iree_vm_variant_list_t* variant_list,
                        std::ostream* os) {
  for (int i = 0; i < iree_vm_variant_list_size(variant_list); ++i) {
    iree_vm_variant_t* variant = iree_vm_variant_list_get(variant_list, i);
    if (!variant) {
      return InvalidArgumentErrorBuilder(IREE_LOC)
             << "variant " << i << "not present";
    }

    const auto& desc = descs[i];
    std::string desc_str;
    desc.ToString(desc_str);
    LOG(INFO) << "result[" << i << "]: " << desc_str;

    switch (desc.type) {
      case RawSignatureParser::Type::kScalar: {
        if (variant->value_type != IREE_VM_VALUE_TYPE_I32) {
          return InvalidArgumentErrorBuilder(IREE_LOC)
                 << "variant " << i << " has value type "
                 << static_cast<int>(variant->value_type)
                 << " but descriptor information " << desc_str;
        }
        if (desc.scalar.type != AbiConstants::ScalarType::kSint32) {
          return UnimplementedErrorBuilder(IREE_LOC)
                 << "Unsupported signature scalar type: " << desc_str;
        }
        *os << "i32=" << variant->i32 << "\n";
        break;
      }
      case RawSignatureParser::Type::kBuffer: {
        if (variant->value_type != IREE_VM_VALUE_TYPE_NONE) {
          return InvalidArgumentErrorBuilder(IREE_LOC)
                 << "variant " << i << " has value type "
                 << static_cast<int>(variant->value_type)
                 << " but descriptor information " << desc_str;
        }
        auto* buffer = iree_hal_buffer_deref(&variant->ref);
        if (!buffer) {
          return InvalidArgumentErrorBuilder(IREE_LOC)
                 << "failed dereferencing variant " << i;
        }

        auto print_mode = BufferDataPrintMode::kFloatingPoint;
        int8_t element_size = 4;
        Shape shape;

        switch (desc.buffer.scalar_type) {
          case AbiConstants::ScalarType::kIeeeFloat16:
          case AbiConstants::ScalarType::kIeeeFloat32:
          case AbiConstants::ScalarType::kIeeeFloat64:
            print_mode = BufferDataPrintMode::kFloatingPoint;
            break;
          case AbiConstants::ScalarType::kSint8:
          case AbiConstants::ScalarType::kSint16:
          case AbiConstants::ScalarType::kSint32:
          case AbiConstants::ScalarType::kSint64:
            print_mode = BufferDataPrintMode::kSignedInteger;
            break;
          case AbiConstants::ScalarType::kUint8:
          case AbiConstants::ScalarType::kUint16:
          case AbiConstants::ScalarType::kUint32:
          case AbiConstants::ScalarType::kUint64:
            print_mode = BufferDataPrintMode::kUnsignedInteger;
            break;
          default:
            print_mode = BufferDataPrintMode::kBinary;
            break;
        }
        element_size = AbiConstants::kScalarTypeSize[static_cast<unsigned>(
            desc.buffer.scalar_type)];
        shape = Shape{desc.dims};

        iree_hal_mapped_memory_t mapped_memory;
        RETURN_IF_ERROR(FromApiStatus(
            iree_hal_buffer_map(buffer, IREE_HAL_MEMORY_ACCESS_READ, 0,
                                IREE_WHOLE_BUFFER, &mapped_memory),
            IREE_LOC))
            << "mapping hal buffer";
        auto contents = absl::MakeConstSpan(mapped_memory.contents.data,
                                            mapped_memory.contents.data_length);
        ShapedBuffer shaped_buffer(
            element_size, shape,
            std::vector<uint8_t>(contents.begin(), contents.end()));
        ASSIGN_OR_RETURN(auto result_str, PrintShapedBufferToString(
                                              shaped_buffer, print_mode, 1024));
        iree_hal_buffer_unmap(buffer, &mapped_memory);
        *os << result_str << "\n";
        break;
      }
      default:
        return UnimplementedErrorBuilder(IREE_LOC)
               << "Unsupported signature type: " << desc_str;
    }
  }

  return OkStatus();
}

Status CreateDevice(absl::string_view driver_name,
                    iree_hal_device_t** out_device) {
  LOG(INFO) << "Creating driver and device for '" << driver_name << "'...";
  iree_hal_driver_t* driver = nullptr;
  RETURN_IF_ERROR(FromApiStatus(
      iree_hal_driver_registry_create_driver(
          iree_string_view_t{driver_name.data(), driver_name.size()},
          IREE_ALLOCATOR_SYSTEM, &driver),
      IREE_LOC))
      << "Creating driver '" << driver_name << "'";
  RETURN_IF_ERROR(FromApiStatus(iree_hal_driver_create_default_device(
                                    driver, IREE_ALLOCATOR_SYSTEM, out_device),
                                IREE_LOC))
      << "Creating default device for driver '" << driver_name << "'";
  RETURN_IF_ERROR(FromApiStatus(iree_hal_driver_release(driver), IREE_LOC))
      << "Releasing driver '" << driver_name << "'";
  return OkStatus();
}

Status CreateHalModule(iree_hal_device_t* device,
                       iree_vm_module_t** out_module) {
  RETURN_IF_ERROR(FromApiStatus(
      iree_hal_module_create(device, IREE_ALLOCATOR_SYSTEM, out_module),
      IREE_LOC))
      << "Creating HAL module";
  return OkStatus();
}

Status LoadBytecodeModule(absl::string_view module_data,
                          iree_vm_module_t** out_module) {
  RETURN_IF_ERROR(FromApiStatus(
      iree_vm_bytecode_module_create(
          iree_const_byte_span_t{
              reinterpret_cast<const uint8_t*>(module_data.data()),
              module_data.size()},
          IREE_ALLOCATOR_NULL, IREE_ALLOCATOR_SYSTEM, out_module),
      IREE_LOC))
      << "Deserializing module";
  return OkStatus();
}
}  // namespace iree
<commit_msg>Switch error code for bad input to InvalidArgument<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "iree/tools/vm_util.h"

#include <ostream>

#include "absl/strings/numbers.h"
#include "absl/strings/string_view.h"
#include "absl/strings/strip.h"
#include "absl/types/span.h"
#include "iree/base/api_util.h"
#include "iree/base/buffer_string_util.h"
#include "iree/base/shape.h"
#include "iree/base/shaped_buffer.h"
#include "iree/base/shaped_buffer_string_util.h"
#include "iree/base/signature_mangle.h"
#include "iree/base/status.h"
#include "iree/modules/hal/hal_module.h"
#include "iree/vm/bytecode_module.h"
#include "iree/vm/module.h"
#include "iree/vm/variant_list.h"

namespace iree {

Status ValidateFunctionAbi(const iree_vm_function_t& function) {
  iree_string_view_t sig_fv =
      iree_vm_function_reflection_attr(&function, iree_make_cstring_view("fv"));
  if (absl::string_view{sig_fv.data, sig_fv.size} != "1") {
    return iree::UnimplementedErrorBuilder(IREE_LOC)
           << "Unsupported function ABI";
  }
  return OkStatus();
}

StatusOr<std::vector<RawSignatureParser::Description>> ParseInputSignature(
    iree_vm_function_t& function) {
  iree_string_view_t sig_f =
      iree_vm_function_reflection_attr(&function, iree_make_cstring_view("f"));
  RawSignatureParser sig_parser;
  std::vector<RawSignatureParser::Description> input_descs;
  sig_parser.VisitInputs(absl::string_view{sig_f.data, sig_f.size},
                         [&](const RawSignatureParser::Description& desc) {
                           input_descs.push_back(desc);
                         });
  if (sig_parser.GetError()) {
    return InvalidArgumentErrorBuilder(IREE_LOC)
           << "Parsing function signature '" << sig_f.data
           << "' failed getting input";
  }
  return input_descs;
}

StatusOr<std::vector<RawSignatureParser::Description>> ParseOutputSignature(
    const iree_vm_function_t& function) {
  iree_string_view_t sig_f =
      iree_vm_function_reflection_attr(&function, iree_make_cstring_view("f"));
  RawSignatureParser sig_parser;
  std::vector<RawSignatureParser::Description> output_descs;
  sig_parser.VisitResults(absl::string_view{sig_f.data, sig_f.size},
                          [&](const RawSignatureParser::Description& desc) {
                            output_descs.push_back(desc);
                          });
  if (sig_parser.GetError()) {
    return InvalidArgumentErrorBuilder(IREE_LOC)
           << "Parsing function signature '" << sig_f.data
           << "' failed getting results";
  }
  return output_descs;
}

StatusOr<iree_vm_variant_list_t*> ParseToVariantList(
    absl::Span<const RawSignatureParser::Description> descs,
    iree_hal_allocator_t* allocator,
    absl::Span<const std::string> input_strings) {
  if (input_strings.size() != descs.size()) {
    return InvalidArgumentErrorBuilder(IREE_LOC)
           << "Signature mismatch; expected " << descs.size()
           << " buffer strings but received " << input_strings.size();
  }
  iree_vm_variant_list_t* variant_list = nullptr;
  RETURN_IF_ERROR(FromApiStatus(
      iree_vm_variant_list_alloc(input_strings.size(), IREE_ALLOCATOR_SYSTEM,
                                 &variant_list),
      IREE_LOC));
  for (int i = 0; i < input_strings.size(); ++i) {
    auto input_string = input_strings[i];
    auto desc = descs[i];
    std::string desc_str;
    desc.ToString(desc_str);
    switch (desc.type) {
      case RawSignatureParser::Type::kScalar: {
        if (desc.scalar.type != AbiConstants::ScalarType::kSint32) {
          return UnimplementedErrorBuilder(IREE_LOC)
                 << "Unsupported signature scalar type: " << desc_str;
        }
        absl::string_view input_view = absl::StripAsciiWhitespace(input_string);
        input_view = absl::StripPrefix(input_view, "\"");
        input_view = absl::StripSuffix(input_view, "\"");
        if (!absl::ConsumePrefix(&input_view, "i32=")) {
          return InvalidArgumentErrorBuilder(IREE_LOC)
                 << "Parsing '" << input_string
                 << "'. Has i32 descriptor but does not start with 'i32='";
        }
        int32_t val;
        if (!absl::SimpleAtoi(input_view, &val)) {
          return InvalidArgumentErrorBuilder(IREE_LOC)
                 << "Converting '" << input_view << "' to i32 when parsing '"
                 << input_string << "'";
        }
        iree_vm_variant_list_append_value(variant_list,
                                          IREE_VM_VALUE_MAKE_I32(val));
        break;
      }
      case RawSignatureParser::Type::kBuffer: {
        ASSIGN_OR_RETURN(auto shaped_buffer,
                         ParseShapedBufferFromString(input_string),
                         _ << "Parsing value '" << input_string << "'");
        iree_hal_buffer_t* buf = nullptr;
        // TODO(benvanik): combined function for linear to optimal upload.
        iree_device_size_t allocation_size =
            shaped_buffer.shape().element_count() *
            shaped_buffer.element_size();
        RETURN_IF_ERROR(FromApiStatus(
            iree_hal_allocator_allocate_buffer(
                allocator,
                static_cast<iree_hal_memory_type_t>(
                    IREE_HAL_MEMORY_TYPE_HOST_LOCAL |
                    IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE),
                static_cast<iree_hal_buffer_usage_t>(
                    IREE_HAL_BUFFER_USAGE_ALL | IREE_HAL_BUFFER_USAGE_CONSTANT),
                allocation_size, &buf),
            IREE_LOC))
            << "Allocating buffer";
        RETURN_IF_ERROR(FromApiStatus(
            iree_hal_buffer_write_data(buf, 0, shaped_buffer.contents().data(),
                                       shaped_buffer.contents().size()),
            IREE_LOC))
            << "Populating buffer contents ";
        auto buf_ref = iree_hal_buffer_move_ref(buf);
        RETURN_IF_ERROR(FromApiStatus(
            iree_vm_variant_list_append_ref_move(variant_list, &buf_ref),
            IREE_LOC));
        break;
      }
      default:
        return UnimplementedErrorBuilder(IREE_LOC)
               << "Unsupported signature type: " << desc_str;
    }
  }
  return variant_list;
}

Status PrintVariantList(absl::Span<const RawSignatureParser::Description> descs,
                        iree_vm_variant_list_t* variant_list,
                        std::ostream* os) {
  for (int i = 0; i < iree_vm_variant_list_size(variant_list); ++i) {
    iree_vm_variant_t* variant = iree_vm_variant_list_get(variant_list, i);
    if (!variant) {
      return InvalidArgumentErrorBuilder(IREE_LOC)
             << "variant " << i << "not present";
    }

    const auto& desc = descs[i];
    std::string desc_str;
    desc.ToString(desc_str);
    LOG(INFO) << "result[" << i << "]: " << desc_str;

    switch (desc.type) {
      case RawSignatureParser::Type::kScalar: {
        if (variant->value_type != IREE_VM_VALUE_TYPE_I32) {
          return InvalidArgumentErrorBuilder(IREE_LOC)
                 << "variant " << i << " has value type "
                 << static_cast<int>(variant->value_type)
                 << " but descriptor information " << desc_str;
        }
        if (desc.scalar.type != AbiConstants::ScalarType::kSint32) {
          return UnimplementedErrorBuilder(IREE_LOC)
                 << "Unsupported signature scalar type: " << desc_str;
        }
        *os << "i32=" << variant->i32 << "\n";
        break;
      }
      case RawSignatureParser::Type::kBuffer: {
        if (variant->value_type != IREE_VM_VALUE_TYPE_NONE) {
          return InvalidArgumentErrorBuilder(IREE_LOC)
                 << "variant " << i << " has value type "
                 << static_cast<int>(variant->value_type)
                 << " but descriptor information " << desc_str;
        }
        auto* buffer = iree_hal_buffer_deref(&variant->ref);
        if (!buffer) {
          return InvalidArgumentErrorBuilder(IREE_LOC)
                 << "failed dereferencing variant " << i;
        }

        auto print_mode = BufferDataPrintMode::kFloatingPoint;
        int8_t element_size = 4;
        Shape shape;

        switch (desc.buffer.scalar_type) {
          case AbiConstants::ScalarType::kIeeeFloat16:
          case AbiConstants::ScalarType::kIeeeFloat32:
          case AbiConstants::ScalarType::kIeeeFloat64:
            print_mode = BufferDataPrintMode::kFloatingPoint;
            break;
          case AbiConstants::ScalarType::kSint8:
          case AbiConstants::ScalarType::kSint16:
          case AbiConstants::ScalarType::kSint32:
          case AbiConstants::ScalarType::kSint64:
            print_mode = BufferDataPrintMode::kSignedInteger;
            break;
          case AbiConstants::ScalarType::kUint8:
          case AbiConstants::ScalarType::kUint16:
          case AbiConstants::ScalarType::kUint32:
          case AbiConstants::ScalarType::kUint64:
            print_mode = BufferDataPrintMode::kUnsignedInteger;
            break;
          default:
            print_mode = BufferDataPrintMode::kBinary;
            break;
        }
        element_size = AbiConstants::kScalarTypeSize[static_cast<unsigned>(
            desc.buffer.scalar_type)];
        shape = Shape{desc.dims};

        iree_hal_mapped_memory_t mapped_memory;
        RETURN_IF_ERROR(FromApiStatus(
            iree_hal_buffer_map(buffer, IREE_HAL_MEMORY_ACCESS_READ, 0,
                                IREE_WHOLE_BUFFER, &mapped_memory),
            IREE_LOC))
            << "mapping hal buffer";
        auto contents = absl::MakeConstSpan(mapped_memory.contents.data,
                                            mapped_memory.contents.data_length);
        ShapedBuffer shaped_buffer(
            element_size, shape,
            std::vector<uint8_t>(contents.begin(), contents.end()));
        ASSIGN_OR_RETURN(auto result_str, PrintShapedBufferToString(
                                              shaped_buffer, print_mode, 1024));
        iree_hal_buffer_unmap(buffer, &mapped_memory);
        *os << result_str << "\n";
        break;
      }
      default:
        return UnimplementedErrorBuilder(IREE_LOC)
               << "Unsupported signature type: " << desc_str;
    }
  }

  return OkStatus();
}

Status CreateDevice(absl::string_view driver_name,
                    iree_hal_device_t** out_device) {
  LOG(INFO) << "Creating driver and device for '" << driver_name << "'...";
  iree_hal_driver_t* driver = nullptr;
  RETURN_IF_ERROR(FromApiStatus(
      iree_hal_driver_registry_create_driver(
          iree_string_view_t{driver_name.data(), driver_name.size()},
          IREE_ALLOCATOR_SYSTEM, &driver),
      IREE_LOC))
      << "Creating driver '" << driver_name << "'";
  RETURN_IF_ERROR(FromApiStatus(iree_hal_driver_create_default_device(
                                    driver, IREE_ALLOCATOR_SYSTEM, out_device),
                                IREE_LOC))
      << "Creating default device for driver '" << driver_name << "'";
  RETURN_IF_ERROR(FromApiStatus(iree_hal_driver_release(driver), IREE_LOC))
      << "Releasing driver '" << driver_name << "'";
  return OkStatus();
}

Status CreateHalModule(iree_hal_device_t* device,
                       iree_vm_module_t** out_module) {
  RETURN_IF_ERROR(FromApiStatus(
      iree_hal_module_create(device, IREE_ALLOCATOR_SYSTEM, out_module),
      IREE_LOC))
      << "Creating HAL module";
  return OkStatus();
}

Status LoadBytecodeModule(absl::string_view module_data,
                          iree_vm_module_t** out_module) {
  RETURN_IF_ERROR(FromApiStatus(
      iree_vm_bytecode_module_create(
          iree_const_byte_span_t{
              reinterpret_cast<const uint8_t*>(module_data.data()),
              module_data.size()},
          IREE_ALLOCATOR_NULL, IREE_ALLOCATOR_SYSTEM, out_module),
      IREE_LOC))
      << "Deserializing module";
  return OkStatus();
}
}  // namespace iree
<|endoftext|>


<commit_before><commit_msg>remove some asserts<commit_after><|endoftext|>


<commit_before><commit_msg>Set enable bit if user requests control of frequency or power.<commit_after><|endoftext|>


<commit_before><commit_msg>Fix ARPA file loading in estimate-lm.<commit_after><|endoftext|>


<commit_before><commit_msg>Re-impelement macro for converting a time object to a string object<commit_after><|endoftext|>


<commit_before>/*
// $Id$
// Fennel is a library of data storage and processing components.
// Copyright (C) 2005-2005 The Eigenbase Project
// Copyright (C) 2005-2005 Disruptive Tech
// Copyright (C) 2005-2005 LucidEra, Inc.
// Portions Copyright (C) 2004-2005 John V. Sichi
//
// This program is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation; either version 2 of the License, or (at your option)
// any later version approved by The Eigenbase Project.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

#include "fennel/common/CommonPreamble.h"
#include "fennel/ftrs/BTreeSortExecStream.h"
#include "fennel/btree/BTreeBuilder.h"
#include "fennel/btree/BTreeWriter.h"
#include "fennel/tuple/TupleDescriptor.h"
#include "fennel/exec/ExecStreamBufAccessor.h"

FENNEL_BEGIN_CPPFILE("$Id$");

void BTreeSortExecStream::prepare(BTreeSortExecStreamParams const &params)
{
    assert(params.rootPageId == NULL_PAGE_ID);
    assert(!params.pRootMap);

    BTreeInsertExecStream::prepare(params);
}

// REVIEW:  do we ever want to save results on restart?
void BTreeSortExecStream::open(bool restart)
{
    sorted = false;
    if (restart) {
        truncateTree(false);
    } else {
        BTreeBuilder builder(
            treeDescriptor,
            treeDescriptor.segmentAccessor.pSegment);
        builder.createEmptyRoot();
        treeDescriptor.rootPageId = builder.getRootPageId();
    }

    // NOTE:  do this last so that rootPageId is available
    BTreeInsertExecStream::open(restart);
}

ExecStreamResult BTreeSortExecStream::execute(
    ExecStreamQuantum const &quantum)
{
    if (!sorted) {
        if (pInAccessor->getState() == EXECBUF_EOS) {
            sorted = true;
            bool found = pWriter->searchFirst();
            if (!found) {
                pWriter->endSearch();
            }
        } else {
            return BTreeInsertExecStream::execute(quantum);
        }
    }

    if (!pWriter->isPositioned()) {
        pOutAccessor->markEOS();
        return EXECRC_EOS;
    }

    if (pOutAccessor->getState() == EXECBUF_OVERFLOW) {
        return EXECRC_BUF_OVERFLOW;
    }

    uint nTuples = 0;
    TupleAccessor const &readAccessor = pWriter->getTupleAccessorForRead();

    do {
        uint cbBuffer = pOutAccessor->getProductionAvailable();
        PBuffer pBuffer = pOutAccessor->getProductionStart();
        uint cbTuple = readAccessor.getCurrentByteCount();
        if (cbBuffer < cbTuple) {
            pOutAccessor->requestConsumption();
            return EXECRC_BUF_OVERFLOW;
        }
        memcpy(
            pBuffer,
            readAccessor.getCurrentTupleBuf(),
            cbTuple);
        pOutAccessor->produceData(pBuffer + cbTuple);
        ++nTuples;
        if (!pWriter->searchNext()) {
            pWriter->endSearch();
        }
        if (nTuples >= quantum.nTuplesMax) {
            return EXECRC_QUANTUM_EXPIRED;
        }
    } while (pWriter->isPositioned());
    return EXECRC_EOS;
}

void BTreeSortExecStream::truncateTree(bool rootless)
{
    BTreeBuilder builder(
        treeDescriptor,
        treeDescriptor.segmentAccessor.pSegment);
    builder.truncate(rootless);
    if (rootless) {
        treeDescriptor.rootPageId = NULL_PAGE_ID;
    }
}

void BTreeSortExecStream::closeImpl()
{
    BTreeInsertExecStream::closeImpl();
    truncateTree(true);
}

FENNEL_END_CPPFILE("$Id$");

// End BTreeSortExecStream.cpp
<commit_msg>FENNEL:  fix FNL-50 (crash in BTreeSortExecStream if closed without ever having been opened)<commit_after>/*
// $Id$
// Fennel is a library of data storage and processing components.
// Copyright (C) 2005-2005 The Eigenbase Project
// Copyright (C) 2005-2005 Disruptive Tech
// Copyright (C) 2005-2005 LucidEra, Inc.
// Portions Copyright (C) 2004-2005 John V. Sichi
//
// This program is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License as published by the Free
// Software Foundation; either version 2 of the License, or (at your option)
// any later version approved by The Eigenbase Project.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*/

#include "fennel/common/CommonPreamble.h"
#include "fennel/ftrs/BTreeSortExecStream.h"
#include "fennel/btree/BTreeBuilder.h"
#include "fennel/btree/BTreeWriter.h"
#include "fennel/tuple/TupleDescriptor.h"
#include "fennel/exec/ExecStreamBufAccessor.h"

FENNEL_BEGIN_CPPFILE("$Id$");

void BTreeSortExecStream::prepare(BTreeSortExecStreamParams const &params)
{
    assert(params.rootPageId == NULL_PAGE_ID);
    assert(!params.pRootMap);

    BTreeInsertExecStream::prepare(params);
}

// REVIEW:  do we ever want to save results on restart?
void BTreeSortExecStream::open(bool restart)
{
    sorted = false;
    if (restart) {
        truncateTree(false);
    } else {
        BTreeBuilder builder(
            treeDescriptor,
            treeDescriptor.segmentAccessor.pSegment);
        builder.createEmptyRoot();
        treeDescriptor.rootPageId = builder.getRootPageId();
    }

    // NOTE:  do this last so that rootPageId is available
    BTreeInsertExecStream::open(restart);
}

ExecStreamResult BTreeSortExecStream::execute(
    ExecStreamQuantum const &quantum)
{
    if (!sorted) {
        if (pInAccessor->getState() == EXECBUF_EOS) {
            sorted = true;
            bool found = pWriter->searchFirst();
            if (!found) {
                pWriter->endSearch();
            }
        } else {
            return BTreeInsertExecStream::execute(quantum);
        }
    }

    if (!pWriter->isPositioned()) {
        pOutAccessor->markEOS();
        return EXECRC_EOS;
    }

    if (pOutAccessor->getState() == EXECBUF_OVERFLOW) {
        return EXECRC_BUF_OVERFLOW;
    }

    uint nTuples = 0;
    TupleAccessor const &readAccessor = pWriter->getTupleAccessorForRead();

    do {
        uint cbBuffer = pOutAccessor->getProductionAvailable();
        PBuffer pBuffer = pOutAccessor->getProductionStart();
        uint cbTuple = readAccessor.getCurrentByteCount();
        if (cbBuffer < cbTuple) {
            pOutAccessor->requestConsumption();
            return EXECRC_BUF_OVERFLOW;
        }
        memcpy(
            pBuffer,
            readAccessor.getCurrentTupleBuf(),
            cbTuple);
        pOutAccessor->produceData(pBuffer + cbTuple);
        ++nTuples;
        if (!pWriter->searchNext()) {
            pWriter->endSearch();
        }
        if (nTuples >= quantum.nTuplesMax) {
            return EXECRC_QUANTUM_EXPIRED;
        }
    } while (pWriter->isPositioned());
    return EXECRC_EOS;
}

void BTreeSortExecStream::truncateTree(bool rootless)
{
    if (treeDescriptor.rootPageId == NULL_PAGE_ID) {
        // nothing to do
        assert(!rootless);
        return;
    }
    BTreeBuilder builder(
        treeDescriptor,
        treeDescriptor.segmentAccessor.pSegment);
    builder.truncate(rootless);
    if (rootless) {
        treeDescriptor.rootPageId = NULL_PAGE_ID;
    }
}

void BTreeSortExecStream::closeImpl()
{
    BTreeInsertExecStream::closeImpl();
    truncateTree(true);
}

FENNEL_END_CPPFILE("$Id$");

// End BTreeSortExecStream.cpp
<|endoftext|>


<commit_before><commit_msg>increase size of nodes with children<commit_after><|endoftext|>


<commit_before>#define OVERRIDE_MATRIX_MULT true

#include <string.h>
#include <thread>
#include <mutex>
#include <future>
#include "../lib/matrix.hpp"
#include "../lib/pool.hpp"
#include "../lib/image.hpp"
#include "../lib/promise-polyfill.hpp"
#include "../lib/queue.hpp"

/* 1a */
matrix operator*(const matrix &x, const matrix &y) {
  if (x.cols != y.rows) {
    throw std::invalid_argument("Invalid arguments");
  }

  matrix z;
  z.create(x.rows, y.cols);

  //Flatten the loops so we can fairly chunk up the work (assume the
  //picture isn't the same width as it is height!)
  parallel_for(0u, x.rows * y.cols, [&](int k) {
    unsigned i = k / y.cols, j = k % y.cols;
    int zz = 0;
    for (unsigned k = 0; k < x.cols; k++) {
      zz += x(i, k) * y(k, j);
    }
    z(i, j) = zz;
  });
  return z;
}

/* 1b */
matrix histogram(const matrix &x) {
  std::mutex mutex;
  matrix h{256, 1};
  parallel_for(0u, x.rows * x.cols, [&](int k) {
    unsigned i = k / x.cols, j = k % x.cols;
    auto value = x(i, j);
    if (value < 0) {
      std::lock_guard<std::mutex> lock(mutex);
      h(0, 0)++;
    } else if (value >= 255) {
      std::lock_guard<std::mutex> lock(mutex);
      h(255, 0)++;
    } else {
      std::lock_guard<std::mutex> lock(mutex);
      h(value, 0)++;
    }
  });
  return h;
}

/*
 * 2. parallel_for uses the queue_work function instead of create_thread.
 * a. Why?
 * - Parallel for loops are typically used in data problems which require the
 * - computation of some value over a loop. As such, these are often short lived
 * - and cause blocks in control flow. To mitigate the delay and increase the
 * - benefit of using parallel_for, the work is pushed into the pool so that thread
 * - creation is not experienced (which takes time), additionally, since the
 * - computation is most likely computationally expensive, the pool will only
 * - schedule the work when threads are available instead of maxing out the system
 * - and causing system lag.
 *
 * b. Give an example of when should you use create_thread instead of queue_work
 * to execute work asynchronously?
 * - Create thread should always be used for work which is *long running* or blocking.
 * - Reason being is that long running work (or threads which will be alive the entire
 * - program life) will utilize all the threads provisioned to the pool and will cease
 * - execution of other work (e.g. a worker thread that works on a concurrent queue).
 * - Knowing this, simply spawn a new thread!
 */

/* 3 */
//Consider the following program
int mainEx() {
  int value = 32;
  queue_work([&] {
    value -= 17;
    printf("%d\n", value);
  });
  return value;
}

/*
* 3a.  What is the result of this program?  (Hint: its not always the same).
* - The result of this program varies depending on how quickly the queued
* - works starts in relation to the execution of the remaining code (pop stack
* - [removing queue_work return], push `value`, and return stack[0]/int).
*
* Possible results:
* 1. mainEx returns 32
* 2. mainEx returns 15
* 3. mainEx returns 15 and prints 15
*/

/* 3b - Correct the defect. */
int mainExFixed() {
  int value = 32;
  auto f = queue_work([&] {
    value -= 17;
    printf("%d\n", value);
  });
  f.wait();//Wait for the async work to complete so our value is consistent every time
  return value;
}

/* 4 completed */
// CONV - START
void conv(matrix &x, const matrix &k) {
  matrix y;
  y.create(x.rows + k.rows, x.cols + k.cols);

  const unsigned xR = x.rows, xC = x.cols;
  parallel_for(0u, xR * xC, [&](auto i) {
    auto row = i % xR, col = i / xR;
    auto yrow = row + k.rows / 2;
    auto ycol = col + k.cols / 2;
    y(yrow, ycol) = x(row, col);
  });

  std::atomic<int> weight(0);
  const unsigned kR = k.rows, kC = k.cols;
  parallel_for(0u, kR * kC, [&](int i) {
    auto row = i % kR, col = i / kR;
    weight += k(row, col);
  });

  parallel_for(0u, xR * xC, [&](int i) {
    auto row = i % xR, col = i / xR;
    int t = 0;
    auto yrow = row;
    for (int krow = k.rows - 1; krow >= 0; krow--, yrow++) {
      auto ycol = col;
      for (int kcol = k.cols - 1; kcol >= 0; kcol--, ycol++) {
        t += y(yrow, ycol) * k(krow, kcol);
      }
    }
    if (weight != 0) {
      t /= weight;
    }
    x(row, col) = t;
  });
}

int binomial_coefficient(int n, int k) {
  if (n <= 1 || k == 0) {
    return 1;
  } else {
    return binomial_coefficient(n - 1, k - 1) * n / k;
  }
}

matrix binomial(int n) {
  if ((n & 1) == 0) {
    throw std::invalid_argument("n must be odd");
  }

  matrix x, y;
  x.create(1, n);
  y.create(n, 1);

  for (int i = 0; i < n / 2; i++) {
    x(0, i) = x(0, n - i - 1) = binomial_coefficient(n - 1, i);
    y(i, 0) = y(n - i - 1, 0) = binomial_coefficient(n - 1, i);
  }

  x(0, n / 2) = binomial_coefficient(n - 1, n / 2);
  y(n / 2, 0) = binomial_coefficient(n - 1, n / 2);

  return y * x;
}
//CONV - END

struct pipeline_container {
  std::promise<matrix> p;
  matrix m;
};

concurrent_queue<pipeline_container> pipeline;

std::future<matrix> blur_image_async(matrix x) {
  pipeline_container c;
  c.m = std::move(x);
  pipeline.push(c);
  return c.p.get_future();
}

int main_q4() {
  auto pipeline_thread = std::thread([] {
    matrix kernel = binomial(3);
    for (;;) {
      auto s = pipeline.pop();
      conv(s.m, kernel);
      s.p.set_value(s.m);
    }
  });

  Promise::then(load_image_async("image.png").share(), [](auto f) {
    Promise::then(blur_image_async(std::move(f.get())), [](auto f) {
      matrix h = histogram(f.get());
    });
  });

  pipeline_thread.join();
  return 0;
}
/* 4 end */

/*
* 5a. How does a semaphore differ from mutex?
* - A semaphore differs from a mutex in that it is not a mutual exclusion lock,
* - a semaphore, unlike a mutex, can be acquired multiple times (depending on how
* - the semaphore is created) unlike a mutex which is strictly locked or unlocked.
* - Additionally, a semaphore lives in the file system and is not restricted to the
* - process in which it was created (like a mutex), which makes it useful to lock
* - access to things like shared memory pools across processes.
*
* 5b. Describe the operation of a bar (sitting down, ordering a drink) in terms
* of a semaphore and mutex.
* = Semaphore:
* - A bar has `n` number of seats, so a semaphore is constructed accordingly.
* - The semaphore (or a bar seat) can be acquired up to `n` times before parties
* - begin waiting for a seat at the bar. Meaning, each person can reserve a seat
* - by sitting (acquiring) it and then leaving (releasing) the seat, where then
* - the fastest person to react can acquire the released seat. This experiences
* - contention when the semaphore is fully acquired, but not at the same level as
* - a mutex.
*
* = Mutex:
* - The bar has a single seat and the bar tender refuses to serve anyone who is
* - not sitting at that seat. The seat can be locked (sat in) and then unlocked
* - (individual leaves). When the seat is unlocked, the fastest person to react
* - can then sit (lock) in the seat. This means that after the seat is taken,
* - every new person who wishes to have service must wait, which creates contention
* - in the form of a large blob of people (many threads waiting in the kernel on
* - the lock).
*/

int main(int argc, char **argv) {
  return 0;
}
<commit_msg>remove the move<commit_after>#define OVERRIDE_MATRIX_MULT true

#include <string.h>
#include <thread>
#include <mutex>
#include <future>
#include "../lib/matrix.hpp"
#include "../lib/pool.hpp"
#include "../lib/image.hpp"
#include "../lib/promise-polyfill.hpp"
#include "../lib/queue.hpp"

/* 1a */
matrix operator*(const matrix &x, const matrix &y) {
  if (x.cols != y.rows) {
    throw std::invalid_argument("Invalid arguments");
  }

  matrix z;
  z.create(x.rows, y.cols);

  //Flatten the loops so we can fairly chunk up the work (assume the
  //picture isn't the same width as it is height!)
  parallel_for(0u, x.rows * y.cols, [&](int k) {
    unsigned i = k / y.cols, j = k % y.cols;
    int zz = 0;
    for (unsigned k = 0; k < x.cols; k++) {
      zz += x(i, k) * y(k, j);
    }
    z(i, j) = zz;
  });
  return z;
}

/* 1b */
matrix histogram(const matrix &x) {
  std::mutex mutex;
  matrix h{256, 1};
  parallel_for(0u, x.rows * x.cols, [&](int k) {
    unsigned i = k / x.cols, j = k % x.cols;
    auto value = x(i, j);
    if (value < 0) {
      std::lock_guard<std::mutex> lock(mutex);
      h(0, 0)++;
    } else if (value >= 255) {
      std::lock_guard<std::mutex> lock(mutex);
      h(255, 0)++;
    } else {
      std::lock_guard<std::mutex> lock(mutex);
      h(value, 0)++;
    }
  });
  return h;
}

/*
 * 2. parallel_for uses the queue_work function instead of create_thread.
 * a. Why?
 * - Parallel for loops are typically used in data problems which require the
 * - computation of some value over a loop. As such, these are often short lived
 * - and cause blocks in control flow. To mitigate the delay and increase the
 * - benefit of using parallel_for, the work is pushed into the pool so that thread
 * - creation is not experienced (which takes time), additionally, since the
 * - computation is most likely computationally expensive, the pool will only
 * - schedule the work when threads are available instead of maxing out the system
 * - and causing system lag.
 *
 * b. Give an example of when should you use create_thread instead of queue_work
 * to execute work asynchronously?
 * - Create thread should always be used for work which is *long running* or blocking.
 * - Reason being is that long running work (or threads which will be alive the entire
 * - program life) will utilize all the threads provisioned to the pool and will cease
 * - execution of other work (e.g. a worker thread that works on a concurrent queue).
 * - Knowing this, simply spawn a new thread!
 */

/* 3 */
//Consider the following program
int mainEx() {
  int value = 32;
  queue_work([&] {
    value -= 17;
    printf("%d\n", value);
  });
  return value;
}

/*
* 3a.  What is the result of this program?  (Hint: its not always the same).
* - The result of this program varies depending on how quickly the queued
* - works starts in relation to the execution of the remaining code (pop stack
* - [removing queue_work return], push `value`, and return stack[0]/int).
*
* Possible results:
* 1. mainEx returns 32
* 2. mainEx returns 15
* 3. mainEx returns 15 and prints 15
*/

/* 3b - Correct the defect. */
int mainExFixed() {
  int value = 32;
  auto f = queue_work([&] {
    value -= 17;
    printf("%d\n", value);
  });
  f.wait();//Wait for the async work to complete so our value is consistent every time
  return value;
}

/* 4 completed */
// CONV - START
void conv(matrix &x, const matrix &k) {
  matrix y;
  y.create(x.rows + k.rows, x.cols + k.cols);

  const unsigned xR = x.rows, xC = x.cols;
  parallel_for(0u, xR * xC, [&](auto i) {
    auto row = i % xR, col = i / xR;
    auto yrow = row + k.rows / 2;
    auto ycol = col + k.cols / 2;
    y(yrow, ycol) = x(row, col);
  });

  std::atomic<int> weight(0);
  const unsigned kR = k.rows, kC = k.cols;
  parallel_for(0u, kR * kC, [&](int i) {
    auto row = i % kR, col = i / kR;
    weight += k(row, col);
  });

  parallel_for(0u, xR * xC, [&](int i) {
    auto row = i % xR, col = i / xR;
    int t = 0;
    auto yrow = row;
    for (int krow = k.rows - 1; krow >= 0; krow--, yrow++) {
      auto ycol = col;
      for (int kcol = k.cols - 1; kcol >= 0; kcol--, ycol++) {
        t += y(yrow, ycol) * k(krow, kcol);
      }
    }
    if (weight != 0) {
      t /= weight;
    }
    x(row, col) = t;
  });
}

int binomial_coefficient(int n, int k) {
  if (n <= 1 || k == 0) {
    return 1;
  } else {
    return binomial_coefficient(n - 1, k - 1) * n / k;
  }
}

matrix binomial(int n) {
  if ((n & 1) == 0) {
    throw std::invalid_argument("n must be odd");
  }

  matrix x, y;
  x.create(1, n);
  y.create(n, 1);

  for (int i = 0; i < n / 2; i++) {
    x(0, i) = x(0, n - i - 1) = binomial_coefficient(n - 1, i);
    y(i, 0) = y(n - i - 1, 0) = binomial_coefficient(n - 1, i);
  }

  x(0, n / 2) = binomial_coefficient(n - 1, n / 2);
  y(n / 2, 0) = binomial_coefficient(n - 1, n / 2);

  return y * x;
}
//CONV - END

struct pipeline_container {
  std::promise<matrix> p;
  matrix m;
};

concurrent_queue<pipeline_container> pipeline;

std::future<matrix> blur_image_async(matrix x) {
  pipeline_container c;
  c.m = std::move(x);
  pipeline.push(c);
  return c.p.get_future();
}

int main_q4() {
  auto pipeline_thread = std::thread([] {
    matrix kernel = binomial(3);
    for (;;) {
      auto s = pipeline.pop();
      conv(s.m, kernel);
      s.p.set_value(s.m);
    }
  });

  Promise::then(load_image_async("image.png").share(), [](auto f) {
    Promise::then(blur_image_async(f.get()), [](auto f) {
      matrix h = histogram(f.get());
    });
  });

  pipeline_thread.join();
  return 0;
}
/* 4 end */

/*
* 5a. How does a semaphore differ from mutex?
* - A semaphore differs from a mutex in that it is not a mutual exclusion lock,
* - a semaphore, unlike a mutex, can be acquired multiple times (depending on how
* - the semaphore is created) unlike a mutex which is strictly locked or unlocked.
* - Additionally, a semaphore lives in the file system and is not restricted to the
* - process in which it was created (like a mutex), which makes it useful to lock
* - access to things like shared memory pools across processes.
*
* 5b. Describe the operation of a bar (sitting down, ordering a drink) in terms
* of a semaphore and mutex.
* = Semaphore:
* - A bar has `n` number of seats, so a semaphore is constructed accordingly.
* - The semaphore (or a bar seat) can be acquired up to `n` times before parties
* - begin waiting for a seat at the bar. Meaning, each person can reserve a seat
* - by sitting (acquiring) it and then leaving (releasing) the seat, where then
* - the fastest person to react can acquire the released seat. This experiences
* - contention when the semaphore is fully acquired, but not at the same level as
* - a mutex.
*
* = Mutex:
* - The bar has a single seat and the bar tender refuses to serve anyone who is
* - not sitting at that seat. The seat can be locked (sat in) and then unlocked
* - (individual leaves). When the seat is unlocked, the fastest person to react
* - can then sit (lock) in the seat. This means that after the seat is taken,
* - every new person who wishes to have service must wait, which creates contention
* - in the form of a large blob of people (many threads waiting in the kernel on
* - the lock).
*/

int main(int argc, char **argv) {
  return 0;
}
<|endoftext|>