<commit_before>/*
 * The MIT License (MIT)
 *
 * Copyright (c) 2015 Microsoft Corporation
 *
 * -=- Robust Distributed System Nucleus (rDSN) -=-
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

/*
 * Description:
 *     the meta server's date structure, impl file
 *
 * Revision history:
 *     2016-04-25, Weijie Sun(sunweijie at xiaomi.com), first version
 *     xxxx-xx-xx, author, fix bug about xxx
 */
#include <dsn/service_api_cpp.h>
#include "meta_data.h"

namespace dsn { namespace replication {

void maintain_drops(/*inout*/ std::vector<rpc_address>& drops, const rpc_address& node, bool is_add)
{
    auto it = std::find(drops.begin(), drops.end(), node);
    if (is_add)
    {
        if (it != drops.end())
            drops.erase(it);
    }
    else
    {
        if (it == drops.end())
        {
            drops.push_back(node);
            if (drops.size() > 3)
            {
                drops.erase(drops.begin());
            }
        }
        else
        {
            dassert(false, "the node cannot be in drops set before this update", node.to_string());
        }
    }
}

void config_context::cancel_sync()
{
    if (config_status::pending_remote_sync == stage)
    {
        pending_sync_task->cancel(false);
        pending_sync_task = nullptr;
        pending_sync_request.reset();
    }
    if (msg)
    {
        dsn_msg_release_ref(msg);
    }
    msg = nullptr;
    stage = config_status::not_pending;
}

void config_context::clear_proposal()
{
    if (config_status::pending_remote_sync != stage)
        stage = config_status::not_pending;
    balancer_proposal->action_list.clear();
}

void app_state_helper::on_init_partitions()
{
    config_context context;
    context.stage = config_status::not_pending;
    context.pending_sync_task = nullptr;
    context.msg = nullptr;

    context.is_cure_proposal = false;
    contexts.assign(owner->partition_count, context);
    for (unsigned int i=0; i!=owner->partition_count; ++i)
    {
        contexts[i].balancer_proposal = std::make_shared<configuration_balancer_request>();
    }
}

app_state::app_state(const app_info &info): app_info(info), helpers(new app_state_helper())
{
    helpers->owner = this;

    partition_configuration config;
    config.ballot = 0;
    config.pid.set_app_id(app_id);
    config.last_committed_decree = 0;
    config.last_drops.clear();
    config.max_replica_count = app_info::max_replica_count;
    config.primary.set_invalid();
    config.secondaries.clear();
    partitions.assign(app_info::partition_count, config);
    for (int i=0; i!=app_info::partition_count; ++i)
        partitions[i].pid.set_partition_index(i);

    helpers->on_init_partitions();
}

std::shared_ptr<app_state> app_state::create(const app_info& info)
{
    std::shared_ptr<app_state> result = std::make_shared<app_state>(info);
    result->helpers->owner = result.get();
    return result;
}

node_state::node_state():
    total_primaries(0),
    total_partitions(0),
    is_alive(false)
{
}

bool node_state::for_each_partition(const std::function<bool (const gpid &)> &f) const
{
    for (const auto& pair: app_partitions)
    {
        const partition_set& ps = pair.second;
        for (const auto& gpid: ps)
        {
            if (!f(gpid))
                return false;
        }
    }
    return true;
}

const partition_set* node_state::get_partitions(int app_id, bool only_primary) const
{
    const std::map<int32_t, partition_set>* all_partitions;
    if (only_primary)
        all_partitions = &app_primaries;
    else
        all_partitions = &app_partitions;

    auto iter = all_partitions->find(app_id);
    if (iter == all_partitions->end())
        return nullptr;
    else
        return &(iter->second);
}

partition_set* node_state::get_partitions(app_id id, bool only_primary, bool create_new)
{
    std::map<int32_t, partition_set>* all_partitions;
    if (only_primary)
        all_partitions = &app_primaries;
    else
        all_partitions = &app_partitions;

    if (create_new)
    {
        return &((*all_partitions)[id]);
    }
    else
    {
        auto iter = all_partitions->find(id);
        if (iter == all_partitions->end())
            return nullptr;
        else
            return &(iter->second);
    }
}

partition_set* node_state::partitions(app_id id, bool only_primary)
{
    return const_cast<partition_set*>(get_partitions(id, only_primary));
}

const partition_set* node_state::partitions(app_id id, bool only_primary) const
{
    return get_partitions(id, only_primary);
}

void node_state::put_partition(const gpid &pid, bool is_primary)
{
    partition_set* all = get_partitions(pid.get_app_id(), false, true);
    if ((all->insert(pid)).second)
        total_partitions++;
    if (is_primary)
    {
        partition_set* pri = get_partitions(pid.get_app_id(), true, true);
        if ((pri->insert(pid)).second)
            total_primaries++;
    }
}

void node_state::remove_partition(const gpid &pid, bool only_primary)
{
    partition_set* pri = get_partitions(pid.get_app_id(), true, true);
    total_primaries -= pri->erase(pid);
    if (!only_primary)
    {
        partition_set* all = get_partitions(pid.get_app_id(), false, true);
        total_partitions -= all->erase(pid);
    }
}

bool node_state::for_each_primary(app_id id, const std::function<bool (const gpid &)> &f) const
{
    const partition_set* pri = partitions(id, true);
    if (pri == nullptr)
        return false;
    for (const gpid& pid: *pri)
    {
        dassert(id == pid.get_app_id(), "");
        if (!f(pid))
            return false;
    }
    return true;
}

unsigned node_state::primary_count(app_id id) const
{
    const partition_set* pri = partitions(id, true);
    if (pri == nullptr)
        return 0;
    return pri->size();
}

unsigned node_state::partition_count(app_id id) const
{
    const partition_set* pri = partitions(id, false);
    if (pri == nullptr)
        return 0;
    return pri->size();
}

partition_status::type node_state::served_as(const gpid &pid) const
{
    const partition_set* ps1 = partitions(pid.get_app_id(), true);
    if (ps1 != nullptr && ps1->find(pid) != ps1->end())
        return partition_status::PS_PRIMARY;
    const partition_set* ps2 = partitions(pid.get_app_id(), false);
    if (ps2 != nullptr && ps2->find(pid) != ps2->end())
        return partition_status::PS_SECONDARY;
    return partition_status::PS_INACTIVE;
}

}}
<commit_msg>fix logging print bug<commit_after>/*
 * The MIT License (MIT)
 *
 * Copyright (c) 2015 Microsoft Corporation
 *
 * -=- Robust Distributed System Nucleus (rDSN) -=-
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 */

/*
 * Description:
 *     the meta server's date structure, impl file
 *
 * Revision history:
 *     2016-04-25, Weijie Sun(sunweijie at xiaomi.com), first version
 *     xxxx-xx-xx, author, fix bug about xxx
 */
#include <dsn/service_api_cpp.h>
#include "meta_data.h"

namespace dsn { namespace replication {

void maintain_drops(/*inout*/ std::vector<rpc_address>& drops, const rpc_address& node, bool is_add)
{
    auto it = std::find(drops.begin(), drops.end(), node);
    if (is_add)
    {
        if (it != drops.end())
            drops.erase(it);
    }
    else
    {
        if (it == drops.end())
        {
            drops.push_back(node);
            if (drops.size() > 3)
            {
                drops.erase(drops.begin());
            }
        }
        else
        {
            dassert(false, "the node %s cannot be in drops set before this update", node.to_string());
        }
    }
}

void config_context::cancel_sync()
{
    if (config_status::pending_remote_sync == stage)
    {
        pending_sync_task->cancel(false);
        pending_sync_task = nullptr;
        pending_sync_request.reset();
    }
    if (msg)
    {
        dsn_msg_release_ref(msg);
    }
    msg = nullptr;
    stage = config_status::not_pending;
}

void config_context::clear_proposal()
{
    if (config_status::pending_remote_sync != stage)
        stage = config_status::not_pending;
    balancer_proposal->action_list.clear();
}

void app_state_helper::on_init_partitions()
{
    config_context context;
    context.stage = config_status::not_pending;
    context.pending_sync_task = nullptr;
    context.msg = nullptr;

    context.is_cure_proposal = false;
    contexts.assign(owner->partition_count, context);
    for (unsigned int i=0; i!=owner->partition_count; ++i)
    {
        contexts[i].balancer_proposal = std::make_shared<configuration_balancer_request>();
    }
}

app_state::app_state(const app_info &info): app_info(info), helpers(new app_state_helper())
{
    helpers->owner = this;

    partition_configuration config;
    config.ballot = 0;
    config.pid.set_app_id(app_id);
    config.last_committed_decree = 0;
    config.last_drops.clear();
    config.max_replica_count = app_info::max_replica_count;
    config.primary.set_invalid();
    config.secondaries.clear();
    partitions.assign(app_info::partition_count, config);
    for (int i=0; i!=app_info::partition_count; ++i)
        partitions[i].pid.set_partition_index(i);

    helpers->on_init_partitions();
}

std::shared_ptr<app_state> app_state::create(const app_info& info)
{
    std::shared_ptr<app_state> result = std::make_shared<app_state>(info);
    result->helpers->owner = result.get();
    return result;
}

node_state::node_state():
    total_primaries(0),
    total_partitions(0),
    is_alive(false)
{
}

bool node_state::for_each_partition(const std::function<bool (const gpid &)> &f) const
{
    for (const auto& pair: app_partitions)
    {
        const partition_set& ps = pair.second;
        for (const auto& gpid: ps)
        {
            if (!f(gpid))
                return false;
        }
    }
    return true;
}

const partition_set* node_state::get_partitions(int app_id, bool only_primary) const
{
    const std::map<int32_t, partition_set>* all_partitions;
    if (only_primary)
        all_partitions = &app_primaries;
    else
        all_partitions = &app_partitions;

    auto iter = all_partitions->find(app_id);
    if (iter == all_partitions->end())
        return nullptr;
    else
        return &(iter->second);
}

partition_set* node_state::get_partitions(app_id id, bool only_primary, bool create_new)
{
    std::map<int32_t, partition_set>* all_partitions;
    if (only_primary)
        all_partitions = &app_primaries;
    else
        all_partitions = &app_partitions;

    if (create_new)
    {
        return &((*all_partitions)[id]);
    }
    else
    {
        auto iter = all_partitions->find(id);
        if (iter == all_partitions->end())
            return nullptr;
        else
            return &(iter->second);
    }
}

partition_set* node_state::partitions(app_id id, bool only_primary)
{
    return const_cast<partition_set*>(get_partitions(id, only_primary));
}

const partition_set* node_state::partitions(app_id id, bool only_primary) const
{
    return get_partitions(id, only_primary);
}

void node_state::put_partition(const gpid &pid, bool is_primary)
{
    partition_set* all = get_partitions(pid.get_app_id(), false, true);
    if ((all->insert(pid)).second)
        total_partitions++;
    if (is_primary)
    {
        partition_set* pri = get_partitions(pid.get_app_id(), true, true);
        if ((pri->insert(pid)).second)
            total_primaries++;
    }
}

void node_state::remove_partition(const gpid &pid, bool only_primary)
{
    partition_set* pri = get_partitions(pid.get_app_id(), true, true);
    total_primaries -= pri->erase(pid);
    if (!only_primary)
    {
        partition_set* all = get_partitions(pid.get_app_id(), false, true);
        total_partitions -= all->erase(pid);
    }
}

bool node_state::for_each_primary(app_id id, const std::function<bool (const gpid &)> &f) const
{
    const partition_set* pri = partitions(id, true);
    if (pri == nullptr)
        return false;
    for (const gpid& pid: *pri)
    {
        dassert(id == pid.get_app_id(), "");
        if (!f(pid))
            return false;
    }
    return true;
}

unsigned node_state::primary_count(app_id id) const
{
    const partition_set* pri = partitions(id, true);
    if (pri == nullptr)
        return 0;
    return pri->size();
}

unsigned node_state::partition_count(app_id id) const
{
    const partition_set* pri = partitions(id, false);
    if (pri == nullptr)
        return 0;
    return pri->size();
}

partition_status::type node_state::served_as(const gpid &pid) const
{
    const partition_set* ps1 = partitions(pid.get_app_id(), true);
    if (ps1 != nullptr && ps1->find(pid) != ps1->end())
        return partition_status::PS_PRIMARY;
    const partition_set* ps2 = partitions(pid.get_app_id(), false);
    if (ps2 != nullptr && ps2->find(pid) != ps2->end())
        return partition_status::PS_SECONDARY;
    return partition_status::PS_INACTIVE;
}

}}
<|endoftext|>


<commit_before>/*
 * The MIT License (MIT)
 *
 * Copyright (c) 2016 Nathan Osman
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <nitroshare/device.h>

#include "device_p.h"

DevicePrivate::DevicePrivate(QObject *parent)
    : QObject(parent)
{
}

Device::Device(const QString &uuid, QObject *parent)
    : QObject(parent)
    , d(new DevicePrivate(this))
{
    d->uuid = uuid;
}

QString Device::uuid() const
{
    return d->uuid;
}

QString Device::name() const
{
    return d->name;
}

void Device::setName(const QString &name)
{
    emit nameChanged(d->name = name);
}

QStringList Device::addresses() const
{
    return d->addresses;
}

void Device::addAddress(const QString &address)
{
    if (!d->addresses.contains(address)) {
        d->addresses.append(address);
        emit addressesChanged(d->addresses);
    }
}

quint16 Device::port() const
{
    return d->port;
}

void Device::setPort(quint16 port)
{
    emit portChanged(d->port = port);
}
<commit_msg>Eliminate useless signal firing.<commit_after>/*
 * The MIT License (MIT)
 *
 * Copyright (c) 2016 Nathan Osman
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <nitroshare/device.h>

#include "device_p.h"

DevicePrivate::DevicePrivate(QObject *parent)
    : QObject(parent)
{
}

Device::Device(const QString &uuid, QObject *parent)
    : QObject(parent)
    , d(new DevicePrivate(this))
{
    d->uuid = uuid;
}

QString Device::uuid() const
{
    return d->uuid;
}

QString Device::name() const
{
    return d->name;
}

void Device::setName(const QString &name)
{
    if (d->name != name) {
        emit nameChanged(d->name = name);
    }
}

QStringList Device::addresses() const
{
    return d->addresses;
}

void Device::addAddress(const QString &address)
{
    if (!d->addresses.contains(address)) {
        d->addresses.append(address);
        emit addressesChanged(d->addresses);
    }
}

quint16 Device::port() const
{
    return d->port;
}

void Device::setPort(quint16 port)
{
    if (d->port != port) {
        emit portChanged(d->port = port);
    }
}
<|endoftext|>


<commit_before>/* -*- mode: C++; tab-width: 4; indent-tabs-mode: t; c-basic-offset: 4 -*- */
// vim:sts=4:sw=4:ts=4:noet:sr:cino=>s,f0,{0,g0,(0,\:0,t0,+0,=s

/* fflas/fflas_pfgemv.inl
 *
 * ========LICENCE========
 * This file is part of the library FFLAS-FFPACK.
 *
 * FFLAS-FFPACK is free software: you can redistribute it and/or modify
 * it under the terms of the  GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 * ========LICENCE========
 *.
 */

namespace FFLAS
{

	template<class Field, class AlgoT, class FieldTrait>
	typename Field::Element_ptr
	pfgemv(const Field& F,
		   const FFLAS_TRANSPOSE ta,
		   const size_t m,
		   const size_t n,
		   const typename Field::Element alpha,
		   const typename Field::ConstElement_ptr A, const size_t lda,
		   const typename Field::ConstElement_ptr X, const size_t incX,
		   const typename Field::Element beta,
		   typename Field::Element_ptr Y, const size_t incY,
		   MMHelper<Field, AlgoT, FieldTrait, ParSeqHelper::Parallel<CuttingStrategy::Recursive, StrategyParameter::Threads> > & H){

		if (m<2){
			/*
			  PAR_BLOCK{
			  SYNCH_GROUP(
							TASK(CONSTREFERENCE(F) MODE( READ(A,X) READWRITE(Y)),
							{fgemv(F, ta,  m, n,  alpha, A, lda, X, incX, beta, Y, incY);}
							);
							);
							}
			*/
			FFLAS::fdot (F, lda, A, incX, X, incX); //Here A represents a line of the original matrix with lda elements
		}else{
			typedef MMHelper<Field,AlgoT,FieldTrait,ParSeqHelper::Parallel<CuttingStrategy::Recursive, StrategyParameter::Threads> > MMH_t;
			MMH_t H1(H);
			MMH_t H2(H);
			size_t M2= m>>1;
			PAR_BLOCK{

				H1.parseq.set_numthreads(H1.parseq.numthreads() >> 1);
				H2.parseq.set_numthreads(H.parseq.numthreads() - H1.parseq.numthreads());

				typename Field::ConstElement_ptr A1= A;
				typename Field::ConstElement_ptr A2= A+(M2+1)*lda;
				typename Field::Element_ptr C1= Y;
				typename Field::Element_ptr C2= Y+(M2+1)*incY;

				// 2 multiply (1 split on dimension m)
				TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(C1)),
					 {pfgemv( F, ta,  M2, n, alpha, A1, lda, X, incX, beta, C1, incY, H1);}
					 );

				TASK(MODE(CONSTREFERENCE(F) READ(A2,X) READWRITE(C2)),
					 {pfgemv(F, ta, m-M2, n, alpha, A2, lda, X, incX, beta, C2, incY, H2);}
					 );
			}

		}
		return Y;

	}

	template<class Field, class AlgoT, class FieldTrait>
	typename Field::Element_ptr
	pfgemv(const Field& F,
		   const FFLAS_TRANSPOSE ta,
		   const size_t m,
		   const size_t n,
		   const typename Field::Element alpha,
		   const typename Field::ConstElement_ptr A, const size_t lda,
		   const typename Field::ConstElement_ptr X, const size_t incX,
		   const typename Field::Element beta,
		   typename Field::Element_ptr Y, const size_t incY,
		   MMHelper<Field, AlgoT, FieldTrait, ParSeqHelper::Parallel<CuttingStrategy::Row, StrategyParameter::Threads> > & H){

		typedef MMHelper<Field,AlgoT,FieldTrait,ParSeqHelper::Parallel<CuttingStrategy::Row, StrategyParameter::Threads> > MMH_t;
		MMH_t HH(H);
		size_t BS;
		if(BS<H.parseq.numthreads()||BS==0) BS = m/(H.parseq.numthreads());
		size_t Niter = std::max(m, H.parseq.numthreads());
		if(m<Niter){
			PAR_BLOCK{

				HH.parseq.set_numthreads(H.parseq.numthreads());
				using FFLAS::CuttingStrategy::Row;
				using FFLAS::StrategyParameter::Threads;

				FORBLOCK1D(iter, Niter,	SPLITTER(BS,Row,Threads),
						   TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(YY)),
								{fgemv( F, ta, 1, n, alpha, A + iter.blockindex()*lda, lda, X, incX, beta, Y+ iter.blockindex(), incY);} )
						   );
			}

		}else{
			PAR_BLOCK{

				HH.parseq.set_numthreads(H.parseq.numthreads());
				using FFLAS::CuttingStrategy::Row;
				using FFLAS::StrategyParameter::Threads;

				FORBLOCK1D(iter, Niter,	SPLITTER(BS,Row,Threads),
						   TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(YY)),
								{fgemv( F, ta, 1, n, alpha, A + iter.blockindex()*(H.parseq.numthreads()-iter.blockindex())*lda, lda, X, incX, beta, Y, incY);}
								)

						   );
			}
		}

		return Y;

	}

	template<class Field, class AlgoT, class FieldTrait>
	typename Field::Element_ptr
	pfgemv(const Field& F,
		   const FFLAS_TRANSPOSE ta,
		   const size_t m,
		   const size_t n,
		   const typename Field::Element alpha,
		   const typename Field::ConstElement_ptr A, const size_t lda,
		   const typename Field::ConstElement_ptr X, const size_t incX,
		   const typename Field::Element beta,
		   typename Field::Element_ptr Y, const size_t incY,
		   size_t BS,
		   MMHelper<Field, AlgoT, FieldTrait, ParSeqHelper::Parallel<CuttingStrategy::Row, StrategyParameter::Grain> > & H){

		typedef MMHelper<Field,AlgoT,FieldTrait,ParSeqHelper::Parallel<CuttingStrategy::Row, StrategyParameter::Grain> > MMH_t;
		MMH_t HH(H);

		//if m/BS>0 then split each line of the matrix into m/BS blocks
		size_t NbBlocs;
		if(BS>0) NbBlocs = n/BS;
		//else split the matrix into m/(H.parseq.numthreads()) line-wise blocks
		else  NbBlocs = n/(H.parseq.numthreads());
		//Put each block for one thread
		if(NbBlocs>0){

			PAR_BLOCK{
				for(size_t j=0;j<m;j++)
					{

						HH.parseq.set_numthreads(H.parseq.numthreads());
						using FFLAS::CuttingStrategy::Row;
						using FFLAS::StrategyParameter::Grain;

						//for(size_t iter=0;iter<NbBlocs;iter++)
						FORBLOCK1D(iter, NbBlocs,	SPLITTER(BS,Row,Grain),
								   TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(Y)),
										{
											fgemv( F, ta, 1, BS, alpha, (A+j) + iter.blockindex()*BS, BS, X+ iter.blockindex()*BS, incX, beta, Y+j, incY);
											fadd (F, 1, BS, Y+j, incY, Y+j, incY, Y+j, incY);
										}
										)
								   );
					}
			}

		}else{
			size_t Niter = std::max(m, H.parseq.numthreads());
			if(m<Niter){
				PAR_BLOCK{

					HH.parseq.set_numthreads(H.parseq.numthreads());
					using FFLAS::CuttingStrategy::Row;
					using FFLAS::StrategyParameter::Grain;

					FORBLOCK1D(iter, Niter,	SPLITTER(BS,Row,Grain),
							   TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(Y)),
									{fgemv( F, ta, 1, n, alpha, A + iter.blockindex()*lda, lda, X, incX, beta, Y+ iter.blockindex(), incY);}
									)
							   );
				}

			}else{
				PAR_BLOCK{

					HH.parseq.set_numthreads(H.parseq.numthreads());
					using FFLAS::CuttingStrategy::Row;
					using FFLAS::StrategyParameter::Grain;

					FORBLOCK1D(iter, Niter,	SPLITTER(BS,Row,Grain),
							   TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(Y)),
									{fgemv( F, ta, 1, n, alpha, A + iter.blockindex()*BS*lda, lda, X, incX, beta, Y+ iter.blockindex(), incY);}
									)
							   );
				}
			}
		}

		return Y;

	}

} // FFLAS
<commit_msg>Take into considertation of the corner case of splitting matrix with StrategyParameter as Grain while the grain size is not a factor of the matrix dimension.<commit_after>/* -*- mode: C++; tab-width: 4; indent-tabs-mode: t; c-basic-offset: 4 -*- */
// vim:sts=4:sw=4:ts=4:noet:sr:cino=>s,f0,{0,g0,(0,\:0,t0,+0,=s

/* fflas/fflas_pfgemv.inl
 *
 * ========LICENCE========
 * This file is part of the library FFLAS-FFPACK.
 *
 * FFLAS-FFPACK is free software: you can redistribute it and/or modify
 * it under the terms of the  GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 * ========LICENCE========
 *.
 */

namespace FFLAS
{

	template<class Field, class AlgoT, class FieldTrait>
	typename Field::Element_ptr
	pfgemv(const Field& F,
		   const FFLAS_TRANSPOSE ta,
		   const size_t m,
		   const size_t n,
		   const typename Field::Element alpha,
		   const typename Field::ConstElement_ptr A, const size_t lda,
		   const typename Field::ConstElement_ptr X, const size_t incX,
		   const typename Field::Element beta,
		   typename Field::Element_ptr Y, const size_t incY,
		   MMHelper<Field, AlgoT, FieldTrait, ParSeqHelper::Parallel<CuttingStrategy::Recursive, StrategyParameter::Threads> > & H){

		if (m<2){
			/*
			  PAR_BLOCK{
			  SYNCH_GROUP(
							TASK(CONSTREFERENCE(F) MODE( READ(A,X) READWRITE(Y)),
							{fgemv(F, ta,  m, n,  alpha, A, lda, X, incX, beta, Y, incY);}
							);
							);
							}
			*/
			FFLAS::fdot (F, lda, A, incX, X, incX); //Here A represents a line of the original matrix with lda elements
		}else{
			typedef MMHelper<Field,AlgoT,FieldTrait,ParSeqHelper::Parallel<CuttingStrategy::Recursive, StrategyParameter::Threads> > MMH_t;
			MMH_t H1(H);
			MMH_t H2(H);
			size_t M2= m>>1;
			PAR_BLOCK{

				H1.parseq.set_numthreads(H1.parseq.numthreads() >> 1);
				H2.parseq.set_numthreads(H.parseq.numthreads() - H1.parseq.numthreads());

				typename Field::ConstElement_ptr A1= A;
				typename Field::ConstElement_ptr A2= A+(M2+1)*lda;
				typename Field::Element_ptr C1= Y;
				typename Field::Element_ptr C2= Y+(M2+1)*incY;

				// 2 multiply (1 split on dimension m)
				TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(C1)),
					 {pfgemv( F, ta,  M2, n, alpha, A1, lda, X, incX, beta, C1, incY, H1);}
					 );

				TASK(MODE(CONSTREFERENCE(F) READ(A2,X) READWRITE(C2)),
					 {pfgemv(F, ta, m-M2, n, alpha, A2, lda, X, incX, beta, C2, incY, H2);}
					 );
			}

		}
		return Y;

	}

	template<class Field, class AlgoT, class FieldTrait>
	typename Field::Element_ptr
	pfgemv(const Field& F,
		   const FFLAS_TRANSPOSE ta,
		   const size_t m,
		   const size_t n,
		   const typename Field::Element alpha,
		   const typename Field::ConstElement_ptr A, const size_t lda,
		   const typename Field::ConstElement_ptr X, const size_t incX,
		   const typename Field::Element beta,
		   typename Field::Element_ptr Y, const size_t incY,
		   MMHelper<Field, AlgoT, FieldTrait, ParSeqHelper::Parallel<CuttingStrategy::Row, StrategyParameter::Threads> > & H){

		typedef MMHelper<Field,AlgoT,FieldTrait,ParSeqHelper::Parallel<CuttingStrategy::Row, StrategyParameter::Threads> > MMH_t;
		MMH_t HH(H);
		size_t BS;
		if(BS<H.parseq.numthreads()||BS==0) BS = m/(H.parseq.numthreads());
		size_t Niter = std::max(m, H.parseq.numthreads());
		if(m<Niter){
			PAR_BLOCK{

				HH.parseq.set_numthreads(H.parseq.numthreads());
				using FFLAS::CuttingStrategy::Row;
				using FFLAS::StrategyParameter::Threads;

				FORBLOCK1D(iter, Niter,	SPLITTER(BS,Row,Threads),
						   TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(YY)),
								{fgemv( F, ta, 1, n, alpha, A + iter.blockindex()*lda, lda, X, incX, beta, Y+ iter.blockindex(), incY);} )
						   );
			}

		}else{
			PAR_BLOCK{

				HH.parseq.set_numthreads(H.parseq.numthreads());
				using FFLAS::CuttingStrategy::Row;
				using FFLAS::StrategyParameter::Threads;

				FORBLOCK1D(iter, Niter,	SPLITTER(BS,Row,Threads),
						   TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(YY)),
								{fgemv( F, ta, 1, n, alpha, A + iter.blockindex()*(H.parseq.numthreads()-iter.blockindex())*lda, lda, X, incX, beta, Y, incY);}
								)

						   );
			}
		}

		return Y;

	}

	template<class Field, class AlgoT, class FieldTrait>
	typename Field::Element_ptr
	pfgemv(const Field& F,
		   const FFLAS_TRANSPOSE ta,
		   const size_t m,
		   const size_t n,
		   const typename Field::Element alpha,
		   const typename Field::ConstElement_ptr A, const size_t lda,
		   const typename Field::ConstElement_ptr X, const size_t incX,
		   const typename Field::Element beta,
		   typename Field::Element_ptr Y, const size_t incY,
		   size_t BS,
		   MMHelper<Field, AlgoT, FieldTrait, ParSeqHelper::Parallel<CuttingStrategy::Row, StrategyParameter::Grain> > & H){

		typedef MMHelper<Field,AlgoT,FieldTrait,ParSeqHelper::Parallel<CuttingStrategy::Row, StrategyParameter::Grain> > MMH_t;
		MMH_t HH(H);

		//if m/BS>0 then split each line of the matrix into m/BS blocks
		size_t NbBlocs;
		if(BS>0) NbBlocs = n/BS;
		//else split the matrix into m/(H.parseq.numthreads()) line-wise blocks
		else  NbBlocs = n/(H.parseq.numthreads());
		//Put each block for one thread
		if(NbBlocs>0){

			PAR_BLOCK{
				for(size_t j=0;j<m;j++)
					{

						HH.parseq.set_numthreads(H.parseq.numthreads());
						using FFLAS::CuttingStrategy::Row;
						using FFLAS::StrategyParameter::Grain;

						//for(size_t iter=0;iter<NbBlocs;iter++)
						FORBLOCK1D(iter, NbBlocs,	SPLITTER(BS,Row,Grain),
								   TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(Y)),
										{
											fgemv( F, ta, 1, BS, alpha, (A+j) + iter.blockindex()*BS, BS, X+ iter.blockindex()*BS, incX, beta, Y+j, incY);
											fadd (F, 1, BS, Y+j, incY, Y+j, incY, Y+j, incY);
										}
										)
								   );
						if(n - NbBlocs*n>0){
								   TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(Y)),
										{
											fgemv( F, ta, 1, BS, alpha, (A+j) + (n - NbBlocs), (n - NbBlocs), X+ (n - NbBlocs), incX, beta, Y+j, incY);
											fadd (F, 1, BS, Y+j, incY, Y+j, incY, Y+j, incY);
										}
										)
								   );
						}
					}
			}

		}else{
			size_t Niter = std::max(m, H.parseq.numthreads());
			if(m<Niter){
				PAR_BLOCK{

					HH.parseq.set_numthreads(H.parseq.numthreads());
					using FFLAS::CuttingStrategy::Row;
					using FFLAS::StrategyParameter::Grain;

					FORBLOCK1D(iter, Niter,	SPLITTER(BS,Row,Grain),
							   TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(Y)),
									{fgemv( F, ta, 1, n, alpha, A + iter.blockindex()*lda, lda, X, incX, beta, Y+ iter.blockindex(), incY);}
									)
							   );
				}

			}else{
				PAR_BLOCK{

					HH.parseq.set_numthreads(H.parseq.numthreads());
					using FFLAS::CuttingStrategy::Row;
					using FFLAS::StrategyParameter::Grain;

					FORBLOCK1D(iter, Niter,	SPLITTER(BS,Row,Grain),
							   TASK(CONSTREFERENCE(F) MODE( READ(A1,X) READWRITE(Y)),
									{fgemv( F, ta, 1, n, alpha, A + iter.blockindex()*BS*lda, lda, X, incX, beta, Y+ iter.blockindex(), incY);}
									)
							   );
				}
			}
		}

		return Y;

	}

} // FFLAS
<|endoftext|>


<commit_before>/*******************************************************************************
* Copyright 2017-2018 Intel Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/

#include <fstream>
#include <map>

#include "cpu_tracing.hpp"

void ngraph::runtime::cpu::to_json(nlohmann::json& json, const TraceEvent& event)
{
    std::map<std::string, std::string> args;
    for (size_t i = 0; i < event.Inputs.size(); i++)
    {
        args["Input" + std::to_string(i + 1)] = event.Inputs[i];
    }
    for (size_t i = 0; i < event.Outputs.size(); i++)
    {
        args["Output" + std::to_string(i + 1)] = event.Outputs[i];
    }

    json = nlohmann::json{{"ph", event.Phase},
                          {"cat", event.Category},
                          {"name", event.Name},
                          {"pid", event.PID},
                          {"tid", event.TID},
                          {"ts", event.Timestamp},
                          {"dur", event.Duration},
                          {"args", args}};
}

void ngraph::runtime::cpu::GenerateTimeline(const std::vector<OpAttributes>& op_attrs,
                                            int64_t* op_durations,
                                            const std::string& file_name)
{
    nlohmann::json timeline;
    std::list<TraceEvent> trace;
    std::ofstream out(file_name);

    int64_t ts = 0;
    for (size_t i = 0; i < op_attrs.size(); i++)
    {
        trace.emplace_back("X",
                           "Op",
                           op_attrs[i].Description,
                           0,
                           0,
                           ts,
                           op_durations[i],
                           op_attrs[i].Outputs,
                           op_attrs[i].Inputs);
        ts += op_durations[i];
    }

    timeline["traceEvents"] = trace;
    out << timeline;
    out.close();

    return;
}

bool ngraph::runtime::cpu::IsTracingEnabled()
{
    return (std::getenv("NGRAPH_CPU_TRACING") != nullptr);
}
<commit_msg>Disallow tracing control outside of init to save some microseconds (#1353)<commit_after>/*******************************************************************************
* Copyright 2017-2018 Intel Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/

#include <fstream>
#include <map>

#include "cpu_tracing.hpp"

void ngraph::runtime::cpu::to_json(nlohmann::json& json, const TraceEvent& event)
{
    std::map<std::string, std::string> args;
    for (size_t i = 0; i < event.Inputs.size(); i++)
    {
        args["Input" + std::to_string(i + 1)] = event.Inputs[i];
    }
    for (size_t i = 0; i < event.Outputs.size(); i++)
    {
        args["Output" + std::to_string(i + 1)] = event.Outputs[i];
    }

    json = nlohmann::json{{"ph", event.Phase},
                          {"cat", event.Category},
                          {"name", event.Name},
                          {"pid", event.PID},
                          {"tid", event.TID},
                          {"ts", event.Timestamp},
                          {"dur", event.Duration},
                          {"args", args}};
}

void ngraph::runtime::cpu::GenerateTimeline(const std::vector<OpAttributes>& op_attrs,
                                            int64_t* op_durations,
                                            const std::string& file_name)
{
    nlohmann::json timeline;
    std::list<TraceEvent> trace;
    std::ofstream out(file_name);

    int64_t ts = 0;
    for (size_t i = 0; i < op_attrs.size(); i++)
    {
        trace.emplace_back("X",
                           "Op",
                           op_attrs[i].Description,
                           0,
                           0,
                           ts,
                           op_durations[i],
                           op_attrs[i].Outputs,
                           op_attrs[i].Inputs);
        ts += op_durations[i];
    }

    timeline["traceEvents"] = trace;
    out << timeline;
    out.close();

    return;
}

bool ngraph::runtime::cpu::IsTracingEnabled()
{
    static bool enabled = (std::getenv("NGRAPH_CPU_TRACING") != nullptr);
    return enabled;
}
<|endoftext|>


<commit_before>/* ---------------------------------------------------------------------
 * Numenta Platform for Intelligent Computing (NuPIC)
 * Copyright (C) 2013-2015, Numenta, Inc.  Unless you have an agreement
 * with Numenta, Inc., for a separate license for this software code, the
 * following terms and conditions apply:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero Public License version 3 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU Affero Public License for more details.
 *
 * You should have received a copy of the GNU Affero Public License
 * along with this program.  If not, see http://www.gnu.org/licenses.
 *
 * http://numenta.org/licenses/
 * ---------------------------------------------------------------------
 */

#include <stdexcept>

#include <capnp/any.h>

#include <nupic/engine/RegionImplFactory.hpp>
#include <nupic/engine/RegionImpl.hpp>
#include <nupic/engine/Region.hpp>
#include <nupic/engine/RegisteredRegionImpl.hpp>
#include <nupic/engine/Spec.hpp>
#include <nupic/os/DynamicLibrary.hpp>
#include <nupic/os/Path.hpp>
#include <nupic/os/OS.hpp>
#include <nupic/os/Env.hpp>
#include <nupic/ntypes/Value.hpp>
#include <nupic/ntypes/BundleIO.hpp>
#include <nupic/engine/YAMLUtils.hpp>
#include <nupic/engine/TestNode.hpp>
#include <nupic/regions/PyRegion.hpp>
#include <nupic/regions/VectorFileEffector.hpp>
#include <nupic/regions/VectorFileSensor.hpp>
#include <nupic/utils/Log.hpp>
#include <nupic/utils/StringUtils.hpp>

// from http://stackoverflow.com/a/9096509/1781435
#define stringify(x)  #x
#define expand_and_stringify(x) stringify(x)

namespace nupic
{
  // Path from site-packages to packages that contain NuPIC Python regions
  static std::map<const std::string, std::set<std::string>> pyRegions;

  // Mappings for C++ regions
  static std::map<const std::string, GenericRegisteredRegionImpl*> cppRegions;

  bool initializedRegions = false;

  // Allows the user to add custom regions
  void RegionImplFactory::registerPyRegion(const std::string module, const std::string className)
  {
    // Verify that no regions exist with the same className
    for (auto pyRegion : pyRegions)
    {
      if (pyRegion.second.find(className) != pyRegion.second.end())
      {
        NTA_THROW << "A pyRegion with name '" << className << "' already exists. "
                  << "Unregister the existing region or register the new region using a "
                  << "different name.";
      }
    }

    // Module hasn't been added yet
    if (pyRegions.find(module) == pyRegions.end())
    {
      pyRegions[module] = std::set<std::string>();
    }

    pyRegions[module].insert(className);
  }

  void RegionImplFactory::registerCPPRegion(const std::string name,
                                            GenericRegisteredRegionImpl * wrapper)
  {
    if (cppRegions.find(name) != cppRegions.end())
    {
      NTA_WARN << "A CPPRegion already exists with the name '"
               << name << "'. Overwriting it...";
    }
    cppRegions[name] = wrapper;
  }

  void RegionImplFactory::unregisterPyRegion(const std::string className)
  {
    for (auto pyRegion : pyRegions)
    {
      if (pyRegion.second.find(className) != pyRegion.second.end())
      {
        pyRegions.erase(pyRegion.first);
        return;
      }
    }
    NTA_WARN << "A pyRegion with name '" << className <<
    "' doesn't exist. Nothing to unregister...";
  }

  void RegionImplFactory::unregisterCPPRegion(const std::string name)
  {
    if (cppRegions.find(name) != cppRegions.end())
    {
      cppRegions.erase(name);
      return;
    }
  }

  class DynamicPythonLibrary
  {
    typedef void (*initPythonFunc)();
    typedef void (*finalizePythonFunc)();
    typedef void * (*createSpecFunc)(const char *, void **, const char *);
    typedef int (*destroySpecFunc)(const char *, const char *);
    typedef void * (*createPyNodeFunc)(const char *, void *, void *, void **, const char *);
    typedef void * (*deserializePyNodeFunc)(const char *, void *, void *, void *, const char *);
    typedef void * (*deserializePyNodeProtoFunc)(const char *, void *, void *, void *, const char *);
  public:
    DynamicPythonLibrary() :
      initPython_(nullptr),
      finalizePython_(nullptr),
      createSpec_(nullptr),
      destroySpec_(nullptr),
      createPyNode_(nullptr)
    {
      initPython_ = (initPythonFunc) PyRegion::NTA_initPython;
      finalizePython_ = (finalizePythonFunc) PyRegion::NTA_finalizePython;
      createPyNode_ = (createPyNodeFunc) PyRegion::NTA_createPyNode;
      deserializePyNode_ = (deserializePyNodeFunc) PyRegion::NTA_deserializePyNode;
      deserializePyNodeProto_ = (deserializePyNodeProtoFunc) PyRegion::NTA_deserializePyNodeProto;
      createSpec_ = (createSpecFunc) PyRegion::NTA_createSpec;
      destroySpec_ = (destroySpecFunc) PyRegion::NTA_destroySpec;

      (*initPython_)();
    }

    ~DynamicPythonLibrary()
    {
      if (finalizePython_)
        finalizePython_();
    }

    void * createSpec(std::string nodeType, void ** exception, std::string className)
    {
      return (*createSpec_)(nodeType.c_str(), exception, className.c_str());
    }

    int destroySpec(std::string nodeType, std::string& className)
    {
      NTA_INFO << "destroySpec(" << nodeType << ")";
      return (*destroySpec_)(nodeType.c_str(), className.c_str());
    }

    void * createPyNode(const std::string& nodeType,
                        ValueMap * nodeParams,
                        Region * region,
                        void ** exception,
                        const std::string& className)
    {
      return (*createPyNode_)(nodeType.c_str(),
                              reinterpret_cast<void *>(nodeParams),
                              reinterpret_cast<void*>(region),
                              exception,
                              className.c_str());

    }

    void * deserializePyNode(const std::string& nodeType,
                             BundleIO* bundle,
                             Region * region,
                             void ** exception,
                             const std::string& className)
    {
      return (*deserializePyNode_)(nodeType.c_str(),
                                   reinterpret_cast<void*>(bundle),
                                   reinterpret_cast<void*>(region),
                                   exception,
                                   className.c_str());
    }

    void * deserializePyNodeProto(const std::string& nodeType,
                                  capnp::AnyPointer::Reader* proto,
                                  Region * region,
                                  void ** exception,
                                  const std::string& className)
    {
      return (*deserializePyNodeProto_)(nodeType.c_str(),
                                        reinterpret_cast<void*>(proto),
                                        reinterpret_cast<void*>(region),
                                        exception,
                                        className.c_str());
    }

    const std::string& getRootDir() const
    {
      return rootDir_;
    }

  private:
    std::string rootDir_;
    boost::shared_ptr<DynamicLibrary> pynodeLibrary_;
    initPythonFunc initPython_;
    finalizePythonFunc finalizePython_;
    createSpecFunc createSpec_;
    destroySpecFunc destroySpec_;
    createPyNodeFunc createPyNode_;
    deserializePyNodeFunc deserializePyNode_;
    deserializePyNodeProtoFunc deserializePyNodeProto_;
  };

RegionImplFactory & RegionImplFactory::getInstance()
{
  static RegionImplFactory instance;

  // Initialize Regions
  if (!initializedRegions)
  {
    // Create C++ regions
    cppRegions["TestNode"] = new RegisteredRegionImpl<TestNode>();
    cppRegions["VectorFileEffector"] = new RegisteredRegionImpl<VectorFileEffector>();
    cppRegions["VectorFileSensor"] = new RegisteredRegionImpl<VectorFileSensor>();

    initializedRegions = true;
  }

  return instance;
}

// This function creates either a NuPIC 2 or NuPIC 1 Python node
static RegionImpl * createPyNode(DynamicPythonLibrary * pyLib,
                                 const std::string & nodeType,
                                 ValueMap * nodeParams,
                                 Region * region)
{
  std::string className(nodeType.c_str() + 3);
  for (auto pyr=pyRegions.begin(); pyr!=pyRegions.end(); pyr++)
  {
    const std::string module = pyr->first;
    std::set<std::string> classes = pyr->second;

    // This module contains the class
    if (classes.find(className) != classes.end())
    {
      void * exception = nullptr;
      void * node = pyLib->createPyNode(module, nodeParams, region, &exception, className);
      if (node)
      {
        return static_cast<RegionImpl*>(node);
      }
    }
  }

  NTA_THROW << "Unable to create region " << region->getName() << " of type " << className;
  return nullptr;
}

// This function deserializes either a NuPIC 2 or NuPIC 1 Python node
static RegionImpl * deserializePyNode(DynamicPythonLibrary * pyLib,
                                      const std::string & nodeType,
                                      BundleIO & bundle,
                                      Region * region)
{
  std::string className(nodeType.c_str() + 3);
  for (auto pyr=pyRegions.begin(); pyr!=pyRegions.end(); pyr++)
  {
    const std::string module = pyr->first;
    std::set<std::string> classes = pyr->second;

    // This module contains the class
    if (classes.find(className) != classes.end())
    {
      void * exception = nullptr;
      void * node = pyLib->deserializePyNode(module, &bundle, region, &exception, className);
      if (node)
      {
        return static_cast<RegionImpl*>(node);
      }
    }
  }

  NTA_THROW << "Unable to deserialize region " << region->getName() << " of type " << className;
  return nullptr;

}

static RegionImpl * deserializePyNode(DynamicPythonLibrary * pyLib,
                                      const std::string & nodeType,
                                      capnp::AnyPointer::Reader& proto,
                                      Region * region)
{
  std::string className(nodeType.c_str() + 3);
  for (auto pyr=pyRegions.begin(); pyr!=pyRegions.end(); pyr++)
  {
    const std::string module = pyr->first;
    std::set<std::string> classes = pyr->second;

    // This module contains the class
    if (classes.find(className) != classes.end())
    {
      void * exception = nullptr;
      void * node = pyLib->deserializePyNodeProto(module, &proto, region, &exception, className);
      if (node)
      {
        return static_cast<RegionImpl*>(node);
      }
    }
  }

  NTA_THROW << "Unable to deserialize region " << region->getName() << " of type " << className;
  return nullptr;
}

RegionImpl* RegionImplFactory::createRegionImpl(const std::string nodeType,
                                                const std::string nodeParams,
                                                Region* region)
{

  RegionImpl *impl = nullptr;
  Spec *ns = getSpec(nodeType);
  ValueMap vm = YAMLUtils::toValueMap(
    nodeParams.c_str(),
    ns->parameters,
    nodeType,
    region->getName());

  if (cppRegions.find(nodeType) != cppRegions.end())
  {
    impl = cppRegions[nodeType]->createRegionImpl(vm, region);
  }
  else if ((nodeType.find(std::string("py.")) == 0))
  {
    if (!pyLib_)
      pyLib_ = boost::shared_ptr<DynamicPythonLibrary>(new DynamicPythonLibrary());

    impl = createPyNode(pyLib_.get(), nodeType, &vm, region);
  } else
  {
    NTA_THROW << "Unsupported node type '" << nodeType << "'";
  }

  return impl;

}

RegionImpl* RegionImplFactory::deserializeRegionImpl(const std::string nodeType,
                                                     BundleIO& bundle,
                                                     Region* region)
{

  RegionImpl *impl = nullptr;

  if (cppRegions.find(nodeType) != cppRegions.end())
  {
    impl = cppRegions[nodeType]->deserializeRegionImpl(bundle, region);
  }
  else if (StringUtils::startsWith(nodeType, "py."))
  {
    if (!pyLib_)
      pyLib_ = boost::shared_ptr<DynamicPythonLibrary>(new DynamicPythonLibrary());

    impl = deserializePyNode(pyLib_.get(), nodeType, bundle, region);
  } else
  {
    NTA_THROW << "Unsupported node type '" << nodeType << "'";
  }
  return impl;

}

RegionImpl* RegionImplFactory::deserializeRegionImpl(
    const std::string nodeType,
    capnp::AnyPointer::Reader& proto,
    Region* region)
{
  RegionImpl *impl = nullptr;

  if (cppRegions.find(nodeType) != cppRegions.end())
  {
    impl = cppRegions[nodeType]->deserializeRegionImpl(proto, region);
  }
  else if (StringUtils::startsWith(nodeType, "py."))
  {
    if (!pyLib_)
     pyLib_ = boost::shared_ptr<DynamicPythonLibrary>(new DynamicPythonLibrary());

    impl = deserializePyNode(pyLib_.get(), nodeType, proto, region);
  }
  else
  {
    NTA_THROW << "Unsupported node type '" << nodeType << "'";
  }
  return impl;
}

// This function returns the node spec of a NuPIC 2 or NuPIC 1 Python node
static Spec * getPySpec(DynamicPythonLibrary * pyLib,
                                const std::string & nodeType)
{
  std::string className(nodeType.c_str() + 3);
  for (auto pyr=pyRegions.begin(); pyr!=pyRegions.end(); pyr++)
  {
    const std::string module = pyr->first;
    std::set<std::string> classes = pyr->second;

    // This module contains the class
    if (classes.find(className) != classes.end())
    {
      void * exception = nullptr;
      void * ns = pyLib->createSpec(module, &exception, className);
      if (ns)
      {
        return (Spec *)ns;
      }
    }
  }

  NTA_THROW << "Matching Python module for " << className << " not found.";
}

Spec * RegionImplFactory::getSpec(const std::string nodeType)
{
  std::map<std::string, Spec*>::iterator it;
  // return from cache if we already have it
  it = nodespecCache_.find(nodeType);
  if (it != nodespecCache_.end())
    return it->second;

  // grab the nodespec and cache it
  // one entry per supported node type
  Spec * ns = nullptr;
  if (cppRegions.find(nodeType) != cppRegions.end())
  {
    ns = cppRegions[nodeType]->createSpec();
  }
  else if (nodeType.find(std::string("py.")) == 0)
  {
    if (!pyLib_)
      pyLib_ = boost::shared_ptr<DynamicPythonLibrary>(new DynamicPythonLibrary());

    ns = getPySpec(pyLib_.get(), nodeType);
  }
  else
  {
    NTA_THROW << "getSpec() -- Unsupported node type '" << nodeType << "'";
  }

  if (!ns)
    NTA_THROW << "Unable to get node spec for: " << nodeType;

  nodespecCache_[nodeType] = ns;
  return ns;
}

void RegionImplFactory::cleanup()
{
  std::map<std::string, Spec*>::iterator ns;
  // destroy all nodespecs
  for (ns = nodespecCache_.begin(); ns != nodespecCache_.end(); ns++)
  {
    assert(ns->second != nullptr);
    // PyNode node specs are destroyed by the C++ PyNode
    if (ns->first.substr(0, 3) == "py.")
    {
      std::string noClass = "";
      pyLib_->destroySpec(ns->first, noClass);
    }
    else
    {
      delete ns->second;
    }

    ns->second = nullptr;
  }

  nodespecCache_.clear();

  // destroy all RegisteredRegionImpls
  for (auto rri = cppRegions.begin(); rri != cppRegions.end(); rri++)
  {
    NTA_ASSERT(rri->second != nullptr);
    delete rri->second;
    rri->second = nullptr;
  }

  cppRegions.clear();
  initializedRegions = false;

  // Never release the Python dynamic library!
  // This is due to cleanup issues of Python itself
  // See: http://docs.python.org/c-api/init.html#Py_Finalize
  //pyLib_.reset();
}

}
<commit_msg>Fix line length<commit_after>/* ---------------------------------------------------------------------
 * Numenta Platform for Intelligent Computing (NuPIC)
 * Copyright (C) 2013-2015, Numenta, Inc.  Unless you have an agreement
 * with Numenta, Inc., for a separate license for this software code, the
 * following terms and conditions apply:
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero Public License version 3 as
 * published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 * See the GNU Affero Public License for more details.
 *
 * You should have received a copy of the GNU Affero Public License
 * along with this program.  If not, see http://www.gnu.org/licenses.
 *
 * http://numenta.org/licenses/
 * ---------------------------------------------------------------------
 */

#include <stdexcept>

#include <capnp/any.h>

#include <nupic/engine/RegionImplFactory.hpp>
#include <nupic/engine/RegionImpl.hpp>
#include <nupic/engine/Region.hpp>
#include <nupic/engine/RegisteredRegionImpl.hpp>
#include <nupic/engine/Spec.hpp>
#include <nupic/os/DynamicLibrary.hpp>
#include <nupic/os/Path.hpp>
#include <nupic/os/OS.hpp>
#include <nupic/os/Env.hpp>
#include <nupic/ntypes/Value.hpp>
#include <nupic/ntypes/BundleIO.hpp>
#include <nupic/engine/YAMLUtils.hpp>
#include <nupic/engine/TestNode.hpp>
#include <nupic/regions/PyRegion.hpp>
#include <nupic/regions/VectorFileEffector.hpp>
#include <nupic/regions/VectorFileSensor.hpp>
#include <nupic/utils/Log.hpp>
#include <nupic/utils/StringUtils.hpp>

// from http://stackoverflow.com/a/9096509/1781435
#define stringify(x)  #x
#define expand_and_stringify(x) stringify(x)

namespace nupic
{
  // Path from site-packages to packages that contain NuPIC Python regions
  static std::map<const std::string, std::set<std::string>> pyRegions;

  // Mappings for C++ regions
  static std::map<const std::string, GenericRegisteredRegionImpl*> cppRegions;

  bool initializedRegions = false;

  // Allows the user to add custom regions
  void RegionImplFactory::registerPyRegion(const std::string module, const std::string className)
  {
    // Verify that no regions exist with the same className
    for (auto pyRegion : pyRegions)
    {
      if (pyRegion.second.find(className) != pyRegion.second.end())
      {
        NTA_THROW << "A pyRegion with name '" << className << "' already exists. "
                  << "Unregister the existing region or register the new region using a "
                  << "different name.";
      }
    }

    // Module hasn't been added yet
    if (pyRegions.find(module) == pyRegions.end())
    {
      pyRegions[module] = std::set<std::string>();
    }

    pyRegions[module].insert(className);
  }

  void RegionImplFactory::registerCPPRegion(const std::string name,
                                            GenericRegisteredRegionImpl * wrapper)
  {
    if (cppRegions.find(name) != cppRegions.end())
    {
      NTA_WARN << "A CPPRegion already exists with the name '"
               << name << "'. Overwriting it...";
    }
    cppRegions[name] = wrapper;
  }

  void RegionImplFactory::unregisterPyRegion(const std::string className)
  {
    for (auto pyRegion : pyRegions)
    {
      if (pyRegion.second.find(className) != pyRegion.second.end())
      {
        pyRegions.erase(pyRegion.first);
        return;
      }
    }
    NTA_WARN << "A pyRegion with name '" << className <<
    "' doesn't exist. Nothing to unregister...";
  }

  void RegionImplFactory::unregisterCPPRegion(const std::string name)
  {
    if (cppRegions.find(name) != cppRegions.end())
    {
      cppRegions.erase(name);
      return;
    }
  }

  class DynamicPythonLibrary
  {
    typedef void (*initPythonFunc)();
    typedef void (*finalizePythonFunc)();
    typedef void * (*createSpecFunc)(const char *, void **, const char *);
    typedef int (*destroySpecFunc)(const char *, const char *);
    typedef void * (*createPyNodeFunc)(const char *, void *, void *, void **, const char *);
    typedef void * (*deserializePyNodeFunc)(const char *, void *, void *, void *, const char *);
    typedef void * (*deserializePyNodeProtoFunc)(const char *, void *, void *, void *, const char *);
  public:
    DynamicPythonLibrary() :
      initPython_(nullptr),
      finalizePython_(nullptr),
      createSpec_(nullptr),
      destroySpec_(nullptr),
      createPyNode_(nullptr)
    {
      initPython_ = (initPythonFunc) PyRegion::NTA_initPython;
      finalizePython_ = (finalizePythonFunc) PyRegion::NTA_finalizePython;
      createPyNode_ = (createPyNodeFunc) PyRegion::NTA_createPyNode;
      deserializePyNode_ = (deserializePyNodeFunc) PyRegion::NTA_deserializePyNode;
      deserializePyNodeProto_ = (deserializePyNodeProtoFunc) PyRegion::NTA_deserializePyNodeProto;
      createSpec_ = (createSpecFunc) PyRegion::NTA_createSpec;
      destroySpec_ = (destroySpecFunc) PyRegion::NTA_destroySpec;

      (*initPython_)();
    }

    ~DynamicPythonLibrary()
    {
      if (finalizePython_)
        finalizePython_();
    }

    void * createSpec(std::string nodeType, void ** exception, std::string className)
    {
      return (*createSpec_)(nodeType.c_str(), exception, className.c_str());
    }

    int destroySpec(std::string nodeType, std::string& className)
    {
      NTA_INFO << "destroySpec(" << nodeType << ")";
      return (*destroySpec_)(nodeType.c_str(), className.c_str());
    }

    void * createPyNode(const std::string& nodeType,
                        ValueMap * nodeParams,
                        Region * region,
                        void ** exception,
                        const std::string& className)
    {
      return (*createPyNode_)(nodeType.c_str(),
                              reinterpret_cast<void *>(nodeParams),
                              reinterpret_cast<void*>(region),
                              exception,
                              className.c_str());

    }

    void * deserializePyNode(const std::string& nodeType,
                             BundleIO* bundle,
                             Region * region,
                             void ** exception,
                             const std::string& className)
    {
      return (*deserializePyNode_)(nodeType.c_str(),
                                   reinterpret_cast<void*>(bundle),
                                   reinterpret_cast<void*>(region),
                                   exception,
                                   className.c_str());
    }

    void * deserializePyNodeProto(const std::string& nodeType,
                                  capnp::AnyPointer::Reader* proto,
                                  Region * region,
                                  void ** exception,
                                  const std::string& className)
    {
      return (*deserializePyNodeProto_)(nodeType.c_str(),
                                        reinterpret_cast<void*>(proto),
                                        reinterpret_cast<void*>(region),
                                        exception,
                                        className.c_str());
    }

    const std::string& getRootDir() const
    {
      return rootDir_;
    }

  private:
    std::string rootDir_;
    boost::shared_ptr<DynamicLibrary> pynodeLibrary_;
    initPythonFunc initPython_;
    finalizePythonFunc finalizePython_;
    createSpecFunc createSpec_;
    destroySpecFunc destroySpec_;
    createPyNodeFunc createPyNode_;
    deserializePyNodeFunc deserializePyNode_;
    deserializePyNodeProtoFunc deserializePyNodeProto_;
  };

RegionImplFactory & RegionImplFactory::getInstance()
{
  static RegionImplFactory instance;

  // Initialize Regions
  if (!initializedRegions)
  {
    // Create C++ regions
    cppRegions["TestNode"] = new RegisteredRegionImpl<TestNode>();
    cppRegions["VectorFileEffector"] = new RegisteredRegionImpl<VectorFileEffector>();
    cppRegions["VectorFileSensor"] = new RegisteredRegionImpl<VectorFileSensor>();

    initializedRegions = true;
  }

  return instance;
}

// This function creates either a NuPIC 2 or NuPIC 1 Python node
static RegionImpl * createPyNode(DynamicPythonLibrary * pyLib,
                                 const std::string & nodeType,
                                 ValueMap * nodeParams,
                                 Region * region)
{
  std::string className(nodeType.c_str() + 3);
  for (auto pyr=pyRegions.begin(); pyr!=pyRegions.end(); pyr++)
  {
    const std::string module = pyr->first;
    std::set<std::string> classes = pyr->second;

    // This module contains the class
    if (classes.find(className) != classes.end())
    {
      void * exception = nullptr;
      void * node = pyLib->createPyNode(module, nodeParams, region, &exception, className);
      if (node)
      {
        return static_cast<RegionImpl*>(node);
      }
    }
  }

  NTA_THROW << "Unable to create region " << region->getName() << " of type " << className;
  return nullptr;
}

// This function deserializes either a NuPIC 2 or NuPIC 1 Python node
static RegionImpl * deserializePyNode(DynamicPythonLibrary * pyLib,
                                      const std::string & nodeType,
                                      BundleIO & bundle,
                                      Region * region)
{
  std::string className(nodeType.c_str() + 3);
  for (auto pyr=pyRegions.begin(); pyr!=pyRegions.end(); pyr++)
  {
    const std::string module = pyr->first;
    std::set<std::string> classes = pyr->second;

    // This module contains the class
    if (classes.find(className) != classes.end())
    {
      void * exception = nullptr;
      void * node = pyLib->deserializePyNode(module, &bundle, region, &exception, className);
      if (node)
      {
        return static_cast<RegionImpl*>(node);
      }
    }
  }

  NTA_THROW << "Unable to deserialize region " << region->getName() << " of type " << className;
  return nullptr;

}

static RegionImpl * deserializePyNode(DynamicPythonLibrary * pyLib,
                                      const std::string & nodeType,
                                      capnp::AnyPointer::Reader& proto,
                                      Region * region)
{
  std::string className(nodeType.c_str() + 3);
  for (auto pyr=pyRegions.begin(); pyr!=pyRegions.end(); pyr++)
  {
    const std::string module = pyr->first;
    std::set<std::string> classes = pyr->second;

    // This module contains the class
    if (classes.find(className) != classes.end())
    {
      void * exception = nullptr;
      void * node = pyLib->deserializePyNodeProto(module, &proto, region,
                                                  &exception, className);
      if (node)
      {
        return static_cast<RegionImpl*>(node);
      }
    }
  }

  NTA_THROW << "Unable to deserialize region " << region->getName() <<
            " of type " << className;
  return nullptr;
}

RegionImpl* RegionImplFactory::createRegionImpl(const std::string nodeType,
                                                const std::string nodeParams,
                                                Region* region)
{

  RegionImpl *impl = nullptr;
  Spec *ns = getSpec(nodeType);
  ValueMap vm = YAMLUtils::toValueMap(
    nodeParams.c_str(),
    ns->parameters,
    nodeType,
    region->getName());

  if (cppRegions.find(nodeType) != cppRegions.end())
  {
    impl = cppRegions[nodeType]->createRegionImpl(vm, region);
  }
  else if ((nodeType.find(std::string("py.")) == 0))
  {
    if (!pyLib_)
      pyLib_ = boost::shared_ptr<DynamicPythonLibrary>(new DynamicPythonLibrary());

    impl = createPyNode(pyLib_.get(), nodeType, &vm, region);
  } else
  {
    NTA_THROW << "Unsupported node type '" << nodeType << "'";
  }

  return impl;

}

RegionImpl* RegionImplFactory::deserializeRegionImpl(const std::string nodeType,
                                                     BundleIO& bundle,
                                                     Region* region)
{

  RegionImpl *impl = nullptr;

  if (cppRegions.find(nodeType) != cppRegions.end())
  {
    impl = cppRegions[nodeType]->deserializeRegionImpl(bundle, region);
  }
  else if (StringUtils::startsWith(nodeType, "py."))
  {
    if (!pyLib_)
      pyLib_ = boost::shared_ptr<DynamicPythonLibrary>(new DynamicPythonLibrary());

    impl = deserializePyNode(pyLib_.get(), nodeType, bundle, region);
  } else
  {
    NTA_THROW << "Unsupported node type '" << nodeType << "'";
  }
  return impl;

}

RegionImpl* RegionImplFactory::deserializeRegionImpl(
    const std::string nodeType,
    capnp::AnyPointer::Reader& proto,
    Region* region)
{
  RegionImpl *impl = nullptr;

  if (cppRegions.find(nodeType) != cppRegions.end())
  {
    impl = cppRegions[nodeType]->deserializeRegionImpl(proto, region);
  }
  else if (StringUtils::startsWith(nodeType, "py."))
  {
    if (!pyLib_)
     pyLib_ = boost::shared_ptr<DynamicPythonLibrary>(new DynamicPythonLibrary());

    impl = deserializePyNode(pyLib_.get(), nodeType, proto, region);
  }
  else
  {
    NTA_THROW << "Unsupported node type '" << nodeType << "'";
  }
  return impl;
}

// This function returns the node spec of a NuPIC 2 or NuPIC 1 Python node
static Spec * getPySpec(DynamicPythonLibrary * pyLib,
                                const std::string & nodeType)
{
  std::string className(nodeType.c_str() + 3);
  for (auto pyr=pyRegions.begin(); pyr!=pyRegions.end(); pyr++)
  {
    const std::string module = pyr->first;
    std::set<std::string> classes = pyr->second;

    // This module contains the class
    if (classes.find(className) != classes.end())
    {
      void * exception = nullptr;
      void * ns = pyLib->createSpec(module, &exception, className);
      if (ns)
      {
        return (Spec *)ns;
      }
    }
  }

  NTA_THROW << "Matching Python module for " << className << " not found.";
}

Spec * RegionImplFactory::getSpec(const std::string nodeType)
{
  std::map<std::string, Spec*>::iterator it;
  // return from cache if we already have it
  it = nodespecCache_.find(nodeType);
  if (it != nodespecCache_.end())
    return it->second;

  // grab the nodespec and cache it
  // one entry per supported node type
  Spec * ns = nullptr;
  if (cppRegions.find(nodeType) != cppRegions.end())
  {
    ns = cppRegions[nodeType]->createSpec();
  }
  else if (nodeType.find(std::string("py.")) == 0)
  {
    if (!pyLib_)
      pyLib_ = boost::shared_ptr<DynamicPythonLibrary>(new DynamicPythonLibrary());

    ns = getPySpec(pyLib_.get(), nodeType);
  }
  else
  {
    NTA_THROW << "getSpec() -- Unsupported node type '" << nodeType << "'";
  }

  if (!ns)
    NTA_THROW << "Unable to get node spec for: " << nodeType;

  nodespecCache_[nodeType] = ns;
  return ns;
}

void RegionImplFactory::cleanup()
{
  std::map<std::string, Spec*>::iterator ns;
  // destroy all nodespecs
  for (ns = nodespecCache_.begin(); ns != nodespecCache_.end(); ns++)
  {
    assert(ns->second != nullptr);
    // PyNode node specs are destroyed by the C++ PyNode
    if (ns->first.substr(0, 3) == "py.")
    {
      std::string noClass = "";
      pyLib_->destroySpec(ns->first, noClass);
    }
    else
    {
      delete ns->second;
    }

    ns->second = nullptr;
  }

  nodespecCache_.clear();

  // destroy all RegisteredRegionImpls
  for (auto rri = cppRegions.begin(); rri != cppRegions.end(); rri++)
  {
    NTA_ASSERT(rri->second != nullptr);
    delete rri->second;
    rri->second = nullptr;
  }

  cppRegions.clear();
  initializedRegions = false;

  // Never release the Python dynamic library!
  // This is due to cleanup issues of Python itself
  // See: http://docs.python.org/c-api/init.html#Py_Finalize
  //pyLib_.reset();
}

}
<|endoftext|>


<commit_before>#ifndef LIGHTGBM_OBJECTIVE_REGRESSION_OBJECTIVE_HPP_
#define LIGHTGBM_OBJECTIVE_REGRESSION_OBJECTIVE_HPP_

#include <LightGBM/meta.h>

#include <LightGBM/objective_function.h>
#include <LightGBM/utils/common.h>

namespace LightGBM {
/*!
* \brief Objective function for regression
*/
class RegressionL2loss: public ObjectiveFunction {
public:
  explicit RegressionL2loss(const ObjectiveConfig&) {
  }

  explicit RegressionL2loss(const std::vector<std::string>&) {

  }

  ~RegressionL2loss() {
  }

  void Init(const Metadata& metadata, data_size_t num_data) override {
    num_data_ = num_data;
    label_ = metadata.label();
    weights_ = metadata.weights();
  }

  void GetGradients(const double* score, score_t* gradients,
                    score_t* hessians) const override {
    if (weights_ == nullptr) {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        gradients[i] = static_cast<score_t>(score[i] - label_[i]);
        hessians[i] = 1.0f;
      }
    } else {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        gradients[i] = static_cast<score_t>(score[i] - label_[i]) * weights_[i];
        hessians[i] = weights_[i];
      }
    }
  }

  const char* GetName() const override {
    return "regression";
  }

  std::string ToString() const override {
    std::stringstream str_buf;
    str_buf << GetName();
    return str_buf.str();
  }

  bool IsConstantHessian() const override {
    if (weights_ == nullptr) {
      return true;
    } else {
      return false;
    }
  }

  bool BoostFromAverage() const override { return true; }

private:
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Pointer of label */
  const float* label_;
  /*! \brief Pointer of weights */
  const float* weights_;
};

/*!
* \brief L1 regression loss
*/
class RegressionL1loss: public ObjectiveFunction {
public:
  explicit RegressionL1loss(const ObjectiveConfig& config) {
    eta_ = static_cast<double>(config.gaussian_eta);
  }

  explicit RegressionL1loss(const std::vector<std::string>&) {

  }

  ~RegressionL1loss() {}

  void Init(const Metadata& metadata, data_size_t num_data) override {
    num_data_ = num_data;
    label_ = metadata.label();
    weights_ = metadata.weights();
  }

  void GetGradients(const double* score, score_t* gradients,
                    score_t* hessians) const override {
    if (weights_ == nullptr) {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double diff = score[i] - label_[i];
        if (diff >= 0.0f) {
          gradients[i] = 1.0f;
        } else {
          gradients[i] = -1.0f;
        }
        hessians[i] = static_cast<score_t>(Common::ApproximateHessianWithGaussian(score[i], label_[i], gradients[i], eta_));
      }
    } else {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double diff = score[i] - label_[i];
        if (diff >= 0.0f) {
          gradients[i] = weights_[i];
        } else {
          gradients[i] = -weights_[i];
        }
        hessians[i] = static_cast<score_t>(Common::ApproximateHessianWithGaussian(score[i], label_[i], gradients[i], eta_, weights_[i]));
      }
    }
  }

  const char* GetName() const override {
    return "regression_l1";
  }

  std::string ToString() const override {
    std::stringstream str_buf;
    str_buf << GetName();
    return str_buf.str();
  }

  bool BoostFromAverage() const override { return true; }

private:
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Pointer of label */
  const float* label_;
  /*! \brief Pointer of weights */
  const float* weights_;
  /*! \brief a parameter to control the width of Gaussian function to approximate hessian */
  double eta_;
};

/*!
* \brief Huber regression loss
*/
class RegressionHuberLoss: public ObjectiveFunction {
public:
  explicit RegressionHuberLoss(const ObjectiveConfig& config) {
    delta_ = static_cast<double>(config.huber_delta);
    eta_ = static_cast<double>(config.gaussian_eta);
  }

  explicit RegressionHuberLoss(const std::vector<std::string>&) {

  }

  ~RegressionHuberLoss() {
  }

  void Init(const Metadata& metadata, data_size_t num_data) override {
    num_data_ = num_data;
    label_ = metadata.label();
    weights_ = metadata.weights();
  }

  void GetGradients(const double* score, score_t* gradients,
                    score_t* hessians) const override {
    if (weights_ == nullptr) {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double diff = score[i] - label_[i];

        if (std::abs(diff) <= delta_) {
          gradients[i] = static_cast<score_t>(diff);
          hessians[i] = 1.0f;
        } else {
          if (diff >= 0.0f) {
            gradients[i] = static_cast<score_t>(delta_);
          } else {
            gradients[i] = static_cast<score_t>(-delta_);
          }
          hessians[i] = static_cast<score_t>(Common::ApproximateHessianWithGaussian(score[i], label_[i], gradients[i], eta_));
        }
      }
    } else {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double diff = score[i] - label_[i];

        if (std::abs(diff) <= delta_) {
          gradients[i] = static_cast<score_t>(diff * weights_[i]);
          hessians[i] = weights_[i];
        } else {
          if (diff >= 0.0f) {
            gradients[i] = static_cast<score_t>(delta_ * weights_[i]);
          } else {
            gradients[i] = static_cast<score_t>(-delta_ * weights_[i]);
          }
          hessians[i] = static_cast<score_t>(Common::ApproximateHessianWithGaussian(score[i], label_[i], gradients[i], eta_, weights_[i]));
        }
      }
    }
  }

  const char* GetName() const override {
    return "huber";
  }

  std::string ToString() const override {
    std::stringstream str_buf;
    str_buf << GetName();
    return str_buf.str();
  }

  bool BoostFromAverage() const override { return true; }

private:
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Pointer of label */
  const float* label_;
  /*! \brief Pointer of weights */
  const float* weights_;
  /*! \brief delta for Huber loss */
  double delta_;
  /*! \brief a parameter to control the width of Gaussian function to approximate hessian */
  double eta_;
};

// http://research.microsoft.com/en-us/um/people/zhang/INRIA/Publis/Tutorial-Estim/node24.html
class RegressionFairLoss: public ObjectiveFunction {
public:
  explicit RegressionFairLoss(const ObjectiveConfig& config) {
    c_ = static_cast<double>(config.fair_c);
  }

  explicit RegressionFairLoss(const std::vector<std::string>&) {

  }

  ~RegressionFairLoss() {}

  void Init(const Metadata& metadata, data_size_t num_data) override {
    num_data_ = num_data;
    label_ = metadata.label();
    weights_ = metadata.weights();
  }

  void GetGradients(const double* score, score_t* gradients,
                    score_t* hessians) const override {
    if (weights_ == nullptr) {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double x = score[i] - label_[i];
        gradients[i] = static_cast<score_t>(c_ * x / (std::fabs(x) + c_));
        hessians[i] = static_cast<score_t>(c_ * c_ / ((std::fabs(x) + c_) * (std::fabs(x) + c_)));
      }
    } else {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double x = score[i] - label_[i];
        gradients[i] = static_cast<score_t>(c_ * x / (std::fabs(x) + c_) * weights_[i]);
        hessians[i] = static_cast<score_t>(c_ * c_ / ((std::fabs(x) + c_) * (std::fabs(x) + c_)) * weights_[i]);
      }
    }
  }

  const char* GetName() const override {
    return "fair";
  }

  std::string ToString() const override {
    std::stringstream str_buf;
    str_buf << GetName();
    return str_buf.str();
  }

  bool BoostFromAverage() const override { return true; }

private:
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Pointer of label */
  const float* label_;
  /*! \brief Pointer of weights */
  const float* weights_;
  /*! \brief c for Fair loss */
  double c_;
};

/*!
* \brief Objective function for Poisson regression
*/
class RegressionPoissonLoss: public ObjectiveFunction {
public:
  explicit RegressionPoissonLoss(const ObjectiveConfig& config) {
    max_delta_step_ = static_cast<double>(config.poisson_max_delta_step);
  }

  explicit RegressionPoissonLoss(const std::vector<std::string>&) {

  }

  ~RegressionPoissonLoss() {}

  void Init(const Metadata& metadata, data_size_t num_data) override {
    num_data_ = num_data;
    label_ = metadata.label();
    weights_ = metadata.weights();
  }

  /* Parametrize with unbounded internal score "f"; then
   *  loss = exp(f) - label * f
   *  grad = exp(f) - label
   *  hess = exp(f)
   *
   * And the output is exp(f); so the associated metric get s=exp(f)
   * so that its loss = s - label * log(s); a little awkward maybe.
   *
   */
  void GetGradients(const double* score, score_t* gradients,
                    score_t* hessians) const override {
    if (weights_ == nullptr) {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double ef = std::exp(score[i]);
        gradients[i] = static_cast<score_t>(ef - label_[i]);
        hessians[i] = static_cast<score_t>(ef);
      }
    } else {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double ef = std::exp(score[i]);
        gradients[i] = static_cast<score_t>((ef - label_[i]) * weights_[i]);
        hessians[i] = static_cast<score_t>(ef * weights_[i]);
      }
    }
  }

  void ConvertOutput(const double* input, double* output) const override {
    output[0] = std::exp(input[0]);
  }

  const char* GetName() const override {
    return "poisson";
  }

  std::string ToString() const override {
    std::stringstream str_buf;
    str_buf << GetName();
    return str_buf.str();
  }

  bool BoostFromAverage() const override { return true; }

  bool GetCustomAverage(double *initscore) const override {
    if (initscore == nullptr) return false;
    double sumw = 0.0f;
    double sumy = 0.0f;
    if (weights_ == nullptr) {
      for (data_size_t i = 0; i < num_data_; i++) {
        sumy += label_[i];
      }
      sumw = static_cast<double>(num_data_);
    } else {
      for (data_size_t i = 0; i < num_data_; i++) {
        sumy += weights_[i] * label_[i];
        sumw += weights_[i];
      }
    }
    const double yavg = sumy / sumw;
    *initscore = std::log(yavg);
    Log::Info("[%s:%s]: yavg=%f -> initscore=%f",  GetName(), __func__, yavg, *initscore);
    return true;
  }

private:
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Pointer of label */
  const float* label_;
  /*! \brief Pointer of weights */
  const float* weights_;
  /*! \brief used to safeguard optimization */
  double max_delta_step_;
};

}  // namespace LightGBM
#endif   // LightGBM_OBJECTIVE_REGRESSION_OBJECTIVE_HPP_
<commit_msg>Added label non-negative safety check.<commit_after>#ifndef LIGHTGBM_OBJECTIVE_REGRESSION_OBJECTIVE_HPP_
#define LIGHTGBM_OBJECTIVE_REGRESSION_OBJECTIVE_HPP_

#include <LightGBM/meta.h>

#include <LightGBM/objective_function.h>
#include <LightGBM/utils/common.h>

namespace LightGBM {
/*!
* \brief Objective function for regression
*/
class RegressionL2loss: public ObjectiveFunction {
public:
  explicit RegressionL2loss(const ObjectiveConfig&) {
  }

  explicit RegressionL2loss(const std::vector<std::string>&) {

  }

  ~RegressionL2loss() {
  }

  void Init(const Metadata& metadata, data_size_t num_data) override {
    num_data_ = num_data;
    label_ = metadata.label();
    weights_ = metadata.weights();
  }

  void GetGradients(const double* score, score_t* gradients,
                    score_t* hessians) const override {
    if (weights_ == nullptr) {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        gradients[i] = static_cast<score_t>(score[i] - label_[i]);
        hessians[i] = 1.0f;
      }
    } else {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        gradients[i] = static_cast<score_t>(score[i] - label_[i]) * weights_[i];
        hessians[i] = weights_[i];
      }
    }
  }

  const char* GetName() const override {
    return "regression";
  }

  std::string ToString() const override {
    std::stringstream str_buf;
    str_buf << GetName();
    return str_buf.str();
  }

  bool IsConstantHessian() const override {
    if (weights_ == nullptr) {
      return true;
    } else {
      return false;
    }
  }

  bool BoostFromAverage() const override { return true; }

private:
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Pointer of label */
  const float* label_;
  /*! \brief Pointer of weights */
  const float* weights_;
};

/*!
* \brief L1 regression loss
*/
class RegressionL1loss: public ObjectiveFunction {
public:
  explicit RegressionL1loss(const ObjectiveConfig& config) {
    eta_ = static_cast<double>(config.gaussian_eta);
  }

  explicit RegressionL1loss(const std::vector<std::string>&) {

  }

  ~RegressionL1loss() {}

  void Init(const Metadata& metadata, data_size_t num_data) override {
    num_data_ = num_data;
    label_ = metadata.label();
    weights_ = metadata.weights();
  }

  void GetGradients(const double* score, score_t* gradients,
                    score_t* hessians) const override {
    if (weights_ == nullptr) {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double diff = score[i] - label_[i];
        if (diff >= 0.0f) {
          gradients[i] = 1.0f;
        } else {
          gradients[i] = -1.0f;
        }
        hessians[i] = static_cast<score_t>(Common::ApproximateHessianWithGaussian(score[i], label_[i], gradients[i], eta_));
      }
    } else {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double diff = score[i] - label_[i];
        if (diff >= 0.0f) {
          gradients[i] = weights_[i];
        } else {
          gradients[i] = -weights_[i];
        }
        hessians[i] = static_cast<score_t>(Common::ApproximateHessianWithGaussian(score[i], label_[i], gradients[i], eta_, weights_[i]));
      }
    }
  }

  const char* GetName() const override {
    return "regression_l1";
  }

  std::string ToString() const override {
    std::stringstream str_buf;
    str_buf << GetName();
    return str_buf.str();
  }

  bool BoostFromAverage() const override { return true; }

private:
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Pointer of label */
  const float* label_;
  /*! \brief Pointer of weights */
  const float* weights_;
  /*! \brief a parameter to control the width of Gaussian function to approximate hessian */
  double eta_;
};

/*!
* \brief Huber regression loss
*/
class RegressionHuberLoss: public ObjectiveFunction {
public:
  explicit RegressionHuberLoss(const ObjectiveConfig& config) {
    delta_ = static_cast<double>(config.huber_delta);
    eta_ = static_cast<double>(config.gaussian_eta);
  }

  explicit RegressionHuberLoss(const std::vector<std::string>&) {

  }

  ~RegressionHuberLoss() {
  }

  void Init(const Metadata& metadata, data_size_t num_data) override {
    num_data_ = num_data;
    label_ = metadata.label();
    weights_ = metadata.weights();
  }

  void GetGradients(const double* score, score_t* gradients,
                    score_t* hessians) const override {
    if (weights_ == nullptr) {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double diff = score[i] - label_[i];

        if (std::abs(diff) <= delta_) {
          gradients[i] = static_cast<score_t>(diff);
          hessians[i] = 1.0f;
        } else {
          if (diff >= 0.0f) {
            gradients[i] = static_cast<score_t>(delta_);
          } else {
            gradients[i] = static_cast<score_t>(-delta_);
          }
          hessians[i] = static_cast<score_t>(Common::ApproximateHessianWithGaussian(score[i], label_[i], gradients[i], eta_));
        }
      }
    } else {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double diff = score[i] - label_[i];

        if (std::abs(diff) <= delta_) {
          gradients[i] = static_cast<score_t>(diff * weights_[i]);
          hessians[i] = weights_[i];
        } else {
          if (diff >= 0.0f) {
            gradients[i] = static_cast<score_t>(delta_ * weights_[i]);
          } else {
            gradients[i] = static_cast<score_t>(-delta_ * weights_[i]);
          }
          hessians[i] = static_cast<score_t>(Common::ApproximateHessianWithGaussian(score[i], label_[i], gradients[i], eta_, weights_[i]));
        }
      }
    }
  }

  const char* GetName() const override {
    return "huber";
  }

  std::string ToString() const override {
    std::stringstream str_buf;
    str_buf << GetName();
    return str_buf.str();
  }

  bool BoostFromAverage() const override { return true; }

private:
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Pointer of label */
  const float* label_;
  /*! \brief Pointer of weights */
  const float* weights_;
  /*! \brief delta for Huber loss */
  double delta_;
  /*! \brief a parameter to control the width of Gaussian function to approximate hessian */
  double eta_;
};

// http://research.microsoft.com/en-us/um/people/zhang/INRIA/Publis/Tutorial-Estim/node24.html
class RegressionFairLoss: public ObjectiveFunction {
public:
  explicit RegressionFairLoss(const ObjectiveConfig& config) {
    c_ = static_cast<double>(config.fair_c);
  }

  explicit RegressionFairLoss(const std::vector<std::string>&) {

  }

  ~RegressionFairLoss() {}

  void Init(const Metadata& metadata, data_size_t num_data) override {
    num_data_ = num_data;
    label_ = metadata.label();
    weights_ = metadata.weights();
  }

  void GetGradients(const double* score, score_t* gradients,
                    score_t* hessians) const override {
    if (weights_ == nullptr) {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double x = score[i] - label_[i];
        gradients[i] = static_cast<score_t>(c_ * x / (std::fabs(x) + c_));
        hessians[i] = static_cast<score_t>(c_ * c_ / ((std::fabs(x) + c_) * (std::fabs(x) + c_)));
      }
    } else {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double x = score[i] - label_[i];
        gradients[i] = static_cast<score_t>(c_ * x / (std::fabs(x) + c_) * weights_[i]);
        hessians[i] = static_cast<score_t>(c_ * c_ / ((std::fabs(x) + c_) * (std::fabs(x) + c_)) * weights_[i]);
      }
    }
  }

  const char* GetName() const override {
    return "fair";
  }

  std::string ToString() const override {
    std::stringstream str_buf;
    str_buf << GetName();
    return str_buf.str();
  }

  bool BoostFromAverage() const override { return true; }

private:
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Pointer of label */
  const float* label_;
  /*! \brief Pointer of weights */
  const float* weights_;
  /*! \brief c for Fair loss */
  double c_;
};

/*!
* \brief Objective function for Poisson regression
*/
class RegressionPoissonLoss: public ObjectiveFunction {
public:
  explicit RegressionPoissonLoss(const ObjectiveConfig& config) {
    max_delta_step_ = static_cast<double>(config.poisson_max_delta_step);
  }

  explicit RegressionPoissonLoss(const std::vector<std::string>&) {

  }

  ~RegressionPoissonLoss() {}

  void Init(const Metadata& metadata, data_size_t num_data) override {
    num_data_ = num_data;
    label_ = metadata.label();
    weights_ = metadata.weights();

    // Safety check of labels
    float miny;
    double sumy;
    Common::obtain_min_max_sum(label_, num_data_, &miny, nullptr, &sumy);
    if (miny < 0.0f) {
      Log::Fatal("[%s]: at least one target label is negative.", GetName());
    }
    if (sumy == 0.0f) {
      Log::Fatal("[%s]: sum of labels is zero.", GetName());
    }
  }

  /* Parametrize with unbounded internal score "f"; then
   *  loss = exp(f) - label * f
   *  grad = exp(f) - label
   *  hess = exp(f)
   *
   * And the output is exp(f); so the associated metric get s=exp(f)
   * so that its loss = s - label * log(s); a little awkward maybe.
   *
   */
  void GetGradients(const double* score, score_t* gradients,
                    score_t* hessians) const override {
    if (weights_ == nullptr) {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double ef = std::exp(score[i]);
        gradients[i] = static_cast<score_t>(ef - label_[i]);
        hessians[i] = static_cast<score_t>(ef);
      }
    } else {
      #pragma omp parallel for schedule(static)
      for (data_size_t i = 0; i < num_data_; ++i) {
        const double ef = std::exp(score[i]);
        gradients[i] = static_cast<score_t>((ef - label_[i]) * weights_[i]);
        hessians[i] = static_cast<score_t>(ef * weights_[i]);
      }
    }
  }

  void ConvertOutput(const double* input, double* output) const override {
    output[0] = std::exp(input[0]);
  }

  const char* GetName() const override {
    return "poisson";
  }

  std::string ToString() const override {
    std::stringstream str_buf;
    str_buf << GetName();
    return str_buf.str();
  }

  bool BoostFromAverage() const override { return true; }

  bool GetCustomAverage(double *initscore) const override {
    if (initscore == nullptr) return false;
    double sumw = 0.0f;
    double sumy = 0.0f;
    if (weights_ == nullptr) {
      for (data_size_t i = 0; i < num_data_; i++) {
        sumy += label_[i];
      }
      sumw = static_cast<double>(num_data_);
    } else {
      for (data_size_t i = 0; i < num_data_; i++) {
        sumy += weights_[i] * label_[i];
        sumw += weights_[i];
      }
    }
    const double yavg = sumy / sumw;
    *initscore = std::log(yavg);
    Log::Info("[%s:%s]: yavg=%f -> initscore=%f",  GetName(), __func__, yavg, *initscore);
    return true;
  }

private:
  /*! \brief Number of data */
  data_size_t num_data_;
  /*! \brief Pointer of label */
  const float* label_;
  /*! \brief Pointer of weights */
  const float* weights_;
  /*! \brief used to safeguard optimization */
  double max_delta_step_;
};

}  // namespace LightGBM
#endif   // LightGBM_OBJECTIVE_REGRESSION_OBJECTIVE_HPP_
<|endoftext|>