<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "parameters.hpp"
#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/vespalib/objects/hexdump.h>
#include <vespa/vespalib/stllike/hash_map.hpp>
#include <vespa/vespalib/util/xmlstream.h>

using namespace vdslib;

Parameters::Parameters() : _parameters() { }

Parameters::Parameters(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer)
    : _parameters()
{
    deserialize(repo, buffer);
}

Parameters::~Parameters()
{
}

size_t Parameters::getSerializedSize() const
{
    size_t mysize = sizeof(int32_t);
    for (const auto & entry : _parameters) {
        mysize += entry.first.size() + 4 + 4 + entry.second.size();
    }
    return mysize;
}

void Parameters::onSerialize(document::ByteBuffer& buffer) const
{
    buffer.putIntNetwork(_parameters.size());
    for (const auto & entry : _parameters) {
        buffer.putIntNetwork(entry.first.size());
        buffer.putBytes(entry.first.c_str(), entry.first.size());
        buffer.putIntNetwork(entry.second.size());
        buffer.putBytes(entry.second.c_str(), entry.second.size());
    }
}

void Parameters::onDeserialize(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer)
{
    (void) repo;
    _parameters.clear();
    int32_t mysize;
    buffer.getIntNetwork(mysize);
    for (int i=0; i<mysize; i++) {
        int32_t keylen = 0;
        buffer.getIntNetwork(keylen);
        vespalib::stringref key(buffer.getBufferAtPos(), keylen);
        buffer.incPos(keylen);
        int32_t sz(0);
        buffer.getIntNetwork(sz);
        _parameters[key] = Value(buffer.getBufferAtPos(), sz);
        buffer.incPos(sz);
    }
}

void
Parameters::printXml(document::XmlOutputStream& xos) const
{
    using namespace vespalib::xml;
    xos << XmlTag("parameters");
    for (const auto & entry : _parameters) {
        xos << XmlTag("item")
                << XmlTag("name") << entry.first << XmlEndTag()
                << XmlTag("value") << entry.second << XmlEndTag()
            << XmlEndTag();
    }
    xos << XmlEndTag();
}

/*
void
Parameters::deserializeXml(const document::XmlElement & xml)
{
    ParametersMap params;
    for(document::XmlElement::ElementList::const_iterator it = xml.elements().begin(), mt = xml.elements().end(); it != mt; it++) {
        const document::XmlElement & elem = *it;
        assert( elem.elements().size() == 2);

        const document::XmlElement & name = (*it).elements()[0];
        const document::XmlElement & value = (*it).elements()[1];
        params[name.value()]  = Value(value.value().c_str(), value.value().size());
    }
    _parameters.swap(params);
}
*/

bool
Parameters::operator==(const Parameters &other) const
{
    if (size() != other.size()) {
        return false;
    }

    for (ParametersMap::const_iterator a(_parameters.begin()), b(other._parameters.begin()), am(_parameters.begin()); (a != am); ++a, ++b) {
        if ((a->first != b->first)) {
            return false;
        }
    }

    return true;
}

Parameters* Parameters::clone() const
{
    return new Parameters(*this);
}

vespalib::stringref Parameters::get(const vespalib::stringref& id, const vespalib::stringref& def) const
{
    ParametersMap::const_iterator it = _parameters.find(id);
    if (it == _parameters.end()) return def;
    return it->second;
}

bool Parameters::get(const KeyT & id, ValueRef & v ) const
{
    ParametersMap::const_iterator it = _parameters.find(id);
    if (it == _parameters.end()) return false;
    v = ValueRef(it->second.c_str(), it->second.size());
    return true;
}

void Parameters::print(std::ostream& out, bool verbose, const std::string& indent) const
{
    out << "Parameters(";
    if (!verbose) {
        out << _parameters.size() << " values";
    } else {
        for (const auto & entry : _parameters) {
            bool isPrintable(true);
            for (size_t i(0), m(entry.second.size()); isPrintable && (i < m); i++) {
                isPrintable = isprint(entry.second[i]);
            }
            out << "\n" << indent << "           " << entry.first << " = ";
            if (!entry.second.empty() && isPrintable && (entry.second[entry.second.size()-1] == 0)) {
                out  << entry.second.c_str();
            } else {
                out  << vespalib::HexDump(entry.second.c_str(), entry.second.size());
            }
        }
    }
    out << ")";
}

std::string Parameters::toString() const
{
    vespalib::string ret;
    for (const auto & entry : _parameters) {
        ret += entry.first;
        ret += '=';
        ret += entry.second;
        ret += '|';
    }
    return ret;
}

template void vdslib::Parameters::set(const vespalib::stringref &, int32_t);
template void vdslib::Parameters::set(const vespalib::stringref &, int64_t);
template void vdslib::Parameters::set(const vespalib::stringref &, uint64_t);
template void vdslib::Parameters::set(const vespalib::stringref &, double);
template void vdslib::Parameters::set(const vespalib::stringref &, const char *);
template void vdslib::Parameters::set(const vespalib::stringref &, vespalib::string);
template void vdslib::Parameters::set(const vespalib::stringref &, std::string);
template int32_t vdslib::Parameters::get(const vespalib::stringref &, int32_t) const;
template int64_t vdslib::Parameters::get(const vespalib::stringref &, int64_t) const;
template uint64_t vdslib::Parameters::get(const vespalib::stringref &, uint64_t) const;
template double vdslib::Parameters::get(const vespalib::stringref &, double) const;
template std::string vdslib::Parameters::get(const vespalib::stringref &, std::string) const;

VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, vdslib::Parameters::Value);
<commit_msg>pre-size hashtable to avoid reallocation<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.

#include "parameters.hpp"
#include <vespa/vespalib/objects/nbostream.h>
#include <vespa/vespalib/objects/hexdump.h>
#include <vespa/vespalib/stllike/hash_map.hpp>
#include <vespa/vespalib/util/xmlstream.h>

using namespace vdslib;

Parameters::Parameters() : _parameters() { }

Parameters::Parameters(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer)
    : _parameters()
{
    deserialize(repo, buffer);
}

Parameters::~Parameters()
{
}

size_t Parameters::getSerializedSize() const
{
    size_t mysize = sizeof(int32_t);
    for (const auto & entry : _parameters) {
        mysize += entry.first.size() + 4 + 4 + entry.second.size();
    }
    return mysize;
}

void Parameters::onSerialize(document::ByteBuffer& buffer) const
{
    buffer.putIntNetwork(_parameters.size());
    for (const auto & entry : _parameters) {
        buffer.putIntNetwork(entry.first.size());
        buffer.putBytes(entry.first.c_str(), entry.first.size());
        buffer.putIntNetwork(entry.second.size());
        buffer.putBytes(entry.second.c_str(), entry.second.size());
    }
}

void Parameters::onDeserialize(const document::DocumentTypeRepo &repo, document::ByteBuffer& buffer)
{
    (void) repo;
    _parameters.clear();
    int32_t mysize;
    buffer.getIntNetwork(mysize);
    _parameters.resize(mysize);
    for (int i=0; i<mysize; i++) {
        int32_t keylen = 0;
        buffer.getIntNetwork(keylen);
        vespalib::stringref key(buffer.getBufferAtPos(), keylen);
        buffer.incPos(keylen);
        int32_t sz(0);
        buffer.getIntNetwork(sz);
        _parameters[key] = Value(buffer.getBufferAtPos(), sz);
        buffer.incPos(sz);
    }
}

void
Parameters::printXml(document::XmlOutputStream& xos) const
{
    using namespace vespalib::xml;
    xos << XmlTag("parameters");
    for (const auto & entry : _parameters) {
        xos << XmlTag("item")
                << XmlTag("name") << entry.first << XmlEndTag()
                << XmlTag("value") << entry.second << XmlEndTag()
            << XmlEndTag();
    }
    xos << XmlEndTag();
}

/*
void
Parameters::deserializeXml(const document::XmlElement & xml)
{
    ParametersMap params;
    for(document::XmlElement::ElementList::const_iterator it = xml.elements().begin(), mt = xml.elements().end(); it != mt; it++) {
        const document::XmlElement & elem = *it;
        assert( elem.elements().size() == 2);

        const document::XmlElement & name = (*it).elements()[0];
        const document::XmlElement & value = (*it).elements()[1];
        params[name.value()]  = Value(value.value().c_str(), value.value().size());
    }
    _parameters.swap(params);
}
*/

bool
Parameters::operator==(const Parameters &other) const
{
    if (size() != other.size()) {
        return false;
    }

    for (ParametersMap::const_iterator a(_parameters.begin()), b(other._parameters.begin()), am(_parameters.begin()); (a != am); ++a, ++b) {
        if ((a->first != b->first)) {
            return false;
        }
    }

    return true;
}

Parameters* Parameters::clone() const
{
    return new Parameters(*this);
}

vespalib::stringref Parameters::get(const vespalib::stringref& id, const vespalib::stringref& def) const
{
    ParametersMap::const_iterator it = _parameters.find(id);
    if (it == _parameters.end()) return def;
    return it->second;
}

bool Parameters::get(const KeyT & id, ValueRef & v ) const
{
    ParametersMap::const_iterator it = _parameters.find(id);
    if (it == _parameters.end()) return false;
    v = ValueRef(it->second.c_str(), it->second.size());
    return true;
}

void Parameters::print(std::ostream& out, bool verbose, const std::string& indent) const
{
    out << "Parameters(";
    if (!verbose) {
        out << _parameters.size() << " values";
    } else {
        for (const auto & entry : _parameters) {
            bool isPrintable(true);
            for (size_t i(0), m(entry.second.size()); isPrintable && (i < m); i++) {
                isPrintable = isprint(entry.second[i]);
            }
            out << "\n" << indent << "           " << entry.first << " = ";
            if (!entry.second.empty() && isPrintable && (entry.second[entry.second.size()-1] == 0)) {
                out  << entry.second.c_str();
            } else {
                out  << vespalib::HexDump(entry.second.c_str(), entry.second.size());
            }
        }
    }
    out << ")";
}

std::string Parameters::toString() const
{
    vespalib::string ret;
    for (const auto & entry : _parameters) {
        ret += entry.first;
        ret += '=';
        ret += entry.second;
        ret += '|';
    }
    return ret;
}

template void vdslib::Parameters::set(const vespalib::stringref &, int32_t);
template void vdslib::Parameters::set(const vespalib::stringref &, int64_t);
template void vdslib::Parameters::set(const vespalib::stringref &, uint64_t);
template void vdslib::Parameters::set(const vespalib::stringref &, double);
template void vdslib::Parameters::set(const vespalib::stringref &, const char *);
template void vdslib::Parameters::set(const vespalib::stringref &, vespalib::string);
template void vdslib::Parameters::set(const vespalib::stringref &, std::string);
template int32_t vdslib::Parameters::get(const vespalib::stringref &, int32_t) const;
template int64_t vdslib::Parameters::get(const vespalib::stringref &, int64_t) const;
template uint64_t vdslib::Parameters::get(const vespalib::stringref &, uint64_t) const;
template double vdslib::Parameters::get(const vespalib::stringref &, double) const;
template std::string vdslib::Parameters::get(const vespalib::stringref &, std::string) const;

VESPALIB_HASH_MAP_INSTANTIATE(vespalib::string, vdslib::Parameters::Value);
<|endoftext|>


<commit_before>/*******************************************************************************
 * thrill/data/channel.hpp
 *
 * Part of Project c7a.
 *
 * Copyright (C) 2015 Timo Bingmann <tb@panthema.net>
 * Copyright (C) 2015 Tobias Sturm <mail@tobiassturm.de>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef C7A_DATA_CHANNEL_HEADER
#define C7A_DATA_CHANNEL_HEADER

#include <thrill/common/stats_counter.hpp>
#include <thrill/common/stats_timer.hpp>
#include <thrill/data/block_queue.hpp>
#include <thrill/data/channel_sink.hpp>
#include <thrill/data/concat_block_source.hpp>
#include <thrill/data/file.hpp>
#include <thrill/data/multiplexer.hpp>
#include <thrill/data/multiplexer_header.hpp>
#include <thrill/net/connection.hpp>
#include <thrill/net/group.hpp>

#include <sstream>
#include <string>
#include <vector>

namespace c7a {
namespace data {

//! \addtogroup data Data Subsystem
//! \{

using ChannelId = size_t;

/*!
 * A Channel is a virtual set of connections to all other worker instances,
 * hence a "Channel" bundles them to a logical communication context. We call an
 * individual connection from a worker to another worker a "Host".
 *
 * To use a Channel, one can get a vector of BlockWriter via OpenWriters() of
 * outbound Channel. The vector is of size of workers in the system.
 * One can then write items destined to the
 * corresponding worker. The written items are buffered into a Block and only
 * sent when the Block is full. To force a send, use BlockWriter::Flush(). When
 * all items are sent, the BlockWriters **must** be closed using
 * BlockWriter::Close().
 *
 * To read the inbound Connection items, one can get a vector of BlockReader via
 * OpenReaders(), which can then be used to read items sent by individual
 * workers.
 *
 * Alternatively, one can use OpenReader() to get a BlockReader which delivers
 * all items from *all* worker in worker order (concatenating all inbound
 * Connections).
 *
 * As soon as all attached streams of the Channel have been Close() the number of
 * expected streams is reached, the channel is marked as finished and no more
 * data will arrive.
 */
class Channel
{
public:
    using BlockQueueReader = BlockReader<BlockQueueSource>;
    using ConcatBlockSource = data::ConcatBlockSource<BlockQueueSource>;
    using ConcatBlockReader = BlockReader<ConcatBlockSource>;

    using CachingConcatBlockSource = data::ConcatBlockSource<CachingBlockQueueSource>;
    using CachingConcatBlockReader = BlockReader<CachingConcatBlockSource>;

    using Reader = BlockQueueReader;
    using ConcatReader = ConcatBlockReader;
    using CachingConcatReader = CachingConcatBlockReader;

    using StatsCounter = common::StatsCounter<size_t, common::g_enable_stats>;
    using StatsTimer = common::StatsTimer<common::g_enable_stats>;

    using ClosedCallback = std::function<void()>;

    //! Creates a new channel instance
    Channel(data::Multiplexer& multiplexer, const ChannelId& id,
            size_t my_local_worker_id)
        : tx_lifetime_(true), rx_lifetime_(true),
          tx_timespan_(), rx_timespan_(),
          id_(id),
          multiplexer_(multiplexer),
          queues_(multiplexer_.num_workers()),
          cache_files_(multiplexer_.num_workers()),
          expected_closing_blocks_(
              (multiplexer_.num_hosts() - 1) * multiplexer_.num_workers_per_host_),
          received_closing_blocks_(0) {
        // construct ChannelSink array
        for (size_t host = 0; host < multiplexer_.num_hosts(); ++host) {
            for (size_t worker = 0; worker < multiplexer_.num_workers_per_host_; worker++) {
                if (host == multiplexer_.my_host_rank()) {
                    sinks_.emplace_back();
                }
                else {
                    sinks_.emplace_back(
                        &multiplexer_.dispatcher_,
                        &multiplexer_.group_.connection(host),
                        id,
                        multiplexer_.my_host_rank(), my_local_worker_id, worker,
                        &outgoing_bytes_, &outgoing_blocks_, &tx_timespan_);
                }
            }
        }
    }

    //! non-copyable: delete copy-constructor
    Channel(const Channel&) = delete;
    //! non-copyable: delete assignment operator
    Channel& operator = (const Channel&) = delete;

    //! move-constructor
    Channel(Channel&&) = default;

    const ChannelId & id() const {
        return id_;
    }

    //! Creates BlockWriters for each worker. BlockWriter can only be opened
    //! once, otherwise the block sequence is incorrectly interleaved!
    std::vector<BlockWriter> OpenWriters(size_t block_size = default_block_size) {
        tx_timespan_.StartEventually();

        std::vector<BlockWriter> result;

        for (size_t host = 0; host < multiplexer_.num_hosts(); ++host) {
            for (size_t local_worker_id = 0; local_worker_id < multiplexer_.num_workers_per_host_; ++local_worker_id) {
                size_t worker_id = host * multiplexer_.num_workers_per_host_ + local_worker_id;
                if (host == multiplexer_.my_host_rank()) {
                    result.emplace_back(&queues_[worker_id], block_size);
                }
                else {
                    result.emplace_back(&sinks_[worker_id], block_size);
                }
            }
        }

        assert(result.size() == multiplexer_.num_workers());
        return result;
    }

    //! Creates a BlockReader for each worker. The BlockReaders are attached to
    //! the BlockQueues in the Channel and wait for further Blocks to arrive or
    //! the Channel's remote close.
    std::vector<BlockQueueReader> OpenReaders() {
        rx_timespan_.StartEventually();

        std::vector<BlockQueueReader> result;

        for (size_t worker = 0; worker < multiplexer_.num_workers(); ++worker) {
            result.emplace_back(BlockQueueSource(queues_[worker]));
        }

        assert(result.size() == multiplexer_.num_workers());
        return result;
    }

    //! Creates a BlockReader for all workers. The BlockReader is attached to
    //! one \ref ConcatBlockSource which includes all incoming queues of
    //! this channel.
    ConcatBlockReader OpenReader() {
        rx_timespan_.StartEventually();

        // construct vector of BlockQueueSources to read from queues_.
        std::vector<BlockQueueSource> result;

        for (size_t worker = 0; worker < multiplexer_.num_workers(); ++worker) {
            result.emplace_back(queues_[worker]);
        }
        // move BlockQueueSources into concatenation BlockSource, and to Reader.
        return ConcatBlockReader(ConcatBlockSource(result));
    }

    //! Creates a BlockReader for all workers. The BlockReader is attached to
    //! one \ref ConcatBlockSource which includes all incoming queues of this
    //! channel. The received Blocks are also cached in the Channel, hence this
    //! function can be called multiple times to read the items again.
    CachingConcatBlockReader OpenCachingReader() {
        rx_timespan_.StartEventually();

        // construct vector of CachingBlockQueueSources to read from queues_.
        std::vector<CachingBlockQueueSource> result;
        for (size_t worker = 0; worker < multiplexer_.num_workers(); ++worker) {
            result.emplace_back(queues_[worker], cache_files_[worker]);
        }
        // move CachingBlockQueueSources into concatenation BlockSource, and to
        // Reader.
        return CachingConcatBlockReader(CachingConcatBlockSource(result));
    }

    /*!
     * Scatters a File to many worker
     *
     * elements from 0..offset[0] are sent to the first worker,
     * elements from (offset[0] + 1)..offset[1] are sent to the second worker.
     * elements from (offset[my_rank - 1] + 1)..(offset[my_rank]) are copied
     * The offset values range from 0..Manager::GetNumElements().
     * The number of given offsets must be equal to the net::Group::num_workers() * workers_per_host_.
     *
     * /param source File containing the data to be scattered.
     *
     * /param offsets - as described above. offsets.size must be equal to group.size
     */
    template <typename ItemType>
    void Scatter(const File& source, const std::vector<size_t>& offsets) {
        tx_timespan_.StartEventually();

        // current item offset in Reader
        size_t current = 0;
        typename File::Reader reader = source.GetReader();

        std::vector<BlockWriter> writers = OpenWriters();

        for (size_t worker = 0; worker < multiplexer_.num_workers(); ++worker) {
            // write [current,limit) to this worker
            size_t limit = offsets[worker];
            assert(current <= limit);
#if 0
            for ( ; current < limit; ++current) {
                assert(reader.HasNext());
                // move over one item (with deserialization and serialization)
                writers[worker](reader.template Next<ItemType>());
            }
#else
            if (current != limit) {
                writers[worker].AppendBlocks(
                    reader.template GetItemBatch<ItemType>(limit - current));
                current = limit;
            }
#endif
            writers[worker].Close();
        }

        tx_timespan_.Stop();
    }

    //! shuts the channel down.
    void Close() {
        // close all sinks, this should emit sentinel to all other worker.
        for (size_t i = 0; i != sinks_.size(); ++i) {
            if (sinks_[i].closed()) continue;
            sinks_[i].Close();
        }

        // close self-loop queues
        for (size_t my_worker = 0; my_worker < multiplexer_.num_workers_per_host_; my_worker++) {
            size_t local_worker_id = multiplexer_.my_host_rank() + my_worker;
            if (!queues_[local_worker_id].write_closed())
                queues_[local_worker_id].Close();
        }

        // wait for close packets to arrive (this is a busy waiting loop, try to
        // do it better -tb)
        for (size_t i = 0; i != queues_.size(); ++i) {
            while (!queues_[i].write_closed()) {
                LOG << "wait for close from worker" << i;
                std::this_thread::sleep_for(std::chrono::milliseconds(10));
            }
        }
        tx_lifetime_.StopEventually();
        tx_timespan_.StopEventually();
        CallClosedCallbacksEventually();
    }

    void CallClosedCallbacksEventually() {
        if (closed()) {
            for (const auto& cb : closed_callbacks_)
                cb();
            closed_callbacks_.clear();
        }
    }

    //! Indicates if the channel is closed - meaning all remite streams have
    //! been closed. This does *not* include the loopback stream
    bool closed() const {
        bool closed = true;
        for (auto& q : queues_) {
            closed = closed && q.write_closed();
        }
        return closed;
    }

    //! Adds a Callback that is called when the channel is closed (r+w)
    void OnClose(ClosedCallback cb) {
        closed_callbacks_.push_back(cb);
    }

    ///////// expse these members - getters would be too java-ish /////////////

    //! StatsCounter for incoming data transfer
    //! Do not include loopback data transfer
    StatsCounter incoming_bytes_, incoming_blocks_;

    //! StatsCounters for outgoing data transfer - shared by all sinks
    //! Do not include loopback data transfer
    StatsCounter outgoing_bytes_, outgoing_blocks_;

    //! Timers from creation of channel until rx / tx direction is closed.
    StatsTimer tx_lifetime_, rx_lifetime_;

    //! Timers from first rx / tx package until rx / tx direction is closed.
    StatsTimer tx_timespan_, rx_timespan_;

    ///////////////////////////////////////////////////////////////////////////

protected:
    static const bool debug = false;

    ChannelId id_;

    //! reference to multiplexer
    data::Multiplexer& multiplexer_;

    //! ChannelSink objects are receivers of Blocks outbound for other worker.
    std::vector<ChannelSink> sinks_;

    //! BlockQueues to store incoming Blocks with no attached destination.
    std::vector<BlockQueue> queues_;

    //! Vector of Files to cache inbound Blocks needed for OpenCachingReader().
    std::vector<File> cache_files_;

    //! number of expected / received stream closing operations. Required to know when to
    //! stop rx_lifetime
    size_t expected_closing_blocks_, received_closing_blocks_;

    //! Callbacks that are called once when the channel is closed (r+w)
    std::vector<ClosedCallback> closed_callbacks_;

    //! for calling methods to deliver blocks
    friend class Multiplexer;

    //! called from Multiplexer when there is a new Block on a
    //! Channel.
    //! \param from the worker rank (host rank * num_workers/host + worker id)
    void OnChannelBlock(size_t from, Block&& b) {
        assert(from < queues_.size());
        rx_timespan_.StartEventually();
        incoming_bytes_ += b.size();
        incoming_blocks_++;

        sLOG << "OnChannelBlock" << b;

        if (debug) {
            sLOG << "channel" << id_ << "receive from" << from << ":"
                 << common::hexdump(b.ToString());
        }

        queues_[from].AppendBlock(b);
    }

    //! called from Multiplexer when a Channel closed notification was
    //! received.
    //! \param from the worker rank (host rank * num_workers/host + worker id)
    void OnCloseChannel(size_t from) {
        assert(from < queues_.size());
        assert(!queues_[from].write_closed());
        queues_[from].Close();
        if (expected_closing_blocks_ == ++received_closing_blocks_) {
            rx_lifetime_.StopEventually();
            rx_timespan_.StopEventually();
            CallClosedCallbacksEventually();
        }
    }
};

using ChannelPtr = std::shared_ptr<Channel>;

//! \}

} // namespace data
} // namespace c7a

#endif // !C7A_DATA_CHANNEL_HEADER

/******************************************************************************/
<commit_msg>attempt to fix race condition in CallClosedCallbacksEventually<commit_after>/*******************************************************************************
 * thrill/data/channel.hpp
 *
 * Part of Project c7a.
 *
 * Copyright (C) 2015 Timo Bingmann <tb@panthema.net>
 * Copyright (C) 2015 Tobias Sturm <mail@tobiassturm.de>
 *
 * This file has no license. Only Chuck Norris can compile it.
 ******************************************************************************/

#pragma once
#ifndef C7A_DATA_CHANNEL_HEADER
#define C7A_DATA_CHANNEL_HEADER

#include <thrill/common/stats_counter.hpp>
#include <thrill/common/stats_timer.hpp>
#include <thrill/data/block_queue.hpp>
#include <thrill/data/channel_sink.hpp>
#include <thrill/data/concat_block_source.hpp>
#include <thrill/data/file.hpp>
#include <thrill/data/multiplexer.hpp>
#include <thrill/data/multiplexer_header.hpp>
#include <thrill/net/connection.hpp>
#include <thrill/net/group.hpp>

#include <sstream>
#include <string>
#include <vector>
#include <mutex>

namespace c7a {
namespace data {

//! \addtogroup data Data Subsystem
//! \{

using ChannelId = size_t;

/*!
 * A Channel is a virtual set of connections to all other worker instances,
 * hence a "Channel" bundles them to a logical communication context. We call an
 * individual connection from a worker to another worker a "Host".
 *
 * To use a Channel, one can get a vector of BlockWriter via OpenWriters() of
 * outbound Channel. The vector is of size of workers in the system.
 * One can then write items destined to the
 * corresponding worker. The written items are buffered into a Block and only
 * sent when the Block is full. To force a send, use BlockWriter::Flush(). When
 * all items are sent, the BlockWriters **must** be closed using
 * BlockWriter::Close().
 *
 * To read the inbound Connection items, one can get a vector of BlockReader via
 * OpenReaders(), which can then be used to read items sent by individual
 * workers.
 *
 * Alternatively, one can use OpenReader() to get a BlockReader which delivers
 * all items from *all* worker in worker order (concatenating all inbound
 * Connections).
 *
 * As soon as all attached streams of the Channel have been Close() the number of
 * expected streams is reached, the channel is marked as finished and no more
 * data will arrive.
 */
class Channel
{
public:
    using BlockQueueReader = BlockReader<BlockQueueSource>;
    using ConcatBlockSource = data::ConcatBlockSource<BlockQueueSource>;
    using ConcatBlockReader = BlockReader<ConcatBlockSource>;

    using CachingConcatBlockSource = data::ConcatBlockSource<CachingBlockQueueSource>;
    using CachingConcatBlockReader = BlockReader<CachingConcatBlockSource>;

    using Reader = BlockQueueReader;
    using ConcatReader = ConcatBlockReader;
    using CachingConcatReader = CachingConcatBlockReader;

    using StatsCounter = common::StatsCounter<size_t, common::g_enable_stats>;
    using StatsTimer = common::StatsTimer<common::g_enable_stats>;

    using ClosedCallback = std::function<void()>;

    //! Creates a new channel instance
    Channel(data::Multiplexer& multiplexer, const ChannelId& id,
            size_t my_local_worker_id)
        : tx_lifetime_(true), rx_lifetime_(true),
          tx_timespan_(), rx_timespan_(),
          id_(id),
          multiplexer_(multiplexer),
          queues_(multiplexer_.num_workers()),
          cache_files_(multiplexer_.num_workers()),
          expected_closing_blocks_(
              (multiplexer_.num_hosts() - 1) * multiplexer_.num_workers_per_host_),
          received_closing_blocks_(0) {
        // construct ChannelSink array
        for (size_t host = 0; host < multiplexer_.num_hosts(); ++host) {
            for (size_t worker = 0; worker < multiplexer_.num_workers_per_host_; worker++) {
                if (host == multiplexer_.my_host_rank()) {
                    sinks_.emplace_back();
                }
                else {
                    sinks_.emplace_back(
                        &multiplexer_.dispatcher_,
                        &multiplexer_.group_.connection(host),
                        id,
                        multiplexer_.my_host_rank(), my_local_worker_id, worker,
                        &outgoing_bytes_, &outgoing_blocks_, &tx_timespan_);
                }
            }
        }
    }

    //! non-copyable: delete copy-constructor
    Channel(const Channel&) = delete;
    //! non-copyable: delete assignment operator
    Channel& operator = (const Channel&) = delete;

    //! move-constructor
    Channel(Channel&&) = default;

    const ChannelId & id() const {
        return id_;
    }

    //! Creates BlockWriters for each worker. BlockWriter can only be opened
    //! once, otherwise the block sequence is incorrectly interleaved!
    std::vector<BlockWriter> OpenWriters(size_t block_size = default_block_size) {
        tx_timespan_.StartEventually();

        std::vector<BlockWriter> result;

        for (size_t host = 0; host < multiplexer_.num_hosts(); ++host) {
            for (size_t local_worker_id = 0; local_worker_id < multiplexer_.num_workers_per_host_; ++local_worker_id) {
                size_t worker_id = host * multiplexer_.num_workers_per_host_ + local_worker_id;
                if (host == multiplexer_.my_host_rank()) {
                    result.emplace_back(&queues_[worker_id], block_size);
                }
                else {
                    result.emplace_back(&sinks_[worker_id], block_size);
                }
            }
        }

        assert(result.size() == multiplexer_.num_workers());
        return result;
    }

    //! Creates a BlockReader for each worker. The BlockReaders are attached to
    //! the BlockQueues in the Channel and wait for further Blocks to arrive or
    //! the Channel's remote close.
    std::vector<BlockQueueReader> OpenReaders() {
        rx_timespan_.StartEventually();

        std::vector<BlockQueueReader> result;

        for (size_t worker = 0; worker < multiplexer_.num_workers(); ++worker) {
            result.emplace_back(BlockQueueSource(queues_[worker]));
        }

        assert(result.size() == multiplexer_.num_workers());
        return result;
    }

    //! Creates a BlockReader for all workers. The BlockReader is attached to
    //! one \ref ConcatBlockSource which includes all incoming queues of
    //! this channel.
    ConcatBlockReader OpenReader() {
        rx_timespan_.StartEventually();

        // construct vector of BlockQueueSources to read from queues_.
        std::vector<BlockQueueSource> result;

        for (size_t worker = 0; worker < multiplexer_.num_workers(); ++worker) {
            result.emplace_back(queues_[worker]);
        }
        // move BlockQueueSources into concatenation BlockSource, and to Reader.
        return ConcatBlockReader(ConcatBlockSource(result));
    }

    //! Creates a BlockReader for all workers. The BlockReader is attached to
    //! one \ref ConcatBlockSource which includes all incoming queues of this
    //! channel. The received Blocks are also cached in the Channel, hence this
    //! function can be called multiple times to read the items again.
    CachingConcatBlockReader OpenCachingReader() {
        rx_timespan_.StartEventually();

        // construct vector of CachingBlockQueueSources to read from queues_.
        std::vector<CachingBlockQueueSource> result;
        for (size_t worker = 0; worker < multiplexer_.num_workers(); ++worker) {
            result.emplace_back(queues_[worker], cache_files_[worker]);
        }
        // move CachingBlockQueueSources into concatenation BlockSource, and to
        // Reader.
        return CachingConcatBlockReader(CachingConcatBlockSource(result));
    }

    /*!
     * Scatters a File to many worker
     *
     * elements from 0..offset[0] are sent to the first worker,
     * elements from (offset[0] + 1)..offset[1] are sent to the second worker.
     * elements from (offset[my_rank - 1] + 1)..(offset[my_rank]) are copied
     * The offset values range from 0..Manager::GetNumElements().
     * The number of given offsets must be equal to the net::Group::num_workers() * workers_per_host_.
     *
     * /param source File containing the data to be scattered.
     *
     * /param offsets - as described above. offsets.size must be equal to group.size
     */
    template <typename ItemType>
    void Scatter(const File& source, const std::vector<size_t>& offsets) {
        tx_timespan_.StartEventually();

        // current item offset in Reader
        size_t current = 0;
        typename File::Reader reader = source.GetReader();

        std::vector<BlockWriter> writers = OpenWriters();

        for (size_t worker = 0; worker < multiplexer_.num_workers(); ++worker) {
            // write [current,limit) to this worker
            size_t limit = offsets[worker];
            assert(current <= limit);
#if 0
            for ( ; current < limit; ++current) {
                assert(reader.HasNext());
                // move over one item (with deserialization and serialization)
                writers[worker](reader.template Next<ItemType>());
            }
#else
            if (current != limit) {
                writers[worker].AppendBlocks(
                    reader.template GetItemBatch<ItemType>(limit - current));
                current = limit;
            }
#endif
            writers[worker].Close();
        }

        tx_timespan_.Stop();
    }

    //! shuts the channel down.
    void Close() {
        // close all sinks, this should emit sentinel to all other worker.
        for (size_t i = 0; i != sinks_.size(); ++i) {
            if (sinks_[i].closed()) continue;
            sinks_[i].Close();
        }

        // close self-loop queues
        for (size_t my_worker = 0; my_worker < multiplexer_.num_workers_per_host_; my_worker++) {
            size_t local_worker_id = multiplexer_.my_host_rank() + my_worker;
            if (!queues_[local_worker_id].write_closed())
                queues_[local_worker_id].Close();
        }

        // wait for close packets to arrive (this is a busy waiting loop, try to
        // do it better -tb)
        for (size_t i = 0; i != queues_.size(); ++i) {
            while (!queues_[i].write_closed()) {
                LOG << "wait for close from worker" << i;
                std::this_thread::sleep_for(std::chrono::milliseconds(10));
            }
        }
        tx_lifetime_.StopEventually();
        tx_timespan_.StopEventually();
        CallClosedCallbacksEventually();
    }

    void CallClosedCallbacksEventually() {
        std::lock_guard<std::mutex> lock(mutex_);
        if (closed()) {
            for (const auto& cb : closed_callbacks_)
                cb();
            closed_callbacks_.clear();
        }
    }

    //! Indicates if the channel is closed - meaning all remite streams have
    //! been closed. This does *not* include the loopback stream
    bool closed() const {
        bool closed = true;
        for (auto& q : queues_) {
            closed = closed && q.write_closed();
        }
        return closed;
    }

    //! Adds a Callback that is called when the channel is closed (r+w)
    void OnClose(ClosedCallback cb) {
        closed_callbacks_.push_back(cb);
    }

    ///////// expse these members - getters would be too java-ish /////////////

    //! StatsCounter for incoming data transfer
    //! Do not include loopback data transfer
    StatsCounter incoming_bytes_, incoming_blocks_;

    //! StatsCounters for outgoing data transfer - shared by all sinks
    //! Do not include loopback data transfer
    StatsCounter outgoing_bytes_, outgoing_blocks_;

    //! Timers from creation of channel until rx / tx direction is closed.
    StatsTimer tx_lifetime_, rx_lifetime_;

    //! Timers from first rx / tx package until rx / tx direction is closed.
    StatsTimer tx_timespan_, rx_timespan_;

    ///////////////////////////////////////////////////////////////////////////

protected:
    static const bool debug = false;

    ChannelId id_;

    //! reference to multiplexer
    data::Multiplexer& multiplexer_;

    //! ChannelSink objects are receivers of Blocks outbound for other worker.
    std::vector<ChannelSink> sinks_;

    //! BlockQueues to store incoming Blocks with no attached destination.
    std::vector<BlockQueue> queues_;

    //! Vector of Files to cache inbound Blocks needed for OpenCachingReader().
    std::vector<File> cache_files_;

    //! number of expected / received stream closing operations. Required to know when to
    //! stop rx_lifetime
    size_t expected_closing_blocks_, received_closing_blocks_;

    //! Callbacks that are called once when the channel is closed (r+w)
    std::vector<ClosedCallback> closed_callbacks_;

    // protects against race conditions in closed_callbacks_ loop
    std::mutex mutex_;

    //! for calling methods to deliver blocks
    friend class Multiplexer;

    //! called from Multiplexer when there is a new Block on a
    //! Channel.
    //! \param from the worker rank (host rank * num_workers/host + worker id)
    void OnChannelBlock(size_t from, Block&& b) {
        assert(from < queues_.size());
        rx_timespan_.StartEventually();
        incoming_bytes_ += b.size();
        incoming_blocks_++;

        sLOG << "OnChannelBlock" << b;

        if (debug) {
            sLOG << "channel" << id_ << "receive from" << from << ":"
                 << common::hexdump(b.ToString());
        }

        queues_[from].AppendBlock(b);
    }

    //! called from Multiplexer when a Channel closed notification was
    //! received.
    //! \param from the worker rank (host rank * num_workers/host + worker id)
    void OnCloseChannel(size_t from) {
        assert(from < queues_.size());
        assert(!queues_[from].write_closed());
        queues_[from].Close();
        if (expected_closing_blocks_ == ++received_closing_blocks_) {
            rx_lifetime_.StopEventually();
            rx_timespan_.StopEventually();
            CallClosedCallbacksEventually();
        }
    }
};

using ChannelPtr = std::shared_ptr<Channel>;

//! \}

} // namespace data
} // namespace c7a

#endif // !C7A_DATA_CHANNEL_HEADER

/******************************************************************************/
<|endoftext|>


<commit_before>/*=========================================================================

  Program:   Monteverdi2
  Language:  C++

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See Copyright.txt for details.

  Monteverdi2 is distributed under the CeCILL licence version 2. See
  Licence_CeCILL_V2-en.txt or
  http://www.cecill.info/licences/Licence_CeCILL_V2-en.txt for more details.

  This software is distributed WITHOUT ANY WARRANTY; without even
  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
  PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "mvdImageViewManipulator.h"

//
// Qt includes (sorted by alphabetic order)
//// Must be included before system/custom includes.

//
// System includes (sorted by alphabetic order)

//
// ITK includes (sorted by alphabetic order)

//
// OTB includes (sorted by alphabetic order)

//
// Monteverdi includes (sorted by alphabetic order)

namespace mvd
{
/*
  TRANSLATOR mvd::ImageViewManipulator

  Necessary for lupdate to be aware of C++ namespaces.

  Context comment for translator.
*/

/*****************************************************************************/
ImageViewManipulator
::ImageViewManipulator( QObject* parent ) :
  QObject( parent ),
  m_NavigationContext(),
  m_MouseContext(),
  m_IsotropicZoom(1)
{
  // TODO: Remove later because initialized in struct's default constructor and resizeEvent().
  this->InitializeContext(1,1);
}

/*****************************************************************************/
ImageViewManipulator
::~ImageViewManipulator()
{
}

/*******************************************************************************/
void
ImageViewManipulator
::InitializeContext(int width, int height)
{
  ImageRegionType::SizeType  initialSize;
  initialSize[0] = width;
  initialSize[1] = height;

  // initialize with the given size
  m_NavigationContext.m_ViewportImageRegion.SetSize(initialSize);
}

/******************************************************************************/
void
ImageViewManipulator
::mousePressEvent(QMouseEvent * event)
{
  // Update the context with the pressed position
  m_MouseContext.x = event->x();
  m_MouseContext.y = event->y();

  // Update the context with the pressed position for the mouseMoveEvent
  m_MouseContext.xMove = event->x();
  m_MouseContext.yMove = event->y();
}

/******************************************************************************/
void
ImageViewManipulator
::mouseMoveEvent( QMouseEvent * event)
{
  // Update the mouse context
  m_MouseContext.dx = -event->x() + m_MouseContext.xMove;
  m_MouseContext.dy = -event->y() + m_MouseContext.yMove;

  // moveRegion
  this->moveRegion( m_MouseContext.dx,  m_MouseContext.dy);

  // Update the position of the first press to take into account the
  // last drag
  m_MouseContext.xMove -=  m_MouseContext.dx;
  m_MouseContext.yMove -=  m_MouseContext.dy;
}

/******************************************************************************/
void
ImageViewManipulator
::moveRegion(double dx, double dy)
{
  // Update the navigation context
  ImageRegionType & currentRegion = m_NavigationContext.m_ViewportImageRegion;

  // Apply the offset to the (start) index of the stored region
  ImageRegionType::OffsetType offset;
  offset[0] = dx/ m_IsotropicZoom;
  offset[1] = dy/m_IsotropicZoom;

  // Apply the offset to the (start) index of the stored region
  IndexType    index = currentRegion.GetIndex() + offset;
  currentRegion.SetIndex(index);

  // Constraint the region to the largestPossibleRegion
  this->ConstrainRegion(currentRegion, m_NavigationContext.m_ModelImageRegion);
}

/******************************************************************************/
void
ImageViewManipulator
::mouseReleaseEvent(  QMouseEvent * event)
{
  //TODO: Implement mouseReleaseEvent.
  //std::cout <<" Not Implemented yet ..." << std::endl;
}

/******************************************************************************/
void ImageViewManipulator
::resizeEvent( QResizeEvent * event )
{
  // Update the navigation context
  ImageRegionType & currentRegion = m_NavigationContext.m_ViewportImageRegion;

  // Get the new widget size
  ImageRegionType::SizeType size;
  size[0] = event->size().width();
  size[1] = event->size().height();

  // Update the stored region with the new size
  currentRegion.SetSize(size);

  // Recompute the size before
  m_NavigationContext.m_SizeXBeforeConstrain = (double)size[0] / this->GetIsotropicZoom();
  m_NavigationContext.m_SizeYBeforeConstrain = (double)size[1] / this->GetIsotropicZoom();

  // Constraint this region to the LargestPossibleRegion
  this->ConstrainRegion(currentRegion, m_NavigationContext.m_ModelImageRegion);

  // call the rescale method with the same zoom as before (scale = 1)
  this->Zoom(1.);
}

/******************************************************************************/
void
ImageViewManipulator
::wheelEvent(  QWheelEvent * event)
{
  // compute the new scale
  double scaleRatio = 1.25;
  double numDegrees = event->delta() / 8.;
  int    nbSteps    = static_cast<int>(numDegrees / 15.);

  double scale = vcl_pow(scaleRatio, nbSteps);

  // center the region on the center of the previous region
  this->CenterRegion(scale);

  // rescale the viewport region
  this->Zoom(scale);
}

/******************************************************************************/
void
ImageViewManipulator
::CenterRegion(double scale)
{
  if( m_IsotropicZoom * scale > 0.01 && m_IsotropicZoom * scale < 10.)
    {
    // The viewPort Region must be adapted to this zoom ratio
    ImageRegionType & currentRegion = m_NavigationContext.m_ViewportImageRegion;

    // center the region on the position under the cursor
    IndexType        origin = currentRegion.GetIndex();
    double centerX = origin[0] + currentRegion.GetSize()[0]/2.;
    double centerY = origin[1] + currentRegion.GetSize()[1]/2.;

    // new index
    IndexType  newIndex;
    newIndex[0] = centerX - currentRegion.GetSize()[0]/scale/2.;
    newIndex[1] = centerY - currentRegion.GetSize()[1]/scale/2.;
    currentRegion.SetIndex(newIndex);

    // Constraint this region to the LargestPossibleRegion
    this->ConstrainRegion(currentRegion, m_NavigationContext.m_ModelImageRegion);
    }
}

/******************************************************************************/
void
ImageViewManipulator
::Zoom(const double scale)
{
  // compute the new size
  double sizeX = m_NavigationContext.m_SizeXBeforeConstrain / scale;
  double sizeY = m_NavigationContext.m_SizeYBeforeConstrain / scale;

  // check that the new size is greater than 30x30
  // check that the new isoZoom is not too low and not too large
  // TODO : compute automatically the minSize and the isoZoom range ???
  if (sizeX > 30 && sizeY > 30  &&
      m_IsotropicZoom * scale > 0.01 &&
      m_IsotropicZoom * scale < 10.)
    {
    // Update the the sizeBeforeConstrain
    m_NavigationContext.m_SizeXBeforeConstrain = sizeX;
    m_NavigationContext.m_SizeYBeforeConstrain = sizeY;

    // Update the viewort region with the new size
    ImageRegionType::SizeType size;
    size[0] = static_cast<unsigned int>(sizeX);
    size[1] = static_cast<unsigned int>(sizeY);

    // The viewPort Region must be adapted to this zoom ratio
    ImageRegionType & currentRegion = m_NavigationContext.m_ViewportImageRegion;

    // Update the stored region with the new size
    currentRegion.SetSize(size);

    // Constraint this region to the LargestPossibleRegion
    this->ConstrainRegion(currentRegion, m_NavigationContext.m_ModelImageRegion);

    // Update the isotropicZoom
    m_IsotropicZoom *= scale;
    }
}

/******************************************************************************/
void
ImageViewManipulator
::ConstrainRegion( ImageRegionType& region, const ImageRegionType& largest)
{
  ImageRegionType::SizeType zeroSize;
  zeroSize.Fill(0);

  if (largest.GetSize() != zeroSize)
    {
    // Else we can constrain it
    IndexType                     index = region.GetIndex();
    ImageRegionType::SizeType size = region.GetSize();

    // If region is larger than big, then crop
    if (region.GetSize()[0] > largest.GetSize()[0])
      {
      size[0] = largest.GetSize()[0];
      }
    if (region.GetSize()[1] > largest.GetSize()[1])
      {
      size[1] = largest.GetSize()[1];
      }

    // Else we can constrain it
    // For each dimension
    for (unsigned int dim = 0; dim < ImageRegionType::ImageDimension; ++dim)
      {
      // push left if necessary
      if (region.GetIndex()[dim] < largest.GetIndex()[dim])
        {
        index[dim] = largest.GetIndex()[dim];
        }
      // push right if necessary
      if (index[dim] + size[dim] >= largest.GetIndex()[dim] + largest.GetSize()[dim])
        {
        index[dim] = largest.GetIndex()[dim] + largest.GetSize()[dim] - size[dim];
        }
      }
    region.SetSize(size);
    region.SetIndex(index);
    }
}

/******************************************************************************/
void
ImageViewManipulator
::keyPressEvent( QKeyEvent * event )
{

  switch(event->key())
    {
    case Qt::Key_Minus:
      CenterRegion(0.8);
      Zoom(0.8);
      break;
    case Qt::Key_Plus:
      CenterRegion(1.25);
      Zoom(1.25);
      break;
    case Qt::Key_Left:
      moveRegion(10,0);
      break;
    case Qt::Key_Right:
      moveRegion(-10,0);
      break;
    case Qt::Key_Up:
      moveRegion(0,10);
      break;
    case Qt::Key_Down:
      moveRegion(0,-10);
      break;
    default:
      break;
    }
}

/*****************************************************************************/
/* SLOTS                                                                     */
/*****************************************************************************/

/*****************************************************************************/

} // end namespace 'mvd'
<commit_msg>ENH: avoid setting negative index + warnings<commit_after>/*=========================================================================

  Program:   Monteverdi2
  Language:  C++

  Copyright (c) Centre National d'Etudes Spatiales. All rights reserved.
  See Copyright.txt for details.

  Monteverdi2 is distributed under the CeCILL licence version 2. See
  Licence_CeCILL_V2-en.txt or
  http://www.cecill.info/licences/Licence_CeCILL_V2-en.txt for more details.

  This software is distributed WITHOUT ANY WARRANTY; without even
  the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
  PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "mvdImageViewManipulator.h"

//
// Qt includes (sorted by alphabetic order)
//// Must be included before system/custom includes.

//
// System includes (sorted by alphabetic order)

//
// ITK includes (sorted by alphabetic order)

//
// OTB includes (sorted by alphabetic order)

//
// Monteverdi includes (sorted by alphabetic order)

namespace mvd
{
/*
  TRANSLATOR mvd::ImageViewManipulator

  Necessary for lupdate to be aware of C++ namespaces.

  Context comment for translator.
*/

/*****************************************************************************/
ImageViewManipulator
::ImageViewManipulator( QObject* parent ) :
  QObject( parent ),
  m_NavigationContext(),
  m_MouseContext(),
  m_IsotropicZoom(1)
{
  // TODO: Remove later because initialized in struct's default constructor and resizeEvent().
  this->InitializeContext(1,1);
}

/*****************************************************************************/
ImageViewManipulator
::~ImageViewManipulator()
{
}

/*******************************************************************************/
void
ImageViewManipulator
::InitializeContext(int width, int height)
{
  ImageRegionType::SizeType  initialSize;
  initialSize[0] = width;
  initialSize[1] = height;

  // initialize with the given size
  m_NavigationContext.m_ViewportImageRegion.SetSize(initialSize);
}

/******************************************************************************/
void
ImageViewManipulator
::mousePressEvent(QMouseEvent * event)
{
  // Update the context with the pressed position
  m_MouseContext.x = event->x();
  m_MouseContext.y = event->y();

  // Update the context with the pressed position for the mouseMoveEvent
  m_MouseContext.xMove = event->x();
  m_MouseContext.yMove = event->y();
}

/******************************************************************************/
void
ImageViewManipulator
::mouseMoveEvent( QMouseEvent * event)
{
  // Update the mouse context
  m_MouseContext.dx = -event->x() + m_MouseContext.xMove;
  m_MouseContext.dy = -event->y() + m_MouseContext.yMove;

  // moveRegion
  this->moveRegion( m_MouseContext.dx,  m_MouseContext.dy);

  // Update the position of the first press to take into account the
  // last drag
  m_MouseContext.xMove -=  m_MouseContext.dx;
  m_MouseContext.yMove -=  m_MouseContext.dy;
}

/******************************************************************************/
void
ImageViewManipulator
::moveRegion(double dx, double dy)
{
  // Update the navigation context
  ImageRegionType & currentRegion = m_NavigationContext.m_ViewportImageRegion;

  // Apply the offset to the (start) index of the stored region
  ImageRegionType::OffsetType offset;
  offset[0] = dx/ m_IsotropicZoom;
  offset[1] = dy/m_IsotropicZoom;

  // Apply the offset to the (start) index of the stored region
  IndexType    index = currentRegion.GetIndex() + offset;
  currentRegion.SetIndex(index);

  // Constraint the region to the largestPossibleRegion
  this->ConstrainRegion(currentRegion, m_NavigationContext.m_ModelImageRegion);
}

/******************************************************************************/
void
ImageViewManipulator
::mouseReleaseEvent(  QMouseEvent * event)
{
  //TODO: Implement mouseReleaseEvent.
  //std::cout <<" Not Implemented yet ..." << std::endl;
}

/******************************************************************************/
void ImageViewManipulator
::resizeEvent( QResizeEvent * event )
{
  // Update the navigation context
  ImageRegionType & currentRegion = m_NavigationContext.m_ViewportImageRegion;

  // Get the new widget size
  ImageRegionType::SizeType size;
  size[0] = event->size().width();
  size[1] = event->size().height();

  // Update the stored region with the new size
  currentRegion.SetSize(size);

  // Recompute the size before
  m_NavigationContext.m_SizeXBeforeConstrain = (double)size[0] / this->GetIsotropicZoom();
  m_NavigationContext.m_SizeYBeforeConstrain = (double)size[1] / this->GetIsotropicZoom();

  // Constraint this region to the LargestPossibleRegion
  this->ConstrainRegion(currentRegion, m_NavigationContext.m_ModelImageRegion);

  // call the rescale method with the same zoom as before (scale = 1)
  this->Zoom(1.);
}

/******************************************************************************/
void
ImageViewManipulator
::wheelEvent(  QWheelEvent * event)
{
  // compute the new scale
  double scaleRatio = 1.25;
  double numDegrees = event->delta() / 8.;
  int    nbSteps    = static_cast<int>(numDegrees / 15.);

  double scale = vcl_pow(scaleRatio, nbSteps);

  // center the region on the center of the previous region
  this->CenterRegion(scale);

  // rescale the viewport region
  this->Zoom(scale);
}

/******************************************************************************/
void
ImageViewManipulator
::CenterRegion(double scale)
{
  if( m_IsotropicZoom * scale > 0.01 && m_IsotropicZoom * scale < 10.)
    {
    // The viewPort Region must be adapted to this zoom ratio
    ImageRegionType & currentRegion = m_NavigationContext.m_ViewportImageRegion;

    // center the region on the position under the cursor
    IndexType        origin = currentRegion.GetIndex();
    double centerX = (double)(origin[0]) + (double)(currentRegion.GetSize()[0])/2.;
    double centerY = (double)(origin[1]) + (double)(currentRegion.GetSize()[1])/2.;

    // new origin
    IndexType  newIndex;
    newIndex[0] = centerX - currentRegion.GetSize()[0]/scale/2.;
    if (newIndex[0] < 0) newIndex[0] = 0;

    newIndex[1] = centerY - currentRegion.GetSize()[1]/scale/2.;
    if (newIndex[1] < 0) newIndex[1] = 0;

    // set the new origin
    currentRegion.SetIndex(newIndex);

    // Constraint this region to the LargestPossibleRegion
    this->ConstrainRegion(currentRegion, m_NavigationContext.m_ModelImageRegion);
    }
}

/******************************************************************************/
void
ImageViewManipulator
::Zoom(const double scale)
{
  // compute the new size
  double sizeX = m_NavigationContext.m_SizeXBeforeConstrain / scale;
  double sizeY = m_NavigationContext.m_SizeYBeforeConstrain / scale;

  // check that the new size is greater than 30x30
  // check that the new isoZoom is not too low and not too large
  // TODO : compute automatically the minSize and the isoZoom range ???
  if (sizeX > 30 && sizeY > 30  &&
      m_IsotropicZoom * scale > 0.01 &&
      m_IsotropicZoom * scale < 10.)
    {
    // Update the the sizeBeforeConstrain
    m_NavigationContext.m_SizeXBeforeConstrain = sizeX;
    m_NavigationContext.m_SizeYBeforeConstrain = sizeY;

    // Update the viewort region with the new size
    ImageRegionType::SizeType size;
    size[0] = static_cast<unsigned int>(sizeX);
    size[1] = static_cast<unsigned int>(sizeY);

    // The viewPort Region must be adapted to this zoom ratio
    ImageRegionType & currentRegion = m_NavigationContext.m_ViewportImageRegion;

    // Update the stored region with the new size
    currentRegion.SetSize(size);

    // Constraint this region to the LargestPossibleRegion
    this->ConstrainRegion(currentRegion, m_NavigationContext.m_ModelImageRegion);

    // Update the isotropicZoom
    m_IsotropicZoom *= scale;
    }
}

/******************************************************************************/
void
ImageViewManipulator
::ConstrainRegion( ImageRegionType& region, const ImageRegionType& largest)
{
  ImageRegionType::SizeType zeroSize;
  zeroSize.Fill(0);

  if (largest.GetSize() != zeroSize)
    {
    // Else we can constrain it
    IndexType                     index = region.GetIndex();
    ImageRegionType::SizeType size = region.GetSize();

    // If region is larger than big, then crop
    if (region.GetSize()[0] > largest.GetSize()[0])
      {
      size[0] = largest.GetSize()[0];
      }
    if (region.GetSize()[1] > largest.GetSize()[1])
      {
      size[1] = largest.GetSize()[1];
      }

    // Else we can constrain it
    // For each dimension
    for (unsigned int dim = 0; dim < ImageRegionType::ImageDimension; ++dim)
      {
      // push left if necessary
      if (region.GetIndex()[dim] < largest.GetIndex()[dim])
        {
        index[dim] = largest.GetIndex()[dim];
        }
      // push right if necessary
      if (index[dim] + size[dim] >= largest.GetIndex()[dim] + largest.GetSize()[dim])
        {
        index[dim] = largest.GetIndex()[dim] + largest.GetSize()[dim] - size[dim];
        }
      }
    region.SetSize(size);
    region.SetIndex(index);
    }
}

/******************************************************************************/
void
ImageViewManipulator
::keyPressEvent( QKeyEvent * event )
{

  switch(event->key())
    {
    case Qt::Key_Minus:
      CenterRegion(0.8);
      Zoom(0.8);
      break;
    case Qt::Key_Plus:
      CenterRegion(1.25);
      Zoom(1.25);
      break;
    case Qt::Key_Left:
      moveRegion(10,0);
      break;
    case Qt::Key_Right:
      moveRegion(-10,0);
      break;
    case Qt::Key_Up:
      moveRegion(0,10);
      break;
    case Qt::Key_Down:
      moveRegion(0,-10);
      break;
    default:
      break;
    }
}

/*****************************************************************************/
/* SLOTS                                                                     */
/*****************************************************************************/

/*****************************************************************************/

} // end namespace 'mvd'
<|endoftext|>


<commit_before>/*=========================================================================

  Program:   Insight Segmentation & Registration Toolkit
  Module:    itkFEMLightObject.cxx
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Insight Software Consortium. All rights reserved.
  See ITKCopyright.txt or http://www.itk.org/HTML/Copyright.htm for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

// disable debug warnings in MS compiler
#ifdef _MSC_VER
#pragma warning(disable: 4786)
#endif

#include "itkFEMLightObject.h"
#include "itkNumericTraits.h"

namespace itk {
namespace fem {

/*
 * Here we just read the global number from the stream.
 * This should be the first function called when reading object data.
 */
void FEMLightObject::Read( std::istream& f, void* )
{
  int n;

  /** Read and set the global object number */
  this->SkipWhiteSpace(f); f>>n; if(!f) { goto out; }
  this->GN=n;

out:

  if( !f )
  {
    throw FEMExceptionIO(__FILE__,__LINE__,"FEMLightObject::Read","Error reading FEM object!");
  }

}

/*
 * Here we just write the class name and GN.
 * This should be the first function called when writing object data, so
 * every derived class should first call the parent's write function.
 * The Write function in base (this one) class knows which class is
 * being written by calling the virtual ClassID() function and can write
 * the class name properly.
 */
void FEMLightObject::Write( std::ostream& f ) const
{
  // first write the class name
  f<<'<'<<FEMObjectFactory<Self>::ID2ClassName(this->ClassID())<<">\n";

  // then the global object number
  f<<"\t"<<GN<<"\t% Global object number\n";

  // check for errors
  if (!f)
  {
    throw FEMExceptionIO(__FILE__,__LINE__,"FEMLightObject::Write","Error writing FEM object!");
  }
}

/*
 * Read and create object of any derived class from stream
 */
FEMLightObject::Pointer
FEMLightObject::
CreateFromStream( std::istream& f, void *info )
{

// local variables
std::streampos l(0);
char buf[256];
std::string s;
std::string::size_type b,e;
int clID;
FEMLightObject::Pointer a=0;
std::string errorMessage;

start:
#ifndef __sgi
  l=f.tellg();    // remember the stream position
#endif
  SkipWhiteSpace(f);      // skip comments and whitespaces
  if ( f.eof() ) return 0; // end of stream. all was good

  if ( f.get()!='<' )
    {
    errorMessage = "Expected < token not found";
    goto out; // we expect a token
    }
  f.getline(buf,256,'>');  // read up to 256 characters until '>' is reached. we read and discard the '>'
  s=std::string(buf);

  // get rid of the whitespaces in front of and the back of token
  b=s.find_first_not_of(whitespaces); // end of whitespaces in the beginning
  if ( (e=s.find_first_of(whitespaces,b))==std::string::npos )  // beginning of whitespaces at the end
  {
    e=s.size();
  }
  s=s.substr(b,e-b);

  if ( s=="END" )
  {
    /*
     * We can ignore this token. Start again by reading the next object.
     */
    goto start;
  }
  clID=FEMOF::ClassName2ID(s);  // obtain the class ID from FEMObjectFactory
  if (clID<0)
    {
    errorMessage = "Could not obtain class ID from FEMObjectFactory";
    goto out;  // class not found
    }
  // create a new object of the correct class
  a=FEMOF::Create(clID);
  if (!a)
    {
    errorMessage = "Error creating new object of the derived class";
    goto out;    // error creating new object of the derived class
    }
  /*
   * Now we have to read additional data, which is
   * specific to the class of object we just created
   */
  try
  {
    a->Read(f,info);
  }
  /*
   * Catch possible exceptions while
   * reading object's data from stream
   */
  catch (...)
  {
    #ifndef FEM_USE_SMART_POINTERS
    delete a;  // if something went wrong, we need to destroy the already created object
    #endif
    a=0;
    throw;     // rethrow the same exception
  }

  /*
   * Return a pointer to a newly created object if all was OK
   * Technically everithing should be fine here (a!=0), but we
   * check again, just in case.
   */
  if (a) { return a; }

out:

  /*
   * Something went wrong.
   * Reset the stream position to where it was before reading the object.
   */
#ifndef __sgi
  f.seekg(l);
#endif

  /*
   * Throw an IO exception
   */
  throw FEMExceptionIO(__FILE__,__LINE__,"FEMLightObject::ReadAnyObjectFromStream()",errorMessage);

}

// Helper function to skip all whitespaces and comments in input stream
void
FEMLightObject::
SkipWhiteSpace(std::istream& f)
{
  while(f && !f.eof() && (std::ws(f).peek())=='%' )
  {
    f.ignore(NumericTraits<int>::max(), '\n');
  }
}

// string containing all whitespace characters
const std::string
FEMLightObject
::whitespaces=" \t\n\r";

}} // end namespace itk::fem
<commit_msg>COMP: trying to resolve cygwin issues regarding .fem file. Added more meaningful error message. Also, trying getline instead of ignore.<commit_after>/*=========================================================================

  Program:   Insight Segmentation & Registration Toolkit
  Module:    itkFEMLightObject.cxx
  Language:  C++
  Date:      $Date$
  Version:   $Revision$

  Copyright (c) Insight Software Consortium. All rights reserved.
  See ITKCopyright.txt or http://www.itk.org/HTML/Copyright.htm for details.

     This software is distributed WITHOUT ANY WARRANTY; without even
     the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
     PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

// disable debug warnings in MS compiler
#ifdef _MSC_VER
#pragma warning(disable: 4786)
#endif

#include "itkFEMLightObject.h"
#include "itkNumericTraits.h"

namespace itk {
namespace fem {

/*
 * Here we just read the global number from the stream.
 * This should be the first function called when reading object data.
 */
void FEMLightObject::Read( std::istream& f, void* )
{
  int n;

  /** Read and set the global object number */
  this->SkipWhiteSpace(f); f>>n; if(!f) { goto out; }
  this->GN=n;

out:

  if( !f )
  {
    throw FEMExceptionIO(__FILE__,__LINE__,"FEMLightObject::Read","Error reading FEM object!");
  }

}

/*
 * Here we just write the class name and GN.
 * This should be the first function called when writing object data, so
 * every derived class should first call the parent's write function.
 * The Write function in base (this one) class knows which class is
 * being written by calling the virtual ClassID() function and can write
 * the class name properly.
 */
void FEMLightObject::Write( std::ostream& f ) const
{
  // first write the class name
  f<<'<'<<FEMObjectFactory<Self>::ID2ClassName(this->ClassID())<<">\n";

  // then the global object number
  f<<"\t"<<GN<<"\t% Global object number\n";

  // check for errors
  if (!f)
  {
    throw FEMExceptionIO(__FILE__,__LINE__,"FEMLightObject::Write","Error writing FEM object!");
  }
}

/*
 * Read and create object of any derived class from stream
 */
FEMLightObject::Pointer
FEMLightObject::
CreateFromStream( std::istream& f, void *info )
{

// local variables
std::streampos l(0);
char buf[256];
std::string s;
std::string::size_type b,e;
int clID;
FEMLightObject::Pointer a=0;
std::string errorMessage;

start:
#ifndef __sgi
  l=f.tellg();    // remember the stream position
#endif
  SkipWhiteSpace(f);      // skip comments and whitespaces
  if ( f.eof() ) return 0; // end of stream. all was good

  char c;
  if ( (c = f.get())!='<' )
    {
    errorMessage = "Expected < token not found. Instead found '";
    errorMessage += c;
    errorMessage += "'.";
    goto out; // we expect a token
    }
  f.getline(buf,256,'>');  // read up to 256 characters until '>' is reached. we read and discard the '>'
  s=std::string(buf);

  // get rid of the whitespaces in front of and the back of token
  b=s.find_first_not_of(whitespaces); // end of whitespaces in the beginning
  if ( (e=s.find_first_of(whitespaces,b))==std::string::npos )  // beginning of whitespaces at the end
  {
    e=s.size();
  }
  s=s.substr(b,e-b);

  if ( s=="END" )
  {
    /*
     * We can ignore this token. Start again by reading the next object.
     */
    goto start;
  }
  clID=FEMOF::ClassName2ID(s);  // obtain the class ID from FEMObjectFactory
  if (clID<0)
    {
    errorMessage = "Could not obtain class ID from FEMObjectFactory";
    goto out;  // class not found
    }
  // create a new object of the correct class
  a=FEMOF::Create(clID);
  if (!a)
    {
    errorMessage = "Error creating new object of the derived class";
    goto out;    // error creating new object of the derived class
    }
  /*
   * Now we have to read additional data, which is
   * specific to the class of object we just created
   */
  try
  {
    a->Read(f,info);
  }
  /*
   * Catch possible exceptions while
   * reading object's data from stream
   */
  catch (...)
  {
    #ifndef FEM_USE_SMART_POINTERS
    delete a;  // if something went wrong, we need to destroy the already created object
    #endif
    a=0;
    throw;     // rethrow the same exception
  }

  /*
   * Return a pointer to a newly created object if all was OK
   * Technically everithing should be fine here (a!=0), but we
   * check again, just in case.
   */
  if (a) { return a; }

out:

  /*
   * Something went wrong.
   * Reset the stream position to where it was before reading the object.
   */
#ifndef __sgi
  f.seekg(l);
#endif

  /*
   * Throw an IO exception
   */
  throw FEMExceptionIO(__FILE__,__LINE__,"FEMLightObject::ReadAnyObjectFromStream()",errorMessage);

}

// Helper function to skip all whitespaces and comments in input stream
void
FEMLightObject::
SkipWhiteSpace(std::istream& f)
{
  std::string skip;
  while(f && !f.eof() && (std::ws(f).peek())=='%' )
    {
    std::getline(f,skip);
    }
}

// string containing all whitespace characters
const std::string
FEMLightObject
::whitespaces=" \t\n\r";

}} // end namespace itk::fem
<|endoftext|>


<commit_before>#pragma once

#include <stdexcept>
#include <vector>
#include <fstream>
#include <string>
#include <sstream>
#include <algorithm>
#include <cctype>

#include "Utility.hpp"

/*
    Helper Functions
*/

namespace detail
{
    inline bool isTrimChar(char c)
    {
        return false;
    }

    template<class... TrimChar>
    inline bool isTrimChar(char c, char trim_char, TrimChar... other_trim_char)
    {
        return c == trim_char || isTrimChar(c, other_trim_char...);
    }

    template<class T>
    inline void parse(const std::string& cell, T& object)
    {
        std::istringstream iss(cell);

        iss >> object;
    }

    template<>
    inline void parse(const std::string& cell, bool& object)
    {
        std::string buffer = cell;

        std::for_each(buffer.begin(), buffer.end(),
            [](char& c)
        {
            c = std::tolower(c);
        });

        std::istringstream iss(buffer);
        iss.imbue({ iss.getloc(), new boolyesno() });

        iss >> std::boolalpha >> object;
    }

    template<>
    inline void parse(const std::string& cell, std::string& object)
    {
        object.assign(cell.begin(), cell.end());
    }
}

/*
    TrimChars Policy
*/

template<char... TrimChar>
struct TrimChars
{
    static void trim(std::string& line)
    {
        auto found_iter = std::remove_if(line.begin(), line.end(),
            [](char c)
        {
            return detail::isTrimChar(c, TrimChar...);
        });

        found_iter = line.erase(found_iter, line.end());
    }
};

/*
    QuoteEscape Policy
*/

template<char sep>
struct NoQuoteEscape
{
    static std::string split(std::string& line)
    {
        std::string result;

        std::size_t found_pos = line.find_first_of(sep, 0);

        if (found_pos == std::string::npos)
        {
            result = line;

            line.clear();
        }
        else
        {
            result = line.substr(0, found_pos);

            line = line.substr(found_pos + 1);
        }

        return result;
    }

    static void unescape(std::string& line)
    {
    }
};

template<char sep, char quote>
struct DoubleQuoteEscape
{
    static std::string split(std::string& line)
    {
        std::string result;

        std::size_t sep_pos = line.find_first_of(sep, 0);
        std::size_t quote_start_pos = line.find_first_of(quote, 0);
        std::size_t quote_end_pos = line.find_first_of(quote, quote_start_pos + 1);

        if (quote_start_pos != std::string::npos &&
            ((quote_end_pos != std::string::npos && sep_pos < quote_end_pos) ||
                quote_end_pos == std::string::npos))
        {
            throw std::runtime_error("No ending escape string was found!");
        }

        if (sep_pos == std::string::npos)
        {
            result = line;

            line.clear();
        }
        else
        {
            result = line.substr(0, sep_pos);

            line = line.substr(sep_pos + 1);
        }

        return result;
    }

    static void unescape(std::string& line)
    {
        for (std::size_t next = line.find(quote);
            next != std::string::npos;
            next = line.find(quote, next)
            )
        {
            line.erase(next, 1);
            next += 1;
        }
    }
};

/*
    Comment Policy
*/

struct NoComment
{
    static bool isComment(const std::string& line)
    {
        return false;
    }
};

struct EmptyLineComment
{
    static bool isComment(const std::string& line)
    {
        if (line.empty())
        {
            return true;
        }

        std::string buffer = line;

        auto found_iter = std::remove(buffer.begin(), buffer.end(), ' ');
        found_iter = buffer.erase(found_iter, buffer.end());

        found_iter = std::remove(buffer.begin(), buffer.end(), '\t');
        found_iter = buffer.erase(found_iter, buffer.end());

        if (found_iter != buffer.end())
        {
            return true;
        }

        return false;
    }
};

/*
    CSV Parser
*/

template
<
    unsigned int column_num,
    class CommentPolicy = EmptyLineComment,
    class QuotePolicy = NoQuoteEscape<','>,
    class TrimPolicy = TrimChars<'\t'>
>
class CSVParser
{
public:

    enum class IgnoreType : std::size_t
    {
        Invalid = 0,
        NoColumn,
        ExtraColumn,
        MissingColumn,
        MaxCount
    };

    explicit CSVParser(const std::string& data) : CSVParser()
    {
        reader << data;
    }

    explicit CSVParser(const std::ifstream& file) : CSVParser()
    {
        reader << file.rdbuf();
    }

    template<class ...ColumnTypes>
    void readHeader(IgnoreType ignore_column, ColumnTypes&... header_columns)
    {
        constexpr std::size_t column_count = sizeof...(ColumnTypes);

        static_assert(column_count >= column_num, "Not enough columns specified");
        static_assert(column_count <= column_num, "Too many columns specified");

        setHeaderColumns(header_columns...);

        std::string line;

        do
        {
            std::getline(reader, line);

            if (line.empty())
            {
                throw std::runtime_error("Header row is missing!");
            }
        } while (CommentPolicy::isComment(line));

        std::vector<bool> found_columns(column_count);
        std::fill(found_columns.begin(), found_columns.end(), false);

        while (!line.empty())
        {
            std::string column = QuotePolicy::split(line);

            TrimPolicy::trim(column);
            QuotePolicy::unescape(column);

            auto column_iter = std::find(this->header_columns.begin(), this->header_columns.end(), column);

            if (column_iter == this->header_columns.end())
            {
                if (ignore_column == IgnoreType::ExtraColumn)
                {
                    column_skip.push_back(-1);
                }
                else
                {
                    throw std::runtime_error("Column not found!");
                }
            }

            std::size_t index = std::distance(this->header_columns.begin(), column_iter);

            found_columns[index] = true;

            column_skip.push_back(index);
        }

        if (ignore_column != IgnoreType::MissingColumn)
        {
            std::for_each(found_columns.begin(), found_columns.end(),
                [](bool found_column)
            {
                if (!found_column)
                {
                    throw std::runtime_error("Missing column in header file");
                }
            });
        }
    }

    template<class ...ColumnTypes>
    bool readRow(ColumnTypes&... column_types)
    {
        constexpr std::size_t column_cout = sizeof...(ColumnTypes);

        static_assert(column_cout >= column_num, "Not enough columns specified");
        static_assert(column_cout <= column_num, "Too many columns specified");

        std::string line;

        do
        {
            std::getline(reader, line);

            if (line.empty())
            {
                return false;
            }
        } while (CommentPolicy::isComment(line));

        std::for_each(column_skip.begin(), column_skip.end(),
            [&](std::int32_t skip_column)
        {
            std::string cell = QuotePolicy::split(line);

            if (skip_column < 0)
            {
                row_columns.push_back(std::string());
            }
            else
            {
                TrimPolicy::trim(cell);
                QuotePolicy::unescape(cell);

                row_columns.push_back(cell);
            }
        });

        setRowColumns(0, column_types...);

        row_columns.clear();

        return true;
    }

private:

    std::stringstream reader;

    std::vector<std::string> header_columns;

    std::vector<std::int32_t> column_skip;

    std::vector<std::string> row_columns;

    CSVParser()
    {
        header_columns.resize(column_num);
    }

    template<class ...ColumnTypes>
    void setHeaderColumns(const std::string& column_name, ColumnTypes... column_types)
    {
        constexpr std::size_t column_left = sizeof...(ColumnTypes);

        header_columns[column_num - column_left - 1] = std::move(column_name);

        setHeaderColumns(std::forward<ColumnTypes>(column_types)...);
    }

    void setHeaderColumns()
    {
    }

    template<class T, class ...ColumnTypes>
    void setRowColumns(std::size_t index, T& object, ColumnTypes&... column_types)
    {
        std::string cell = row_columns.at(index);

        if (column_skip.at(index) != -1)
        {
            detail::parse<T>(cell, object);
        }

        setRowColumns(index + 1, column_types...);
    }

    void setRowColumns(std::size_t index)
    {
    }

};
<commit_msg>Remove default constructor.<commit_after>#pragma once

#include <stdexcept>
#include <vector>
#include <fstream>
#include <string>
#include <sstream>
#include <algorithm>
#include <cctype>

#include "Utility.hpp"

/*
    Helper Functions
*/

namespace detail
{
    inline bool isTrimChar(char c)
    {
        return false;
    }

    template<class... TrimChar>
    inline bool isTrimChar(char c, char trim_char, TrimChar... other_trim_char)
    {
        return c == trim_char || isTrimChar(c, other_trim_char...);
    }

    template<class T>
    inline void parse(const std::string& cell, T& object)
    {
        std::istringstream iss(cell);

        iss >> object;
    }

    template<>
    inline void parse(const std::string& cell, bool& object)
    {
        std::string buffer = cell;

        std::for_each(buffer.begin(), buffer.end(),
            [](char& c)
        {
            c = std::tolower(c);
        });

        std::istringstream iss(buffer);
        iss.imbue({ iss.getloc(), new boolyesno() });

        iss >> std::boolalpha >> object;
    }

    template<>
    inline void parse(const std::string& cell, std::string& object)
    {
        object.assign(cell.begin(), cell.end());
    }
}

/*
    TrimChars Policy
*/

template<char... TrimChar>
struct TrimChars
{
    static void trim(std::string& line)
    {
        auto found_iter = std::remove_if(line.begin(), line.end(),
            [](char c)
        {
            return detail::isTrimChar(c, TrimChar...);
        });

        found_iter = line.erase(found_iter, line.end());
    }
};

/*
    QuoteEscape Policy
*/

template<char sep>
struct NoQuoteEscape
{
    static std::string split(std::string& line)
    {
        std::string result;

        std::size_t found_pos = line.find_first_of(sep, 0);

        if (found_pos == std::string::npos)
        {
            result = line;

            line.clear();
        }
        else
        {
            result = line.substr(0, found_pos);

            line = line.substr(found_pos + 1);
        }

        return result;
    }

    static void unescape(std::string& line)
    {
    }
};

template<char sep, char quote>
struct DoubleQuoteEscape
{
    static std::string split(std::string& line)
    {
        std::string result;

        std::size_t sep_pos = line.find_first_of(sep, 0);
        std::size_t quote_start_pos = line.find_first_of(quote, 0);
        std::size_t quote_end_pos = line.find_first_of(quote, quote_start_pos + 1);

        if (quote_start_pos != std::string::npos &&
            ((quote_end_pos != std::string::npos && sep_pos < quote_end_pos) ||
                quote_end_pos == std::string::npos))
        {
            throw std::runtime_error("No ending escape string was found!");
        }

        if (sep_pos == std::string::npos)
        {
            result = line;

            line.clear();
        }
        else
        {
            result = line.substr(0, sep_pos);

            line = line.substr(sep_pos + 1);
        }

        return result;
    }

    static void unescape(std::string& line)
    {
        for (std::size_t next = line.find(quote);
            next != std::string::npos;
            next = line.find(quote, next)
            )
        {
            line.erase(next, 1);
            next += 1;
        }
    }
};

/*
    Comment Policy
*/

struct NoComment
{
    static bool isComment(const std::string& line)
    {
        return false;
    }
};

struct EmptyLineComment
{
    static bool isComment(const std::string& line)
    {
        if (line.empty())
        {
            return true;
        }

        std::string buffer = line;

        auto found_iter = std::remove(buffer.begin(), buffer.end(), ' ');
        found_iter = buffer.erase(found_iter, buffer.end());

        found_iter = std::remove(buffer.begin(), buffer.end(), '\t');
        found_iter = buffer.erase(found_iter, buffer.end());

        if (found_iter != buffer.end())
        {
            return true;
        }

        return false;
    }
};

/*
    CSV Parser
*/

template
<
    unsigned int column_num,
    class CommentPolicy = EmptyLineComment,
    class QuotePolicy = NoQuoteEscape<','>,
    class TrimPolicy = TrimChars<'\t'>
>
class CSVParser
{
public:

    enum class IgnoreType : std::size_t
    {
        Invalid = 0,
        NoColumn,
        ExtraColumn,
        MissingColumn,
        MaxCount
    };

    explicit CSVParser(const std::string& data)
    {
        header_columns.resize(column_num);

        reader << data;
    }

    explicit CSVParser(const std::ifstream& file)
    {
        header_columns.resize(column_num);

        reader << file.rdbuf();
    }

    template<class ...ColumnTypes>
    void readHeader(IgnoreType ignore_column, ColumnTypes&... header_columns)
    {
        constexpr std::size_t column_count = sizeof...(ColumnTypes);

        static_assert(column_count >= column_num, "Not enough columns specified");
        static_assert(column_count <= column_num, "Too many columns specified");

        setHeaderColumns(header_columns...);

        std::string line;

        do
        {
            std::getline(reader, line);

            if (line.empty())
            {
                throw std::runtime_error("Header row is missing!");
            }
        } while (CommentPolicy::isComment(line));

        std::vector<bool> found_columns(column_count);
        std::fill(found_columns.begin(), found_columns.end(), false);

        while (!line.empty())
        {
            std::string column = QuotePolicy::split(line);

            TrimPolicy::trim(column);
            QuotePolicy::unescape(column);

            auto column_iter = std::find(this->header_columns.begin(), this->header_columns.end(), column);

            if (column_iter == this->header_columns.end())
            {
                if (ignore_column == IgnoreType::ExtraColumn)
                {
                    column_skip.push_back(-1);
                }
                else
                {
                    throw std::runtime_error("Column not found!");
                }
            }

            std::size_t index = std::distance(this->header_columns.begin(), column_iter);

            found_columns[index] = true;

            column_skip.push_back(index);
        }

        if (ignore_column != IgnoreType::MissingColumn)
        {
            std::for_each(found_columns.begin(), found_columns.end(),
                [](bool found_column)
            {
                if (!found_column)
                {
                    throw std::runtime_error("Missing column in header file");
                }
            });
        }
    }

    template<class ...ColumnTypes>
    bool readRow(ColumnTypes&... column_types)
    {
        constexpr std::size_t column_cout = sizeof...(ColumnTypes);

        static_assert(column_cout >= column_num, "Not enough columns specified");
        static_assert(column_cout <= column_num, "Too many columns specified");

        std::string line;

        do
        {
            std::getline(reader, line);

            if (line.empty())
            {
                return false;
            }
        } while (CommentPolicy::isComment(line));

        std::for_each(column_skip.begin(), column_skip.end(),
            [&](std::int32_t skip_column)
        {
            std::string cell = QuotePolicy::split(line);

            if (skip_column < 0)
            {
                row_columns.push_back(std::string());
            }
            else
            {
                TrimPolicy::trim(cell);
                QuotePolicy::unescape(cell);

                row_columns.push_back(cell);
            }
        });

        setRowColumns(0, column_types...);

        row_columns.clear();

        return true;
    }

private:

    std::stringstream reader;

    std::vector<std::string> header_columns;

    std::vector<std::int32_t> column_skip;

    std::vector<std::string> row_columns;

    template<class ...ColumnTypes>
    void setHeaderColumns(const std::string& column_name, ColumnTypes... column_types)
    {
        constexpr std::size_t column_left = sizeof...(ColumnTypes);

        header_columns[column_num - column_left - 1] = std::move(column_name);

        setHeaderColumns(std::forward<ColumnTypes>(column_types)...);
    }

    void setHeaderColumns()
    {
    }

    template<class T, class ...ColumnTypes>
    void setRowColumns(std::size_t index, T& object, ColumnTypes&... column_types)
    {
        std::string cell = row_columns.at(index);

        if (column_skip.at(index) != -1)
        {
            detail::parse<T>(cell, object);
        }

        setRowColumns(index + 1, column_types...);
    }

    void setRowColumns(std::size_t index)
    {
    }

};
<|endoftext|>


<commit_before>/*=========================================================================

Program:   Medical Imaging & Interaction Toolkit
Module:    $RCSfile$
Language:  C++
Date:      $Date$
Version:   $Revision$

Copyright (c) German Cancer Research Center, Division of Medical and
Biological Informatics. All rights reserved.
See MITKCopyright.txt or http://www.mitk.org/copyright.html for details.

This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "mitkOperationEvent.h"
#include <itkCommand.h>

int mitk::UndoStackItem::m_CurrObjectEventId = 0;
int mitk::UndoStackItem::m_CurrGroupEventId = 0;

bool mitk::UndoStackItem::m_IncrObjectEventId = false;
bool mitk::UndoStackItem::m_IncrGroupEventId = false;

mitk::UndoStackItem::UndoStackItem(std::string description)
: m_Reversed(false),
  m_Description(description)
{
  m_ObjectEventId = GetCurrObjectEventId();
  m_GroupEventId = GetCurrGroupEventId();
}

mitk::UndoStackItem::~UndoStackItem()
{
  // nothing to do
}

void mitk::UndoStackItem::ExecuteIncrement()
{
  if (m_IncrObjectEventId)
  {
    ++m_CurrObjectEventId;
    m_IncrObjectEventId = false;
  }

  if (m_IncrGroupEventId)
  {
    ++m_CurrGroupEventId;
    m_IncrGroupEventId = false;
  }
}

int mitk::UndoStackItem::GetCurrObjectEventId()
{
  return m_CurrObjectEventId;
}

int mitk::UndoStackItem::GetCurrGroupEventId()
{
  return m_CurrGroupEventId;
}

void mitk::UndoStackItem::IncCurrObjectEventId()
{
  m_IncrObjectEventId = true;
}

void mitk::UndoStackItem::IncCurrGroupEventId()
{
  m_IncrGroupEventId = true;
}

int mitk::UndoStackItem::GetObjectEventId()
{
  return m_ObjectEventId;
}

int mitk::UndoStackItem::GetGroupEventId()
{
  return m_GroupEventId;
}

std::string mitk::UndoStackItem::GetDescription()
{
  return m_Description;
}

void mitk::UndoStackItem::ReverseOperations()
{
  m_Reversed = !m_Reversed;
}

void mitk::UndoStackItem::ReverseAndExecute()
{
  ReverseOperations();
}

// ******************** mitk::OperationEvent ********************

mitk::Operation* mitk::OperationEvent::GetOperation()
{
  return m_Operation;
}

mitk::OperationEvent::OperationEvent(OperationActor* destination,
                                     Operation* operation, Operation* undoOperation,
                                     std::string description)
: UndoStackItem(description),
  m_Destination(destination),
  m_Operation(operation),
  m_UndoOperation(undoOperation),
  m_Invalid(false)
{
  // nothing to do
  if (itk::Object* object = dynamic_cast<itk::Object*>( m_Destination ))
  {
    itk::SimpleMemberCommand< OperationEvent >::Pointer command = itk::SimpleMemberCommand< OperationEvent >::New();
    command->SetCallbackFunction( this, &OperationEvent::OnObjectDeleted );
    m_DeleteTag = object->AddObserver( itk::DeleteEvent(), command );
  }
}

mitk::OperationEvent::~OperationEvent()
{
  if (itk::Object* object = dynamic_cast<itk::Object*>( m_Destination ))
  {
    if (!m_Invalid)
    {
      object->RemoveObserver( m_DeleteTag );
    }
  }

  delete m_Operation;
  delete m_UndoOperation;
}

//##Documentation
//##  swaps the Undo and Redo- operation and changes m_Reversed
void mitk::OperationEvent::ReverseOperations()
{
  if (m_Operation == NULL)
    return;

  Operation *tempOperation = m_Operation;
  m_Operation = m_UndoOperation;
  m_UndoOperation = tempOperation;

  UndoStackItem::ReverseOperations();
}

void mitk::OperationEvent::ReverseAndExecute()
{
  ReverseOperations();
  if (m_Destination && m_Operation)
    m_Destination->ExecuteOperation( m_Operation );
}

mitk::OperationActor* mitk::OperationEvent::GetDestination()
{
  return m_Destination;
}

void mitk::OperationEvent::OnObjectDeleted()
{
  m_Invalid = true;
}

bool mitk::OperationEvent::IsValid()
{
  return !m_Invalid;
}
<commit_msg>Check m_Invalid<commit_after>/*=========================================================================

Program:   Medical Imaging & Interaction Toolkit
Module:    $RCSfile$
Language:  C++
Date:      $Date$
Version:   $Revision$

Copyright (c) German Cancer Research Center, Division of Medical and
Biological Informatics. All rights reserved.
See MITKCopyright.txt or http://www.mitk.org/copyright.html for details.

This software is distributed WITHOUT ANY WARRANTY; without even
the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
PURPOSE.  See the above copyright notices for more information.

=========================================================================*/

#include "mitkOperationEvent.h"
#include <itkCommand.h>

int mitk::UndoStackItem::m_CurrObjectEventId = 0;
int mitk::UndoStackItem::m_CurrGroupEventId = 0;

bool mitk::UndoStackItem::m_IncrObjectEventId = false;
bool mitk::UndoStackItem::m_IncrGroupEventId = false;

mitk::UndoStackItem::UndoStackItem(std::string description)
: m_Reversed(false),
  m_Description(description)
{
  m_ObjectEventId = GetCurrObjectEventId();
  m_GroupEventId = GetCurrGroupEventId();
}

mitk::UndoStackItem::~UndoStackItem()
{
  // nothing to do
}

void mitk::UndoStackItem::ExecuteIncrement()
{
  if (m_IncrObjectEventId)
  {
    ++m_CurrObjectEventId;
    m_IncrObjectEventId = false;
  }

  if (m_IncrGroupEventId)
  {
    ++m_CurrGroupEventId;
    m_IncrGroupEventId = false;
  }
}

int mitk::UndoStackItem::GetCurrObjectEventId()
{
  return m_CurrObjectEventId;
}

int mitk::UndoStackItem::GetCurrGroupEventId()
{
  return m_CurrGroupEventId;
}

void mitk::UndoStackItem::IncCurrObjectEventId()
{
  m_IncrObjectEventId = true;
}

void mitk::UndoStackItem::IncCurrGroupEventId()
{
  m_IncrGroupEventId = true;
}

int mitk::UndoStackItem::GetObjectEventId()
{
  return m_ObjectEventId;
}

int mitk::UndoStackItem::GetGroupEventId()
{
  return m_GroupEventId;
}

std::string mitk::UndoStackItem::GetDescription()
{
  return m_Description;
}

void mitk::UndoStackItem::ReverseOperations()
{
  m_Reversed = !m_Reversed;
}

void mitk::UndoStackItem::ReverseAndExecute()
{
  ReverseOperations();
}

// ******************** mitk::OperationEvent ********************

mitk::Operation* mitk::OperationEvent::GetOperation()
{
  return m_Operation;
}

mitk::OperationEvent::OperationEvent(OperationActor* destination,
                                     Operation* operation, Operation* undoOperation,
                                     std::string description)
: UndoStackItem(description),
  m_Destination(destination),
  m_Operation(operation),
  m_UndoOperation(undoOperation),
  m_Invalid(false)
{
  // nothing to do
  if (itk::Object* object = dynamic_cast<itk::Object*>( m_Destination ))
  {
    itk::SimpleMemberCommand< OperationEvent >::Pointer command = itk::SimpleMemberCommand< OperationEvent >::New();
    command->SetCallbackFunction( this, &OperationEvent::OnObjectDeleted );
    m_DeleteTag = object->AddObserver( itk::DeleteEvent(), command );
  }
}

mitk::OperationEvent::~OperationEvent()
{
  if (itk::Object* object = dynamic_cast<itk::Object*>( m_Destination ))
  {
    if (!m_Invalid)
    {
      object->RemoveObserver( m_DeleteTag );
    }
  }

  delete m_Operation;
  delete m_UndoOperation;
}

//##Documentation
//##  swaps the Undo and Redo- operation and changes m_Reversed
void mitk::OperationEvent::ReverseOperations()
{
  if (m_Operation == NULL)
    return;

  Operation *tempOperation = m_Operation;
  m_Operation = m_UndoOperation;
  m_UndoOperation = tempOperation;

  UndoStackItem::ReverseOperations();
}

void mitk::OperationEvent::ReverseAndExecute()
{
  ReverseOperations();
  if (m_Destination && m_Operation && !m_Invalid)
    m_Destination->ExecuteOperation( m_Operation );
}

mitk::OperationActor* mitk::OperationEvent::GetDestination()
{
  return m_Destination;
}

void mitk::OperationEvent::OnObjectDeleted()
{
  m_Invalid = true;
}

bool mitk::OperationEvent::IsValid()
{
  return !m_Invalid;
}
<|endoftext|>