<commit_before>// Copyright (c) 2012-2013 The PPCoin developers
// Distributed under the MIT/X11 software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.

#include <boost/assign/list_of.hpp>

#include "kernel.h"
#include "txdb.h"

using namespace std;

typedef std::map<int, unsigned int> MapModifierCheckpoints;

// Hard checkpoints of stake modifiers to ensure they are deterministic
static std::map<int, unsigned int> mapStakeModifierCheckpoints =
    boost::assign::map_list_of
        ( 0, 0x0e00670bu )
    ;

// Hard checkpoints of stake modifiers to ensure they are deterministic (testNet)
static std::map<int, unsigned int> mapStakeModifierCheckpointsTestNet =
    boost::assign::map_list_of
        ( 0, 0x0e00670bu )
    ;

// Get time weight
int64_t GetWeight(int64_t nIntervalBeginning, int64_t nIntervalEnd)
{
    // Kernel hash weight starts from 0 at the min age
    // this change increases active coins participating the hash and helps
    // to secure the network when proof-of-stake difficulty is low

    return min(nIntervalEnd - nIntervalBeginning - nStakeMinAge, (int64_t)nStakeMaxAge);
}

// Get the last stake modifier and its generation time from a given block
static bool GetLastStakeModifier(const CBlockIndex* pindex, uint64_t& nStakeModifier, int64_t& nModifierTime)
{
    if (!pindex)
        return error("GetLastStakeModifier: null pindex");
    while (pindex && pindex->pprev && !pindex->GeneratedStakeModifier())
        pindex = pindex->pprev;
    if (!pindex->GeneratedStakeModifier())
        return error("GetLastStakeModifier: no generation at genesis block");
    nStakeModifier = pindex->nStakeModifier;
    nModifierTime = pindex->GetBlockTime();
    return true;
}

// Get selection interval section (in seconds)
static int64_t GetStakeModifierSelectionIntervalSection(int nSection)
{
    assert (nSection >= 0 && nSection < 64);
    return (nModifierInterval * 63 / (63 + ((63 - nSection) * (MODIFIER_INTERVAL_RATIO - 1))));
}

// Get stake modifier selection interval (in seconds)
static int64_t GetStakeModifierSelectionInterval()
{
    int64_t nSelectionInterval = 0;
    for (int nSection=0; nSection<64; nSection++)
        nSelectionInterval += GetStakeModifierSelectionIntervalSection(nSection);
    return nSelectionInterval;
}

// select a block from the candidate blocks in vSortedByTimestamp, excluding
// already selected blocks in vSelectedBlocks, and with timestamp up to
// nSelectionIntervalStop.
static bool SelectBlockFromCandidates(vector<pair<int64_t, uint256> >& vSortedByTimestamp, map<uint256, const CBlockIndex*>& mapSelectedBlocks,
    int64_t nSelectionIntervalStop, uint64_t nStakeModifierPrev, const CBlockIndex** pindexSelected)
{
    bool fSelected = false;
    uint256 hashBest = 0;
    *pindexSelected = (const CBlockIndex*) 0;
    BOOST_FOREACH(const PAIRTYPE(int64_t, uint256)& item, vSortedByTimestamp)
    {
        if (!mapBlockIndex.count(item.second))
            return error("SelectBlockFromCandidates: failed to find block index for candidate block %s", item.second.ToString().c_str());
        const CBlockIndex* pindex = mapBlockIndex[item.second];
        if (fSelected && pindex->GetBlockTime() > nSelectionIntervalStop)
            break;
        if (mapSelectedBlocks.count(pindex->GetBlockHash()) > 0)
            continue;
        // compute the selection hash by hashing its proof-hash and the
        // previous proof-of-stake modifier
        CDataStream ss(SER_GETHASH, 0);
        ss << pindex->hashProof << nStakeModifierPrev;
        uint256 hashSelection = Hash(ss.begin(), ss.end());
        // the selection hash is divided by 2**32 so that proof-of-stake block
        // is always favored over proof-of-work block. this is to preserve
        // the energy efficiency property
        if (pindex->IsProofOfStake())
            hashSelection >>= 32;
        if (fSelected && hashSelection < hashBest)
        {
            hashBest = hashSelection;
            *pindexSelected = (const CBlockIndex*) pindex;
        }
        else if (!fSelected)
        {
            fSelected = true;
            hashBest = hashSelection;
            *pindexSelected = (const CBlockIndex*) pindex;
        }
    }
    if (fDebug && GetBoolArg("-printstakemodifier"))
        printf("SelectBlockFromCandidates: selection hash=%s\n", hashBest.ToString().c_str());
    return fSelected;
}

// Stake Modifier (hash modifier of proof-of-stake):
// The purpose of stake modifier is to prevent a txout (coin) owner from
// computing future proof-of-stake generated by this txout at the time
// of transaction confirmation. To meet kernel protocol, the txout
// must hash with a future stake modifier to generate the proof.
// Stake modifier consists of bits each of which is contributed from a
// selected block of a given block group in the past.
// The selection of a block is based on a hash of the block's proof-hash and
// the previous stake modifier.
// Stake modifier is recomputed at a fixed time interval instead of every
// block. This is to make it difficult for an attacker to gain control of
// additional bits in the stake modifier, even after generating a chain of
// blocks.
bool ComputeNextStakeModifier(const CBlockIndex* pindexPrev, uint64_t& nStakeModifier, bool& fGeneratedStakeModifier)
{
    nStakeModifier = 0;
    fGeneratedStakeModifier = false;
    if (!pindexPrev)
    {
        fGeneratedStakeModifier = true;
        return true;  // genesis block's modifier is 0
    }
    // First find current stake modifier and its generation block time
    // if it's not old enough, return the same stake modifier
    int64_t nModifierTime = 0;
    if (!GetLastStakeModifier(pindexPrev, nStakeModifier, nModifierTime))
        return error("ComputeNextStakeModifier: unable to get last modifier");
    if (fDebug)
    {
        printf("ComputeNextStakeModifier: prev modifier=0x%016"PRIx64" time=%s\n", nStakeModifier, DateTimeStrFormat(nModifierTime).c_str());
    }
    if (nModifierTime / nModifierInterval >= pindexPrev->GetBlockTime() / nModifierInterval)
        return true;

    // Sort candidate blocks by timestamp
    vector<pair<int64_t, uint256> > vSortedByTimestamp;
    vSortedByTimestamp.reserve(64 * nModifierInterval / nTargetSpacing);
    int64_t nSelectionInterval = GetStakeModifierSelectionInterval();
    int64_t nSelectionIntervalStart = (pindexPrev->GetBlockTime() / nModifierInterval) * nModifierInterval - nSelectionInterval;
    const CBlockIndex* pindex = pindexPrev;
    while (pindex && pindex->GetBlockTime() >= nSelectionIntervalStart)
    {
        vSortedByTimestamp.push_back(make_pair(pindex->GetBlockTime(), pindex->GetBlockHash()));
        pindex = pindex->pprev;
    }
    int nHeightFirstCandidate = pindex ? (pindex->nHeight + 1) : 0;
    reverse(vSortedByTimestamp.begin(), vSortedByTimestamp.end());
    sort(vSortedByTimestamp.begin(), vSortedByTimestamp.end());

    // Select 64 blocks from candidate blocks to generate stake modifier
    uint64_t nStakeModifierNew = 0;
    int64_t nSelectionIntervalStop = nSelectionIntervalStart;
    map<uint256, const CBlockIndex*> mapSelectedBlocks;
    for (int nRound=0; nRound<min(64, (int)vSortedByTimestamp.size()); nRound++)
    {
        // add an interval section to the current selection round
        nSelectionIntervalStop += GetStakeModifierSelectionIntervalSection(nRound);
        // select a block from the candidates of current round
        if (!SelectBlockFromCandidates(vSortedByTimestamp, mapSelectedBlocks, nSelectionIntervalStop, nStakeModifier, &pindex))
            return error("ComputeNextStakeModifier: unable to select block at round %d", nRound);
        // write the entropy bit of the selected block
        nStakeModifierNew |= (((uint64_t)pindex->GetStakeEntropyBit()) << nRound);
        // add the selected block from candidates to selected list
        mapSelectedBlocks.insert(make_pair(pindex->GetBlockHash(), pindex));
        if (fDebug && GetBoolArg("-printstakemodifier"))
            printf("ComputeNextStakeModifier: selected round %d stop=%s height=%d bit=%d\n", nRound, DateTimeStrFormat(nSelectionIntervalStop).c_str(), pindex->nHeight, pindex->GetStakeEntropyBit());
    }

    // Print selection map for visualization of the selected blocks
    if (fDebug && GetBoolArg("-printstakemodifier"))
    {
        string strSelectionMap = "";
        // '-' indicates proof-of-work blocks not selected
        strSelectionMap.insert(0, pindexPrev->nHeight - nHeightFirstCandidate + 1, '-');
        pindex = pindexPrev;
        while (pindex && pindex->nHeight >= nHeightFirstCandidate)
        {
            // '=' indicates proof-of-stake blocks not selected
            if (pindex->IsProofOfStake())
                strSelectionMap.replace(pindex->nHeight - nHeightFirstCandidate, 1, "=");
            pindex = pindex->pprev;
        }
        BOOST_FOREACH(const PAIRTYPE(uint256, const CBlockIndex*)& item, mapSelectedBlocks)
        {
            // 'S' indicates selected proof-of-stake blocks
            // 'W' indicates selected proof-of-work blocks
            strSelectionMap.replace(item.second->nHeight - nHeightFirstCandidate, 1, item.second->IsProofOfStake()? "S" : "W");
        }
        printf("ComputeNextStakeModifier: selection height [%d, %d] map %s\n", nHeightFirstCandidate, pindexPrev->nHeight, strSelectionMap.c_str());
    }
    if (fDebug)
    {
        printf("ComputeNextStakeModifier: new modifier=0x%016"PRIx64" time=%s\n", nStakeModifierNew, DateTimeStrFormat(pindexPrev->GetBlockTime()).c_str());
    }

    nStakeModifier = nStakeModifierNew;
    fGeneratedStakeModifier = true;
    return true;
}

// The stake modifier used to hash for a stake kernel is chosen as the stake
// modifier about a selection interval later than the coin generating the kernel
static bool GetKernelStakeModifier(uint256 hashBlockFrom, uint64_t& nStakeModifier, int& nStakeModifierHeight, int64_t& nStakeModifierTime, bool fPrintProofOfStake)
{
    nStakeModifier = 0;
    if (!mapBlockIndex.count(hashBlockFrom))
        return error("GetKernelStakeModifier() : block not indexed");
    const CBlockIndex* pindexFrom = mapBlockIndex[hashBlockFrom];
    nStakeModifierHeight = pindexFrom->nHeight;
    nStakeModifierTime = pindexFrom->GetBlockTime();
    int64_t nStakeModifierSelectionInterval = GetStakeModifierSelectionInterval();
    const CBlockIndex* pindex = pindexFrom;
    // loop to find the stake modifier later by a selection interval
    while (nStakeModifierTime < pindexFrom->GetBlockTime() + nStakeModifierSelectionInterval)
    {
        if (!pindex->pnext)
        {   // reached best block; may happen if node is behind on block chain
            if (fPrintProofOfStake || (pindex->GetBlockTime() + nStakeMinAge - nStakeModifierSelectionInterval > GetAdjustedTime()))
                return error("GetKernelStakeModifier() : reached best block %s at height %d from block %s",
                    pindex->GetBlockHash().ToString().c_str(), pindex->nHeight, hashBlockFrom.ToString().c_str());
            else
                return false;
        }
        pindex = pindex->pnext;
        if (pindex->GeneratedStakeModifier())
        {
            nStakeModifierHeight = pindex->nHeight;
            nStakeModifierTime = pindex->GetBlockTime();
        }
    }
    nStakeModifier = pindex->nStakeModifier;
    return true;
}

// ppcoin kernel protocol
// coinstake must meet hash target according to the protocol:
// kernel (input 0) must meet the formula
//     hash(nStakeModifier + txPrev.block.nTime + txPrev.offset + txPrev.nTime + txPrev.vout.n + nTime) < bnTarget * nCoinDayWeight
// this ensures that the chance of getting a coinstake is proportional to the
// amount of coin age one owns.
// The reason this hash is chosen is the following:
//   nStakeModifier: scrambles computation to make it very difficult to precompute
//                  future proof-of-stake at the time of the coin's confirmation
//   txPrev.block.nTime: prevent nodes from guessing a good timestamp to
//                       generate transaction for future advantage
//   txPrev.offset: offset of txPrev inside block, to reduce the chance of
//                  nodes generating coinstake at the same time
//   txPrev.nTime: reduce the chance of nodes generating coinstake at the same
//                 time
//   txPrev.vout.n: output number of txPrev, to reduce the chance of nodes
//                  generating coinstake at the same time
//   block/tx hash should not be used here as they can be generated in vast
//   quantities so as to generate blocks faster, degrading the system back into
//   a proof-of-work situation.
//
bool CheckStakeKernelHash(unsigned int nBits, const CBlock& blockFrom, unsigned int nTxPrevOffset, const CTransaction& txPrev, const COutPoint& prevout, unsigned int nTimeTx, uint256& hashProofOfStake, uint256& targetProofOfStake, bool fPrintProofOfStake)
{
    if (nTimeTx < txPrev.nTime)  // Transaction timestamp violation
        return error("CheckStakeKernelHash() : nTime violation");

    unsigned int nTimeBlockFrom = blockFrom.GetBlockTime();
    if (nTimeBlockFrom + nStakeMinAge > nTimeTx) // Min age requirement
        return error("CheckStakeKernelHash() : min age violation");

    CBigNum bnTargetPerCoinDay;
    bnTargetPerCoinDay.SetCompact(nBits);
    int64_t nValueIn = txPrev.vout[prevout.n].nValue;

    uint256 hashBlockFrom = blockFrom.GetHash();

    CBigNum bnCoinDayWeight = CBigNum(nValueIn) * GetWeight((int64_t)txPrev.nTime, (int64_t)nTimeTx) / COIN / (24 * 60 * 60);
    targetProofOfStake = (bnCoinDayWeight * bnTargetPerCoinDay).getuint256();

    // Calculate hash
    CDataStream ss(SER_GETHASH, 0);
    uint64_t nStakeModifier = 0;
    int nStakeModifierHeight = 0;
    int64_t nStakeModifierTime = 0;

    if (!GetKernelStakeModifier(hashBlockFrom, nStakeModifier, nStakeModifierHeight, nStakeModifierTime, fPrintProofOfStake))
        return false;
    ss << nStakeModifier;

    ss << nTimeBlockFrom << nTxPrevOffset << txPrev.nTime << prevout.n << nTimeTx;
    hashProofOfStake = Hash(ss.begin(), ss.end());
    if (fPrintProofOfStake)
    {
        printf("CheckStakeKernelHash() : using modifier 0x%016"PRIx64" at height=%d timestamp=%s for block from height=%d timestamp=%s\n",
            nStakeModifier, nStakeModifierHeight,
            DateTimeStrFormat(nStakeModifierTime).c_str(),
            mapBlockIndex[hashBlockFrom]->nHeight,
            DateTimeStrFormat(blockFrom.GetBlockTime()).c_str());
        printf("CheckStakeKernelHash() : check modifier=0x%016"PRIx64" nTimeBlockFrom=%u nTxPrevOffset=%u nTimeTxPrev=%u nPrevout=%u nTimeTx=%u hashProof=%s\n",
            nStakeModifier,
            nTimeBlockFrom, nTxPrevOffset, txPrev.nTime, prevout.n, nTimeTx,
            hashProofOfStake.ToString().c_str());
    }

    // Now check if proof-of-stake hash meets target protocol
    if (CBigNum(hashProofOfStake) > bnCoinDayWeight * bnTargetPerCoinDay)
        return false;
    if (fDebug && !fPrintProofOfStake)
    {
        printf("CheckStakeKernelHash() : using modifier 0x%016"PRIx64" at height=%d timestamp=%s for block from height=%d timestamp=%s\n",
            nStakeModifier, nStakeModifierHeight,
            DateTimeStrFormat(nStakeModifierTime).c_str(),
            mapBlockIndex[hashBlockFrom]->nHeight,
            DateTimeStrFormat(blockFrom.GetBlockTime()).c_str());
        printf("CheckStakeKernelHash() : pass modifier=0x%016"PRIx64" nTimeBlockFrom=%u nTxPrevOffset=%u nTimeTxPrev=%u nPrevout=%u nTimeTx=%u hashProof=%s\n",
            nStakeModifier,
            nTimeBlockFrom, nTxPrevOffset, txPrev.nTime, prevout.n, nTimeTx,
            hashProofOfStake.ToString().c_str());
    }
    return true;
}

// Check kernel hash target and coinstake signature
bool CheckProofOfStake(const CTransaction& tx, unsigned int nBits, uint256& hashProofOfStake, uint256& targetProofOfStake)
{
    if (!tx.IsCoinStake())
        return error("CheckProofOfStake() : called on non-coinstake %s", tx.GetHash().ToString().c_str());

    // Kernel (input 0) must match the stake hash target per coin age (nBits)
    const CTxIn& txin = tx.vin[0];

    // First try finding the previous transaction in database
    CTxDB txdb("r");
    CTransaction txPrev;
    CTxIndex txindex;
    if (!txPrev.ReadFromDisk(txdb, txin.prevout, txindex))
        return tx.DoS(1, error("CheckProofOfStake() : INFO: read txPrev failed"));  // previous transaction not in main chain, may occur during initial download

    // Verify signature
    if (!VerifySignature(txPrev, tx, 0, 0))
        return tx.DoS(100, error("CheckProofOfStake() : VerifySignature failed on coinstake %s", tx.GetHash().ToString().c_str()));

    // Read block header
    CBlock block;
    if (!block.ReadFromDisk(txindex.pos.nFile, txindex.pos.nBlockPos, false))
        return fDebug? error("CheckProofOfStake() : read block failed") : false; // unable to read block of previous transaction

    if (!CheckStakeKernelHash(nBits, block, txindex.pos.nTxPos - txindex.pos.nBlockPos, txPrev, txin.prevout, tx.nTime, hashProofOfStake, targetProofOfStake, fDebug))
        return tx.DoS(1, error("CheckProofOfStake() : INFO: check kernel failed on coinstake %s, hashProof=%s", tx.GetHash().ToString().c_str(), hashProofOfStake.ToString().c_str())); // may occur during initial download or if behind on block chain sync

    return true;
}

// Check whether the coinstake timestamp meets protocol
bool CheckCoinStakeTimestamp(int64_t nTimeBlock, int64_t nTimeTx)
{
    // v0.3 protocol
    return (nTimeBlock == nTimeTx);
}

// Get stake modifier checksum
unsigned int GetStakeModifierChecksum(const CBlockIndex* pindex)
{
    assert (pindex->pprev || pindex->GetBlockHash() == (!fTestNet ? hashGenesisBlock : hashGenesisBlockTestNet));
    // Hash previous checksum with flags, hashProofOfStake and nStakeModifier
    CDataStream ss(SER_GETHASH, 0);
    if (pindex->pprev)
        ss << pindex->pprev->nStakeModifierChecksum;
    ss << pindex->nFlags << (pindex->IsProofOfStake() ? pindex->hashProof : 0) << pindex->nStakeModifier;
    uint256 hashChecksum = Hash(ss.begin(), ss.end());
    hashChecksum >>= (256 - 32);
    return hashChecksum.Get64();
}

// Check stake modifier hard checkpoints
bool CheckStakeModifierCheckpoints(int nHeight, unsigned int nStakeModifierChecksum)
{
    MapModifierCheckpoints& checkpoints = (fTestNet ? mapStakeModifierCheckpointsTestNet : mapStakeModifierCheckpoints);

    if (checkpoints.count(nHeight))
        return nStakeModifierChecksum == checkpoints[nHeight];
    return true;
}
<commit_msg>Delete kernel.cpp<commit_after><|endoftext|>


<commit_before>#include "private/qhttpclient_private.hpp"

#include <QTimerEvent>
///////////////////////////////////////////////////////////////////////////////
namespace qhttp {
namespace client {
///////////////////////////////////////////////////////////////////////////////
QHttpClient::QHttpClient(QObject *parent)
    : QTcpSocket(parent), d_ptr(new QHttpClientPrivate(this)) {
    d_ptr->initialize();
    QHTTP_LINE_LOG
}

QHttpClient::QHttpClient(QHttpClientPrivate &dd, QObject *parent)
    : QTcpSocket(parent), d_ptr(&dd) {
    d_ptr->initialize();
    QHTTP_LINE_LOG
}

QHttpClient::~QHttpClient() {
    QHTTP_LINE_LOG
}

quint32
QHttpClient::timeOut() const {
    return d_func()->itimeOut;
}

void
QHttpClient::setTimeOut(quint32 t) {
    d_func()->itimeOut = t;
}

bool
QHttpClient::isOpen() const {
    return QTcpSocket::isOpen()    &&    state() == QTcpSocket::ConnectedState;
}

bool
QHttpClient::request(THttpMethod method, QUrl url,
                     const TRequstHandler &reqHandler, const TResponseHandler &resHandler) {
    Q_D(QHttpClient);

    d->ireqHandler   = nullptr;
    d->irespHandler  = nullptr;

    if ( !url.isValid()    ||    url.isEmpty()    ||    url.host().isEmpty() )
        return false;

    d->ilastMethod  = method;
    d->ilastUrl     = url;

    connectToHost(url.host(), url.port(80));

    // process handlers
    if ( resHandler ) {
        d->irespHandler = resHandler;

        if ( reqHandler )
            d->ireqHandler = reqHandler;
        else
            d->ireqHandler = [](QHttpRequest* req) ->void { req->end(); };
    }

    return true;

}

void
QHttpClient::timerEvent(QTimerEvent *e) {
    Q_D(QHttpClient);

    if ( e->timerId() == d->itimer.timerId() ) {
        close();
    }
}

void
QHttpClient::onRequestReady(QHttpRequest *req) {
    emit httpConnected(req);
}

void
QHttpClient::onResponseReady(QHttpResponse *res) {
    emit newResponse(res);
}

///////////////////////////////////////////////////////////////////////////////

void
QHttpClientPrivate::onConnected() {
    QHttpRequest *request = new QHttpRequest(isocket);

    request->d_func()->imethod  = ilastMethod;
    request->d_func()->iurl     = ilastUrl;

    if ( itimeOut > 0 )
        itimer.start(itimeOut, Qt::CoarseTimer, q_func());

#   if QHTTP_MESSAGES_LOG > 0
    iinputBuffer.clear();
#   endif

    if ( ireqHandler )
        ireqHandler(request);
    else
        q_func()->onRequestReady(request);
}

void
QHttpClientPrivate::onReadyRead() {
    while ( isocket->bytesAvailable() > 0 ) {
        char buffer[4097] = {0};
        size_t readLength = isocket->read(buffer, 4096);

        parse(buffer, readLength);

#       if QHTTP_MESSAGES_LOG > 0
        iinputBuffer.append(buffer);
#       endif
    }
}

///////////////////////////////////////////////////////////////////////////////

int
QHttpClientPrivate::messageBegin(http_parser*) {
    itempHeaderField.clear();
    itempHeaderValue.clear();

    return 0;
}

int
QHttpClientPrivate::status(http_parser* parser, const char* at, size_t length) {

    ilastResponse = new QHttpResponse(isocket);
    ilastResponse->d_func()->istatus  = static_cast<TStatusCode>(parser->status_code);
    ilastResponse->d_func()->iversion = QString("%1.%2")
                                        .arg(parser->http_major)
                                        .arg(parser->http_minor);
    ilastResponse->d_func()->icustomeStatusMessage = QString::fromUtf8(at, length);

    return 0;
}

int
QHttpClientPrivate::headerField(http_parser*, const char* at, size_t length) {
    Q_ASSERT(ilastResponse);

    // insert the header we parsed previously
    // into the header map
    if ( !itempHeaderField.isEmpty() && !itempHeaderValue.isEmpty() ) {
        // header names are always lower-cased
        ilastResponse->d_func()->iheaders.insert(
                    itempHeaderField.toLower(),
                    itempHeaderValue.toLower()
                    );
        // clear header value. this sets up a nice
        // feedback loop where the next time
        // HeaderValue is called, it can simply append
        itempHeaderField.clear();
        itempHeaderValue.clear();
    }

    itempHeaderField.append(at, length);
    return 0;
}

int
QHttpClientPrivate::headerValue(http_parser*, const char* at, size_t length) {
    Q_ASSERT(ilastResponse);

    itempHeaderValue.append(at, length);
    return 0;
}

int
QHttpClientPrivate::headersComplete(http_parser*) {
    Q_ASSERT(ilastResponse);

    // Insert last remaining header
    ilastResponse->d_func()->iheaders.insert(
                itempHeaderField.toLower(),
                itempHeaderValue.toLower()
                );

    if ( irespHandler )
        irespHandler(ilastResponse);
    else
        q_func()->onResponseReady(ilastResponse);

    return 0;
}

int
QHttpClientPrivate::body(http_parser*, const char* at, size_t length) {
    Q_ASSERT(ilastResponse);

    if ( ilastResponse->idataHandler )
        ilastResponse->idataHandler(QByteArray(at, length));
    else
        emit ilastResponse->data(QByteArray(at, length));

    return 0;
}

int
QHttpClientPrivate::messageComplete(http_parser*) {
    Q_ASSERT(ilastResponse);

    ilastResponse->d_func()->isuccessful = true;

    if ( ilastResponse->iendHandler )
        ilastResponse->iendHandler();
    else
        emit ilastResponse->end();

    return 0;
}

///////////////////////////////////////////////////////////////////////////////
} // namespace client
} // namespace qhttp
///////////////////////////////////////////////////////////////////////////////
<commit_msg>fix a bug in qhttpclient.<commit_after>#include "private/qhttpclient_private.hpp"

#include <QTimerEvent>
///////////////////////////////////////////////////////////////////////////////
namespace qhttp {
namespace client {
///////////////////////////////////////////////////////////////////////////////
QHttpClient::QHttpClient(QObject *parent)
    : QTcpSocket(parent), d_ptr(new QHttpClientPrivate(this)) {
    d_ptr->initialize();
    QHTTP_LINE_LOG
}

QHttpClient::QHttpClient(QHttpClientPrivate &dd, QObject *parent)
    : QTcpSocket(parent), d_ptr(&dd) {
    d_ptr->initialize();
    QHTTP_LINE_LOG
}

QHttpClient::~QHttpClient() {
    QHTTP_LINE_LOG
}

quint32
QHttpClient::timeOut() const {
    return d_func()->itimeOut;
}

void
QHttpClient::setTimeOut(quint32 t) {
    d_func()->itimeOut = t;
}

bool
QHttpClient::isOpen() const {
    return QTcpSocket::isOpen()    &&    state() == QTcpSocket::ConnectedState;
}

bool
QHttpClient::request(THttpMethod method, QUrl url,
                     const TRequstHandler &reqHandler, const TResponseHandler &resHandler) {
    Q_D(QHttpClient);

    d->ireqHandler   = nullptr;
    d->irespHandler  = nullptr;

    if ( !url.isValid()    ||    url.isEmpty()    ||    url.host().isEmpty() )
        return false;

    d->ilastMethod  = method;
    d->ilastUrl     = url;

    connectToHost(url.host(), url.port(80));

    // process handlers
    if ( resHandler ) {
        d->irespHandler = resHandler;

        if ( reqHandler )
            d->ireqHandler = reqHandler;
        else
            d->ireqHandler = [](QHttpRequest* req) ->void { req->end(); };
    }

    return true;

}

void
QHttpClient::timerEvent(QTimerEvent *e) {
    Q_D(QHttpClient);

    if ( e->timerId() == d->itimer.timerId() ) {
        close();
    }
}

void
QHttpClient::onRequestReady(QHttpRequest *req) {
    emit httpConnected(req);
}

void
QHttpClient::onResponseReady(QHttpResponse *res) {
    emit newResponse(res);
}

///////////////////////////////////////////////////////////////////////////////

void
QHttpClientPrivate::onConnected() {
    QHttpRequest *request = new QHttpRequest(isocket);

    request->d_func()->imethod  = ilastMethod;
    request->d_func()->iurl     = ilastUrl;

    if ( itimeOut > 0 )
        itimer.start(itimeOut, Qt::CoarseTimer, q_func());

#   if QHTTP_MESSAGES_LOG > 0
    iinputBuffer.clear();
#   endif

    if ( ireqHandler )
        ireqHandler(request);
    else
        q_func()->onRequestReady(request);
}

void
QHttpClientPrivate::onReadyRead() {
    while ( isocket->bytesAvailable() > 0 ) {
        char buffer[4097] = {0};
        size_t readLength = isocket->read(buffer, 4096);

        parse(buffer, readLength);

#       if QHTTP_MESSAGES_LOG > 0
        iinputBuffer.append(buffer);
#       endif
    }
}

///////////////////////////////////////////////////////////////////////////////

// if user closes the connection, ends the response or by any other reason
//  the socket be disconnected, then the iresponse instance may has been deleted.
//  In these situations reading more http body or emitting end() for incoming response
//  is not possible.
#define CHECK_FOR_DISCONNECTED  if ( ilastResponse == nullptr ) \
    return 0;

int
QHttpClientPrivate::messageBegin(http_parser*) {
    itempHeaderField.clear();
    itempHeaderValue.clear();

    return 0;
}

int
QHttpClientPrivate::status(http_parser* parser, const char* at, size_t length) {
    CHECK_FOR_DISCONNECTED

    ilastResponse = new QHttpResponse(isocket);
    ilastResponse->d_func()->istatus  = static_cast<TStatusCode>(parser->status_code);
    ilastResponse->d_func()->iversion = QString("%1.%2")
                                        .arg(parser->http_major)
                                        .arg(parser->http_minor);
    ilastResponse->d_func()->icustomeStatusMessage = QString::fromUtf8(at, length);

    return 0;
}

int
QHttpClientPrivate::headerField(http_parser*, const char* at, size_t length) {
    CHECK_FOR_DISCONNECTED

    // insert the header we parsed previously
    // into the header map
    if ( !itempHeaderField.isEmpty() && !itempHeaderValue.isEmpty() ) {
        // header names are always lower-cased
        ilastResponse->d_func()->iheaders.insert(
                    itempHeaderField.toLower(),
                    itempHeaderValue.toLower()
                    );
        // clear header value. this sets up a nice
        // feedback loop where the next time
        // HeaderValue is called, it can simply append
        itempHeaderField.clear();
        itempHeaderValue.clear();
    }

    itempHeaderField.append(at, length);
    return 0;
}

int
QHttpClientPrivate::headerValue(http_parser*, const char* at, size_t length) {

    itempHeaderValue.append(at, length);
    return 0;
}

int
QHttpClientPrivate::headersComplete(http_parser*) {
    CHECK_FOR_DISCONNECTED

    // Insert last remaining header
    ilastResponse->d_func()->iheaders.insert(
                itempHeaderField.toLower(),
                itempHeaderValue.toLower()
                );

    if ( irespHandler )
        irespHandler(ilastResponse);
    else
        q_func()->onResponseReady(ilastResponse);

    return 0;
}

int
QHttpClientPrivate::body(http_parser*, const char* at, size_t length) {
    CHECK_FOR_DISCONNECTED

    if ( ilastResponse->idataHandler )
        ilastResponse->idataHandler(QByteArray(at, length));
    else
        emit ilastResponse->data(QByteArray(at, length));

    return 0;
}

int
QHttpClientPrivate::messageComplete(http_parser*) {
    CHECK_FOR_DISCONNECTED

    ilastResponse->d_func()->isuccessful = true;

    if ( ilastResponse->iendHandler )
        ilastResponse->iendHandler();
    else
        emit ilastResponse->end();

    return 0;
}

///////////////////////////////////////////////////////////////////////////////
} // namespace client
} // namespace qhttp
///////////////////////////////////////////////////////////////////////////////
<|endoftext|>


<commit_before>#include <iostream>
#include <string>
#include <vector>
using namespace std;

#include "libsqasm/assembler.hpp"
#include "libsqrun/runner.hpp"

static void assemble(const string &fileName, const bool debug = false)
{
    auto compilationData(Assembler::assemble(fileName));
    Runner::run(compilationData->ops());

    if (debug)
    {
        if (!compilationData->symbolTable().empty())
        {
            cout << "Symbol table" << endl;
            for (auto &pair : compilationData->symbolTable())
            {
                cout << "  symbol: " << pair.first << " = " << pair.second << endl;
            }
        }

        if (!compilationData->cells().empty())
        {
            cout << "Cells" << endl;
            for (auto &cell : compilationData->cells())
            {
                cout << "  cell: " << cell->toString() << endl;
            }
        }
    }
}

int main(int argc, char *argv[])
{
    try
    {
        for (auto i = 1; i < argc; ++i)
        {
            assemble(argv[i]);
        }
        return EXIT_SUCCESS;
    }
    catch (const exception &ex)
    {
        cout << "Message: " << ex.what() << endl;
        return EXIT_FAILURE;
    }
}

<commit_msg>Parse --debug switch from command line<commit_after>#include <iostream>
#include <string>
#include <vector>
using namespace std;

#include "libsqasm/assembler.hpp"
#include "libsqrun/runner.hpp"

static void assemble(const string &fileName, const bool debug)
{
    auto compilationData(Assembler::assemble(fileName));
    Runner::run(compilationData->ops(), debug);
}

int main(int argc, char *argv[])
{
    try
    {
        bool debug = false;

        for (auto i = 1; i < argc; ++i)
        {
            string arg(argv[i]);

            if (i == 1 && arg.compare("--debug") == 0)
            {
                debug = true;
                continue;
            }

            assemble(arg, debug);
        }
        return EXIT_SUCCESS;
    }
    catch (const exception &ex)
    {
        cout << "Message: " << ex.what() << endl;
        return EXIT_FAILURE;
    }
}

<|endoftext|>


<commit_before>/* Copyright (C) 2000 MySQL AB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

/*
  UNION  of select's
  UNION's  were introduced by Monty and Sinisa <sinisa@mysql.com>
*/

#include "mysql_priv.h"
#include "sql_select.h"

int mysql_union(THD *thd, LEX *lex,select_result *result)
{
  SELECT_LEX *sl;
  SELECT_LEX_UNIT *unit= &(lex->unit);
  List<Item> item_list;
  TABLE *table;
  int describe=(lex->select_lex.options & SELECT_DESCRIBE) ? 1 : 0;
  int res;
  bool found_rows_for_union=false;
  TABLE_LIST result_table_list;
  TABLE_LIST *first_table=(TABLE_LIST *)lex->select_lex.table_list.first;
  TMP_TABLE_PARAM tmp_table_param;
  select_union *union_result;
  DBUG_ENTER("mysql_union");
  st_select_lex_node * global;

  /* Fix tables 'to-be-unioned-from' list to point at opened tables */
  for (sl= &lex->select_lex;
       sl;
       sl= (SELECT_LEX *) sl->next)
  {
    for (TABLE_LIST *cursor= (TABLE_LIST *)sl->table_list.first;
	 cursor;
	 cursor=cursor->next)
      cursor->table= ((TABLE_LIST*) cursor->table)->table;
  }

  /* Global option */
  if (((void*)(global= unit->global_parameters)) == ((void*)unit))
  {
    found_rows_for_union = lex->select_lex.options & OPTION_FOUND_ROWS &&
      !describe && global->select_limit;
    if (found_rows_for_union)
      lex->select_lex.options ^=  OPTION_FOUND_ROWS;
  }

  if (describe)
  {
    Item *item;
    item_list.push_back(new Item_empty_string("table",NAME_LEN));
    item_list.push_back(new Item_empty_string("type",10));
    item_list.push_back(item=new Item_empty_string("possible_keys",
						  NAME_LEN*MAX_KEY));
    item->maybe_null=1;
    item_list.push_back(item=new Item_empty_string("key",NAME_LEN));
    item->maybe_null=1;
    item_list.push_back(item=new Item_int("key_len",0,3));
    item->maybe_null=1;
    item_list.push_back(item=new Item_empty_string("ref",
						    NAME_LEN*MAX_REF_PARTS));
    item->maybe_null=1;
    item_list.push_back(new Item_real("rows",0.0,0,10));
    item_list.push_back(new Item_empty_string("Extra",255));
  }
  else
  {
    Item *item;
    List_iterator<Item> it(lex->select_lex.item_list);
    TABLE_LIST *first_table= (TABLE_LIST*) lex->select_lex.table_list.first;

    /* Create a list of items that will be in the result set */
    while ((item= it++))
      if (item_list.push_back(item))
	DBUG_RETURN(-1);
    if (setup_fields(thd,first_table,item_list,0,0,1))
      DBUG_RETURN(-1);
  }

  bzero((char*) &tmp_table_param,sizeof(tmp_table_param));
  tmp_table_param.field_count=item_list.elements;
  if (!(table= create_tmp_table(thd, &tmp_table_param, item_list,
				(ORDER*) 0, !describe & !lex->union_option,
				1, 0,
				(lex->select_lex.options | thd->options |
				 TMP_TABLE_ALL_COLUMNS),
				unit)))
    DBUG_RETURN(-1);
  table->file->extra(HA_EXTRA_WRITE_CACHE);
  table->file->extra(HA_EXTRA_IGNORE_DUP_KEY);
  bzero((char*) &result_table_list,sizeof(result_table_list));
  result_table_list.db= (char*) "";
  result_table_list.real_name=result_table_list.name=(char*) "union";
  result_table_list.table=table;

  if (!(union_result=new select_union(table)))
  {
    res= -1;
    goto exit;
  }
  union_result->save_time_stamp=!describe;

  for (sl= &lex->select_lex; sl; sl= (SELECT_LEX*) sl->next)
  {
    lex->select=sl;
    unit->offset_limit_cnt= sl->offset_limit;
    unit->select_limit_cnt= sl->select_limit+sl->offset_limit;
    if (unit->select_limit_cnt < sl->select_limit)
      unit->select_limit_cnt= HA_POS_ERROR;		// no limit
    if (unit->select_limit_cnt == HA_POS_ERROR)
      sl->options&= ~OPTION_FOUND_ROWS;

    res= mysql_select(thd,
		      (describe && sl->linkage==GLOBAL_OPTIONS_TYPE) ?
		      first_table :  (TABLE_LIST*) sl->table_list.first,
		      sl->item_list,
		      sl->where,
		      (sl->braces) ?
		      (ORDER *)sl->order_list.first : (ORDER *) 0,
		      (ORDER*) sl->group_list.first,
		      sl->having,
		      (ORDER*) NULL,
		      sl->options | thd->options |
		      SELECT_NO_UNLOCK | ((describe) ? SELECT_DESCRIBE : 0),
		      union_result, unit);
    if (res)
      goto exit;
  }
  if (union_result->flush())
  {
    res= 1;					// Error is already sent
    goto exit;
  }
  delete union_result;

  /* Send result to 'result' */
  lex->select = &lex->select_lex;
  res =-1;
  {
    /* Create a list of fields in the temporary table */
    List_iterator<Item> it(item_list);
    Field **field;
#if 0
    List<Item_func_match> ftfunc_list;
    ftfunc_list.empty();
#else
    thd->lex.select_lex.ftfunc_list.empty();
#endif

    for (field=table->field ; *field ; field++)
    {
      (void) it++;
      (void) it.replace(new Item_field(*field));
    }
    if (!thd->fatal_error)			// Check if EOM
    {
      st_select_lex_node * global= unit->global_parameters;
      unit->offset_limit_cnt= global->offset_limit;
      unit->select_limit_cnt= global->select_limit+global->offset_limit;
      if (unit->select_limit_cnt < global->select_limit)
	unit->select_limit_cnt= HA_POS_ERROR;		// no limit
      if (unit->select_limit_cnt == HA_POS_ERROR)
	thd->options&= ~OPTION_FOUND_ROWS;
      if (describe)
	unit->select_limit_cnt= HA_POS_ERROR;		// no limit
      res= mysql_select(thd,&result_table_list,
			item_list, NULL,
			(describe) ? 0 : (ORDER*)global->order_list.first,
			(ORDER*) NULL, NULL, (ORDER*) NULL,
			thd->options, result, unit);
      if (found_rows_for_union && !res)
	thd->limit_found_rows = (ulonglong)table->file->records;
    }
  }

exit:
  free_tmp_table(thd,table);
  DBUG_RETURN(res);
}

/***************************************************************************
** store records in temporary table for UNION
***************************************************************************/

select_union::select_union(TABLE *table_par)
    :table(table_par)
{
  bzero((char*) &info,sizeof(info));
  /*
    We can always use DUP_IGNORE because the temporary table will only
    contain a unique key if we are using not using UNION ALL
  */
  info.handle_duplicates= DUP_IGNORE;
}

select_union::~select_union()
{
}

int select_union::prepare(List<Item> &list, SELECT_LEX_UNIT *u)
{
  unit= u;
  if (save_time_stamp && list.elements != table->fields)
  {
    my_message(ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT,
	       ER(ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT),MYF(0));
    return -1;
  }
  return 0;
}

bool select_union::send_data(List<Item> &values)
{
  if (unit->offset_limit_cnt)
  {						// using limit offset,count
    unit->offset_limit_cnt--;
    return 0;
  }
  fill_record(table->field,values);
  return write_record(table,&info) ? 1 : 0;
}

bool select_union::send_eof()
{
  return 0;
}

bool select_union::flush()
{
  int error;
  if ((error=table->file->extra(HA_EXTRA_NO_CACHE)))
  {
    table->file->print_error(error,MYF(0));
    ::send_error(&thd->net);
    return 1;
  }
  return 0;
}
<commit_msg>removed fake description (EXPLAIN) of first table for last SELECT_LEX with global parameters, because now it is absent<commit_after>/* Copyright (C) 2000 MySQL AB

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

/*
  UNION  of select's
  UNION's  were introduced by Monty and Sinisa <sinisa@mysql.com>
*/

#include "mysql_priv.h"
#include "sql_select.h"

int mysql_union(THD *thd, LEX *lex,select_result *result)
{
  SELECT_LEX *sl;
  SELECT_LEX_UNIT *unit= &(lex->unit);
  List<Item> item_list;
  TABLE *table;
  int describe=(lex->select_lex.options & SELECT_DESCRIBE) ? 1 : 0;
  int res;
  bool found_rows_for_union=false;
  TABLE_LIST result_table_list;
  TMP_TABLE_PARAM tmp_table_param;
  select_union *union_result;
  DBUG_ENTER("mysql_union");
  st_select_lex_node * global;

  /* Fix tables 'to-be-unioned-from' list to point at opened tables */
  for (sl= &lex->select_lex;
       sl;
       sl= (SELECT_LEX *) sl->next)
  {
    for (TABLE_LIST *cursor= (TABLE_LIST *)sl->table_list.first;
	 cursor;
	 cursor=cursor->next)
      cursor->table= ((TABLE_LIST*) cursor->table)->table;
  }

  /* Global option */
  if (((void*)(global= unit->global_parameters)) == ((void*)unit))
  {
    found_rows_for_union = lex->select_lex.options & OPTION_FOUND_ROWS &&
      !describe && global->select_limit;
    if (found_rows_for_union)
      lex->select_lex.options ^=  OPTION_FOUND_ROWS;
  }

  if (describe)
  {
    Item *item;
    item_list.push_back(new Item_empty_string("table",NAME_LEN));
    item_list.push_back(new Item_empty_string("type",10));
    item_list.push_back(item=new Item_empty_string("possible_keys",
						  NAME_LEN*MAX_KEY));
    item->maybe_null=1;
    item_list.push_back(item=new Item_empty_string("key",NAME_LEN));
    item->maybe_null=1;
    item_list.push_back(item=new Item_int("key_len",0,3));
    item->maybe_null=1;
    item_list.push_back(item=new Item_empty_string("ref",
						    NAME_LEN*MAX_REF_PARTS));
    item->maybe_null=1;
    item_list.push_back(new Item_real("rows",0.0,0,10));
    item_list.push_back(new Item_empty_string("Extra",255));
  }
  else
  {
    Item *item;
    List_iterator<Item> it(lex->select_lex.item_list);
    TABLE_LIST *first_table= (TABLE_LIST*) lex->select_lex.table_list.first;

    /* Create a list of items that will be in the result set */
    while ((item= it++))
      if (item_list.push_back(item))
	DBUG_RETURN(-1);
    if (setup_fields(thd,first_table,item_list,0,0,1))
      DBUG_RETURN(-1);
  }

  bzero((char*) &tmp_table_param,sizeof(tmp_table_param));
  tmp_table_param.field_count=item_list.elements;
  if (!(table= create_tmp_table(thd, &tmp_table_param, item_list,
				(ORDER*) 0, !describe & !lex->union_option,
				1, 0,
				(lex->select_lex.options | thd->options |
				 TMP_TABLE_ALL_COLUMNS),
				unit)))
    DBUG_RETURN(-1);
  table->file->extra(HA_EXTRA_WRITE_CACHE);
  table->file->extra(HA_EXTRA_IGNORE_DUP_KEY);
  bzero((char*) &result_table_list,sizeof(result_table_list));
  result_table_list.db= (char*) "";
  result_table_list.real_name=result_table_list.name=(char*) "union";
  result_table_list.table=table;

  if (!(union_result=new select_union(table)))
  {
    res= -1;
    goto exit;
  }
  union_result->save_time_stamp=!describe;

  for (sl= &lex->select_lex; sl; sl= (SELECT_LEX*) sl->next)
  {
    lex->select=sl;
    unit->offset_limit_cnt= sl->offset_limit;
    unit->select_limit_cnt= sl->select_limit+sl->offset_limit;
    if (unit->select_limit_cnt < sl->select_limit)
      unit->select_limit_cnt= HA_POS_ERROR;		// no limit
    if (unit->select_limit_cnt == HA_POS_ERROR)
      sl->options&= ~OPTION_FOUND_ROWS;

    res= mysql_select(thd,
		      (TABLE_LIST*) sl->table_list.first,
		      sl->item_list,
		      sl->where,
		      (sl->braces) ?
		      (ORDER *)sl->order_list.first : (ORDER *) 0,
		      (ORDER*) sl->group_list.first,
		      sl->having,
		      (ORDER*) NULL,
		      sl->options | thd->options |
		      SELECT_NO_UNLOCK | ((describe) ? SELECT_DESCRIBE : 0),
		      union_result, unit);
    if (res)
      goto exit;
  }
  if (union_result->flush())
  {
    res= 1;					// Error is already sent
    goto exit;
  }
  delete union_result;

  /* Send result to 'result' */
  lex->select = &lex->select_lex;
  res =-1;
  {
    /* Create a list of fields in the temporary table */
    List_iterator<Item> it(item_list);
    Field **field;
#if 0
    List<Item_func_match> ftfunc_list;
    ftfunc_list.empty();
#else
    thd->lex.select_lex.ftfunc_list.empty();
#endif

    for (field=table->field ; *field ; field++)
    {
      (void) it++;
      (void) it.replace(new Item_field(*field));
    }
    if (!thd->fatal_error)			// Check if EOM
    {
      st_select_lex_node * global= unit->global_parameters;
      unit->offset_limit_cnt= global->offset_limit;
      unit->select_limit_cnt= global->select_limit+global->offset_limit;
      if (unit->select_limit_cnt < global->select_limit)
	unit->select_limit_cnt= HA_POS_ERROR;		// no limit
      if (unit->select_limit_cnt == HA_POS_ERROR)
	thd->options&= ~OPTION_FOUND_ROWS;
      if (describe)
	unit->select_limit_cnt= HA_POS_ERROR;		// no limit
      res= mysql_select(thd,&result_table_list,
			item_list, NULL,
			(describe) ? 0 : (ORDER*)global->order_list.first,
			(ORDER*) NULL, NULL, (ORDER*) NULL,
			thd->options, result, unit);
      if (found_rows_for_union && !res)
	thd->limit_found_rows = (ulonglong)table->file->records;
    }
  }

exit:
  free_tmp_table(thd,table);
  DBUG_RETURN(res);
}

/***************************************************************************
** store records in temporary table for UNION
***************************************************************************/

select_union::select_union(TABLE *table_par)
    :table(table_par)
{
  bzero((char*) &info,sizeof(info));
  /*
    We can always use DUP_IGNORE because the temporary table will only
    contain a unique key if we are using not using UNION ALL
  */
  info.handle_duplicates= DUP_IGNORE;
}

select_union::~select_union()
{
}

int select_union::prepare(List<Item> &list, SELECT_LEX_UNIT *u)
{
  unit= u;
  if (save_time_stamp && list.elements != table->fields)
  {
    my_message(ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT,
	       ER(ER_WRONG_NUMBER_OF_COLUMNS_IN_SELECT),MYF(0));
    return -1;
  }
  return 0;
}

bool select_union::send_data(List<Item> &values)
{
  if (unit->offset_limit_cnt)
  {						// using limit offset,count
    unit->offset_limit_cnt--;
    return 0;
  }
  fill_record(table->field,values);
  return write_record(table,&info) ? 1 : 0;
}

bool select_union::send_eof()
{
  return 0;
}

bool select_union::flush()
{
  int error;
  if ((error=table->file->extra(HA_EXTRA_NO_CACHE)))
  {
    table->file->print_error(error,MYF(0));
    ::send_error(&thd->net);
    return 1;
  }
  return 0;
}
<|endoftext|>


<commit_before>
#include "IntegrateOverRadius.h"

#include "GslWrappers.h"
#include "RadialFunction.h"

#include <cmath>
#include <gsl/gsl_spline.h>

namespace {

  RadialFunction weighByVolumeElement(const RadialFunction& f) {
    std::vector<double> weighted(f.data);
    for (size_t r=0; r<weighted.size(); ++r) {
      weighted[r] *= 4.0*M_PI * f.radii[r] * f.radii[r];
    }
    return RadialFunction(f.radii, weighted);
  }

  class RadialInterpFunction : public GSL::FunctionObject {
    private:
      gsl_interp_accel* acc;
      gsl_spline* spline;
    public:
      RadialInterpFunction(const RadialFunction& f) {
        acc = gsl_interp_accel_alloc();
        spline = gsl_spline_alloc(gsl_interp_cspline, f.radii.size());
        gsl_spline_init(spline, f.radii.data(), f.data.data(), f.radii.size());
      }
      ~RadialInterpFunction() {
        gsl_spline_free(spline);
        gsl_interp_accel_free(acc);
      }
      double operator()(double x) const {
        return gsl_spline_eval(spline, x, acc);
      }
  };
} // helper namespace

double integrateOverRadius(const RadialFunction& input) {

  // multiply data values by volume element (4 pi r^2) before integration
  const RadialFunction integrand = weighByVolumeElement(input);

  // construct a GSL cubic spline approximation to the data
  // TODO: see if GSL interfacing can be cleaned up
  RadialInterpFunction interp_integrand(integrand);

  // perform integration with GSL calls
  // TODO:
  //   consider other algorithm possibilities:
  //   1- use the spline's exact integral
  //   2- use a spectral type approach
  //   3- ?
  const double r_min = integrand.radii.front();
  const double r_max = integrand.radii.back();
  const double eps_abs = 1.0e-6;
  const double eps_rel = eps_abs;
  return GSL::integrate(interp_integrand, r_min, r_max, eps_abs, eps_rel);
}

<commit_msg>IntegrateOverRadius: Use new spline wrapper<commit_after>
#include "IntegrateOverRadius.h"

#include "GslWrappers.h"
#include "RadialFunction.h"

#include <cmath>

namespace {

  RadialFunction weighByVolumeElement(const RadialFunction& f) {
    std::vector<double> weighted(f.data);
    for (size_t r=0; r<weighted.size(); ++r) {
      weighted[r] *= 4.0*M_PI * f.radii[r] * f.radii[r];
    }
    return RadialFunction(f.radii, weighted);
  }

  class RadialInterpFunction : public GSL::FunctionObject {
    private:
      GSL::Spline spline;
    public:
      RadialInterpFunction(const RadialFunction& f) : spline(f) {}
      double operator()(double x) const {
        return spline.eval(x);
      }
  };
} // helper namespace

double integrateOverRadius(const RadialFunction& input) {

  // multiply data values by volume element (4 pi r^2) before integration
  const RadialFunction integrand = weighByVolumeElement(input);

  // construct a GSL cubic spline approximation to the data
  // TODO: see if GSL interfacing can be cleaned up
  RadialInterpFunction interp_integrand(integrand);

  // perform integration with GSL calls
  // TODO:
  //   consider other algorithm possibilities:
  //   1- use the spline's exact integral
  //   2- use a spectral type approach
  //   3- ?
  const double r_min = integrand.radii.front();
  const double r_max = integrand.radii.back();
  const double eps_abs = 1.0e-6;
  const double eps_rel = eps_abs;
  return GSL::integrate(interp_integrand, r_min, r_max, eps_abs, eps_rel);
}

<|endoftext|>


<commit_before>// ffmpeg player output (portaudio)
// part of MusicPlayer, https://github.com/albertz/music-player
// Copyright (c) 2012, Albert Zeyer, www.az2000.de
// All rights reserved.
// This code is under the 2-clause BSD license, see License.txt in the root directory of this project.

#include "ffmpeg.h"
#include "PyUtils.h"

#include <portaudio.h>
#include <boost/bind.hpp>
#include <boost/atomic.hpp>

#ifdef __APPLE__
// PortAudio specific Mac stuff
#include "pa_mac_core.h"
#endif

// For setting the thread priority to realtime on MacOSX.
#ifdef __APPLE__
#include <mach/mach_init.h>
#include <mach/thread_policy.h>
#include <mach/thread_act.h> // the doc says mach/sched.h but that seems outdated...
#include <pthread.h>
#include <mach/mach_error.h>
#include <mach/mach_time.h>
#endif
static void setRealtime();

/*
The implementation with the PortAudio callback was there first.
For testing, I implemented also the blocking PortAudio interface.
I had some issues which small hiccups in the audio output -
maybe the blocking PortAudio implementation can avoid them better.
For the callback, we need to minimize the locks - or better, avoid
them fully. I'm not sure it is good to depend on thread context
switches in case it is locked. This however needs some heavy redesign.
*/
#define USE_PORTAUDIO_CALLBACK 1

int initPlayerOutput() {
	PaError ret = Pa_Initialize();
	if(ret != paNoError)
		Py_FatalError("PortAudio init failed");
	return 0;
}

void reinitPlayerOutput() {
	if(Pa_Terminate() != paNoError)
		printf("Warning: Pa_Terminate() failed\n");
	initPlayerOutput();
}

template<typename T> struct OutPaSampleFormat{};
template<> struct OutPaSampleFormat<int16_t> {
	static const PaSampleFormat format = paInt16;
};
template<> struct OutPaSampleFormat<float32_t> {
	static const PaSampleFormat format = paFloat32;
};

#define LATENCY_IN_MS 100

struct PlayerObject::OutStream {
	PlayerObject* const player;
	PaStream* stream;
	boost::atomic<bool> needRealtimeReset; // PortAudio callback thread must set itself to realtime
	boost::atomic<bool> setThreadName;

	OutStream(PlayerObject* p) : player(p), stream(NULL), needRealtimeReset(false) {
		mlock(this, sizeof(*this));
	}
	~OutStream() { close(); }

#if USE_PORTAUDIO_CALLBACK
	static int paStreamCallback(
		const void *input, void *output,
		unsigned long frameCount,
		const PaStreamCallbackTimeInfo* timeInfo,
		PaStreamCallbackFlags statusFlags,
		void *userData )
	{
		OutStream* outStream = (OutStream*) userData;

		if(outStream->needRealtimeReset.exchange(false))
			setRealtime();

		if(outStream->setThreadName.exchange(false))
			setCurThreadName("audio callback");

		if(statusFlags & paOutputUnderflow)
			printf("audio: paOutputUnderflow\n");
		if(statusFlags & paOutputOverflow)
			printf("audio: paOutputOverflow\n");

		// We must not hold the PyGIL here!
		// Also no need to hold the player lock, all is safe!

		outStream->player->readOutStream((OUTSAMPLE_t*) output, frameCount * outStream->player->outNumChannels, NULL);
		return paContinue;
	}
#else
	PyThread audioThread;
	void audioThreadProc(PyMutex& threadLock, bool& stopSignal) {
		while(true) {
			{
				PyScopedLock l(threadLock);
				if(stopSignal) return;
			}

			if(needRealtimeReset) {
				needRealtimeReset = false;
				setRealtime();
			}

			OUTSAMPLE_t buffer[48 * 2 * 10]; // 10ms stereo in 48khz
			size_t frameCount = 0;
			{
				PyScopedLock lock(player->lock);
				if(stopSignal) return;
				player->readOutStream(buffer, sizeof(buffer)/sizeof(OUTSAMPLE_t), NULL);
				frameCount = sizeof(buffer)/sizeof(OUTSAMPLE_t) / player->outNumChannels;
			}

			PaError ret = Pa_WriteStream(stream, buffer, frameCount);
			if(ret == paOutputUnderflowed)
				printf("warning: paOutputUnderflowed\n");
			else if(ret != paNoError) {
				printf("Pa_WriteStream error %i (%s)\n", ret, Pa_GetErrorText(ret));
				// sleep half a second to avoid spamming
				for(int i = 0; i < 50; ++i) {
					usleep(10 * 1000);
					PyScopedLock l(threadLock);
					if(stopSignal) return;
				}
			}
		}
	}
#endif

	bool open() {
		if(stream) close();
		assert(stream == NULL);

		// For reference:
		// Mixxx code: http://bazaar.launchpad.net/~mixxxdevelopers/mixxx/trunk/view/head:/mixxx/src/sounddeviceportaudio.cpp

		PaStreamParameters outputParameters;

#if defined(__APPLE__)
		PaMacCoreStreamInfo macInfo;
		PaMacCore_SetupStreamInfo( &macInfo,
			paMacCorePlayNice | paMacCorePro );
		outputParameters.hostApiSpecificStreamInfo = &macInfo;
#elif defined(__LINUX__)
		// TODO: if we have PaAlsa_EnableRealtimeScheduling in portaudio,
		// we can call that to enable RT priority with ALSA.
		// We could check dynamically via dsym.
		outputParameters.hostApiSpecificStreamInfo = NULL;
#else
		outputParameters.hostApiSpecificStreamInfo = NULL;
#endif

		outputParameters.device = Pa_GetDefaultOutputDevice();
		if (outputParameters.device == paNoDevice) {
			PyScopedGIL gil;
			PyErr_SetString(PyExc_RuntimeError, "Pa_GetDefaultOutputDevice didn't returned a device");
			return false;
		}
		outputParameters.channelCount = player->outNumChannels;
		outputParameters.sampleFormat = OutPaSampleFormat<OUTSAMPLE_t>::format;

		//unsigned long bufferSize = (player->outSamplerate * player->outNumChannels / 1000) * LATENCY_IN_MS / 4;
		unsigned long bufferSize = paFramesPerBufferUnspecified; // support any buffer size
		if(bufferSize == paFramesPerBufferUnspecified)
			outputParameters.suggestedLatency = Pa_GetDeviceInfo( outputParameters.device )->defaultHighOutputLatency;
		else
			outputParameters.suggestedLatency = LATENCY_IN_MS / 1000.0;

		// Note about framesPerBuffer:
		// Earlier, we used (2048 * 5 * OUTSAMPLEBYTELEN) which caused
		// some random more or less rare cracking noises.
		// See here: https://github.com/albertz/music-player/issues/35
		// This doesn't seem to happen with paFramesPerBufferUnspecified.

		PaError ret = Pa_OpenStream(
			&stream,
			NULL, // no input
			&outputParameters,
			player->outSamplerate, // sampleRate
			bufferSize,
			paClipOff | paDitherOff,
#if USE_PORTAUDIO_CALLBACK
			&paStreamCallback,
#else
			NULL,
#endif
			this //void *userData
			);

		if(ret != paNoError) {
			{
				PyScopedGIL gil;
				PyErr_Format(PyExc_RuntimeError, "Pa_OpenStream failed: (err %i) %s", ret, Pa_GetErrorText(ret));
			}
			if(stream)
				close();
			return false;
		}

		needRealtimeReset = true;
		setThreadName = true;
		Pa_StartStream(stream);

#if !USE_PORTAUDIO_CALLBACK
		audioThread.func = boost::bind(&OutStream::audioThreadProc, this, _1, _2);
		audioThread.start();
#endif
		return true;
	}

	void close() {
		if(this->stream == NULL) return;
		// we expect that we have the player lock here.
		// reset fader.
		player->fader.change(0,0);
		// we must release the player lock so that any thread-join can be done.
		PaStream* stream = NULL;
		std::swap(stream, this->stream);
		PyScopedUnlock unlock(player->lock);
#if !USE_PORTAUDIO_CALLBACK
		audioThread.stop();
#endif
		Pa_CloseStream(stream);
	}

	bool isOpen() const { return stream != NULL; }

};

int PlayerObject::setPlaying(bool playing) {
	PlayerObject* player = this;
	bool oldplayingstate = player->playing;

	if(oldplayingstate != playing)
		outOfSync = true;

	PyScopedGIL gil;
	{
		PyScopedGIUnlock gunlock;

		player->workerThread.start(); // if not running yet, start
		if(!player->outStream.get())
			player->outStream.reset(new OutStream(this));
		assert(player->outStream.get() != NULL);

		if(soundcardOutputEnabled) {
			if(playing && !player->outStream->isOpen()) {
				if(!player->outStream->open())
					playing = false;
			}
		}

		if(soundcardOutputEnabled && player->outStream.get() && player->outStream->isOpen() && oldplayingstate != playing)
			fader.change(playing ? 1 : -1, outSamplerate);
		player->playing = playing;
	}

	if(!PyErr_Occurred() && player->dict) {
		Py_INCREF(player->dict);
		PyObject* onPlayingStateChange = PyDict_GetItemString(player->dict, "onPlayingStateChange");
		if(onPlayingStateChange && onPlayingStateChange != Py_None) {
			Py_INCREF(onPlayingStateChange);

			PyObject* kwargs = PyDict_New();
			assert(kwargs);
			PyDict_SetItemString_retain(kwargs, "oldState", PyBool_FromLong(oldplayingstate));
			PyDict_SetItemString_retain(kwargs, "newState", PyBool_FromLong(playing));

			PyObject* retObj = PyEval_CallObjectWithKeywords(onPlayingStateChange, NULL, kwargs);
			Py_XDECREF(retObj);

			// errors are not fatal from the callback, so handle it now and go on
			if(PyErr_Occurred())
				PyErr_Print();

			Py_DECREF(kwargs);
			Py_DECREF(onPlayingStateChange);
		}
		Py_DECREF(player->dict);
	}

	return PyErr_Occurred() ? -1 : 0;
}

void PlayerObject::resetPlaying() {
	if(this->playing)
		this->setPlaying(false);
	if(this->outStream.get() != NULL)
		this->outStream.reset();
	reinitPlayerOutput();
}

#ifdef __APPLE__
// https://developer.apple.com/library/mac/#documentation/Darwin/Conceptual/KernelProgramming/scheduler/scheduler.html
// Also, from Google Native Client, osx/nacl_thread_nice.c has some related code.
// Or, from Google Chrome, platform_thread_mac.mm. http://src.chromium.org/svn/trunk/src/base/threading/platform_thread_mac.mm
void setRealtime() {
	kern_return_t ret;
	thread_port_t threadport = pthread_mach_thread_np(pthread_self());

	thread_extended_policy_data_t policy;
	policy.timeshare = 0;
	ret = thread_policy_set(threadport,
							THREAD_EXTENDED_POLICY,
							(thread_policy_t)&policy,
							THREAD_EXTENDED_POLICY_COUNT);
	if(ret != KERN_SUCCESS) {
		fprintf(stderr, "setRealtime() THREAD_EXTENDED_POLICY failed: %d, %s\n", ret, mach_error_string(ret));
		return;
	}

	thread_precedence_policy_data_t precedence;
	precedence.importance = 63;
	ret = thread_policy_set(threadport,
							THREAD_PRECEDENCE_POLICY,
							(thread_policy_t)&precedence,
							THREAD_PRECEDENCE_POLICY_COUNT);
	if(ret != KERN_SUCCESS) {
		fprintf(stderr, "setRealtime() THREAD_PRECEDENCE_POLICY failed: %d, %s\n", ret, mach_error_string(ret));
		return;
	}

	// Get the conversion factor from milliseconds to absolute time
	// which is what the time-constraints call needs.
	mach_timebase_info_data_t tb_info;
	mach_timebase_info(&tb_info);
	double timeFact = ((double)tb_info.denom / (double)tb_info.numer) * 1000000;

	thread_time_constraint_policy_data_t ttcpolicy;
	ttcpolicy.period = 2.9 * timeFact; // in Chrome: 2.9ms ~= 128 frames @44.1KHz
	ttcpolicy.computation = 0.75 * ttcpolicy.period;
	ttcpolicy.constraint = 0.85 * ttcpolicy.period;
	ttcpolicy.preemptible = 0;

	ret = thread_policy_set(threadport,
							THREAD_TIME_CONSTRAINT_POLICY,
							(thread_policy_t)&ttcpolicy,
							THREAD_TIME_CONSTRAINT_POLICY_COUNT);
	if(ret != KERN_SUCCESS) {
		fprintf(stderr, "setRealtime() THREAD_TIME_CONSTRAINT_POLICY failed: %d, %s\n", ret, mach_error_string(ret));
		return;
	}
}
#else
void setRealtime() {} // not implemented yet
#endif

<commit_msg>more on realtime param<commit_after>// ffmpeg player output (portaudio)
// part of MusicPlayer, https://github.com/albertz/music-player
// Copyright (c) 2012, Albert Zeyer, www.az2000.de
// All rights reserved.
// This code is under the 2-clause BSD license, see License.txt in the root directory of this project.

#include "ffmpeg.h"
#include "PyUtils.h"

#include <portaudio.h>
#include <boost/bind.hpp>
#include <boost/atomic.hpp>

#ifdef __APPLE__
// PortAudio specific Mac stuff
#include "pa_mac_core.h"
#endif

// For setting the thread priority to realtime on MacOSX.
#ifdef __APPLE__
#include <mach/mach_init.h>
#include <mach/thread_policy.h>
#include <mach/thread_act.h> // the doc says mach/sched.h but that seems outdated...
#include <pthread.h>
#include <mach/mach_error.h>
#include <mach/mach_time.h>
#endif
static void setRealtime(double dutyCicleMs);

/*
The implementation with the PortAudio callback was there first.
For testing, I implemented also the blocking PortAudio interface.
I had some issues which small hiccups in the audio output -
maybe the blocking PortAudio implementation can avoid them better.
For the callback, we need to minimize the locks - or better, avoid
them fully. I'm not sure it is good to depend on thread context
switches in case it is locked. This however needs some heavy redesign.
*/
#define USE_PORTAUDIO_CALLBACK 1

int initPlayerOutput() {
	PaError ret = Pa_Initialize();
	if(ret != paNoError)
		Py_FatalError("PortAudio init failed");
	return 0;
}

void reinitPlayerOutput() {
	if(Pa_Terminate() != paNoError)
		printf("Warning: Pa_Terminate() failed\n");
	initPlayerOutput();
}

template<typename T> struct OutPaSampleFormat{};
template<> struct OutPaSampleFormat<int16_t> {
	static const PaSampleFormat format = paInt16;
};
template<> struct OutPaSampleFormat<float32_t> {
	static const PaSampleFormat format = paFloat32;
};

#define LATENCY_IN_MS 100

struct PlayerObject::OutStream {
	PlayerObject* const player;
	PaStream* stream;
	boost::atomic<bool> needRealtimeReset; // PortAudio callback thread must set itself to realtime
	boost::atomic<bool> setThreadName;

	OutStream(PlayerObject* p) : player(p), stream(NULL), needRealtimeReset(false) {
		mlock(this, sizeof(*this));
	}
	~OutStream() { close(); }

#if USE_PORTAUDIO_CALLBACK
	static int paStreamCallback(
		const void *input, void *output,
		unsigned long frameCount,
		const PaStreamCallbackTimeInfo* timeInfo,
		PaStreamCallbackFlags statusFlags,
		void *userData )
	{
		OutStream* outStream = (OutStream*) userData;
		PlayerObject* player = outStream->player;

		if(outStream->needRealtimeReset.exchange(false))
			setRealtime(1000.0 * frameCount / player->outSamplerate);

		if(outStream->setThreadName.exchange(false))
			setCurThreadName("audio callback");

		if(statusFlags & paOutputUnderflow)
			printf("audio: paOutputUnderflow\n");
		if(statusFlags & paOutputOverflow)
			printf("audio: paOutputOverflow\n");

		// We must not hold the PyGIL here!
		// Also no need to hold the player lock, all is safe!

		player->readOutStream((OUTSAMPLE_t*) output, frameCount * outStream->player->outNumChannels, NULL);
		return paContinue;
	}
#else
	PyThread audioThread;
	void audioThreadProc(PyMutex& threadLock, bool& stopSignal) {
		while(true) {
			{
				PyScopedLock l(threadLock);
				if(stopSignal) return;
			}

			if(needRealtimeReset) {
				needRealtimeReset = false;
				setRealtime();
			}

			OUTSAMPLE_t buffer[48 * 2 * 10]; // 10ms stereo in 48khz
			size_t frameCount = 0;
			{
				PyScopedLock lock(player->lock);
				if(stopSignal) return;
				player->readOutStream(buffer, sizeof(buffer)/sizeof(OUTSAMPLE_t), NULL);
				frameCount = sizeof(buffer)/sizeof(OUTSAMPLE_t) / player->outNumChannels;
			}

			PaError ret = Pa_WriteStream(stream, buffer, frameCount);
			if(ret == paOutputUnderflowed)
				printf("warning: paOutputUnderflowed\n");
			else if(ret != paNoError) {
				printf("Pa_WriteStream error %i (%s)\n", ret, Pa_GetErrorText(ret));
				// sleep half a second to avoid spamming
				for(int i = 0; i < 50; ++i) {
					usleep(10 * 1000);
					PyScopedLock l(threadLock);
					if(stopSignal) return;
				}
			}
		}
	}
#endif

	bool open() {
		if(stream) close();
		assert(stream == NULL);

		// For reference:
		// Mixxx code: http://bazaar.launchpad.net/~mixxxdevelopers/mixxx/trunk/view/head:/mixxx/src/sounddeviceportaudio.cpp

		PaStreamParameters outputParameters;

#if defined(__APPLE__)
		PaMacCoreStreamInfo macInfo;
		PaMacCore_SetupStreamInfo( &macInfo,
			paMacCorePlayNice | paMacCorePro );
		outputParameters.hostApiSpecificStreamInfo = &macInfo;
#elif defined(__LINUX__)
		// TODO: if we have PaAlsa_EnableRealtimeScheduling in portaudio,
		// we can call that to enable RT priority with ALSA.
		// We could check dynamically via dsym.
		outputParameters.hostApiSpecificStreamInfo = NULL;
#else
		outputParameters.hostApiSpecificStreamInfo = NULL;
#endif

		outputParameters.device = Pa_GetDefaultOutputDevice();
		if (outputParameters.device == paNoDevice) {
			PyScopedGIL gil;
			PyErr_SetString(PyExc_RuntimeError, "Pa_GetDefaultOutputDevice didn't returned a device");
			return false;
		}
		outputParameters.channelCount = player->outNumChannels;
		outputParameters.sampleFormat = OutPaSampleFormat<OUTSAMPLE_t>::format;

		//unsigned long bufferSize = (player->outSamplerate * player->outNumChannels / 1000) * LATENCY_IN_MS / 4;
		unsigned long bufferSize = paFramesPerBufferUnspecified; // support any buffer size
		if(bufferSize == paFramesPerBufferUnspecified)
			outputParameters.suggestedLatency = Pa_GetDeviceInfo( outputParameters.device )->defaultHighOutputLatency;
		else
			outputParameters.suggestedLatency = LATENCY_IN_MS / 1000.0;

		// Note about framesPerBuffer:
		// Earlier, we used (2048 * 5 * OUTSAMPLEBYTELEN) which caused
		// some random more or less rare cracking noises.
		// See here: https://github.com/albertz/music-player/issues/35
		// This doesn't seem to happen with paFramesPerBufferUnspecified.

		PaError ret = Pa_OpenStream(
			&stream,
			NULL, // no input
			&outputParameters,
			player->outSamplerate, // sampleRate
			bufferSize,
			paClipOff | paDitherOff,
#if USE_PORTAUDIO_CALLBACK
			&paStreamCallback,
#else
			NULL,
#endif
			this //void *userData
			);

		if(ret != paNoError) {
			{
				PyScopedGIL gil;
				PyErr_Format(PyExc_RuntimeError, "Pa_OpenStream failed: (err %i) %s", ret, Pa_GetErrorText(ret));
			}
			if(stream)
				close();
			return false;
		}

		needRealtimeReset = true;
		setThreadName = true;
		Pa_StartStream(stream);

#if !USE_PORTAUDIO_CALLBACK
		audioThread.func = boost::bind(&OutStream::audioThreadProc, this, _1, _2);
		audioThread.start();
#endif
		return true;
	}

	void close() {
		if(this->stream == NULL) return;
		// we expect that we have the player lock here.
		// reset fader.
		player->fader.change(0,0);
		// we must release the player lock so that any thread-join can be done.
		PaStream* stream = NULL;
		std::swap(stream, this->stream);
		PyScopedUnlock unlock(player->lock);
#if !USE_PORTAUDIO_CALLBACK
		audioThread.stop();
#endif
		Pa_CloseStream(stream);
	}

	bool isOpen() const { return stream != NULL; }

};

int PlayerObject::setPlaying(bool playing) {
	PlayerObject* player = this;
	bool oldplayingstate = player->playing;

	if(oldplayingstate != playing)
		outOfSync = true;

	PyScopedGIL gil;
	{
		PyScopedGIUnlock gunlock;

		player->workerThread.start(); // if not running yet, start
		if(!player->outStream.get())
			player->outStream.reset(new OutStream(this));
		assert(player->outStream.get() != NULL);

		if(soundcardOutputEnabled) {
			if(playing && !player->outStream->isOpen()) {
				if(!player->outStream->open())
					playing = false;
			}
		}

		if(soundcardOutputEnabled && player->outStream.get() && player->outStream->isOpen() && oldplayingstate != playing)
			fader.change(playing ? 1 : -1, outSamplerate);
		player->playing = playing;
	}

	if(!PyErr_Occurred() && player->dict) {
		Py_INCREF(player->dict);
		PyObject* onPlayingStateChange = PyDict_GetItemString(player->dict, "onPlayingStateChange");
		if(onPlayingStateChange && onPlayingStateChange != Py_None) {
			Py_INCREF(onPlayingStateChange);

			PyObject* kwargs = PyDict_New();
			assert(kwargs);
			PyDict_SetItemString_retain(kwargs, "oldState", PyBool_FromLong(oldplayingstate));
			PyDict_SetItemString_retain(kwargs, "newState", PyBool_FromLong(playing));

			PyObject* retObj = PyEval_CallObjectWithKeywords(onPlayingStateChange, NULL, kwargs);
			Py_XDECREF(retObj);

			// errors are not fatal from the callback, so handle it now and go on
			if(PyErr_Occurred())
				PyErr_Print();

			Py_DECREF(kwargs);
			Py_DECREF(onPlayingStateChange);
		}
		Py_DECREF(player->dict);
	}

	return PyErr_Occurred() ? -1 : 0;
}

void PlayerObject::resetPlaying() {
	if(this->playing)
		this->setPlaying(false);
	if(this->outStream.get() != NULL)
		this->outStream.reset();
	reinitPlayerOutput();
}

#ifdef __APPLE__
// https://developer.apple.com/library/mac/#documentation/Darwin/Conceptual/KernelProgramming/scheduler/scheduler.html
// Also, from Google Native Client, osx/nacl_thread_nice.c has some related code.
// Or, from Google Chrome, platform_thread_mac.mm. http://src.chromium.org/svn/trunk/src/base/threading/platform_thread_mac.mm
void setRealtime(double dutyCicleMs) {
	kern_return_t ret;
	thread_port_t threadport = pthread_mach_thread_np(pthread_self());

	thread_extended_policy_data_t policy;
	policy.timeshare = 0;
	ret = thread_policy_set(threadport,
							THREAD_EXTENDED_POLICY,
							(thread_policy_t)&policy,
							THREAD_EXTENDED_POLICY_COUNT);
	if(ret != KERN_SUCCESS) {
		fprintf(stderr, "setRealtime() THREAD_EXTENDED_POLICY failed: %d, %s\n", ret, mach_error_string(ret));
		return;
	}

	thread_precedence_policy_data_t precedence;
	precedence.importance = 63;
	ret = thread_policy_set(threadport,
							THREAD_PRECEDENCE_POLICY,
							(thread_policy_t)&precedence,
							THREAD_PRECEDENCE_POLICY_COUNT);
	if(ret != KERN_SUCCESS) {
		fprintf(stderr, "setRealtime() THREAD_PRECEDENCE_POLICY failed: %d, %s\n", ret, mach_error_string(ret));
		return;
	}

	// Get the conversion factor from milliseconds to absolute time
	// which is what the time-constraints call needs.
	mach_timebase_info_data_t tb_info;
	mach_timebase_info(&tb_info);
	double timeFact = ((double)tb_info.denom / (double)tb_info.numer) * 1000000;

	thread_time_constraint_policy_data_t ttcpolicy;
	// in Chrome: period = 2.9ms ~= 128 frames @44.1KHz, comp = 0.75 * period, constr = 0.85 * period.
	ttcpolicy.period = dutyCicleMs * timeFact;
	ttcpolicy.computation = dutyCicleMs * 0.75 * timeFact;
	ttcpolicy.constraint = dutyCicleMs * 0.85 * timeFact;
	ttcpolicy.preemptible = 0;

	ret = thread_policy_set(threadport,
							THREAD_TIME_CONSTRAINT_POLICY,
							(thread_policy_t)&ttcpolicy,
							THREAD_TIME_CONSTRAINT_POLICY_COUNT);
	if(ret != KERN_SUCCESS) {
		fprintf(stderr, "setRealtime() THREAD_TIME_CONSTRAINT_POLICY failed: %d, %s\n", ret, mach_error_string(ret));
		return;
	}
}
#else
void setRealtime(double dutyCicleMs) {} // not implemented yet
#endif

<|endoftext|>


<commit_before><commit_msg>add application/ogg mime type BUG=18000 TEST=unknown which websites this affects.  ask andrew who reported the bug.<commit_after><|endoftext|>


<commit_before>#include "../columns/Communicator.hpp"

#include <assert.h>
#include <gdal_priv.h>
#include <mpi.h>
#include <ogr_spatialref.h>

#undef DEBUG_OUTPUT

static void copyToLocBuffer(int buf[], LayerLoc * loc)
{
	buf[0] = loc->nx;
	buf[1] = loc->ny;
	buf[2] = loc->nxGlobal;
	buf[3] = loc->nyGlobal;
	buf[4] = loc->kx0;
	buf[5] = loc->ky0;
	buf[6] = loc->nPad;
	buf[7] = loc->nBands;
}

static void copyFromLocBuffer(int buf[], LayerLoc * loc)
{
	loc->nx       = buf[0];
	loc->ny       = buf[1];
	loc->nxGlobal = buf[2];
	loc->nyGlobal = buf[3];
	loc->kx0      = buf[4];
	loc->ky0      = buf[5];
	loc->nPad     = buf[6];
	loc->nBands   = buf[7];
}

/**
 * Calculates location information given processor distribution and the
 * size of the image.
 *
 * @filename the name of the image file (in)
 * @ic the inter-column communicator (in)
 * @loc location information (inout) (loc->nx and loc->ny are out)
 */
int getImageInfo(const char* filename, PV::Communicator * comm, LayerLoc * loc)
{
   const int locSize = sizeof(LayerLoc) / sizeof(int);
   int locBuf[locSize];
   int status = 0;

   // LayerLoc should contain 8 ints
   assert(locSize == 8);

   const int nxProcs = comm->numCommColumns();
   const int nyProcs = comm->numCommRows();

   const int icCol = comm->commColumn();
   const int icRow = comm->commRow();

#ifdef DEBUG_OUTPUT
   fprintf(stderr, "[%2d]: nxProcs==%d nyProcs==%d icRow==%d icCol==%d\n",
           comm->commRank(), nxProcs, nyProcs, icRow, icCol);
#endif

   if (comm->commRank() == 0) {
      GDALAllRegister();

      GDALDataset * dataset = (GDALDataset *) GDALOpen(filename, GA_ReadOnly);
      if (dataset == NULL) return 1;

      int xImageSize = dataset->GetRasterXSize();
      int yImageSize = dataset->GetRasterYSize();

      loc->nBands = dataset->GetRasterCount();

      // calculate local layer size

      int nx = xImageSize / nxProcs;
      int ny = yImageSize / nyProcs;

      loc->nx = nx;
      loc->ny = ny;

      loc->nxGlobal = nxProcs * nx;
      loc->nyGlobal = nyProcs * ny;

      copyToLocBuffer(locBuf, loc);

      GDALClose(dataset);
   }

   // broadcast location information
   MPI_Bcast(locBuf, 1+locSize, MPI_INT, 0, comm->communicator());

   copyFromLocBuffer(locBuf, loc);

   // fix up layer indices
   loc->kx0 = loc->nx * icCol;
   loc->ky0 = loc->ny * icRow;

   return status;
}

/**
 * @filename
 */
int scatterImageFile(const char * filename,
                     PV::Communicator * comm, LayerLoc * loc, float * buf)
{
   int status = 0;
   const int tag = 13;
   const int maxBands = 3;

   const MPI_Comm mpi_comm = comm->communicator();

   const int nxProcs = comm->numCommColumns();
   const int nyProcs = comm->numCommRows();

   const int icRank = comm->commRank();

   const int nx = loc->nx;
   const int ny = loc->ny;

   const int numBands = loc->nBands;
   assert(numBands <= maxBands);

   const int nxny     = nx * ny;
   const int numTotal = nxny * numBands;

   if (icRank > 0) {
      const int src = 0;
      for (int b = 0; b < numBands; b++) {
         MPI_Recv(&buf[b*nxny], numTotal, MPI_FLOAT, src, tag, mpi_comm, MPI_STATUS_IGNORE);
      }
#ifdef DEBUG_OUTPUT
      fprintf(stderr, "[%2d]: scatter: received from 0, nx==%d ny==%d size==%d\n",
              comm->commRank(), nx, ny, numTotal);
#endif
   }
   else {
      GDALAllRegister();

      GDALDataset * dataset = (GDALDataset *) GDALOpen(filename, GA_ReadOnly);

      int xImageSize = dataset->GetRasterXSize();
      int yImageSize = dataset->GetRasterYSize();

      int xTotalSize = nx * nxProcs;
      int yTotalSize = ny * nyProcs;

      if (xTotalSize > xImageSize || yTotalSize > yImageSize) {
         fprintf(stderr, "[ 0]: scatterImageFile: image size too small, "
                 "xTotalSize==%d xImageSize==%d yTotalSize==%d yImageSize==%d\n",
                 xTotalSize, xImageSize, yTotalSize, yImageSize);
         fprintf(stderr, "[ 0]: xSize==%d ySize==%d nxProcs==%d nyProcs==%d\n",
                 nx, ny, nxProcs, nyProcs);
         GDALClose(dataset);
         return -1;
      }

      GDALRasterBand * band[maxBands];

      assert(numBands <= dataset->GetRasterCount());

      for (int b = 0; b < numBands; b++) {
         band[b] = dataset->GetRasterBand(b+1);
      }

      int dest = -1;
      for (int py = 0; py < nyProcs; py++) {
         for (int px = 0; px < nxProcs; px++) {
            if (++dest == 0) continue;
            int kx = nx * px;
            int ky = ny * py;
#ifdef DEBUG_OUTPUT
            fprintf(stderr, "[%2d]: scatter: sending to %d xSize==%d"
                    " ySize==%d size==%d total==%d\n",
                    comm->commRank(), dest, nx, ny, nx*ny,
                    nx*ny*comm->commSize());
#endif
            for (int b = 0; b < numBands; b++) {
               band[b]->RasterIO(GF_Read, kx, ky, nx, ny,
                                 &buf[b*nxny], nx, ny, GDT_Float32, 0, 0);
               MPI_Send(&buf[b*nxny], nx*ny, MPI_FLOAT, dest, tag, mpi_comm);
            }
         }
      }

      // get local image portion
      for (int b = 0; b < numBands; b++) {
         band[b]->RasterIO(GF_Read, 0, 0, nx, ny,
                           &buf[b*nxny], nx, ny, GDT_Float32, 0, 0);
      }
      GDALClose(dataset);
   }

   return status;
}

/**
 * @filename
 */
int gatherImageFile(const char * filename,
                    PV::Communicator * comm, LayerLoc * loc, float * buf)
{
   int status = 0;
   const int tag = 14;
   const int maxBands = 3;

   const MPI_Comm mpi_comm = comm->communicator();

   const int nxProcs = comm->numCommColumns();
   const int nyProcs = comm->numCommRows();

   const int icRank = comm->commRank();

   const int nx = loc->nx;
   const int ny = loc->ny;

   const int numBands = loc->nBands;
   assert(numBands <= maxBands);

   const int nxny     = nx * ny;
   const int numTotal = nxny * numBands;

   if (icRank > 0) {
      const int dest = 0;
      for (int b = 0; b < numBands; b++) {
         MPI_Send(&buf[b*nxny], nx*ny, MPI_FLOAT, dest, tag, mpi_comm);
      }
#ifdef DEBUG_OUTPUT
      fprintf(stderr, "[%2d]: gather: sent to 0, nx==%d ny==%d size==%d\n",
              comm->commRank(), nx, ny, nx*ny);
#endif
   }
   else {
//#include "cpl_string.h"

      GDALAllRegister();

      char ** metadata;

      GDALDriver * driver = GetGDALDriverManager()->GetDriverByName("GTiff");

      if( driver == NULL )
          exit( 1 );

      metadata = driver->GetMetadata();
      if( CSLFetchBoolean( metadata, GDAL_DCAP_CREATE, FALSE ) ) {
          // printf("Driver %s supports Create() method.\n", "GTiff");
      }

      GDALDataset * dataset;
      char ** options = NULL;

      int xImageSize = nx * nxProcs;
      int yImageSize = ny * nyProcs;

      dataset = driver->Create(filename, xImageSize, yImageSize, numBands,
                               GDT_Byte, options);

      if (dataset == NULL) {
          fprintf(stderr, "[%2d]: gather: failed to open file %s\n", comm->commRank(), filename);
      }
      else {
#ifdef DEBUG_OUTPUT
          fprintf(stderr, "[%2d]: gather: opened file %s\n", comm->commRank(), filename);
#endif
      }

//      double adfGeoTransform[6] = { 444720, 30, 0, 3751320, 0, -30 };
//      OGRSpatialReference oSRS;
//      char *pszSRS_WKT = NULL;

//      dataset->SetGeoTransform( adfGeoTransform );

//      oSRS.SetUTM( 11, TRUE );
//      oSRS.SetWellKnownGeogCS( "NAD27" );
//      oSRS.exportToWkt( &pszSRS_WKT );
//      dataset->SetProjection( pszSRS_WKT );
//      CPLFree( pszSRS_WKT );

      GDALRasterBand * band[maxBands];

      assert(numBands <= dataset->GetRasterCount());

      for (int b = 0; b < numBands; b++) {
         band[b] = dataset->GetRasterBand(b+1);
      }

      // write local image portion
      for (int b = 0; b < numBands; b++) {
         band[b]->RasterIO(GF_Write, 0, 0, nx, ny,
                           &buf[b*nxny], nx, ny, GDT_Float32, 0, 0);
      }

      int src = -1;
      for (int py = 0; py < nyProcs; py++) {
         for (int px = 0; px < nxProcs; px++) {
            if (++src == 0) continue;
            int kx = nx * px;
            int ky = ny * py;
#ifdef DEBUG_OUTPUT
            fprintf(stderr, "[%2d]: gather: receiving from %d xSize==%d"
                    " ySize==%d size==%d total==%d\n",
                    comm->commRank(), src, nx, ny, numTotal,
                    numTotal*comm->commSize());
#endif
            for (int b = 0; b < numBands; b++) {
               MPI_Recv(&buf[b*nxny], numTotal, MPI_FLOAT, src, tag, mpi_comm, MPI_STATUS_IGNORE);
               band[b]->RasterIO(GF_Write, kx, ky, nx, ny,
                                 &buf[b*nxny], nx, ny, GDT_Float32, 0, 0);
            }
         }
      }
      GDALClose(dataset);
   }

   return status;
}

int scatterImageBlocks(const char* filename,
                       PV::Communicator * comm, LayerLoc * loc, float * buf)
{
   int status = 0;
#ifdef UNIMPLEMENTED

   const MPI_Comm icComm = comm->communicator();

   const int nxProcs = comm->numCommColumns();
   const int nyProcs = comm->numCommRows();

   const int icRank = comm->commRank();
   const int icCol  = comm->commColumn();
   const int icRow  = comm->commRow();

   const int nx = loc->nx;
   const int ny = loc->ny;

   const int nxGlobal = loc->nxGlobal;
   const int nyBorder = loc->nyBorder;

   const int xSize = nx + 2 * nxGlobal;
   const int ySize = ny + 2 * nyBorder;

   if (icRank > 0) {
   }
   else {
      int nxBlocks, nyBlocks, nxBlockSize, nyBlockSize;
      int ixBlock, iyBlock;

      GDALAllRegister();

      GDALDataset * dataset = (GDALDataset *) GDALOpen(filename, GA_ReadOnly);
      GDALRasterBand * band = dataset->GetRasterBand(1);

      CPLAssert(band->GetRasterDataType() == GDT_Byte);

      band->GetBlockSize(&nxBlockSize, &nyBlockSize);
      nxBlocks = (band->GetXSize() + nxBlockSize - 1) / nxBlockSize;
      nyBlocks = (band->GetYSize() + nyBlockSize - 1) / nyBlockSize;

      GByte * data = (GByte *) CPLMalloc(nxBlockSize * nyBlockSize);

      fprintf(stderr, "[ 0]: nxBlockSize==%d nyBlockSize==%d"
              " nxBlocks==%d nyBlocks==%d\n",
              nxBlockSize, nyBlockSize, nxBlocks, nyBlocks);

      for (iyBlock = 0; iyBlock < nyBlocks; iyBlock++) {
         for (ixBlock = 0; ixBlock < nxBlocks; ixBlock++) {
            int nxValid, nyValid;
            band->ReadBlock(ixBlock, ixBlock, data);
         }
      }
   }
#endif

   return status;
}

/**
 * gather relevant portions of buf on root process from all others
 *    NOTE: buf is np times larger on root process
 */
int gather (PV::Communicator * comm, LayerLoc * loc, float * buf)
{
   return -1;
}

/**
 * scatter relevant portions of buf from root process to all others
 *    NOTE: buf is np times larger on root process
 */
int scatter(PV::Communicator * comm, LayerLoc * loc, float * buf)
{
   return -1;
}

int writeWithBorders(const char * filename, LayerLoc * loc, float * buf)
{
   int X = loc->nx + 2 * loc->nPad;
   int Y = loc->ny + 2 * loc->nPad;
   int B = loc->nBands;

   GDALDriver * driver = GetGDALDriverManager()->GetDriverByName("GTiff");
   GDALDataset* layer_file = driver->Create(filename, X, Y, B, GDT_Byte, NULL);

   // TODO - add multiple raster bands
   GDALRasterBand * band = layer_file->GetRasterBand(1);

   band->RasterIO(GF_Write, 0, 0, X, Y, buf, X, Y, GDT_Float32, 0, 0);

   GDALClose(layer_file);

   return 0;
}
<commit_msg>Added optional use of MPI (for PANN).<commit_after>#include "imageio.hpp"

#include <assert.h>
#include <gdal_priv.h>
#include <ogr_spatialref.h>

#undef DEBUG_OUTPUT

static void copyToLocBuffer(int buf[], LayerLoc * loc)
{
	buf[0] = loc->nx;
	buf[1] = loc->ny;
	buf[2] = loc->nxGlobal;
	buf[3] = loc->nyGlobal;
	buf[4] = loc->kx0;
	buf[5] = loc->ky0;
	buf[6] = loc->nPad;
	buf[7] = loc->nBands;
}

static void copyFromLocBuffer(int buf[], LayerLoc * loc)
{
	loc->nx       = buf[0];
	loc->ny       = buf[1];
	loc->nxGlobal = buf[2];
	loc->nyGlobal = buf[3];
	loc->kx0      = buf[4];
	loc->ky0      = buf[5];
	loc->nPad     = buf[6];
	loc->nBands   = buf[7];
}

/**
 * Calculates location information given processor distribution and the
 * size of the image.
 *
 * @filename the name of the image file (in)
 * @ic the inter-column communicator (in)
 * @loc location information (inout) (loc->nx and loc->ny are out)
 */
int getImageInfo(const char* filename, PV::Communicator * comm, LayerLoc * loc)
{
   const int locSize = sizeof(LayerLoc) / sizeof(int);
   int locBuf[locSize];
   int status = 0;

   // LayerLoc should contain 8 ints
   assert(locSize == 8);

   const int nxProcs = comm->numCommColumns();
   const int nyProcs = comm->numCommRows();

   const int icCol = comm->commColumn();
   const int icRow = comm->commRow();

#ifdef DEBUG_OUTPUT
   fprintf(stderr, "[%2d]: nxProcs==%d nyProcs==%d icRow==%d icCol==%d\n",
           comm->commRank(), nxProcs, nyProcs, icRow, icCol);
#endif

   if (comm->commRank() == 0) {
      GDALAllRegister();

      GDALDataset * dataset = (GDALDataset *) GDALOpen(filename, GA_ReadOnly);
      if (dataset == NULL) return 1;

      int xImageSize = dataset->GetRasterXSize();
      int yImageSize = dataset->GetRasterYSize();

      loc->nBands = dataset->GetRasterCount();

      // calculate local layer size

      int nx = xImageSize / nxProcs;
      int ny = yImageSize / nyProcs;

      loc->nx = nx;
      loc->ny = ny;

      loc->nxGlobal = nxProcs * nx;
      loc->nyGlobal = nyProcs * ny;

      copyToLocBuffer(locBuf, loc);

      GDALClose(dataset);
   }

#ifdef PV_USE_MPI
   // broadcast location information
   MPI_Bcast(locBuf, 1+locSize, MPI_INT, 0, comm->communicator());
#endif

   copyFromLocBuffer(locBuf, loc);

   // fix up layer indices
   loc->kx0 = loc->nx * icCol;
   loc->ky0 = loc->ny * icRow;

   return status;
}

/**
 * @filename
 */
int scatterImageFile(const char * filename,
                     PV::Communicator * comm, LayerLoc * loc, float * buf)
{
   int status = 0;
   const int maxBands = 3;

   const int nxProcs = comm->numCommColumns();
   const int nyProcs = comm->numCommRows();

   const int icRank = comm->commRank();

   const int nx = loc->nx;
   const int ny = loc->ny;

   const int numBands = loc->nBands;
   assert(numBands <= maxBands);

   const int nxny     = nx * ny;

   if (icRank > 0) {
#ifdef PV_USE_MPI
      const int numTotal = nxny * numBands;

      const int src = 0;
      const int tag = 13;
      const MPI_Comm mpi_comm = comm->communicator();

      for (int b = 0; b < numBands; b++) {
         MPI_Recv(&buf[b*nxny], numTotal, MPI_FLOAT, src, tag, mpi_comm, MPI_STATUS_IGNORE);
      }
#ifdef DEBUG_OUTPUT
      fprintf(stderr, "[%2d]: scatter: received from 0, nx==%d ny==%d size==%d\n",
              comm->commRank(), nx, ny, numTotal);
#endif
#endif // PV_USE_MPI
   }
   else {
      GDALAllRegister();

      GDALDataset * dataset = (GDALDataset *) GDALOpen(filename, GA_ReadOnly);

      int xImageSize = dataset->GetRasterXSize();
      int yImageSize = dataset->GetRasterYSize();

      int xTotalSize = nx * nxProcs;
      int yTotalSize = ny * nyProcs;

      if (xTotalSize > xImageSize || yTotalSize > yImageSize) {
         fprintf(stderr, "[ 0]: scatterImageFile: image size too small, "
                 "xTotalSize==%d xImageSize==%d yTotalSize==%d yImageSize==%d\n",
                 xTotalSize, xImageSize, yTotalSize, yImageSize);
         fprintf(stderr, "[ 0]: xSize==%d ySize==%d nxProcs==%d nyProcs==%d\n",
                 nx, ny, nxProcs, nyProcs);
         GDALClose(dataset);
         return -1;
      }

      GDALRasterBand * band[maxBands];

      assert(numBands <= dataset->GetRasterCount());

      for (int b = 0; b < numBands; b++) {
         band[b] = dataset->GetRasterBand(b+1);
      }

#ifdef PV_USE_MPI
      int dest = -1;
      const int tag = 13;
      const MPI_Comm mpi_comm = comm->communicator();

      for (int py = 0; py < nyProcs; py++) {
         for (int px = 0; px < nxProcs; px++) {
            if (++dest == 0) continue;
            int kx = nx * px;
            int ky = ny * py;
#ifdef DEBUG_OUTPUT
            fprintf(stderr, "[%2d]: scatter: sending to %d xSize==%d"
                    " ySize==%d size==%d total==%d\n",
                    comm->commRank(), dest, nx, ny, nx*ny,
                    nx*ny*comm->commSize());
#endif
            for (int b = 0; b < numBands; b++) {
               band[b]->RasterIO(GF_Read, kx, ky, nx, ny,
                                 &buf[b*nxny], nx, ny, GDT_Float32, 0, 0);
               MPI_Send(&buf[b*nxny], nx*ny, MPI_FLOAT, dest, tag, mpi_comm);
            }
         }
      }
#endif // PV_USE_MPI

      // get local image portion
      for (int b = 0; b < numBands; b++) {
         band[b]->RasterIO(GF_Read, 0, 0, nx, ny,
                           &buf[b*nxny], nx, ny, GDT_Float32, 0, 0);
      }
      GDALClose(dataset);
   }

   return status;
}

/**
 * @filename
 */
int gatherImageFile(const char * filename,
                    PV::Communicator * comm, LayerLoc * loc, float * buf)
{
   int status = 0;
   const int maxBands = 3;

   const int nxProcs = comm->numCommColumns();
   const int nyProcs = comm->numCommRows();

   const int icRank = comm->commRank();

   const int nx = loc->nx;
   const int ny = loc->ny;

   const int numBands = loc->nBands;
   assert(numBands <= maxBands);

   const int nxny = nx * ny;

#ifdef PV_USE_MPI
   const int tag = 14;
   const int numTotal = nxny * numBands;
   const MPI_Comm mpi_comm = comm->communicator();
#endif // PV_USE_MPI

   if (icRank > 0) {
#ifdef PV_USE_MPI
      const int dest = 0;

      for (int b = 0; b < numBands; b++) {
         MPI_Send(&buf[b*nxny], nx*ny, MPI_FLOAT, dest, tag, mpi_comm);
      }
#ifdef DEBUG_OUTPUT
      fprintf(stderr, "[%2d]: gather: sent to 0, nx==%d ny==%d size==%d\n",
              comm->commRank(), nx, ny, nx*ny);
#endif
#endif // PV_USE_MPI
   }
   else {
      GDALAllRegister();

      char ** metadata;

      GDALDriver * driver = GetGDALDriverManager()->GetDriverByName("GTiff");

      if( driver == NULL )
          exit( 1 );

      metadata = driver->GetMetadata();
      if( CSLFetchBoolean( metadata, GDAL_DCAP_CREATE, FALSE ) ) {
          // printf("Driver %s supports Create() method.\n", "GTiff");
      }

      GDALDataset * dataset;
      char ** options = NULL;

      int xImageSize = nx * nxProcs;
      int yImageSize = ny * nyProcs;

      dataset = driver->Create(filename, xImageSize, yImageSize, numBands,
                               GDT_Byte, options);

      if (dataset == NULL) {
          fprintf(stderr, "[%2d]: gather: failed to open file %s\n", comm->commRank(), filename);
      }
      else {
#ifdef DEBUG_OUTPUT
          fprintf(stderr, "[%2d]: gather: opened file %s\n", comm->commRank(), filename);
#endif
      }

//      double adfGeoTransform[6] = { 444720, 30, 0, 3751320, 0, -30 };
//      OGRSpatialReference oSRS;
//      char *pszSRS_WKT = NULL;

//      dataset->SetGeoTransform( adfGeoTransform );

//      oSRS.SetUTM( 11, TRUE );
//      oSRS.SetWellKnownGeogCS( "NAD27" );
//      oSRS.exportToWkt( &pszSRS_WKT );
//      dataset->SetProjection( pszSRS_WKT );
//      CPLFree( pszSRS_WKT );

      GDALRasterBand * band[maxBands];

      assert(numBands <= dataset->GetRasterCount());

      for (int b = 0; b < numBands; b++) {
         band[b] = dataset->GetRasterBand(b+1);
      }

      // write local image portion
      for (int b = 0; b < numBands; b++) {
         band[b]->RasterIO(GF_Write, 0, 0, nx, ny,
                           &buf[b*nxny], nx, ny, GDT_Float32, 0, 0);
      }

#ifdef PV_USE_MPI
      int src = -1;
      for (int py = 0; py < nyProcs; py++) {
         for (int px = 0; px < nxProcs; px++) {
            if (++src == 0) continue;
            int kx = nx * px;
            int ky = ny * py;
#ifdef DEBUG_OUTPUT
            fprintf(stderr, "[%2d]: gather: receiving from %d xSize==%d"
                    " ySize==%d size==%d total==%d\n",
                    comm->commRank(), src, nx, ny, numTotal,
                    numTotal*comm->commSize());
#endif
            for (int b = 0; b < numBands; b++) {
               MPI_Recv(&buf[b*nxny], numTotal, MPI_FLOAT, src, tag, mpi_comm, MPI_STATUS_IGNORE);
               band[b]->RasterIO(GF_Write, kx, ky, nx, ny,
                                 &buf[b*nxny], nx, ny, GDT_Float32, 0, 0);
            }
         }
      }
#endif // PV_USE_MPI
      GDALClose(dataset);
   }

   return status;
}

int scatterImageBlocks(const char* filename,
                       PV::Communicator * comm, LayerLoc * loc, float * buf)
{
   int status = 0;
#ifdef UNIMPLEMENTED

   const MPI_Comm icComm = comm->communicator();

   const int nxProcs = comm->numCommColumns();
   const int nyProcs = comm->numCommRows();

   const int icRank = comm->commRank();
   const int icCol  = comm->commColumn();
   const int icRow  = comm->commRow();

   const int nx = loc->nx;
   const int ny = loc->ny;

   const int nxGlobal = loc->nxGlobal;
   const int nyBorder = loc->nyBorder;

   const int xSize = nx + 2 * nxGlobal;
   const int ySize = ny + 2 * nyBorder;

   if (icRank > 0) {
   }
   else {
      int nxBlocks, nyBlocks, nxBlockSize, nyBlockSize;
      int ixBlock, iyBlock;

      GDALAllRegister();

      GDALDataset * dataset = (GDALDataset *) GDALOpen(filename, GA_ReadOnly);
      GDALRasterBand * band = dataset->GetRasterBand(1);

      CPLAssert(band->GetRasterDataType() == GDT_Byte);

      band->GetBlockSize(&nxBlockSize, &nyBlockSize);
      nxBlocks = (band->GetXSize() + nxBlockSize - 1) / nxBlockSize;
      nyBlocks = (band->GetYSize() + nyBlockSize - 1) / nyBlockSize;

      GByte * data = (GByte *) CPLMalloc(nxBlockSize * nyBlockSize);

      fprintf(stderr, "[ 0]: nxBlockSize==%d nyBlockSize==%d"
              " nxBlocks==%d nyBlocks==%d\n",
              nxBlockSize, nyBlockSize, nxBlocks, nyBlocks);

      for (iyBlock = 0; iyBlock < nyBlocks; iyBlock++) {
         for (ixBlock = 0; ixBlock < nxBlocks; ixBlock++) {
            int nxValid, nyValid;
            band->ReadBlock(ixBlock, ixBlock, data);
         }
      }
   }
#endif

   return status;
}

/**
 * gather relevant portions of buf on root process from all others
 *    NOTE: buf is np times larger on root process
 */
int gather (PV::Communicator * comm, LayerLoc * loc, float * buf)
{
   return -1;
}

/**
 * scatter relevant portions of buf from root process to all others
 *    NOTE: buf is np times larger on root process
 */
int scatter(PV::Communicator * comm, LayerLoc * loc, float * buf)
{
   return -1;
}

int writeWithBorders(const char * filename, LayerLoc * loc, float * buf)
{
   int X = loc->nx + 2 * loc->nPad;
   int Y = loc->ny + 2 * loc->nPad;
   int B = loc->nBands;

   GDALDriver * driver = GetGDALDriverManager()->GetDriverByName("GTiff");
   GDALDataset* layer_file = driver->Create(filename, X, Y, B, GDT_Byte, NULL);

   // TODO - add multiple raster bands
   GDALRasterBand * band = layer_file->GetRasterBand(1);

   band->RasterIO(GF_Write, 0, 0, X, Y, buf, X, Y, GDT_Float32, 0, 0);

   GDALClose(layer_file);

   return 0;
}
<|endoftext|>