<commit_before>/*
 Copyright 2015 Nervana Systems Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

#include <assert.h>

#include <vector>
#include <cstdio>
#include <iostream>
#include <chrono>
#include <utility>
#include <algorithm>

#include "media.hpp"
#include "matrix.hpp"
#include "device.hpp"
#include "loader.hpp"
#include "batch_loader_cpio_cache.hpp"
#include "sequential_batch_iterator.hpp"
#include "shuffled_batch_iterator.hpp"

using namespace std;

DecodeThreadPool::DecodeThreadPool(int count, int batchSize,
                 int datumSize, int datumTypeSize,
                 int targetSize, int targetTypeSize,
                 const std::shared_ptr<BufferPool>& in, const std::shared_ptr<BufferPool>& out,
                 const std::shared_ptr<Device>& device,
                 MediaParams* mediaParams)
: ThreadPool(count),
  _itemsPerThread((batchSize - 1) / count + 1),
  _in(in), _out(out), _endSignaled(0),
  _manager(0), _stopManager(false), _managerStopped(false), _inputBuf(0),
  _bufferIndex(0), _batchSize(batchSize),
  _datumSize(datumSize), _datumTypeSize(datumTypeSize),
  _targetSize(targetSize), _targetTypeSize(targetTypeSize),
  _datumLen(datumSize * datumTypeSize),
  _targetLen(targetSize * targetTypeSize),
  _device(device) {
    assert(_itemsPerThread * count >= _batchSize);
    assert(_itemsPerThread * (count - 1) < _batchSize);
    for (int i = 0; i < count; i++) {
        _media.push_back(Media::create(mediaParams, 0, i));
        _startSignaled.push_back(0);
        _startInds.push_back(0);
        _endInds.push_back(0);
        _dataOffsets.push_back(0);
        _targetOffsets.push_back(0);
    }
}

DecodeThreadPool::~DecodeThreadPool() {
    if (_manager != 0) {
        _manager->join();
        delete _manager;
    }
    // The other thread objects are freed in the destructor
    // of the parent class.
}

void DecodeThreadPool::start() {
    for (int i = 0; i < _count; i++) {
        _threads.push_back(new thread(&DecodeThreadPool::run, this, i));
    }
    _manager = new thread(&DecodeThreadPool::manage, this);
}

void DecodeThreadPool::stop() {
    ThreadPool::stop();
    while (stopped() == false) {
        std::this_thread::yield();
        _in->advanceWritePos();
        _in->signalNonEmpty();
    }

    _stopManager = true;
    while (_managerStopped == false) {
        std::this_thread::yield();
        _in->advanceWritePos();
        _in->signalNonEmpty();
        _endSignaled++;
        _ended.notify_one();
    }
}

void DecodeThreadPool::run(int id) {
    // Initialize worker threads by computing memory offsets for the
    // data this thread should work on
    assert(id < _count);
    _startInds[id] = id * _itemsPerThread;
    int itemCount = _itemsPerThread;
    if (id == _count - 1) {
        itemCount = _batchSize - id * _itemsPerThread;
    }

    _endInds[id] = _startInds[id] + itemCount;
    _dataOffsets[id] = _startInds[id] * _datumLen;
    _targetOffsets[id] = _startInds[id] * _targetLen;
    while (_done == false) {
        work(id);
    }

    _stopped[id] = true;
}

void DecodeThreadPool::work(int id) {
    // Thread function.
    {
        unique_lock<mutex> lock(_mutex);
        while (_startSignaled[id] == 0) {
            _started.wait(lock);
            if (_done == true) {
                return;
            }
        }
        _startSignaled[id]--;
        assert(_startSignaled[id] == 0);
    }

    int start = _startInds[id];
    int end = _endInds[id];
    // No locking required because threads
    // write into non-overlapping regions.
    BufferPair& outBuf = _out->getForWrite();
    char* dataBuf = outBuf.first->_data + _dataOffsets[id];
    char* targetBuf = outBuf.second->_data + _targetOffsets[id];
    for (int i = start; i < end; i++) {
        // Handle the data.
        int itemSize = 0;
        char* item = _inputBuf->first->getItem(i, itemSize);
        if (item == 0) {
            return;
        }
        _media[id]->transform(item, itemSize, dataBuf, _datumLen);
        dataBuf += _datumLen;

        // Handle the targets.
        int targetLen = 0;
        char* target = _inputBuf->second->getItem(i, targetLen);
        memcpy(targetBuf, target, targetLen);
        if (_targetLen > targetLen) {
            // Pad the rest of the buffer with zeros.
            memset(targetBuf + targetLen, 0, _targetLen - targetLen);
        }
        targetBuf += _targetLen;
    }

    {
        lock_guard<mutex> lock(_mutex);
        _endSignaled++;
        assert(_endSignaled <= _count);
    }
    _ended.notify_one();
}

void DecodeThreadPool::produce() {
    // Produce a minibatch.
    {
        unique_lock<mutex> lock(_out->getMutex());
        while (_out->full() == true) {
            _out->waitForNonFull(lock);
        }
        {
            lock_guard<mutex> lock(_mutex);
            for (unsigned int i = 0; i < _startSignaled.size(); i++) {
                _startSignaled[i] = 1;
            }
        }
        _started.notify_all();
        {
            unique_lock<mutex> lock(_mutex);
            while (_endSignaled < _count) {
                _ended.wait(lock);
            }
            _endSignaled = 0;
        }
        // At this point, we have decoded data for the whole minibatch.
        BufferPair& outBuf = _out->getForWrite();
        Matrix::transpose(outBuf.first->_data, _batchSize,
                          _datumSize, _datumTypeSize);
        Matrix::transpose(outBuf.second->_data, _batchSize,
                          _targetSize, _targetTypeSize);
        // Copy to device.
        _device->copyData(_bufferIndex, outBuf.first->_data,
                          outBuf.first->_size);
        _device->copyLabels(_bufferIndex, outBuf.second->_data,
                            outBuf.second->_size);
        _bufferIndex = (_bufferIndex == 0) ? 1 : 0;
        _out->advanceWritePos();
    }
    _out->signalNonEmpty();
}

void DecodeThreadPool::consume() {
    // Consume an input buffer.
    {
        unique_lock<mutex> lock(_in->getMutex());
        while (_in->empty() == true) {
            _in->waitForNonEmpty(lock);
            if (_stopManager == true) {
                return;
            }
        }
        _inputBuf = &_in->getForRead();
        produce();
        _in->advanceReadPos();
    }
    _in->signalNonFull();
}

void DecodeThreadPool::manage() {
    // Thread function.
    int result = _device->init();
    if (result != 0) {
        _stopManager = true;
    }
    while (_stopManager == false) {
        consume();
    }
    _managerStopped = true;
}

ReadThread::ReadThread(const shared_ptr<BufferPool>& out, const shared_ptr<BatchIterator>& batch_iterator)
: ThreadPool(1), _out(out), _batch_iterator(batch_iterator) {
    assert(_count == 1);
}

void ReadThread::work(int id) {
    // Fill input buffers.
    // TODO: make sure this locking still makes sense with new
    // BatchIterator
    {
        unique_lock<mutex> lock(_out->getMutex());
        while (_out->full() == true) {
            _out->waitForNonFull(lock);
        }
        _batch_iterator->read(_out->getForWrite());
        _out->advanceWritePos();
    }
    _out->signalNonEmpty();
}

Loader::Loader(int* itemCount, int batchSize,
       const char* repoDir, const char* archiveDir,
       const char* indexFile, const char* archivePrefix,
       bool shuffle, bool reshuffle,
       int datumSize, int datumTypeSize,
       int targetSize, int targetTypeSize,
       int subsetPercent,
       MediaParams* mediaParams,
       DeviceParams* deviceParams,
       const char* manifestFilename,
       const char* rootCacheDir)
: _first(true),
  _batchSize(batchSize),
  _datumSize(datumSize), _datumTypeSize(datumTypeSize),
  _targetSize(targetSize), _targetTypeSize(targetTypeSize),
  _readBufs(nullptr), _decodeBufs(nullptr), _readThread(nullptr), _decodeThreads(nullptr),
  _device(nullptr), _batch_iterator(nullptr), _mediaParams(mediaParams)
  {
    // TODO: rename to shuffleManifest and shuffleEveryEpoch
    // TODO: not a constant
    uint _macroBatchSize = 1024;
    // TODO: not a constant
    uint _seed = 0;

    _device = Device::create(deviceParams);

    // the manifest defines which data should be included in the dataset
    auto manifest = shared_ptr<Manifest>(new Manifest(manifestFilename, shuffle));
    *itemCount = manifest->getSize();

    // build cacheDir from rootCacheDir and a hash of the manifest
    stringstream cacheDirStream;
    cacheDirStream << rootCacheDir << '/' << manifest->hash();
    string cacheDir = cacheDirStream.str();

    // batch loader provdes random access to blocks of data in the manifest
    auto batchLoader = shared_ptr<BatchLoaderCPIOCache>(new BatchLoaderCPIOCache(
        cacheDir.c_str(),
        shared_ptr<BatchFileLoader>(new BatchFileLoader(
            manifest, subsetPercent
        ))
    ));

    // _batch_iterator provides an unending iterator (shuffled or not) over
    // the batchLoader
    if(reshuffle) {
        _batch_iterator = shared_ptr<ShuffledBatchIterator>(new ShuffledBatchIterator(
             batchLoader, _macroBatchSize, _seed
        ));
    } else {
        _batch_iterator = shared_ptr<SequentialBatchIterator>(new SequentialBatchIterator(
             batchLoader, _macroBatchSize
        ));
    }
}

Loader::~Loader() {
}

int Loader::start() {
    _first = true;
    try {
        int dataLen = _batchSize * _datumSize * _datumTypeSize;
        int targetLen = _batchSize * _targetSize * _targetTypeSize;
        // Start the read buffers off with a reasonable size. They will
        // get resized as needed.
        _readBufs = shared_ptr<BufferPool>(new BufferPool(dataLen / 8, targetLen));
        _readThread = unique_ptr<ReadThread>(new ReadThread(_readBufs, _batch_iterator));
        bool pinned = (_device->_type != CPU);
        _decodeBufs = shared_ptr<BufferPool>(new BufferPool(dataLen, targetLen, pinned));
        int numCores = thread::hardware_concurrency();
        int itemsPerThread = (_batchSize - 1) /  numCores + 1;
        int threadCount =  (_batchSize - 1) / itemsPerThread + 1;
        threadCount = std::min(threadCount, _batchSize);
        _decodeThreads = unique_ptr<DecodeThreadPool>(new DecodeThreadPool(threadCount, _batchSize,
                _datumSize, _datumTypeSize,
                _targetSize, _targetTypeSize,
                _readBufs, _decodeBufs, _device, _mediaParams));
    } catch(std::bad_alloc&) {
        return -1;
    }
    _decodeThreads->start();
    _readThread->start();
    return 0;
}

void Loader::stop() {
    _readThread->stop();
    while (_readThread->stopped() == false) {
        std::this_thread::yield();
        drain();
    }
    while ((_decodeBufs->empty() == false) ||
           (_readBufs->empty() == false)) {
        drain();
    }
    _decodeThreads->stop();

    _readBufs = nullptr;
    _readThread = nullptr;
    _decodeBufs = nullptr;
    _decodeThreads = nullptr;
}

int Loader::reset() {
    stop();
    _batch_iterator->reset();
    start();
    return 0;
}

void Loader::next(Buffer* dataBuf, Buffer* targetsBuf) {
    // Copy minibatch data into the buffers passed in.
    // Only used for testing purposes.
    {
        unique_lock<mutex> lock(_decodeBufs->getMutex());
        while (_decodeBufs->empty()) {
            _decodeBufs->waitForNonEmpty(lock);
        }
        Buffer* data = _decodeBufs->getForRead().first;
        memcpy(dataBuf->_data, data->_data, dataBuf->_size);
        Buffer* targets = _decodeBufs->getForRead().second;
        memcpy(targetsBuf->_data, targets->_data, targetsBuf->_size);
        _decodeBufs->advanceReadPos();
    }
    _decodeBufs->signalNonFull();
}

void Loader::next() {
    unique_lock<mutex> lock(_decodeBufs->getMutex());
    if (_first == true) {
        _first = false;
    } else {
        // Unlock the buffer used for the previous minibatch.
        _decodeBufs->advanceReadPos();
        _decodeBufs->signalNonFull();
    }
    while (_decodeBufs->empty()) {
        _decodeBufs->waitForNonEmpty(lock);
    }
}

std::shared_ptr<BatchIterator> Loader::getBatchIterator() {
    return _batch_iterator;
}

std::shared_ptr<Device> Loader::getDevice() {
    return _device;
}

void Loader::drain() {
    {
        unique_lock<mutex> lock(_decodeBufs->getMutex());
        if (_decodeBufs->empty() == true) {
            return;
        }
        _decodeBufs->advanceReadPos();
    }
    _decodeBufs->signalNonFull();
}
<commit_msg>replace shared_ptr with make_shared<commit_after>/*
 Copyright 2015 Nervana Systems Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

#include <assert.h>

#include <vector>
#include <cstdio>
#include <iostream>
#include <chrono>
#include <utility>
#include <algorithm>

#include "media.hpp"
#include "matrix.hpp"
#include "device.hpp"
#include "loader.hpp"
#include "batch_loader_cpio_cache.hpp"
#include "sequential_batch_iterator.hpp"
#include "shuffled_batch_iterator.hpp"

using namespace std;

DecodeThreadPool::DecodeThreadPool(int count, int batchSize,
                 int datumSize, int datumTypeSize,
                 int targetSize, int targetTypeSize,
                 const std::shared_ptr<BufferPool>& in, const std::shared_ptr<BufferPool>& out,
                 const std::shared_ptr<Device>& device,
                 MediaParams* mediaParams)
: ThreadPool(count),
  _itemsPerThread((batchSize - 1) / count + 1),
  _in(in), _out(out), _endSignaled(0),
  _manager(0), _stopManager(false), _managerStopped(false), _inputBuf(0),
  _bufferIndex(0), _batchSize(batchSize),
  _datumSize(datumSize), _datumTypeSize(datumTypeSize),
  _targetSize(targetSize), _targetTypeSize(targetTypeSize),
  _datumLen(datumSize * datumTypeSize),
  _targetLen(targetSize * targetTypeSize),
  _device(device) {
    assert(_itemsPerThread * count >= _batchSize);
    assert(_itemsPerThread * (count - 1) < _batchSize);
    for (int i = 0; i < count; i++) {
        _media.push_back(Media::create(mediaParams, 0, i));
        _startSignaled.push_back(0);
        _startInds.push_back(0);
        _endInds.push_back(0);
        _dataOffsets.push_back(0);
        _targetOffsets.push_back(0);
    }
}

DecodeThreadPool::~DecodeThreadPool() {
    if (_manager != 0) {
        _manager->join();
        delete _manager;
    }
    // The other thread objects are freed in the destructor
    // of the parent class.
}

void DecodeThreadPool::start() {
    for (int i = 0; i < _count; i++) {
        _threads.push_back(new thread(&DecodeThreadPool::run, this, i));
    }
    _manager = new thread(&DecodeThreadPool::manage, this);
}

void DecodeThreadPool::stop() {
    ThreadPool::stop();
    while (stopped() == false) {
        std::this_thread::yield();
        _in->advanceWritePos();
        _in->signalNonEmpty();
    }

    _stopManager = true;
    while (_managerStopped == false) {
        std::this_thread::yield();
        _in->advanceWritePos();
        _in->signalNonEmpty();
        _endSignaled++;
        _ended.notify_one();
    }
}

void DecodeThreadPool::run(int id) {
    // Initialize worker threads by computing memory offsets for the
    // data this thread should work on
    assert(id < _count);
    _startInds[id] = id * _itemsPerThread;
    int itemCount = _itemsPerThread;
    if (id == _count - 1) {
        itemCount = _batchSize - id * _itemsPerThread;
    }

    _endInds[id] = _startInds[id] + itemCount;
    _dataOffsets[id] = _startInds[id] * _datumLen;
    _targetOffsets[id] = _startInds[id] * _targetLen;
    while (_done == false) {
        work(id);
    }

    _stopped[id] = true;
}

void DecodeThreadPool::work(int id) {
    // Thread function.
    {
        unique_lock<mutex> lock(_mutex);
        while (_startSignaled[id] == 0) {
            _started.wait(lock);
            if (_done == true) {
                return;
            }
        }
        _startSignaled[id]--;
        assert(_startSignaled[id] == 0);
    }

    int start = _startInds[id];
    int end = _endInds[id];
    // No locking required because threads
    // write into non-overlapping regions.
    BufferPair& outBuf = _out->getForWrite();
    char* dataBuf = outBuf.first->_data + _dataOffsets[id];
    char* targetBuf = outBuf.second->_data + _targetOffsets[id];
    for (int i = start; i < end; i++) {
        // Handle the data.
        int itemSize = 0;
        char* item = _inputBuf->first->getItem(i, itemSize);
        if (item == 0) {
            return;
        }
        _media[id]->transform(item, itemSize, dataBuf, _datumLen);
        dataBuf += _datumLen;

        // Handle the targets.
        int targetLen = 0;
        char* target = _inputBuf->second->getItem(i, targetLen);
        memcpy(targetBuf, target, targetLen);
        if (_targetLen > targetLen) {
            // Pad the rest of the buffer with zeros.
            memset(targetBuf + targetLen, 0, _targetLen - targetLen);
        }
        targetBuf += _targetLen;
    }

    {
        lock_guard<mutex> lock(_mutex);
        _endSignaled++;
        assert(_endSignaled <= _count);
    }
    _ended.notify_one();
}

void DecodeThreadPool::produce() {
    // Produce a minibatch.
    {
        unique_lock<mutex> lock(_out->getMutex());
        while (_out->full() == true) {
            _out->waitForNonFull(lock);
        }
        {
            lock_guard<mutex> lock(_mutex);
            for (unsigned int i = 0; i < _startSignaled.size(); i++) {
                _startSignaled[i] = 1;
            }
        }
        _started.notify_all();
        {
            unique_lock<mutex> lock(_mutex);
            while (_endSignaled < _count) {
                _ended.wait(lock);
            }
            _endSignaled = 0;
        }
        // At this point, we have decoded data for the whole minibatch.
        BufferPair& outBuf = _out->getForWrite();
        Matrix::transpose(outBuf.first->_data, _batchSize,
                          _datumSize, _datumTypeSize);
        Matrix::transpose(outBuf.second->_data, _batchSize,
                          _targetSize, _targetTypeSize);
        // Copy to device.
        _device->copyData(_bufferIndex, outBuf.first->_data,
                          outBuf.first->_size);
        _device->copyLabels(_bufferIndex, outBuf.second->_data,
                            outBuf.second->_size);
        _bufferIndex = (_bufferIndex == 0) ? 1 : 0;
        _out->advanceWritePos();
    }
    _out->signalNonEmpty();
}

void DecodeThreadPool::consume() {
    // Consume an input buffer.
    {
        unique_lock<mutex> lock(_in->getMutex());
        while (_in->empty() == true) {
            _in->waitForNonEmpty(lock);
            if (_stopManager == true) {
                return;
            }
        }
        _inputBuf = &_in->getForRead();
        produce();
        _in->advanceReadPos();
    }
    _in->signalNonFull();
}

void DecodeThreadPool::manage() {
    // Thread function.
    int result = _device->init();
    if (result != 0) {
        _stopManager = true;
    }
    while (_stopManager == false) {
        consume();
    }
    _managerStopped = true;
}

ReadThread::ReadThread(const shared_ptr<BufferPool>& out, const shared_ptr<BatchIterator>& batch_iterator)
: ThreadPool(1), _out(out), _batch_iterator(batch_iterator) {
    assert(_count == 1);
}

void ReadThread::work(int id) {
    // Fill input buffers.
    // TODO: make sure this locking still makes sense with new
    // BatchIterator
    {
        unique_lock<mutex> lock(_out->getMutex());
        while (_out->full() == true) {
            _out->waitForNonFull(lock);
        }
        _batch_iterator->read(_out->getForWrite());
        _out->advanceWritePos();
    }
    _out->signalNonEmpty();
}

Loader::Loader(int* itemCount, int batchSize,
       const char* repoDir, const char* archiveDir,
       const char* indexFile, const char* archivePrefix,
       bool shuffle, bool reshuffle,
       int datumSize, int datumTypeSize,
       int targetSize, int targetTypeSize,
       int subsetPercent,
       MediaParams* mediaParams,
       DeviceParams* deviceParams,
       const char* manifestFilename,
       const char* rootCacheDir)
: _first(true),
  _batchSize(batchSize),
  _datumSize(datumSize), _datumTypeSize(datumTypeSize),
  _targetSize(targetSize), _targetTypeSize(targetTypeSize),
  _readBufs(nullptr), _decodeBufs(nullptr), _readThread(nullptr), _decodeThreads(nullptr),
  _device(nullptr), _batch_iterator(nullptr), _mediaParams(mediaParams)
  {
    // TODO: rename to shuffleManifest and shuffleEveryEpoch
    // TODO: not a constant
    uint _macroBatchSize = 1024;
    // TODO: not a constant
    uint _seed = 0;

    _device = Device::create(deviceParams);

    // the manifest defines which data should be included in the dataset
    auto manifest = make_shared<Manifest>(manifestFilename, shuffle);
    *itemCount = manifest->getSize();

    // build cacheDir from rootCacheDir and a hash of the manifest
    stringstream cacheDirStream;
    cacheDirStream << rootCacheDir << '/' << manifest->hash();
    string cacheDir = cacheDirStream.str();

    // batch loader provdes random access to blocks of data in the manifest
    auto batchLoader = make_shared<BatchLoaderCPIOCache>(
        cacheDir.c_str(),
        make_shared<BatchFileLoader>(manifest, subsetPercent)
    );

    // _batch_iterator provides an unending iterator (shuffled or not) over
    // the batchLoader
    if(reshuffle) {
        _batch_iterator = make_shared<ShuffledBatchIterator>(
             batchLoader, _macroBatchSize, _seed
        );
    } else {
        _batch_iterator = make_shared<SequentialBatchIterator>(
             batchLoader, _macroBatchSize
        );
    }
}

Loader::~Loader() {
}

int Loader::start() {
    _first = true;
    try {
        int dataLen = _batchSize * _datumSize * _datumTypeSize;
        int targetLen = _batchSize * _targetSize * _targetTypeSize;
        // Start the read buffers off with a reasonable size. They will
        // get resized as needed.
        _readBufs = make_shared<BufferPool>(dataLen / 8, targetLen);
        _readThread = unique_ptr<ReadThread>(new ReadThread(_readBufs, _batch_iterator));
        bool pinned = (_device->_type != CPU);
        _decodeBufs = make_shared<BufferPool>(dataLen, targetLen, pinned);
        int numCores = thread::hardware_concurrency();
        int itemsPerThread = (_batchSize - 1) /  numCores + 1;
        int threadCount =  (_batchSize - 1) / itemsPerThread + 1;
        threadCount = std::min(threadCount, _batchSize);
        _decodeThreads = unique_ptr<DecodeThreadPool>(new DecodeThreadPool(threadCount, _batchSize,
                _datumSize, _datumTypeSize,
                _targetSize, _targetTypeSize,
                _readBufs, _decodeBufs, _device, _mediaParams));
    } catch(std::bad_alloc&) {
        return -1;
    }
    _decodeThreads->start();
    _readThread->start();
    return 0;
}

void Loader::stop() {
    _readThread->stop();
    while (_readThread->stopped() == false) {
        std::this_thread::yield();
        drain();
    }
    while ((_decodeBufs->empty() == false) ||
           (_readBufs->empty() == false)) {
        drain();
    }
    _decodeThreads->stop();

    _readBufs = nullptr;
    _readThread = nullptr;
    _decodeBufs = nullptr;
    _decodeThreads = nullptr;
}

int Loader::reset() {
    stop();
    _batch_iterator->reset();
    start();
    return 0;
}

void Loader::next(Buffer* dataBuf, Buffer* targetsBuf) {
    // Copy minibatch data into the buffers passed in.
    // Only used for testing purposes.
    {
        unique_lock<mutex> lock(_decodeBufs->getMutex());
        while (_decodeBufs->empty()) {
            _decodeBufs->waitForNonEmpty(lock);
        }
        Buffer* data = _decodeBufs->getForRead().first;
        memcpy(dataBuf->_data, data->_data, dataBuf->_size);
        Buffer* targets = _decodeBufs->getForRead().second;
        memcpy(targetsBuf->_data, targets->_data, targetsBuf->_size);
        _decodeBufs->advanceReadPos();
    }
    _decodeBufs->signalNonFull();
}

void Loader::next() {
    unique_lock<mutex> lock(_decodeBufs->getMutex());
    if (_first == true) {
        _first = false;
    } else {
        // Unlock the buffer used for the previous minibatch.
        _decodeBufs->advanceReadPos();
        _decodeBufs->signalNonFull();
    }
    while (_decodeBufs->empty()) {
        _decodeBufs->waitForNonEmpty(lock);
    }
}

std::shared_ptr<BatchIterator> Loader::getBatchIterator() {
    return _batch_iterator;
}

std::shared_ptr<Device> Loader::getDevice() {
    return _device;
}

void Loader::drain() {
    {
        unique_lock<mutex> lock(_decodeBufs->getMutex());
        if (_decodeBufs->empty() == true) {
            return;
        }
        _decodeBufs->advanceReadPos();
    }
    _decodeBufs->signalNonFull();
}
<|endoftext|>


<commit_before>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * This file is part of the LibreOffice project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This file incorporates work covered by the following license notice:
 *
 *   Licensed to the Apache Software Foundation (ASF) under one or more
 *   contributor license agreements. See the NOTICE file distributed
 *   with this work for additional information regarding copyright
 *   ownership. The ASF licenses this file to you under the Apache
 *   License, Version 2.0 (the "License"); you may not use this file
 *   except in compliance with the License. You may obtain a copy of
 *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
 */

#ifndef INCLUDED_SD_SOURCE_UI_INC_FUTEXT_HXX
#define INCLUDED_SD_SOURCE_UI_INC_FUTEXT_HXX

#include <editeng/editdata.hxx>
#include "fuconstr.hxx"
#include <svx/svdotext.hxx>

struct StyleRequestData;
class SdrTextObj;
class FontList;
class OutlinerView;

namespace sd {

/**
 * Base class for text functions
 */
class FuText
    : public FuConstruct
{
public:
    TYPEINFO_OVERRIDE();

    static rtl::Reference<FuPoor> Create( ViewShell* pViewSh, ::sd::Window* pWin, ::sd::View* pView, SdDrawDocument* pDoc, SfxRequest& rReq );
    virtual void DoExecute( SfxRequest& rReq ) SAL_OVERRIDE;

    virtual bool KeyInput(const KeyEvent& rKEvt) SAL_OVERRIDE;
    virtual bool MouseMove(const MouseEvent& rMEvt) SAL_OVERRIDE;
    virtual bool MouseButtonUp(const MouseEvent& rMEvt) SAL_OVERRIDE;
    virtual bool MouseButtonDown(const MouseEvent& rMEvt) SAL_OVERRIDE;
    virtual bool Command(const CommandEvent& rCEvt) SAL_OVERRIDE;
    virtual bool RequestHelp(const HelpEvent& rHEvt) SAL_OVERRIDE;
    virtual void ReceiveRequest(SfxRequest& rReq) SAL_OVERRIDE;
    virtual void DoubleClick(const MouseEvent& rMEvt) SAL_OVERRIDE;

    virtual void Activate() SAL_OVERRIDE;           ///< activates the function
    virtual void Deactivate() SAL_OVERRIDE;         ///< deactivates the function

    void    SetInEditMode(const MouseEvent& rMEvt, bool bQuickDrag);
    bool    DeleteDefaultText();
    SdrTextObj* GetTextObj() { return static_cast< SdrTextObj* >( mxTextObj.get() ); }

    virtual SdrObject* CreateDefaultObject(const sal_uInt16 nID, const Rectangle& rRectangle) SAL_OVERRIDE;

    /** is called when the current function should be aborted.

        This is used when a function gets a KEY_ESCAPE but can also
        be called directly.

        @returns true if a active function was aborted
    */
    virtual bool cancel() SAL_OVERRIDE;

    static void ChangeFontSize( bool, OutlinerView*, const FontList*, ::sd::View* );

protected:
    FuText (ViewShell* pViewSh,
        ::sd::Window* pWin,
        ::sd::View* pView,
        SdDrawDocument* pDoc,
        SfxRequest& rReq);

    virtual void disposing() SAL_OVERRIDE;

    SdrObjectWeakRef    mxTextObj;
    Link<>              aOldLink;
    bool                bFirstObjCreated;
    bool                bJustEndedEdit;

    SfxRequest&         rRequest;

private:
    void ImpSetAttributesForNewTextObject(SdrTextObj* pTxtObj);
    void ImpSetAttributesFitToSize(SdrTextObj* pTxtObj);
    void ImpSetAttributesFitToSizeVertical(SdrTextObj* pTxtObj);
    void ImpSetAttributesFitCommon(SdrTextObj* pTxtObj);
};

} // end of namespace sd

#endif

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<commit_msg>Make more FuText members private<commit_after>/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
/*
 * This file is part of the LibreOffice project.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This file incorporates work covered by the following license notice:
 *
 *   Licensed to the Apache Software Foundation (ASF) under one or more
 *   contributor license agreements. See the NOTICE file distributed
 *   with this work for additional information regarding copyright
 *   ownership. The ASF licenses this file to you under the Apache
 *   License, Version 2.0 (the "License"); you may not use this file
 *   except in compliance with the License. You may obtain a copy of
 *   the License at http://www.apache.org/licenses/LICENSE-2.0 .
 */

#ifndef INCLUDED_SD_SOURCE_UI_INC_FUTEXT_HXX
#define INCLUDED_SD_SOURCE_UI_INC_FUTEXT_HXX

#include <editeng/editdata.hxx>
#include "fuconstr.hxx"
#include <svx/svdotext.hxx>

struct StyleRequestData;
class SdrTextObj;
class FontList;
class OutlinerView;

namespace sd {

/**
 * Base class for text functions
 */
class FuText
    : public FuConstruct
{
public:
    TYPEINFO_OVERRIDE();

    static rtl::Reference<FuPoor> Create( ViewShell* pViewSh, ::sd::Window* pWin, ::sd::View* pView, SdDrawDocument* pDoc, SfxRequest& rReq );
    virtual void DoExecute( SfxRequest& rReq ) SAL_OVERRIDE;

    virtual bool KeyInput(const KeyEvent& rKEvt) SAL_OVERRIDE;
    virtual bool MouseMove(const MouseEvent& rMEvt) SAL_OVERRIDE;
    virtual bool MouseButtonUp(const MouseEvent& rMEvt) SAL_OVERRIDE;
    virtual bool MouseButtonDown(const MouseEvent& rMEvt) SAL_OVERRIDE;
    virtual bool Command(const CommandEvent& rCEvt) SAL_OVERRIDE;
    virtual bool RequestHelp(const HelpEvent& rHEvt) SAL_OVERRIDE;
    virtual void ReceiveRequest(SfxRequest& rReq) SAL_OVERRIDE;
    virtual void DoubleClick(const MouseEvent& rMEvt) SAL_OVERRIDE;

    virtual void Activate() SAL_OVERRIDE;           ///< activates the function
    virtual void Deactivate() SAL_OVERRIDE;         ///< deactivates the function

    void    SetInEditMode(const MouseEvent& rMEvt, bool bQuickDrag);
    bool    DeleteDefaultText();
    SdrTextObj* GetTextObj() { return static_cast< SdrTextObj* >( mxTextObj.get() ); }

    virtual SdrObject* CreateDefaultObject(const sal_uInt16 nID, const Rectangle& rRectangle) SAL_OVERRIDE;

    /** is called when the current function should be aborted.

        This is used when a function gets a KEY_ESCAPE but can also
        be called directly.

        @returns true if a active function was aborted
    */
    virtual bool cancel() SAL_OVERRIDE;

    static void ChangeFontSize( bool, OutlinerView*, const FontList*, ::sd::View* );

protected:
    FuText (ViewShell* pViewSh,
        ::sd::Window* pWin,
        ::sd::View* pView,
        SdDrawDocument* pDoc,
        SfxRequest& rReq);

private:
    virtual void disposing() SAL_OVERRIDE;

    SdrObjectWeakRef    mxTextObj;
    bool                bFirstObjCreated;
    bool                bJustEndedEdit;

    SfxRequest&         rRequest;

    void ImpSetAttributesForNewTextObject(SdrTextObj* pTxtObj);
    void ImpSetAttributesFitToSize(SdrTextObj* pTxtObj);
    void ImpSetAttributesFitToSizeVertical(SdrTextObj* pTxtObj);
    void ImpSetAttributesFitCommon(SdrTextObj* pTxtObj);
};

} // end of namespace sd

#endif

/* vim:set shiftwidth=4 softtabstop=4 expandtab: */
<|endoftext|>


<commit_before>/*************************************************************************
 *
 *  OpenOffice.org - a multi-platform office productivity suite
 *
 *  $RCSfile: optdlg.hxx,v $
 *
 *  $Revision: 1.3 $
 *
 *  last change: $Author: kz $ $Date: 2006-12-12 17:45:47 $
 *
 *  The Contents of this file are made available subject to
 *  the terms of GNU Lesser General Public License Version 2.1.
 *
 *
 *    GNU Lesser General Public License Version 2.1
 *    =============================================
 *    Copyright 2005 by Sun Microsystems, Inc.
 *    901 San Antonio Road, Palo Alto, CA 94303, USA
 *
 *    This library is free software; you can redistribute it and/or
 *    modify it under the terms of the GNU Lesser General Public
 *    License version 2.1, as published by the Free Software Foundation.
 *
 *    This library is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *    Lesser General Public License for more details.
 *
 *    You should have received a copy of the GNU Lesser General Public
 *    License along with this library; if not, write to the Free Software
 *    Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 *    MA  02111-1307  USA
 *
 ************************************************************************/

#ifndef SD_OPTDLG_HXX
#define SD_OPTDLG_HXX

#ifndef _SFXTABDLG_HXX //autogen
#include <sfx2/tabdlg.hxx>
#endif

#ifndef _PRESENTATION_HXX
#include "pres.hxx"
#endif

class SfxItemSet;

class SdOptionsDlg : public SfxTabDialog
{
private:
    DocumentType    meDocType;

public:
                SdOptionsDlg( Window* pParent, const SfxItemSet& rInAttrs,
                              DocumentType eDocType );
                ~SdOptionsDlg();

protected:

    virtual void        PageCreated( USHORT nId, SfxTabPage &rPage );
//  virtual SfxItemSet* CreateInputItemSet( USHORT nPageId );
};

#endif // SD_OPTDLG_HXX

<commit_msg>INTEGRATION: CWS changefileheader (1.3.298); FILE MERGED 2008/04/01 15:35:28 thb 1.3.298.3: #i85898# Stripping all external header guards 2008/04/01 12:39:08 thb 1.3.298.2: #i85898# Stripping all external header guards 2008/03/31 13:58:16 rt 1.3.298.1: #i87441# Change license header to LPGL v3.<commit_after>/*************************************************************************
 *
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * Copyright 2008 by Sun Microsystems, Inc.
 *
 * OpenOffice.org - a multi-platform office productivity suite
 *
 * $RCSfile: optdlg.hxx,v $
 * $Revision: 1.4 $
 *
 * This file is part of OpenOffice.org.
 *
 * OpenOffice.org is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License version 3
 * only, as published by the Free Software Foundation.
 *
 * OpenOffice.org is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License version 3 for more details
 * (a copy is included in the LICENSE file that accompanied this code).
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 3 along with OpenOffice.org.  If not, see
 * <http://www.openoffice.org/license.html>
 * for a copy of the LGPLv3 License.
 *
 ************************************************************************/

#ifndef SD_OPTDLG_HXX
#define SD_OPTDLG_HXX

#include <sfx2/tabdlg.hxx>
#include "pres.hxx"

class SfxItemSet;

class SdOptionsDlg : public SfxTabDialog
{
private:
    DocumentType    meDocType;

public:
                SdOptionsDlg( Window* pParent, const SfxItemSet& rInAttrs,
                              DocumentType eDocType );
                ~SdOptionsDlg();

protected:

    virtual void        PageCreated( USHORT nId, SfxTabPage &rPage );
//  virtual SfxItemSet* CreateInputItemSet( USHORT nPageId );
};

#endif // SD_OPTDLG_HXX

<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

#include "manager/stmgr-server.h"
#include <iostream>
#include <unordered_set>
#include <string>
#include <vector>
#include "manager/stmgr.h"
#include "proto/messages.h"
#include "basics/basics.h"
#include "errors/errors.h"
#include "threads/threads.h"
#include "network/network.h"
#include "config/helper.h"
#include "config/heron-internals-config-reader.h"
#include "metrics/metrics.h"

namespace heron {
namespace stmgr {

// The scope the metrics in this file are under
const sp_string SERVER_SCOPE = "__server/";
// Num data tuples received from other stream managers
const sp_string METRIC_DATA_TUPLES_FROM_STMGRS = "__tuples_from_stmgrs";
// Num ack tuples received from other stream managers
const sp_string METRIC_ACK_TUPLES_FROM_STMGRS = "__ack_tuples_from_stmgrs";
// Num fail tuples received from other stream managers
const sp_string METRIC_FAIL_TUPLES_FROM_STMGRS = "__fail_tuples_from_stmgrs";
// Bytes received from other stream managers
const sp_string METRIC_BYTES_FROM_STMGRS = "__bytes_from_stmgrs";

StMgrServer::StMgrServer(EventLoop* eventLoop, const NetworkOptions& _options,
                         const sp_string& _topology_name, const sp_string& _topology_id,
                         const sp_string& _stmgr_id, StMgr* _stmgr,
                         heron::common::MetricsMgrSt* _metrics_manager_client)
    : Server(eventLoop, _options),
      topology_name_(_topology_name),
      topology_id_(_topology_id),
      stmgr_id_(_stmgr_id),
      stmgr_(_stmgr),
      metrics_manager_client_(_metrics_manager_client) {
  // stmgr related handlers
  InstallRequestHandler(&StMgrServer::HandleStMgrHelloRequest);
  InstallMessageHandler(&StMgrServer::HandleTupleStreamMessage);
  InstallMessageHandler(&StMgrServer::HandleStartBackPressureMessage);
  InstallMessageHandler(&StMgrServer::HandleStopBackPressureMessage);
  InstallMessageHandler(&StMgrServer::HandleDownstreamStatefulCheckpointMessage);

  // The metrics need to be registered one by one here because the "__server" scope
  // is already registered in heron::stmgr::InstanceServer. Duplicated registrations
  // will only have one successfully registered.
  tuples_from_stmgrs_metrics_ = new heron::common::CountMetric();
  metrics_manager_client_->register_metric(SERVER_SCOPE + METRIC_DATA_TUPLES_FROM_STMGRS,
                                           tuples_from_stmgrs_metrics_);
  ack_tuples_from_stmgrs_metrics_ = new heron::common::CountMetric();
  metrics_manager_client_->register_metric(SERVER_SCOPE + METRIC_ACK_TUPLES_FROM_STMGRS,
                                           ack_tuples_from_stmgrs_metrics_);
  fail_tuples_from_stmgrs_metrics_ = new heron::common::CountMetric();
  metrics_manager_client_->register_metric(SERVER_SCOPE + METRIC_FAIL_TUPLES_FROM_STMGRS,
                                           fail_tuples_from_stmgrs_metrics_);
  bytes_from_stmgrs_metrics_ = new heron::common::CountMetric();
  metrics_manager_client_->register_metric(SERVER_SCOPE + METRIC_BYTES_FROM_STMGRS,
                                           bytes_from_stmgrs_metrics_);
}

StMgrServer::~StMgrServer() {
  Stop();
  metrics_manager_client_->unregister_metric(SERVER_SCOPE + METRIC_DATA_TUPLES_FROM_STMGRS);
  delete tuples_from_stmgrs_metrics_;
  metrics_manager_client_->unregister_metric(SERVER_SCOPE + METRIC_ACK_TUPLES_FROM_STMGRS);
  delete ack_tuples_from_stmgrs_metrics_;
  metrics_manager_client_->unregister_metric(SERVER_SCOPE + METRIC_FAIL_TUPLES_FROM_STMGRS);
  delete fail_tuples_from_stmgrs_metrics_;
  metrics_manager_client_->unregister_metric(SERVER_SCOPE + METRIC_BYTES_FROM_STMGRS);
  delete bytes_from_stmgrs_metrics_;
}

void StMgrServer::HandleNewConnection(Connection* _conn) {
  // There is nothing to be done here. Instead we wait
  // for the register/hello
  LOG(INFO) << "StMgrServer Got new connection " << _conn << " from "
            << _conn->getIPAddress() << ":" << _conn->getPort();
}

void StMgrServer::HandleConnectionClose(Connection* _conn, NetworkErrorCode) {
  LOG(INFO) << "StMgrServer Got connection close of " << _conn << " from "
            << _conn->getIPAddress() << ":" << _conn->getPort();
  // Find the stmgr who hung up
  auto siter = rstmgrs_.find(_conn);
  if (siter == rstmgrs_.end()) {
    LOG(ERROR) << "StMgrServer could not identity connection " << _conn << " from "
               << _conn->getIPAddress() << ":" << _conn->getPort();
    return;
  }
  // This is a stmgr connection
  LOG(INFO) << "Stmgr " << siter->second << " closed connection";
  sp_string stmgr_id = rstmgrs_[_conn];
  // Did we receive a start back pressure message from this stmgr to
  // begin with?
  if (stmgrs_who_announced_back_pressure_.find(stmgr_id) !=
      stmgrs_who_announced_back_pressure_.end()) {
    stmgrs_who_announced_back_pressure_.erase(stmgr_id);
    if (stmgrs_who_announced_back_pressure_.empty()) {
      stmgr_->AttemptStopBackPressureFromSpouts();
    }
  }
  // Now cleanup the data structures
  stmgrs_.erase(siter->second);
  rstmgrs_.erase(_conn);
}

void StMgrServer::HandleStMgrHelloRequest(REQID _id, Connection* _conn,
                                          proto::stmgr::StrMgrHelloRequest* _request) {
  LOG(INFO) << "Got a hello message from stmgr " << _request->stmgr() << " on connection " << _conn;
  proto::stmgr::StrMgrHelloResponse response;
  // Some basic checks
  if (_request->topology_name() != topology_name_) {
    LOG(ERROR) << "The hello message was from a different topology " << _request->topology_name()
               << std::endl;
    response.mutable_status()->set_status(proto::system::NOTOK);
  } else if (_request->topology_id() != topology_id_) {
    LOG(ERROR) << "The hello message was from a different topology id " << _request->topology_id()
               << std::endl;
    response.mutable_status()->set_status(proto::system::NOTOK);
  } else if (stmgrs_.find(_request->stmgr()) != stmgrs_.end()) {
    LOG(WARNING) << "We already had an active connection from the stmgr " << _request->stmgr()
                 << ". Closing existing connection...";
    // This will free up the slot in the various maps in this class
    // and the next time around we'll be able to add this stmgr.
    // We shouldn't add the new stmgr connection right now because
    // the close could be asynchronous (fired through a 0 timer)
    stmgrs_[_request->stmgr()]->closeConnection();
    response.mutable_status()->set_status(proto::system::NOTOK);
  } else {
    stmgrs_[_request->stmgr()] = _conn;
    rstmgrs_[_conn] = _request->stmgr();
    response.mutable_status()->set_status(proto::system::OK);
  }
  SendResponse(_id, _conn, response);
  delete _request;
}

void StMgrServer::HandleTupleStreamMessage(Connection* _conn,
                                           proto::stmgr::TupleStreamMessage* _message) {
  auto iter = rstmgrs_.find(_conn);
  if (iter == rstmgrs_.end()) {
    LOG(INFO) << "Recieved Tuple messages from unknown streammanager connection";
    __global_protobuf_pool_release__(_message);
  } else {
    proto::system::HeronTupleSet2* tuple_set = nullptr;
    tuple_set = __global_protobuf_pool_acquire__(tuple_set);
    tuple_set->ParsePartialFromString(_message->set());

    bytes_from_stmgrs_metrics_->incr_by(_message->ByteSize());
    if (tuple_set->has_data()) {
      tuples_from_stmgrs_metrics_->incr_by(tuple_set->data().tuples_size());
    } else if (tuple_set->has_control()) {
      ack_tuples_from_stmgrs_metrics_->incr_by(tuple_set->control().acks_size());
      fail_tuples_from_stmgrs_metrics_->incr_by(tuple_set->control().fails_size());
    }

    stmgr_->HandleStreamManagerData(iter->second, _message);
  }
}

void StMgrServer::StartBackPressureClientCb(const sp_string& _other_stmgr_id) {
  if (!stmgr_->DidAnnounceBackPressure()) {
    stmgr_->SendStartBackPressureToOtherStMgrs();
  }
  remote_ends_who_caused_back_pressure_.insert(_other_stmgr_id);
  LOG(INFO) << "We observe back pressure on sending data to remote stream manager "
            << _other_stmgr_id;
  stmgr_->StartBackPressureOnSpouts();
}

void StMgrServer::StopBackPressureClientCb(const sp_string& _other_stmgr_id) {
  CHECK(remote_ends_who_caused_back_pressure_.find(_other_stmgr_id) !=
        remote_ends_who_caused_back_pressure_.end());
  remote_ends_who_caused_back_pressure_.erase(_other_stmgr_id);

  if (!stmgr_->DidAnnounceBackPressure()) {
    stmgr_->SendStopBackPressureToOtherStMgrs();
  }
  LOG(INFO) << "We don't observe back pressure now on sending data to remote "
               "stream manager "
            << _other_stmgr_id;
  if (!stmgr_->DidAnnounceBackPressure() && !stmgr_->DidOthersAnnounceBackPressure()) {
    stmgr_->AttemptStopBackPressureFromSpouts();
  }
}

void StMgrServer::HandleStartBackPressureMessage(Connection* _conn,
                                                 proto::stmgr::StartBackPressureMessage* _message) {
  // Close spouts
  LOG(INFO) << "Received start back pressure from str mgr " << _message->stmgr();
  if (_message->topology_name() != topology_name_ || _message->topology_id() != topology_id_) {
    LOG(ERROR) << "Received start back pressure message from unknown stream manager "
               << _message->topology_name() << " " << _message->topology_id() << " "
               << _message->stmgr() << " " << _message->message_id();

    __global_protobuf_pool_release__(_message);
    return;
  }
  auto iter = rstmgrs_.find(_conn);
  CHECK(iter != rstmgrs_.end());
  sp_string stmgr_id = iter->second;
  stmgrs_who_announced_back_pressure_.insert(stmgr_id);

  stmgr_->StartBackPressureOnSpouts();

  __global_protobuf_pool_release__(_message);
}

void StMgrServer::HandleStopBackPressureMessage(Connection* _conn,
                                                proto::stmgr::StopBackPressureMessage* _message) {
  LOG(INFO) << "Received stop back pressure from str mgr " << _message->stmgr();
  if (_message->topology_name() != topology_name_ || _message->topology_id() != topology_id_) {
    LOG(ERROR) << "Received stop back pressure message from unknown stream manager "
               << _message->topology_name() << " " << _message->topology_id() << " "
               << _message->stmgr();

    __global_protobuf_pool_release__(_message);
    return;
  }
  auto iter = rstmgrs_.find(_conn);
  CHECK(iter != rstmgrs_.end());
  sp_string stmgr_id = iter->second;
  // Did we receive a start back pressure message from this stmgr to
  // begin with? We could have been dead at the time of the announcement
  if (stmgrs_who_announced_back_pressure_.find(stmgr_id) !=
      stmgrs_who_announced_back_pressure_.end()) {
    stmgrs_who_announced_back_pressure_.erase(stmgr_id);
  }
  if (!stmgr_->DidAnnounceBackPressure() && !stmgr_->DidOthersAnnounceBackPressure()) {
    stmgr_->AttemptStopBackPressureFromSpouts();
  }

  __global_protobuf_pool_release__(_message);
}

void StMgrServer::HandleDownstreamStatefulCheckpointMessage(Connection* _conn,
                               proto::ckptmgr::DownstreamStatefulCheckpoint* _message) {
  stmgr_->HandleDownStreamStatefulCheckpoint(*_message);
  __global_protobuf_pool_release__(_message);
}

}  // namespace stmgr
}  // namespace heron
<commit_msg>Fix memory leak in stmgr when measuring tuple data size (#3175)<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

#include "manager/stmgr-server.h"
#include <iostream>
#include <unordered_set>
#include <string>
#include <vector>
#include "manager/stmgr.h"
#include "proto/messages.h"
#include "basics/basics.h"
#include "errors/errors.h"
#include "threads/threads.h"
#include "network/network.h"
#include "config/helper.h"
#include "config/heron-internals-config-reader.h"
#include "metrics/metrics.h"

namespace heron {
namespace stmgr {

// The scope the metrics in this file are under
const sp_string SERVER_SCOPE = "__server/";
// Num data tuples received from other stream managers
const sp_string METRIC_DATA_TUPLES_FROM_STMGRS = "__tuples_from_stmgrs";
// Num ack tuples received from other stream managers
const sp_string METRIC_ACK_TUPLES_FROM_STMGRS = "__ack_tuples_from_stmgrs";
// Num fail tuples received from other stream managers
const sp_string METRIC_FAIL_TUPLES_FROM_STMGRS = "__fail_tuples_from_stmgrs";
// Bytes received from other stream managers
const sp_string METRIC_BYTES_FROM_STMGRS = "__bytes_from_stmgrs";

StMgrServer::StMgrServer(EventLoop* eventLoop, const NetworkOptions& _options,
                         const sp_string& _topology_name, const sp_string& _topology_id,
                         const sp_string& _stmgr_id, StMgr* _stmgr,
                         heron::common::MetricsMgrSt* _metrics_manager_client)
    : Server(eventLoop, _options),
      topology_name_(_topology_name),
      topology_id_(_topology_id),
      stmgr_id_(_stmgr_id),
      stmgr_(_stmgr),
      metrics_manager_client_(_metrics_manager_client) {
  // stmgr related handlers
  InstallRequestHandler(&StMgrServer::HandleStMgrHelloRequest);
  InstallMessageHandler(&StMgrServer::HandleTupleStreamMessage);
  InstallMessageHandler(&StMgrServer::HandleStartBackPressureMessage);
  InstallMessageHandler(&StMgrServer::HandleStopBackPressureMessage);
  InstallMessageHandler(&StMgrServer::HandleDownstreamStatefulCheckpointMessage);

  // The metrics need to be registered one by one here because the "__server" scope
  // is already registered in heron::stmgr::InstanceServer. Duplicated registrations
  // will only have one successfully registered.
  tuples_from_stmgrs_metrics_ = new heron::common::CountMetric();
  metrics_manager_client_->register_metric(SERVER_SCOPE + METRIC_DATA_TUPLES_FROM_STMGRS,
                                           tuples_from_stmgrs_metrics_);
  ack_tuples_from_stmgrs_metrics_ = new heron::common::CountMetric();
  metrics_manager_client_->register_metric(SERVER_SCOPE + METRIC_ACK_TUPLES_FROM_STMGRS,
                                           ack_tuples_from_stmgrs_metrics_);
  fail_tuples_from_stmgrs_metrics_ = new heron::common::CountMetric();
  metrics_manager_client_->register_metric(SERVER_SCOPE + METRIC_FAIL_TUPLES_FROM_STMGRS,
                                           fail_tuples_from_stmgrs_metrics_);
  bytes_from_stmgrs_metrics_ = new heron::common::CountMetric();
  metrics_manager_client_->register_metric(SERVER_SCOPE + METRIC_BYTES_FROM_STMGRS,
                                           bytes_from_stmgrs_metrics_);
}

StMgrServer::~StMgrServer() {
  Stop();
  metrics_manager_client_->unregister_metric(SERVER_SCOPE + METRIC_DATA_TUPLES_FROM_STMGRS);
  delete tuples_from_stmgrs_metrics_;
  metrics_manager_client_->unregister_metric(SERVER_SCOPE + METRIC_ACK_TUPLES_FROM_STMGRS);
  delete ack_tuples_from_stmgrs_metrics_;
  metrics_manager_client_->unregister_metric(SERVER_SCOPE + METRIC_FAIL_TUPLES_FROM_STMGRS);
  delete fail_tuples_from_stmgrs_metrics_;
  metrics_manager_client_->unregister_metric(SERVER_SCOPE + METRIC_BYTES_FROM_STMGRS);
  delete bytes_from_stmgrs_metrics_;
}

void StMgrServer::HandleNewConnection(Connection* _conn) {
  // There is nothing to be done here. Instead we wait
  // for the register/hello
  LOG(INFO) << "StMgrServer Got new connection " << _conn << " from "
            << _conn->getIPAddress() << ":" << _conn->getPort();
}

void StMgrServer::HandleConnectionClose(Connection* _conn, NetworkErrorCode) {
  LOG(INFO) << "StMgrServer Got connection close of " << _conn << " from "
            << _conn->getIPAddress() << ":" << _conn->getPort();
  // Find the stmgr who hung up
  auto siter = rstmgrs_.find(_conn);
  if (siter == rstmgrs_.end()) {
    LOG(ERROR) << "StMgrServer could not identity connection " << _conn << " from "
               << _conn->getIPAddress() << ":" << _conn->getPort();
    return;
  }
  // This is a stmgr connection
  LOG(INFO) << "Stmgr " << siter->second << " closed connection";
  sp_string stmgr_id = rstmgrs_[_conn];
  // Did we receive a start back pressure message from this stmgr to
  // begin with?
  if (stmgrs_who_announced_back_pressure_.find(stmgr_id) !=
      stmgrs_who_announced_back_pressure_.end()) {
    stmgrs_who_announced_back_pressure_.erase(stmgr_id);
    if (stmgrs_who_announced_back_pressure_.empty()) {
      stmgr_->AttemptStopBackPressureFromSpouts();
    }
  }
  // Now cleanup the data structures
  stmgrs_.erase(siter->second);
  rstmgrs_.erase(_conn);
}

void StMgrServer::HandleStMgrHelloRequest(REQID _id, Connection* _conn,
                                          proto::stmgr::StrMgrHelloRequest* _request) {
  LOG(INFO) << "Got a hello message from stmgr " << _request->stmgr() << " on connection " << _conn;
  proto::stmgr::StrMgrHelloResponse response;
  // Some basic checks
  if (_request->topology_name() != topology_name_) {
    LOG(ERROR) << "The hello message was from a different topology " << _request->topology_name()
               << std::endl;
    response.mutable_status()->set_status(proto::system::NOTOK);
  } else if (_request->topology_id() != topology_id_) {
    LOG(ERROR) << "The hello message was from a different topology id " << _request->topology_id()
               << std::endl;
    response.mutable_status()->set_status(proto::system::NOTOK);
  } else if (stmgrs_.find(_request->stmgr()) != stmgrs_.end()) {
    LOG(WARNING) << "We already had an active connection from the stmgr " << _request->stmgr()
                 << ". Closing existing connection...";
    // This will free up the slot in the various maps in this class
    // and the next time around we'll be able to add this stmgr.
    // We shouldn't add the new stmgr connection right now because
    // the close could be asynchronous (fired through a 0 timer)
    stmgrs_[_request->stmgr()]->closeConnection();
    response.mutable_status()->set_status(proto::system::NOTOK);
  } else {
    stmgrs_[_request->stmgr()] = _conn;
    rstmgrs_[_conn] = _request->stmgr();
    response.mutable_status()->set_status(proto::system::OK);
  }
  SendResponse(_id, _conn, response);
  delete _request;
}

void StMgrServer::HandleTupleStreamMessage(Connection* _conn,
                                           proto::stmgr::TupleStreamMessage* _message) {
  auto iter = rstmgrs_.find(_conn);
  if (iter == rstmgrs_.end()) {
    LOG(INFO) << "Recieved Tuple messages from unknown streammanager connection";
    __global_protobuf_pool_release__(_message);
  } else {
    proto::system::HeronTupleSet2* tuple_set = nullptr;
    tuple_set = __global_protobuf_pool_acquire__(tuple_set);
    tuple_set->ParsePartialFromString(_message->set());

    bytes_from_stmgrs_metrics_->incr_by(_message->ByteSize());
    if (tuple_set->has_data()) {
      tuples_from_stmgrs_metrics_->incr_by(tuple_set->data().tuples_size());
    } else if (tuple_set->has_control()) {
      ack_tuples_from_stmgrs_metrics_->incr_by(tuple_set->control().acks_size());
      fail_tuples_from_stmgrs_metrics_->incr_by(tuple_set->control().fails_size());
    }
    __global_protobuf_pool_release__(tuple_set);

    stmgr_->HandleStreamManagerData(iter->second, _message);
  }
}

void StMgrServer::StartBackPressureClientCb(const sp_string& _other_stmgr_id) {
  if (!stmgr_->DidAnnounceBackPressure()) {
    stmgr_->SendStartBackPressureToOtherStMgrs();
  }
  remote_ends_who_caused_back_pressure_.insert(_other_stmgr_id);
  LOG(INFO) << "We observe back pressure on sending data to remote stream manager "
            << _other_stmgr_id;
  stmgr_->StartBackPressureOnSpouts();
}

void StMgrServer::StopBackPressureClientCb(const sp_string& _other_stmgr_id) {
  CHECK(remote_ends_who_caused_back_pressure_.find(_other_stmgr_id) !=
        remote_ends_who_caused_back_pressure_.end());
  remote_ends_who_caused_back_pressure_.erase(_other_stmgr_id);

  if (!stmgr_->DidAnnounceBackPressure()) {
    stmgr_->SendStopBackPressureToOtherStMgrs();
  }
  LOG(INFO) << "We don't observe back pressure now on sending data to remote "
               "stream manager "
            << _other_stmgr_id;
  if (!stmgr_->DidAnnounceBackPressure() && !stmgr_->DidOthersAnnounceBackPressure()) {
    stmgr_->AttemptStopBackPressureFromSpouts();
  }
}

void StMgrServer::HandleStartBackPressureMessage(Connection* _conn,
                                                 proto::stmgr::StartBackPressureMessage* _message) {
  // Close spouts
  LOG(INFO) << "Received start back pressure from str mgr " << _message->stmgr();
  if (_message->topology_name() != topology_name_ || _message->topology_id() != topology_id_) {
    LOG(ERROR) << "Received start back pressure message from unknown stream manager "
               << _message->topology_name() << " " << _message->topology_id() << " "
               << _message->stmgr() << " " << _message->message_id();

    __global_protobuf_pool_release__(_message);
    return;
  }
  auto iter = rstmgrs_.find(_conn);
  CHECK(iter != rstmgrs_.end());
  sp_string stmgr_id = iter->second;
  stmgrs_who_announced_back_pressure_.insert(stmgr_id);

  stmgr_->StartBackPressureOnSpouts();

  __global_protobuf_pool_release__(_message);
}

void StMgrServer::HandleStopBackPressureMessage(Connection* _conn,
                                                proto::stmgr::StopBackPressureMessage* _message) {
  LOG(INFO) << "Received stop back pressure from str mgr " << _message->stmgr();
  if (_message->topology_name() != topology_name_ || _message->topology_id() != topology_id_) {
    LOG(ERROR) << "Received stop back pressure message from unknown stream manager "
               << _message->topology_name() << " " << _message->topology_id() << " "
               << _message->stmgr();

    __global_protobuf_pool_release__(_message);
    return;
  }
  auto iter = rstmgrs_.find(_conn);
  CHECK(iter != rstmgrs_.end());
  sp_string stmgr_id = iter->second;
  // Did we receive a start back pressure message from this stmgr to
  // begin with? We could have been dead at the time of the announcement
  if (stmgrs_who_announced_back_pressure_.find(stmgr_id) !=
      stmgrs_who_announced_back_pressure_.end()) {
    stmgrs_who_announced_back_pressure_.erase(stmgr_id);
  }
  if (!stmgr_->DidAnnounceBackPressure() && !stmgr_->DidOthersAnnounceBackPressure()) {
    stmgr_->AttemptStopBackPressureFromSpouts();
  }

  __global_protobuf_pool_release__(_message);
}

void StMgrServer::HandleDownstreamStatefulCheckpointMessage(Connection* _conn,
                               proto::ckptmgr::DownstreamStatefulCheckpoint* _message) {
  stmgr_->HandleDownStreamStatefulCheckpoint(*_message);
  __global_protobuf_pool_release__(_message);
}

}  // namespace stmgr
}  // namespace heron
<|endoftext|>


<commit_before>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2010-2011 Emmanuel Benazera <ebenazer@seeks-project.info>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "cf_configuration.h"
#include "sweeper.h"
#include "miscutil.h"
#include "urlmatch.h"
#include "errlog.h"

#include <iostream>

using sp::sweeper;
using sp::miscutil;
using sp::urlmatch;
using sp::errlog;

namespace seeks_plugins
{
#define hash_domain_name_weight       1333166351ul  /* "domain-name-weight" */
#define hash_record_cache_timeout     1954675964ul  /* "record-cache-timeout" */
#define hash_cf_peer                  1520012134ul  /* "cf-peer" */
#define hash_dead_peer_check          1043267473ul  /* "dead-peer-check" */
#define hash_dead_peer_retries         681362871ul  /* "dead-peer-retries" */
#define hash_post_url_check           3323226172ul  /* "post-url-check" */
#define hash_post_radius              2436628877ul  /* "post-radius" */
#define hash_post_ua                  1442804836ul  /* "post-ua" */
#define hash_stop_words_filtering     4002206625ul  /* "stop-words-filtering" */
#define hash_remote_post              4059800377ul  /* "remote-post" */
#define hash_use_http_url             1825269331ul  /* "use-http-urls-only" */
#define hash_cf_estimator             1689657696ul  /* "cf-estimator" */

  cf_configuration* cf_configuration::_config = NULL;

  cf_configuration::cf_configuration(const std::string &filename)
    :configuration_spec(filename)
  {
    // external static variables.
    _pl = new peer_list();
    _dpl = new peer_list();
    dead_peer::_pl = _pl;
    dead_peer::_dpl = _dpl;

    load_config();
  }

  cf_configuration::~cf_configuration()
  {
    // XXX: should mutex to avoid crash (this is only called when node is dying, for now).
    dead_peer::_pl = NULL;
    dead_peer::_dpl = NULL;
    delete _pl;

    // dead peers need to be unregistered from sweeper first.
    hash_map<const char*,peer*,hash<const char*>,eqstr>::iterator hit
    = _dpl->_peers.begin();
    while(hit!=_dpl->_peers.end())
      {
        dead_peer *dp = dynamic_cast<dead_peer*>((*hit).second);
        if (dp)
          sweeper::unregister_sweepable(dp);
        ++hit;
      }
    delete _dpl;
  }

  void cf_configuration::set_default_config()
  {
    _domain_name_weight = 0.7;
    _record_cache_timeout = 600; // 10 mins.
    _dead_peer_check = 300; // 5 mins.
    _dead_peer_retries = 3;
    _post_url_check = true;
    _post_radius = 5;
    _post_url_ua = "Mozilla/5.0 (X11; Linux x86_64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"; // default.
    _stop_words_filtering = false;
    _remote_post = true;
    _use_http_urls = true;
    _estimator = "sre";
  }

  void cf_configuration::handle_config_cmd(char *cmd, const uint32_t &cmd_hash, char *arg,
      char *buf, const unsigned long &linenum)
  {
    char tmp[BUFFER_SIZE];
    int vec_count;
    char *vec[4];
    int port;
    std::vector<std::string> elts;
    std::string host, path, address, port_str;

    switch (cmd_hash)
      {
      case hash_domain_name_weight:
        _domain_name_weight = atof(arg);
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Weight given to the domain names in the simple filter");
        break;

      case hash_record_cache_timeout:
        _record_cache_timeout = atoi(arg);
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Timeout on cached remote records, in seconds");
        break;

      case hash_cf_peer:
        strlcpy(tmp,arg,sizeof(tmp));
        vec_count = miscutil::ssplit(tmp," \t",vec,SZ(vec),1,1);
        div_t divresult;
        divresult = div(vec_count,2);
        if (divresult.rem != 0)
          {
            errlog::log_error(LOG_LEVEL_ERROR,"Wrong number of parameter when specifying static collaborative filtering peer");
            break;
          }
        address = vec[0];
        urlmatch::parse_url_host_and_path(address,host,path);
        miscutil::tokenize(host,elts,":");
        port = -1;
        if (elts.size()>1)
          {
            host = elts.at(0);
            port = atoi(elts.at(1).c_str());
          }
        port_str = (port != -1) ? ":" + miscutil::to_string(port) : "";
        errlog::log_error(LOG_LEVEL_DEBUG,"adding peer %s%s%s with resource %s",
                          host.c_str(),port_str.c_str(),path.c_str(),vec[1]);
        _pl->add(host,port,path,std::string(vec[1]));
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Remote peer address for collaborative filtering");
        break;

      case hash_dead_peer_check:
        _dead_peer_check = atoi(arg);
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Interval of time between two dead peer checks");
        break;

      case hash_dead_peer_retries:
        _dead_peer_retries = atoi(arg);
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Number of retries before marking a peer as dead");
        break;

      case hash_post_url_check:
        _post_url_check = static_cast<bool>(atoi(arg));
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Whether to ping and check on posted URLs");
        break;

      case hash_post_radius:
        _post_radius = static_cast<bool>(atoi(arg));
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Query similarity impact radius of posted URLs");
        break;

      case hash_post_ua:
        _post_url_ua = std::string(arg);
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "default 'user-agent' header used to retrieve posted URLs");
        break;

      case hash_stop_words_filtering:
        _stop_words_filtering = static_cast<bool>(atoi(arg));
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Whether to filter similar queries with stop words");
        break;

      case hash_remote_post:
        _remote_post = static_cast<bool>(atoi(arg));
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Whether to allow remote posting of results");
        break;

      case hash_use_http_url:
        _use_http_urls = static_cast<bool>(atoi(arg));
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Whether to allow only HTTP URLs or to allow generic item UIDs");
        break;

      case hash_cf_estimator:
        _estimator = arg;
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Estimator used by the collaborative filter");
        break;

      default:
        break;
      }
  }

  void cf_configuration::finalize_configuration()
  {
  }

} /* end of namespace. */
<commit_msg>fixed broken parsing of collaborative filters' post-radius<commit_after>/**
 * The Seeks proxy and plugin framework are part of the SEEKS project.
 * Copyright (C) 2010-2011 Emmanuel Benazera <ebenazer@seeks-project.info>
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "cf_configuration.h"
#include "sweeper.h"
#include "miscutil.h"
#include "urlmatch.h"
#include "errlog.h"

#include <iostream>

using sp::sweeper;
using sp::miscutil;
using sp::urlmatch;
using sp::errlog;

namespace seeks_plugins
{
#define hash_domain_name_weight       1333166351ul  /* "domain-name-weight" */
#define hash_record_cache_timeout     1954675964ul  /* "record-cache-timeout" */
#define hash_cf_peer                  1520012134ul  /* "cf-peer" */
#define hash_dead_peer_check          1043267473ul  /* "dead-peer-check" */
#define hash_dead_peer_retries         681362871ul  /* "dead-peer-retries" */
#define hash_post_url_check           3323226172ul  /* "post-url-check" */
#define hash_post_radius              2436628877ul  /* "post-radius" */
#define hash_post_ua                  1442804836ul  /* "post-ua" */
#define hash_stop_words_filtering     4002206625ul  /* "stop-words-filtering" */
#define hash_remote_post              4059800377ul  /* "remote-post" */
#define hash_use_http_url             1825269331ul  /* "use-http-urls-only" */
#define hash_cf_estimator             1689657696ul  /* "cf-estimator" */

  cf_configuration* cf_configuration::_config = NULL;

  cf_configuration::cf_configuration(const std::string &filename)
    :configuration_spec(filename)
  {
    // external static variables.
    _pl = new peer_list();
    _dpl = new peer_list();
    dead_peer::_pl = _pl;
    dead_peer::_dpl = _dpl;

    load_config();
  }

  cf_configuration::~cf_configuration()
  {
    // XXX: should mutex to avoid crash (this is only called when node is dying, for now).
    dead_peer::_pl = NULL;
    dead_peer::_dpl = NULL;
    delete _pl;

    // dead peers need to be unregistered from sweeper first.
    hash_map<const char*,peer*,hash<const char*>,eqstr>::iterator hit
    = _dpl->_peers.begin();
    while(hit!=_dpl->_peers.end())
      {
        dead_peer *dp = dynamic_cast<dead_peer*>((*hit).second);
        if (dp)
          sweeper::unregister_sweepable(dp);
        ++hit;
      }
    delete _dpl;
  }

  void cf_configuration::set_default_config()
  {
    _domain_name_weight = 0.7;
    _record_cache_timeout = 600; // 10 mins.
    _dead_peer_check = 300; // 5 mins.
    _dead_peer_retries = 3;
    _post_url_check = true;
    _post_radius = 5;
    _post_url_ua = "Mozilla/5.0 (X11; Linux x86_64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1"; // default.
    _stop_words_filtering = false;
    _remote_post = true;
    _use_http_urls = true;
    _estimator = "sre";
  }

  void cf_configuration::handle_config_cmd(char *cmd, const uint32_t &cmd_hash, char *arg,
      char *buf, const unsigned long &linenum)
  {
    char tmp[BUFFER_SIZE];
    int vec_count;
    char *vec[4];
    int port;
    std::vector<std::string> elts;
    std::string host, path, address, port_str;

    switch (cmd_hash)
      {
      case hash_domain_name_weight:
        _domain_name_weight = atof(arg);
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Weight given to the domain names in the simple filter");
        break;

      case hash_record_cache_timeout:
        _record_cache_timeout = atoi(arg);
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Timeout on cached remote records, in seconds");
        break;

      case hash_cf_peer:
        strlcpy(tmp,arg,sizeof(tmp));
        vec_count = miscutil::ssplit(tmp," \t",vec,SZ(vec),1,1);
        div_t divresult;
        divresult = div(vec_count,2);
        if (divresult.rem != 0)
          {
            errlog::log_error(LOG_LEVEL_ERROR,"Wrong number of parameter when specifying static collaborative filtering peer");
            break;
          }
        address = vec[0];
        urlmatch::parse_url_host_and_path(address,host,path);
        miscutil::tokenize(host,elts,":");
        port = -1;
        if (elts.size()>1)
          {
            host = elts.at(0);
            port = atoi(elts.at(1).c_str());
          }
        port_str = (port != -1) ? ":" + miscutil::to_string(port) : "";
        errlog::log_error(LOG_LEVEL_DEBUG,"adding peer %s%s%s with resource %s",
                          host.c_str(),port_str.c_str(),path.c_str(),vec[1]);
        _pl->add(host,port,path,std::string(vec[1]));
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Remote peer address for collaborative filtering");
        break;

      case hash_dead_peer_check:
        _dead_peer_check = atoi(arg);
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Interval of time between two dead peer checks");
        break;

      case hash_dead_peer_retries:
        _dead_peer_retries = atoi(arg);
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Number of retries before marking a peer as dead");
        break;

      case hash_post_url_check:
        _post_url_check = static_cast<bool>(atoi(arg));
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Whether to ping and check on posted URLs");
        break;

      case hash_post_radius:
        _post_radius = atoi(arg);
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Query similarity impact radius of posted URLs");
        break;

      case hash_post_ua:
        _post_url_ua = std::string(arg);
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "default 'user-agent' header used to retrieve posted URLs");
        break;

      case hash_stop_words_filtering:
        _stop_words_filtering = static_cast<bool>(atoi(arg));
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Whether to filter similar queries with stop words");
        break;

      case hash_remote_post:
        _remote_post = static_cast<bool>(atoi(arg));
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Whether to allow remote posting of results");
        break;

      case hash_use_http_url:
        _use_http_urls = static_cast<bool>(atoi(arg));
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Whether to allow only HTTP URLs or to allow generic item UIDs");
        break;

      case hash_cf_estimator:
        _estimator = arg;
        configuration_spec::html_table_row(_config_args,cmd,arg,
                                           "Estimator used by the collaborative filter");
        break;

      default:
        break;
      }
  }

  void cf_configuration::finalize_configuration()
  {
  }

} /* end of namespace. */
<|endoftext|>


<commit_before>//
// This file is part of the Marble Virtual Globe.
//
// This program is free software licensed under the GNU LGPL. You can
// find a copy of this license in LICENSE.txt in the top directory of
// the source code.
//
// Copyright 2011      Dennis Nienhüser <earthwings@gentoo.org>
//

#include "Routing.h"

#include "MarbleDeclarativeWidget.h"
#include "MarbleModel.h"
#include "MarbleDirs.h"
#include "routing/AlternativeRoutesModel.h"
#include "routing/RoutingManager.h"
#include "routing/RoutingModel.h"
#include "routing/RouteRequest.h"
#include "RouteRequestModel.h"
#include "routing/RoutingProfilesModel.h"

class RoutingPrivate
{
public:
    RoutingPrivate();

    MarbleWidget* m_marbleWidget;

    QAbstractItemModel* m_routeRequestModel;

    QMap<QString, Marble::RoutingProfile> m_profiles;

    QString m_routingProfile;
};

RoutingPrivate::RoutingPrivate() :
    m_marbleWidget( 0 ), m_routeRequestModel( 0 )
{
    // nothing to do
}

Routing::Routing( QObject* parent) :
    QObject( parent ), d( new RoutingPrivate )
{
    // nothing to do
}

Routing::~Routing()
{
    delete d;
}

QObject* Routing::routeRequestModel()
{
    if ( !d->m_routeRequestModel && d->m_marbleWidget ) {
        Marble::RouteRequest* request = d->m_marbleWidget->model()->routingManager()->routeRequest();
        d->m_routeRequestModel = new RouteRequestModel( request, this );
    }

    return d->m_routeRequestModel;
}

QObject* Routing::waypointModel()
{
    return d->m_marbleWidget ? d->m_marbleWidget->model()->routingManager()->routingModel() : 0;
}

void Routing::setMap( MarbleWidget* widget )
{
    d->m_marbleWidget = widget;

    if ( d->m_marbleWidget ) {
        connect( d->m_marbleWidget->model()->routingManager(), SIGNAL(stateChanged(RoutingManager::State)),
                 this, SIGNAL(hasRouteChanged()) );
        QList<Marble::RoutingProfile> profiles = d->m_marbleWidget->model()->routingManager()->profilesModel()->profiles();
        if ( profiles.size() == 4 ) {
            /** @todo FIXME: Restrictive assumptions on available plugins and certain profile loading implementation */
            d->m_profiles["Motorcar"] = profiles.at( 0 );
            d->m_profiles["Bicycle"] = profiles.at( 2 );
            d->m_profiles["Pedestrian"] = profiles.at( 3 );
        } else {
            qDebug() << "Unexpected size of default routing profiles: " << profiles.size();
        }
    }

    emit mapChanged();
}

MarbleWidget *Routing::map()
{
    return d->m_marbleWidget;
}

QString Routing::routingProfile() const
{
    return d->m_routingProfile;
}

void Routing::setRoutingProfile( const QString & profile )
{
    if ( d->m_routingProfile != profile ) {
        d->m_routingProfile = profile;
        if ( d->m_marbleWidget ) {
            d->m_marbleWidget->model()->routingManager()->routeRequest()->setRoutingProfile( d->m_profiles[profile] );
        }
        emit routingProfileChanged();
    }
}

bool Routing::hasRoute() const
{
    return d->m_marbleWidget && d->m_marbleWidget->model()->routingManager()->routingModel()->rowCount() > 0;
}

void Routing::addVia( qreal lon, qreal lat )
{
    if ( d->m_marbleWidget ) {
        Marble::RouteRequest* request = d->m_marbleWidget->model()->routingManager()->routeRequest();
        request->append( Marble::GeoDataCoordinates( lon, lat, 0.0, Marble::GeoDataCoordinates::Degree ) );
        updateRoute();
    }
}

void Routing::setVia( int index, qreal lon, qreal lat )
{
    if ( index < 0 || index > 200 || !d->m_marbleWidget ) {
        return;
    }

    Marble::RouteRequest* request = d->m_marbleWidget->model()->routingManager()->routeRequest();
    Q_ASSERT( request );
    if ( index < request->size() ) {
        request->setPosition( index, Marble::GeoDataCoordinates( lon, lat, 0.0, Marble::GeoDataCoordinates::Degree ) );
    } else {
        for ( int i=request->size(); i<index; ++i ) {
            request->append( Marble::GeoDataCoordinates( 0.0, 0.0 ) );
        }
        request->append( Marble::GeoDataCoordinates( lon, lat, 0.0, Marble::GeoDataCoordinates::Degree ) );
    }

    updateRoute();
}

void Routing::removeVia( int index )
{
    if ( index < 0 || !d->m_marbleWidget ) {
        return;
    }

    Marble::RouteRequest* request = d->m_marbleWidget->model()->routingManager()->routeRequest();
    if ( index < request->size() ) {
        d->m_marbleWidget->model()->routingManager()->routeRequest()->remove( index );
    }
}

void Routing::reverseRoute()
{
    if ( d->m_marbleWidget ) {
        d->m_marbleWidget->model()->routingManager()->reverseRoute();
    }
}

void Routing::clearRoute()
{
    if ( d->m_marbleWidget ) {
        d->m_marbleWidget->model()->routingManager()->clearRoute();
    }
}

void Routing::updateRoute()
{
    if ( d->m_marbleWidget ) {
        d->m_marbleWidget->model()->routingManager()->retrieveRoute();
    }
}

void Routing::openRoute( const QString &fileName )
{
    if ( d->m_marbleWidget ) {
        Marble::RoutingManager * const routingManager = d->m_marbleWidget->model()->routingManager();
        /** @todo FIXME: replace the file:// prefix on QML side */
        routingManager->clearRoute();
        QString target = fileName.startsWith( QLatin1String( "file://" ) ) ? fileName.mid( 7 ) : fileName;
        routingManager->loadRoute( target );
        Marble::GeoDataDocument* route = routingManager->alternativeRoutesModel()->currentRoute();
        if ( route ) {
            Marble::GeoDataLineString* waypoints = routingManager->alternativeRoutesModel()->waypoints( route );
            if ( waypoints ) {
                d->m_marbleWidget->centerOn( waypoints->latLonAltBox() );
            }
        }
    }
}

void Routing::saveRoute( const QString &fileName )
{
    if ( d->m_marbleWidget ) {
        /** @todo FIXME: replace the file:// prefix on QML side */
        QString target = fileName.startsWith( QLatin1String( "file://" ) ) ? fileName.mid( 7 ) : fileName;
        d->m_marbleWidget->model()->routingManager()->saveRoute( target );
    }
}

#include "Routing.moc"
<commit_msg>need to emit a few more signals when assigned MarbleWidget changes<commit_after>//
// This file is part of the Marble Virtual Globe.
//
// This program is free software licensed under the GNU LGPL. You can
// find a copy of this license in LICENSE.txt in the top directory of
// the source code.
//
// Copyright 2011      Dennis Nienhüser <earthwings@gentoo.org>
//

#include "Routing.h"

#include "MarbleDeclarativeWidget.h"
#include "MarbleModel.h"
#include "MarbleDirs.h"
#include "routing/AlternativeRoutesModel.h"
#include "routing/RoutingManager.h"
#include "routing/RoutingModel.h"
#include "routing/RouteRequest.h"
#include "RouteRequestModel.h"
#include "routing/RoutingProfilesModel.h"

class RoutingPrivate
{
public:
    RoutingPrivate();

    MarbleWidget* m_marbleWidget;

    QAbstractItemModel* m_routeRequestModel;

    QMap<QString, Marble::RoutingProfile> m_profiles;

    QString m_routingProfile;
};

RoutingPrivate::RoutingPrivate() :
    m_marbleWidget( 0 ), m_routeRequestModel( 0 )
{
    // nothing to do
}

Routing::Routing( QObject* parent) :
    QObject( parent ), d( new RoutingPrivate )
{
    // nothing to do
}

Routing::~Routing()
{
    delete d;
}

QObject* Routing::routeRequestModel()
{
    if ( !d->m_routeRequestModel && d->m_marbleWidget ) {
        Marble::RouteRequest* request = d->m_marbleWidget->model()->routingManager()->routeRequest();
        d->m_routeRequestModel = new RouteRequestModel( request, this );
    }

    return d->m_routeRequestModel;
}

QObject* Routing::waypointModel()
{
    return d->m_marbleWidget ? d->m_marbleWidget->model()->routingManager()->routingModel() : 0;
}

void Routing::setMap( MarbleWidget* widget )
{
    d->m_marbleWidget = widget;

    if ( d->m_marbleWidget ) {
        connect( d->m_marbleWidget->model()->routingManager(), SIGNAL(stateChanged(RoutingManager::State)),
                 this, SIGNAL(hasRouteChanged()) );
        QList<Marble::RoutingProfile> profiles = d->m_marbleWidget->model()->routingManager()->profilesModel()->profiles();
        if ( profiles.size() == 4 ) {
            /** @todo FIXME: Restrictive assumptions on available plugins and certain profile loading implementation */
            d->m_profiles["Motorcar"] = profiles.at( 0 );
            d->m_profiles["Bicycle"] = profiles.at( 2 );
            d->m_profiles["Pedestrian"] = profiles.at( 3 );
        } else {
            qDebug() << "Unexpected size of default routing profiles: " << profiles.size();
        }
    }

    emit mapChanged();
    emit routingProfileChanged();
    emit hasRouteChanged();
}

MarbleWidget *Routing::map()
{
    return d->m_marbleWidget;
}

QString Routing::routingProfile() const
{
    return d->m_routingProfile;
}

void Routing::setRoutingProfile( const QString & profile )
{
    if ( d->m_routingProfile != profile ) {
        d->m_routingProfile = profile;
        if ( d->m_marbleWidget ) {
            d->m_marbleWidget->model()->routingManager()->routeRequest()->setRoutingProfile( d->m_profiles[profile] );
        }
        emit routingProfileChanged();
    }
}

bool Routing::hasRoute() const
{
    return d->m_marbleWidget && d->m_marbleWidget->model()->routingManager()->routingModel()->rowCount() > 0;
}

void Routing::addVia( qreal lon, qreal lat )
{
    if ( d->m_marbleWidget ) {
        Marble::RouteRequest* request = d->m_marbleWidget->model()->routingManager()->routeRequest();
        request->append( Marble::GeoDataCoordinates( lon, lat, 0.0, Marble::GeoDataCoordinates::Degree ) );
        updateRoute();
    }
}

void Routing::setVia( int index, qreal lon, qreal lat )
{
    if ( index < 0 || index > 200 || !d->m_marbleWidget ) {
        return;
    }

    Marble::RouteRequest* request = d->m_marbleWidget->model()->routingManager()->routeRequest();
    Q_ASSERT( request );
    if ( index < request->size() ) {
        request->setPosition( index, Marble::GeoDataCoordinates( lon, lat, 0.0, Marble::GeoDataCoordinates::Degree ) );
    } else {
        for ( int i=request->size(); i<index; ++i ) {
            request->append( Marble::GeoDataCoordinates( 0.0, 0.0 ) );
        }
        request->append( Marble::GeoDataCoordinates( lon, lat, 0.0, Marble::GeoDataCoordinates::Degree ) );
    }

    updateRoute();
}

void Routing::removeVia( int index )
{
    if ( index < 0 || !d->m_marbleWidget ) {
        return;
    }

    Marble::RouteRequest* request = d->m_marbleWidget->model()->routingManager()->routeRequest();
    if ( index < request->size() ) {
        d->m_marbleWidget->model()->routingManager()->routeRequest()->remove( index );
    }
}

void Routing::reverseRoute()
{
    if ( d->m_marbleWidget ) {
        d->m_marbleWidget->model()->routingManager()->reverseRoute();
    }
}

void Routing::clearRoute()
{
    if ( d->m_marbleWidget ) {
        d->m_marbleWidget->model()->routingManager()->clearRoute();
    }
}

void Routing::updateRoute()
{
    if ( d->m_marbleWidget ) {
        d->m_marbleWidget->model()->routingManager()->retrieveRoute();
    }
}

void Routing::openRoute( const QString &fileName )
{
    if ( d->m_marbleWidget ) {
        Marble::RoutingManager * const routingManager = d->m_marbleWidget->model()->routingManager();
        /** @todo FIXME: replace the file:// prefix on QML side */
        routingManager->clearRoute();
        QString target = fileName.startsWith( QLatin1String( "file://" ) ) ? fileName.mid( 7 ) : fileName;
        routingManager->loadRoute( target );
        Marble::GeoDataDocument* route = routingManager->alternativeRoutesModel()->currentRoute();
        if ( route ) {
            Marble::GeoDataLineString* waypoints = routingManager->alternativeRoutesModel()->waypoints( route );
            if ( waypoints ) {
                d->m_marbleWidget->centerOn( waypoints->latLonAltBox() );
            }
        }
    }
}

void Routing::saveRoute( const QString &fileName )
{
    if ( d->m_marbleWidget ) {
        /** @todo FIXME: replace the file:// prefix on QML side */
        QString target = fileName.startsWith( QLatin1String( "file://" ) ) ? fileName.mid( 7 ) : fileName;
        d->m_marbleWidget->model()->routingManager()->saveRoute( target );
    }
}

#include "Routing.moc"
<|endoftext|>


<commit_before>#include "IRremote.h"
#include "IRremoteInt.h"

//==============================================================================
//                         AAA   IIIII  W   W   AAA
//                        A   A    I    W   W  A   A
//                        AAAAA    I    W W W  AAAAA
//                        A   A    I    W W W  A   A
//                        A   A  IIIII   WWW   A   A
//==============================================================================

// Baszed off the RC-T501 RCU
// Lirc file http://lirc.sourceforge.net/remotes/aiwa/RC-T501

#define AIWA_RC_T501_HZ            38
#define AIWA_RC_T501_BITS          15
#define AIWA_RC_T501_PRE_BITS      26
#define AIWA_RC_T501_POST_BITS      1
#define AIWA_RC_T501_SUM_BITS    (AIWA_RC_T501_PRE_BITS + AIWA_RC_T501_BITS + AIWA_RC_T501_POST_BITS)
#define AIWA_RC_T501_HDR_MARK    8800
#define AIWA_RC_T501_HDR_SPACE   4500
#define AIWA_RC_T501_BIT_MARK     500
#define AIWA_RC_T501_ONE_SPACE    600
#define AIWA_RC_T501_ZERO_SPACE  1700

//+=============================================================================
#if SEND_AIWA_RC_T501
void  IRsend::sendAiwaRCT501 (int code)
{
	unsigned long  pre = 0x0227EEC0;  // 26-bits

	// Set IR carrier frequency
	enableIROut(AIWA_RC_T501_HZ);

	// Header
	mark(AIWA_RC_T501_HDR_MARK);
	space(AIWA_RC_T501_HDR_SPACE);

	// Send "pre" data
	for (unsigned long  mask = 1UL << (26 - 1);  mask;  mask >>= 1) {
		mark(AIWA_RC_T501_BIT_MARK);
		if (pre & mask)  space(AIWA_RC_T501_ONE_SPACE) ;
		else             space(AIWA_RC_T501_ZERO_SPACE) ;
	}

//-v- THIS CODE LOOKS LIKE IT MIGHT BE WRONG - CHECK!
//    it only send 15bits and ignores the top bit
//    then uses TOPBIT which is 0x80000000 to check the bit code
//    I suspect TOPBIT should be changed to 0x00008000

	// Skip first code bit
	code <<= 1;
	// Send code
	for (int  i = 0;  i < 15;  i++) {
		mark(AIWA_RC_T501_BIT_MARK);
		if (code & 0x80000000)  space(AIWA_RC_T501_ONE_SPACE) ;
		else                    space(AIWA_RC_T501_ZERO_SPACE) ;
		code <<= 1;
	}

//-^- THIS CODE LOOKS LIKE IT MIGHT BE WRONG - CHECK!

	// POST-DATA, 1 bit, 0x0
	mark(AIWA_RC_T501_BIT_MARK);
	space(AIWA_RC_T501_ZERO_SPACE);

	mark(AIWA_RC_T501_BIT_MARK);
	space(0);
}
#endif

//+=============================================================================
#if DECODE_AIWA_RC_T501
bool  IRrecv::decodeAiwaRCT501 (decode_results *results)
{
	int  data   = 0;
	int  offset = 1;

	// Check SIZE
	if (irparams.rawlen < 2 * (AIWA_RC_T501_SUM_BITS) + 4)  return false ;

	// Check HDR Mark/Space
	if (!MATCH_MARK (results->rawbuf[offset++], AIWA_RC_T501_HDR_MARK ))  return false ;
	if (!MATCH_SPACE(results->rawbuf[offset++], AIWA_RC_T501_HDR_SPACE))  return false ;

	offset += 26;  // skip pre-data - optional
	while(offset < irparams.rawlen - 4) {
		if (MATCH_MARK(results->rawbuf[offset], AIWA_RC_T501_BIT_MARK))  offset++ ;
		else                                                             return false ;

		// ONE & ZERO
		if      (MATCH_SPACE(results->rawbuf[offset], AIWA_RC_T501_ONE_SPACE))   data = (data << 1) | 1 ;
		else if (MATCH_SPACE(results->rawbuf[offset], AIWA_RC_T501_ZERO_SPACE))  data = (data << 1) | 0 ;
		else                                                                     break ;  // End of one & zero detected
		offset++;
	}

	results->bits = (offset - 1) / 2;
	if (results->bits < 42)  return false ;

	results->value       = data;
	results->decode_type = AIWA_RC_T501;
	return true;
}
#endif
<commit_msg>Update ir_Aiwa.cpp<commit_after>#include "IRremote.h"
#include "IRremoteInt.h"

//==============================================================================
//                         AAA   IIIII  W   W   AAA
//                        A   A    I    W   W  A   A
//                        AAAAA    I    W W W  AAAAA
//                        A   A    I    W W W  A   A
//                        A   A  IIIII   WWW   A   A
//==============================================================================

// Based off the RC-T501 RCU
// Lirc file http://lirc.sourceforge.net/remotes/aiwa/RC-T501

#define AIWA_RC_T501_HZ            38
#define AIWA_RC_T501_BITS          15
#define AIWA_RC_T501_PRE_BITS      26
#define AIWA_RC_T501_POST_BITS      1
#define AIWA_RC_T501_SUM_BITS    (AIWA_RC_T501_PRE_BITS + AIWA_RC_T501_BITS + AIWA_RC_T501_POST_BITS)
#define AIWA_RC_T501_HDR_MARK    8800
#define AIWA_RC_T501_HDR_SPACE   4500
#define AIWA_RC_T501_BIT_MARK     500
#define AIWA_RC_T501_ONE_SPACE    600
#define AIWA_RC_T501_ZERO_SPACE  1700

//+=============================================================================
#if SEND_AIWA_RC_T501
void  IRsend::sendAiwaRCT501 (int code)
{
	unsigned long  pre = 0x0227EEC0;  // 26-bits

	// Set IR carrier frequency
	enableIROut(AIWA_RC_T501_HZ);

	// Header
	mark(AIWA_RC_T501_HDR_MARK);
	space(AIWA_RC_T501_HDR_SPACE);

	// Send "pre" data
	for (unsigned long  mask = 1UL << (26 - 1);  mask;  mask >>= 1) {
		mark(AIWA_RC_T501_BIT_MARK);
		if (pre & mask)  space(AIWA_RC_T501_ONE_SPACE) ;
		else             space(AIWA_RC_T501_ZERO_SPACE) ;
	}

//-v- THIS CODE LOOKS LIKE IT MIGHT BE WRONG - CHECK!
//    it only send 15bits and ignores the top bit
//    then uses TOPBIT which is 0x80000000 to check the bit code
//    I suspect TOPBIT should be changed to 0x00008000

	// Skip first code bit
	code <<= 1;
	// Send code
	for (int  i = 0;  i < 15;  i++) {
		mark(AIWA_RC_T501_BIT_MARK);
		if (code & 0x80000000)  space(AIWA_RC_T501_ONE_SPACE) ;
		else                    space(AIWA_RC_T501_ZERO_SPACE) ;
		code <<= 1;
	}

//-^- THIS CODE LOOKS LIKE IT MIGHT BE WRONG - CHECK!

	// POST-DATA, 1 bit, 0x0
	mark(AIWA_RC_T501_BIT_MARK);
	space(AIWA_RC_T501_ZERO_SPACE);

	mark(AIWA_RC_T501_BIT_MARK);
	space(0);
}
#endif

//+=============================================================================
#if DECODE_AIWA_RC_T501
bool  IRrecv::decodeAiwaRCT501 (decode_results *results)
{
	int  data   = 0;
	int  offset = 1;

	// Check SIZE
	if (irparams.rawlen < 2 * (AIWA_RC_T501_SUM_BITS) + 4)  return false ;

	// Check HDR Mark/Space
	if (!MATCH_MARK (results->rawbuf[offset++], AIWA_RC_T501_HDR_MARK ))  return false ;
	if (!MATCH_SPACE(results->rawbuf[offset++], AIWA_RC_T501_HDR_SPACE))  return false ;

	offset += 26;  // skip pre-data - optional
	while(offset < irparams.rawlen - 4) {
		if (MATCH_MARK(results->rawbuf[offset], AIWA_RC_T501_BIT_MARK))  offset++ ;
		else                                                             return false ;

		// ONE & ZERO
		if      (MATCH_SPACE(results->rawbuf[offset], AIWA_RC_T501_ONE_SPACE))   data = (data << 1) | 1 ;
		else if (MATCH_SPACE(results->rawbuf[offset], AIWA_RC_T501_ZERO_SPACE))  data = (data << 1) | 0 ;
		else                                                                     break ;  // End of one & zero detected
		offset++;
	}

	results->bits = (offset - 1) / 2;
	if (results->bits < 42)  return false ;

	results->value       = data;
	results->decode_type = AIWA_RC_T501;
	return true;
}
#endif
<|endoftext|>