<commit_before>/**
Copyright (c) 2015, Intel Corporation. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of the Intel Corporation nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#pragma once

#include <cstdlib>
#include <vector>
#include <string>

#include "MemoryPool.hpp"
#include "Utils.hpp"

namespace SpMP
{

class CSR {
public:
  int m;
  int n;
  int *rowptr;
  int *colidx;
  double *values;

  double *idiag; /**< inverse of diagonal elements */
  double *diag;
  int *diagptr;
  int *extptr; // points to the beginning of non-local columns (when MPI is used)

  CSR();

  // Following two constructors will make CSR own the data
  CSR(const char *file, int base = 0, bool forceSymmetric = false, int pad = 1);
  CSR(int m, int n, int nnz);
  CSR(const CSR& A);

  // Following constructor will make CSR does not own the data
  CSR(int m, int n, int *rowptr, int *colidx, double *values);

  ~CSR();

  void storeMatrixMarket(const char *fileName) const;
  /**
   * Load PETSc bin format
   */
  void loadBin(const char *fileName, int base = 0);
  /**
   * Load PETSc bin format
   */
  void storeBin(const char *fileName);

  /**
   * permutes current matrix based on permutation vector "perm"
   * and return new permuted matrix
   *
   * @param sort true if we want to sort non-zeros of each row based on colidx
   */
  CSR *permute(const int *columnPerm, const int *rowInversePerm, bool sort = false) const;
  /**
   * permutes rows but not columns
   */
  CSR *permuteRows(const int *inversePerm) const;
  /**
   * just permute rowptr
   */
  CSR *permuteRowptr(const int *inversePerm) const;
  void permuteRowptr(CSR *out, const int *inversePerm) const;

  void permuteColsInPlace(const int *perm);
  void permuteInPlaceIgnoringExtptr(const int *perm);

  /**
   * assuming rowptr is permuted, permute the remaining (colidx and values)
   *
   * @param sort true if we want to sort non-zeros of each row based on colidx
   */
  void permuteMain(
    CSR *out, const int *columnPerm, const int *rowInversePerm,
    bool sort = false) const;
  void permuteRowsMain(
    CSR *out, const int *inversePerm) const;

  /**
   * Compute w = alpha*A*x + beta*y + gamma
   * where A is this matrix
   */
  void multiplyWithVector(
    double *w,
    double alpha, const double *x, double beta, const double *y, double gamma)
    const;

  /**
   * Compute w = A*x
   */
  void multiplyWithVector(double *w, const double *x) const;

  /**
   * Compute W = alpha*A*X + beta*Y + gamma.
   * W, X, and Y are dense matrices with width k.
   */
  void multiplyWithDenseMatrix(
    double *W, int k, int wRowStride, int wColumnStride,
    double alpha,
    const double *X, int xRowStride, int xColumnStride,
    double beta, const double *Y, int yRowStride, int yColumnStride,
    double gamma) const;

  /**
   * Compute W = A*X.
   * W and X are dense matrices with width k, and are in column major
   */
  void multiplyWithDenseMatrix(double *W, int k, const double *X) const;

  /**
   * get reverse Cuthill Mckee permutation that tends to reduce the bandwidth
   *
   * @note only works for a symmetric matrix
   *
   * @param pseudoDiameterSourceSelection true to use heurstic of using a source
   *                                      in a pseudo diameter.
   *                                      Further reduce diameter at the expense
   *                                      of more time on finding permutation.
   */
  void getRCMPermutation(int *perm, int *inversePerm, bool pseudoDiameterSourceSelection = true);
  void getBFSPermutation(int *perm, int *inversePerm);

  CSR *transpose() const;

  bool isSymmetric(bool checkValues = true, bool printFirstNonSymmetry = false) const;

  void make0BasedIndexing();
  void make1BasedIndexing();

  /**
   * Precompute idiag to speedup triangular solver or GS
   */
  void computeInverseDiag();

  void alloc(int m, int nnz, bool createSeparateDiagData = true);
  void dealloc();

  bool useMemoryPool_() const;

  int getBandwidth() const;
  double getAverageWidth(bool sorted = true) const;
  bool equals(const CSR& A, bool print = false) const;
  int getNnz() const { return rowptr[m] - getBase(); }
  int getBase() const { return rowptr[0]; }

  void print() const;

  template<class T> T *allocate_(size_t n) const
  {
    if (useMemoryPool_()) {
      return MemoryPool::getSingleton()->allocate<T>(n);
    }
    else {
      return MALLOC(T, n);
    }
  }

private:
  bool ownData_;
}; // CSR

void generate3D27PtLaplacian(CSR *A, int nx, int ny, int nz, int base = 0);
void generate3D27PtLaplacian(CSR *A, int n, int base = 0);

void splitLU(CSR& A, CSR *L, CSR *U);
bool getSymmetricNnzPattern(
  const CSR *A, int **symRowPtr, int **symDiagPtr, int **symExtPtr, int **symColIdx);

} // namespace SpMP
<commit_msg>assume columns are not sorted by default when computing width<commit_after>/**
Copyright (c) 2015, Intel Corporation. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of the Intel Corporation nor the
      names of its contributors may be used to endorse or promote products
      derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/

#pragma once

#include <cstdlib>
#include <vector>
#include <string>

#include "MemoryPool.hpp"
#include "Utils.hpp"

namespace SpMP
{

class CSR {
public:
  int m;
  int n;
  int *rowptr;
  int *colidx;
  double *values;

  double *idiag; /**< inverse of diagonal elements */
  double *diag;
  int *diagptr;
  int *extptr; // points to the beginning of non-local columns (when MPI is used)

  CSR();

  // Following two constructors will make CSR own the data
  CSR(const char *file, int base = 0, bool forceSymmetric = false, int pad = 1);
  CSR(int m, int n, int nnz);
  CSR(const CSR& A);

  // Following constructor will make CSR does not own the data
  CSR(int m, int n, int *rowptr, int *colidx, double *values);

  ~CSR();

  void storeMatrixMarket(const char *fileName) const;
  /**
   * Load PETSc bin format
   */
  void loadBin(const char *fileName, int base = 0);
  /**
   * Load PETSc bin format
   */
  void storeBin(const char *fileName);

  /**
   * permutes current matrix based on permutation vector "perm"
   * and return new permuted matrix
   *
   * @param sort true if we want to sort non-zeros of each row based on colidx
   */
  CSR *permute(const int *columnPerm, const int *rowInversePerm, bool sort = false) const;
  /**
   * permutes rows but not columns
   */
  CSR *permuteRows(const int *inversePerm) const;
  /**
   * just permute rowptr
   */
  CSR *permuteRowptr(const int *inversePerm) const;
  void permuteRowptr(CSR *out, const int *inversePerm) const;

  void permuteColsInPlace(const int *perm);
  void permuteInPlaceIgnoringExtptr(const int *perm);

  /**
   * assuming rowptr is permuted, permute the remaining (colidx and values)
   *
   * @param sort true if we want to sort non-zeros of each row based on colidx
   */
  void permuteMain(
    CSR *out, const int *columnPerm, const int *rowInversePerm,
    bool sort = false) const;
  void permuteRowsMain(
    CSR *out, const int *inversePerm) const;

  /**
   * Compute w = alpha*A*x + beta*y + gamma
   * where A is this matrix
   */
  void multiplyWithVector(
    double *w,
    double alpha, const double *x, double beta, const double *y, double gamma)
    const;

  /**
   * Compute w = A*x
   */
  void multiplyWithVector(double *w, const double *x) const;

  /**
   * Compute W = alpha*A*X + beta*Y + gamma.
   * W, X, and Y are dense matrices with width k.
   */
  void multiplyWithDenseMatrix(
    double *W, int k, int wRowStride, int wColumnStride,
    double alpha,
    const double *X, int xRowStride, int xColumnStride,
    double beta, const double *Y, int yRowStride, int yColumnStride,
    double gamma) const;

  /**
   * Compute W = A*X.
   * W and X are dense matrices with width k, and are in column major
   */
  void multiplyWithDenseMatrix(double *W, int k, const double *X) const;

  /**
   * get reverse Cuthill Mckee permutation that tends to reduce the bandwidth
   *
   * @note only works for a symmetric matrix
   *
   * @param pseudoDiameterSourceSelection true to use heurstic of using a source
   *                                      in a pseudo diameter.
   *                                      Further reduce diameter at the expense
   *                                      of more time on finding permutation.
   */
  void getRCMPermutation(int *perm, int *inversePerm, bool pseudoDiameterSourceSelection = true);
  void getBFSPermutation(int *perm, int *inversePerm);

  CSR *transpose() const;

  bool isSymmetric(bool checkValues = true, bool printFirstNonSymmetry = false) const;

  void make0BasedIndexing();
  void make1BasedIndexing();

  /**
   * Precompute idiag to speedup triangular solver or GS
   */
  void computeInverseDiag();

  void alloc(int m, int nnz, bool createSeparateDiagData = true);
  void dealloc();

  bool useMemoryPool_() const;

  int getBandwidth() const;
  double getAverageWidth(bool sorted = false) const;
  bool equals(const CSR& A, bool print = false) const;
  int getNnz() const { return rowptr[m] - getBase(); }
  int getBase() const { return rowptr[0]; }

  void print() const;

  template<class T> T *allocate_(size_t n) const
  {
    if (useMemoryPool_()) {
      return MemoryPool::getSingleton()->allocate<T>(n);
    }
    else {
      return MALLOC(T, n);
    }
  }

private:
  bool ownData_;
}; // CSR

void generate3D27PtLaplacian(CSR *A, int nx, int ny, int nz, int base = 0);
void generate3D27PtLaplacian(CSR *A, int n, int base = 0);

void splitLU(CSR& A, CSR *L, CSR *U);
bool getSymmetricNnzPattern(
  const CSR *A, int **symRowPtr, int **symDiagPtr, int **symExtPtr, int **symColIdx);

} // namespace SpMP
<|endoftext|>


<commit_before>#include "vast/partition.h"

#include <cppa/cppa.hpp>
#include "vast/bitmap_index.h"
#include "vast/event.h"
#include "vast/segment.h"
#include "vast/io/serialization.h"

namespace vast {

path const partition::part_meta_file = "partition.meta";
path const partition::event_data_dir = "data";

namespace {

// If the given predicate is a name extractor, extracts the name.
struct name_checker : expr::default_const_visitor
{
  virtual void visit(expr::predicate const& pred)
  {
    pred.lhs().accept(*this);
    if (check_)
      pred.rhs().accept(*this);
  }

  virtual void visit(expr::name_extractor const&)
  {
    check_ = true;
  }

  virtual void visit(expr::constant const& c)
  {
    value_ = c.val;
  }

  bool check_ = false;
  value value_;
};

// Partitions the AST predicates into meta and data predicates. Moreover, it
// assigns each data predicate a name if a it coexists with name-extractor
// predicate that restricts the scope of the data predicate.
struct predicatizer : expr::default_const_visitor
{
  virtual void visit(expr::conjunction const& conj)
  {
    std::vector<expr::ast> flat;
    std::vector<expr::ast> deep;

    // Partition terms into leaves and others.
    for (auto& operand : conj.operands)
    {
      auto op = expr::ast{*operand};
      if (op.is_predicate())
        flat.push_back(std::move(op));
      else
        deep.push_back(std::move(op));
    }

    // Extract the name for all predicates in this conjunction.
    //
    // TODO: add a check to the semantic pass after constructing queries to
    // flag multiple name/offset/time extractors in the same conjunction.
    name_.clear();
    for (auto& pred : flat)
    {
      name_checker checker;
      pred.accept(checker);
      if (checker.check_)
        name_ = checker.value_.get<string>();
    }

    for (auto& pred : flat)
      pred.accept(*this);

    // Proceed with the remaining conjunctions deeper in the tree.
    name_.clear();
    for (auto& n : deep)
      n.accept(*this);
  }

  virtual void visit(expr::disjunction const& disj)
  {
    for (auto& operand : disj.operands)
      operand->accept(*this);
  }

  virtual void visit(expr::predicate const& pred)
  {
    if (! expr::ast{pred}.is_meta_predicate())
      data_predicates_.emplace(name_, pred);
    else
      meta_predicates_.push_back(pred);
  }

  string name_;
  std::multimap<string, expr::ast> data_predicates_;
  std::vector<expr::ast> meta_predicates_;
};

} // namespace <anonymous>

partition::meta_data::meta_data(uuid id)
  : id{id}
{
}

void partition::meta_data::update(segment const& s)
{
  if (first_event == time_range{} || s.first() < first_event)
    first_event = s.first();

  if (last_event == time_range{} || s.last() > last_event)
    last_event = s.last();

  last_modified = now();

  assert(s.coverage());
  coverage |= *s.coverage();
}

void partition::meta_data::serialize(serializer& sink) const
{
  sink << id << first_event << last_event << last_modified << coverage;
}

void partition::meta_data::deserialize(deserializer& source)
{
  source >> id >> first_event >> last_event >> last_modified >> coverage;
}

bool operator==(partition::meta_data const& x, partition::meta_data const& y)
{
  return x.id == y.id
      && x.first_event == y.first_event
      && x.last_event == y.last_event
      && x.last_modified == y.last_modified
      && x.coverage == y.coverage;
}

partition::partition(uuid id)
  : meta_{std::move(id)}
{
}

void partition::update(segment const& s)
{
  meta_.update(s);
}

partition::meta_data const& partition::meta() const
{
  return meta_;
}

void partition::serialize(serializer& sink) const
{
  sink << meta_;
}

void partition::deserialize(deserializer& source)
{
  source >> meta_;
}

using namespace cppa;

namespace {

struct dispatcher : expr::default_const_visitor
{
  dispatcher(partition_actor& pa)
    : actor_{pa}
  {
  }

  virtual void visit(expr::predicate const& pred)
  {
    pred.lhs().accept(*this);
  }

  virtual void visit(expr::timestamp_extractor const&)
  {
    indexes_.push_back(actor_.time_indexer_);
  }

  virtual void visit(expr::name_extractor const&)
  {
    indexes_.push_back(actor_.name_indexer_);
  }

  virtual void visit(expr::type_extractor const& te)
  {
    for (auto& p0 : actor_.indexers_)
      for (auto& p1 : p0.second)
        if (p1.second.type == te.type)
        {
          if (p1.second.actor)
          {
            indexes_.push_back(p1.second.actor);
          }
          else
          {
            auto i = actor_.load_indexer(p0.first, p1.first);
            if (i.failed())
              VAST_LOG_ERROR(i.failure().msg());
            else if (i.empty())
              assert(! "file system inconsistency: index must exist");
            else
              indexes_.push_back(*i);
          }
        }
  }

  virtual void visit(expr::offset_extractor const& oe)
  {
    // TODO: restrict to the right name rather than going through all indexers.
    for (auto& p0 : actor_.indexers_)
    {
      auto i = p0.second.find(oe.off);
      if (i == p0.second.end())
      {
        VAST_LOG_WARN("no index for offset " << oe.off);
      }
      else if (! i->second.actor)
      {
        auto a = actor_.load_indexer(p0.first, oe.off);
        if (a.failed())
          VAST_LOG_ERROR(a.failure().msg());
        else if (a.empty())
          assert(! "file system inconsistency: index must exist");
        else
          indexes_.push_back(*a);
      }
      else
      {
        indexes_.push_back(i->second.actor);
      }
    }
  }

  partition_actor& actor_;
  std::vector<actor_ptr> indexes_;
};

} // namespace <anonymous>

partition_actor::partition_actor(path dir, size_t batch_size, uuid id)
  : dir_{std::move(dir)},
    batch_size_{batch_size},
    partition_{std::move(id)}
{
}

void partition_actor::act()
{
  chaining(false);
  trap_exit(true);

  if (exists(dir_))
  {
    if (! io::unarchive(dir_ / partition::part_meta_file, partition_))
    {
      VAST_LOG_ACTOR_ERROR("failed to load partition meta data from " << dir_);
      quit(exit::error);
      return;
    }

    std::map<string, std::map<offset, value_type>> types;
    if (! io::unarchive(dir_ / "types", types))
    {
      VAST_LOG_ACTOR_ERROR("failed to load type data from " << dir_ / "types");
      quit(exit::error);
      return;
    }

    for (auto& p0 : types)
      for (auto& p1 : p0.second)
        indexers_[p0.first][p1.first].type = p1.second;
  }

  traverse(
      dir_ / partition::event_data_dir,
      [&](path const& event_dir) -> bool
      {
        if (! event_dir.is_directory())
        {
          VAST_LOG_ACTOR_WARN("found unrecognized file in " << event_dir);
          return true;
        }

        auto event_name = event_dir.basename().str();
        traverse(
            event_dir,
            [&](path const& idx_file) -> bool
            {
              if (idx_file.extension() != ".idx")
                return true;

              VAST_LOG_ACTOR_DEBUG("found index " << idx_file);
              auto str = idx_file.basename(true).str();
              auto start = str.begin();
              offset o;
              if (! extract(start, str.end(), o))
              {
                VAST_LOG_ACTOR_ERROR("got invalid offset in " << idx_file);
                quit(exit::error);
                return false;
              }

              if (! indexers_[event_name].count(o))
              {
                VAST_LOG_ACTOR_ERROR("has no meta data for " << idx_file);
                quit(exit::error);
                return false;
              }

              return true;
            });

        return true;
      });

  auto submit_meta = [=](std::vector<cow<event>> events)
  {
    if (! name_indexer_)
    {
      auto p = dir_ / "name.idx";
      name_indexer_ =
        spawn<event_name_indexer<default_bitstream>>(std::move(p));
    }

    if (! time_indexer_)
    {
      auto p = dir_ / "time.idx";
      time_indexer_ =
        spawn<event_time_indexer<default_bitstream>>(std::move(p));
    }

    auto t = make_any_tuple(std::move(events));
    name_indexer_ << t;
    time_indexer_ << t;
  };

  auto submit_data = [=](event const& e, std::vector<cow<event>> events)
  {
    auto t = make_any_tuple(std::move(events));

    e.each_offset(
        [&](value const& v, offset const& o)
        {
          // We can't handle container types (yet).
          if (is_container_type(v.which()))
            return;

          actor_ptr indexer;

          auto i = load_indexer(e.name(), o);
          if (i.failed())
          {
            VAST_LOG_ACTOR_ERROR(i.failure().msg());
            quit(exit::error);
            return;
          }
          else if (i.empty())
          {
            auto a = create_indexer(e.name(), o, v.which());
            if (! a)
            {
              VAST_LOG_ACTOR_ERROR(a.failure().msg());
              quit(exit::error);
              return;
            }

            indexer = *a;
          }
          else
          {
            indexer = *i;
          }

          indexer << t;
        });
  };

  auto flush = [=]
  {
    std::map<string, std::map<offset, value_type>> types;

    send(name_indexer_, atom("flush"));
    send(time_indexer_, atom("flush"));
    for (auto& p0 : indexers_)
      for (auto& p1 : p0.second)
      {
        types[p0.first][p1.first] = p1.second.type;

        if (p1.second.actor)
          send(p1.second.actor, atom("flush"));
      }

    if (! io::archive(dir_ / "types", types))
    {
      VAST_LOG_ACTOR_ERROR("failed to save type data for " << dir_);
      quit(exit::error);
      return;
    }

    if (! io::archive(dir_ / partition::part_meta_file, partition_))
    {
      VAST_LOG_ACTOR_ERROR("failed to save partition " << dir_);
      quit(exit::error);
      return;
    }
  };

  become(
      on(atom("EXIT"), arg_match) >> [=](uint32_t reason)
      {
        if (reason != exit::kill)
          if (partition_.meta().coverage)
            if (! io::archive(dir_ / partition::part_meta_file, partition_))
            {
              VAST_LOG_ACTOR_ERROR("failed to save partition " << dir_);
              quit(exit::error);
              return;
            }

        flush();

        send_exit(time_indexer_, reason);
        send_exit(name_indexer_, reason);
        for (auto& p0 : indexers_)
          for (auto& p1 : p0.second)
            if (p1.second.actor)
              send_exit(p1.second.actor, reason);

        quit(reason);
      },
      on(atom("DOWN"), arg_match) >> [=](uint32_t /* reason */)
      {
        VAST_LOG_ACTOR_DEBUG("got DOWN from " << VAST_ACTOR_ID(last_sender()));

        for (auto& p0 : indexers_)
          for (auto& p1 : p0.second)
            if (p1.second.actor == last_sender())
              p1.second.actor = {};

        stats_.erase(last_sender());
      },
      on_arg_match >> [=](expr::ast const& pred, actor_ptr const& sink)
      {
        VAST_LOG_ACTOR_DEBUG("got predicate " << pred <<
                             " for " << VAST_ACTOR_ID(sink));

        dispatcher d{*this};
        pred.accept(d);

        uint64_t n = d.indexes_.size();
        send(last_sender(), pred, partition_.meta().id, n);

        auto t = make_any_tuple(pred, partition_.meta().id, sink);
        for (auto& a : d.indexes_)
          a << t;
      },
      on(atom("flush")) >> flush,
      on_arg_match >> [=](segment const& s)
      {
        VAST_LOG_ACTOR_DEBUG(
            "processes " << s.events() << " events from segment " << s.id());

        std::vector<cow<event>> all_events;
        all_events.reserve(batch_size_);
        std::unordered_map<string, std::vector<cow<event>>> groups;

        segment::reader r{&s};
        while (auto ev = r.read())
        {
          assert(ev);
          cow<event> e{std::move(*ev)};

          all_events.push_back(e);
          if (all_events.size() == batch_size_)
          {
            submit_meta(std::move(all_events));
            all_events = {};
          }

          auto& group = groups[e->name()];
          group.push_back(e);
          if (group.size() == batch_size_)
          {
            submit_data(*e, std::move(group));
            group = {};
          }
        }

        // Send away final events.
        submit_meta(std::move(all_events));
        for (auto& p : groups)
          if (! p.second.empty())
          {
            auto first = p.second[0];
            submit_data(*first, std::move(p.second));
          }

        // Record segment.
        partition_.update(s);

        // Flush partition meta data and data indexes.
        flush();
        send(self, atom("stats"), atom("show"));
      },
      on(atom("stats"), arg_match) >> [=](uint64_t n, uint64_t rate, uint64_t mean)
      {
        auto& s = stats_[last_sender()];
        s.values += n;
        s.rate = rate;
        s.mean = mean;
      },
      on(atom("stats"), atom("show")) >> [=]
      {
        uint64_t total_values = 0;
        uint64_t total_rate = 0;
        uint64_t total_mean = 0;
        for (auto& p : stats_)
        {
          total_values += p.second.values;
          total_rate += p.second.rate;
          total_mean += p.second.mean;
        }

        if (total_rate > 0)
          VAST_LOG_ACTOR_INFO(
              "indexed " << total_values << " values at rate " <<
              total_rate << " values/sec" << " (mean " << total_mean << ')');
      });
}

char const* partition_actor::description() const
{
  return "partition";
}

} // namespace vast
<commit_msg>Avoid redundant flushing of state.<commit_after>#include "vast/partition.h"

#include <cppa/cppa.hpp>
#include "vast/bitmap_index.h"
#include "vast/event.h"
#include "vast/segment.h"
#include "vast/io/serialization.h"

namespace vast {

path const partition::part_meta_file = "partition.meta";
path const partition::event_data_dir = "data";

namespace {

// If the given predicate is a name extractor, extracts the name.
struct name_checker : expr::default_const_visitor
{
  virtual void visit(expr::predicate const& pred)
  {
    pred.lhs().accept(*this);
    if (check_)
      pred.rhs().accept(*this);
  }

  virtual void visit(expr::name_extractor const&)
  {
    check_ = true;
  }

  virtual void visit(expr::constant const& c)
  {
    value_ = c.val;
  }

  bool check_ = false;
  value value_;
};

// Partitions the AST predicates into meta and data predicates. Moreover, it
// assigns each data predicate a name if a it coexists with name-extractor
// predicate that restricts the scope of the data predicate.
struct predicatizer : expr::default_const_visitor
{
  virtual void visit(expr::conjunction const& conj)
  {
    std::vector<expr::ast> flat;
    std::vector<expr::ast> deep;

    // Partition terms into leaves and others.
    for (auto& operand : conj.operands)
    {
      auto op = expr::ast{*operand};
      if (op.is_predicate())
        flat.push_back(std::move(op));
      else
        deep.push_back(std::move(op));
    }

    // Extract the name for all predicates in this conjunction.
    //
    // TODO: add a check to the semantic pass after constructing queries to
    // flag multiple name/offset/time extractors in the same conjunction.
    name_.clear();
    for (auto& pred : flat)
    {
      name_checker checker;
      pred.accept(checker);
      if (checker.check_)
        name_ = checker.value_.get<string>();
    }

    for (auto& pred : flat)
      pred.accept(*this);

    // Proceed with the remaining conjunctions deeper in the tree.
    name_.clear();
    for (auto& n : deep)
      n.accept(*this);
  }

  virtual void visit(expr::disjunction const& disj)
  {
    for (auto& operand : disj.operands)
      operand->accept(*this);
  }

  virtual void visit(expr::predicate const& pred)
  {
    if (! expr::ast{pred}.is_meta_predicate())
      data_predicates_.emplace(name_, pred);
    else
      meta_predicates_.push_back(pred);
  }

  string name_;
  std::multimap<string, expr::ast> data_predicates_;
  std::vector<expr::ast> meta_predicates_;
};

} // namespace <anonymous>

partition::meta_data::meta_data(uuid id)
  : id{id}
{
}

void partition::meta_data::update(segment const& s)
{
  if (first_event == time_range{} || s.first() < first_event)
    first_event = s.first();

  if (last_event == time_range{} || s.last() > last_event)
    last_event = s.last();

  last_modified = now();

  assert(s.coverage());
  coverage |= *s.coverage();
}

void partition::meta_data::serialize(serializer& sink) const
{
  sink << id << first_event << last_event << last_modified << coverage;
}

void partition::meta_data::deserialize(deserializer& source)
{
  source >> id >> first_event >> last_event >> last_modified >> coverage;
}

bool operator==(partition::meta_data const& x, partition::meta_data const& y)
{
  return x.id == y.id
      && x.first_event == y.first_event
      && x.last_event == y.last_event
      && x.last_modified == y.last_modified
      && x.coverage == y.coverage;
}

partition::partition(uuid id)
  : meta_{std::move(id)}
{
}

void partition::update(segment const& s)
{
  meta_.update(s);
}

partition::meta_data const& partition::meta() const
{
  return meta_;
}

void partition::serialize(serializer& sink) const
{
  sink << meta_;
}

void partition::deserialize(deserializer& source)
{
  source >> meta_;
}

using namespace cppa;

namespace {

struct dispatcher : expr::default_const_visitor
{
  dispatcher(partition_actor& pa)
    : actor_{pa}
  {
  }

  virtual void visit(expr::predicate const& pred)
  {
    pred.lhs().accept(*this);
  }

  virtual void visit(expr::timestamp_extractor const&)
  {
    indexes_.push_back(actor_.time_indexer_);
  }

  virtual void visit(expr::name_extractor const&)
  {
    indexes_.push_back(actor_.name_indexer_);
  }

  virtual void visit(expr::type_extractor const& te)
  {
    for (auto& p0 : actor_.indexers_)
      for (auto& p1 : p0.second)
        if (p1.second.type == te.type)
        {
          if (p1.second.actor)
          {
            indexes_.push_back(p1.second.actor);
          }
          else
          {
            auto i = actor_.load_indexer(p0.first, p1.first);
            if (i.failed())
              VAST_LOG_ERROR(i.failure().msg());
            else if (i.empty())
              assert(! "file system inconsistency: index must exist");
            else
              indexes_.push_back(*i);
          }
        }
  }

  virtual void visit(expr::offset_extractor const& oe)
  {
    // TODO: restrict to the right name rather than going through all indexers.
    for (auto& p0 : actor_.indexers_)
    {
      auto i = p0.second.find(oe.off);
      if (i == p0.second.end())
      {
        VAST_LOG_WARN("no index for offset " << oe.off);
      }
      else if (! i->second.actor)
      {
        auto a = actor_.load_indexer(p0.first, oe.off);
        if (a.failed())
          VAST_LOG_ERROR(a.failure().msg());
        else if (a.empty())
          assert(! "file system inconsistency: index must exist");
        else
          indexes_.push_back(*a);
      }
      else
      {
        indexes_.push_back(i->second.actor);
      }
    }
  }

  partition_actor& actor_;
  std::vector<actor_ptr> indexes_;
};

} // namespace <anonymous>

partition_actor::partition_actor(path dir, size_t batch_size, uuid id)
  : dir_{std::move(dir)},
    batch_size_{batch_size},
    partition_{std::move(id)}
{
}

void partition_actor::act()
{
  chaining(false);
  trap_exit(true);

  if (exists(dir_))
  {
    if (! io::unarchive(dir_ / partition::part_meta_file, partition_))
    {
      VAST_LOG_ACTOR_ERROR("failed to load partition meta data from " << dir_);
      quit(exit::error);
      return;
    }

    std::map<string, std::map<offset, value_type>> types;
    if (! io::unarchive(dir_ / "types", types))
    {
      VAST_LOG_ACTOR_ERROR("failed to load type data from " << dir_ / "types");
      quit(exit::error);
      return;
    }

    for (auto& p0 : types)
      for (auto& p1 : p0.second)
        indexers_[p0.first][p1.first].type = p1.second;
  }

  traverse(
      dir_ / partition::event_data_dir,
      [&](path const& event_dir) -> bool
      {
        if (! event_dir.is_directory())
        {
          VAST_LOG_ACTOR_WARN("found unrecognized file in " << event_dir);
          return true;
        }

        auto event_name = event_dir.basename().str();
        traverse(
            event_dir,
            [&](path const& idx_file) -> bool
            {
              if (idx_file.extension() != ".idx")
                return true;

              VAST_LOG_ACTOR_DEBUG("found index " << idx_file);
              auto str = idx_file.basename(true).str();
              auto start = str.begin();
              offset o;
              if (! extract(start, str.end(), o))
              {
                VAST_LOG_ACTOR_ERROR("got invalid offset in " << idx_file);
                quit(exit::error);
                return false;
              }

              if (! indexers_[event_name].count(o))
              {
                VAST_LOG_ACTOR_ERROR("has no meta data for " << idx_file);
                quit(exit::error);
                return false;
              }

              return true;
            });

        return true;
      });

  auto submit_meta = [=](std::vector<cow<event>> events)
  {
    if (! name_indexer_)
    {
      auto p = dir_ / "name.idx";
      name_indexer_ =
        spawn<event_name_indexer<default_bitstream>>(std::move(p));
    }

    if (! time_indexer_)
    {
      auto p = dir_ / "time.idx";
      time_indexer_ =
        spawn<event_time_indexer<default_bitstream>>(std::move(p));
    }

    auto t = make_any_tuple(std::move(events));
    name_indexer_ << t;
    time_indexer_ << t;
  };

  auto submit_data = [=](event const& e, std::vector<cow<event>> events)
  {
    auto t = make_any_tuple(std::move(events));

    e.each_offset(
        [&](value const& v, offset const& o)
        {
          // We can't handle container types (yet).
          if (is_container_type(v.which()))
            return;

          actor_ptr indexer;

          auto i = load_indexer(e.name(), o);
          if (i.failed())
          {
            VAST_LOG_ACTOR_ERROR(i.failure().msg());
            quit(exit::error);
            return;
          }
          else if (i.empty())
          {
            auto a = create_indexer(e.name(), o, v.which());
            if (! a)
            {
              VAST_LOG_ACTOR_ERROR(a.failure().msg());
              quit(exit::error);
              return;
            }

            indexer = *a;
          }
          else
          {
            indexer = *i;
          }

          indexer << t;
        });
  };

  auto flush = [=]
  {
    std::map<string, std::map<offset, value_type>> types;

    send(name_indexer_, atom("flush"));
    send(time_indexer_, atom("flush"));
    for (auto& p0 : indexers_)
      for (auto& p1 : p0.second)
      {
        types[p0.first][p1.first] = p1.second.type;

        if (p1.second.actor)
          send(p1.second.actor, atom("flush"));
      }

    if (! io::archive(dir_ / "types", types))
    {
      VAST_LOG_ACTOR_ERROR("failed to save type data for " << dir_);
      quit(exit::error);
      return;
    }

    if (! io::archive(dir_ / partition::part_meta_file, partition_))
    {
      VAST_LOG_ACTOR_ERROR("failed to save partition " << dir_);
      quit(exit::error);
      return;
    }
  };

  become(
      on(atom("EXIT"), arg_match) >> [=](uint32_t reason)
      {
        if (reason != exit::kill)
          flush();

        send_exit(time_indexer_, reason);
        send_exit(name_indexer_, reason);
        for (auto& p0 : indexers_)
          for (auto& p1 : p0.second)
            if (p1.second.actor)
              send_exit(p1.second.actor, reason);

        quit(reason);
      },
      on(atom("DOWN"), arg_match) >> [=](uint32_t /* reason */)
      {
        VAST_LOG_ACTOR_DEBUG("got DOWN from " << VAST_ACTOR_ID(last_sender()));

        for (auto& p0 : indexers_)
          for (auto& p1 : p0.second)
            if (p1.second.actor == last_sender())
              p1.second.actor = {};

        stats_.erase(last_sender());
      },
      on_arg_match >> [=](expr::ast const& pred, actor_ptr const& sink)
      {
        VAST_LOG_ACTOR_DEBUG("got predicate " << pred <<
                             " for " << VAST_ACTOR_ID(sink));

        dispatcher d{*this};
        pred.accept(d);

        uint64_t n = d.indexes_.size();
        send(last_sender(), pred, partition_.meta().id, n);

        auto t = make_any_tuple(pred, partition_.meta().id, sink);
        for (auto& a : d.indexes_)
          a << t;
      },
      on(atom("flush")) >> flush,
      on_arg_match >> [=](segment const& s)
      {
        VAST_LOG_ACTOR_DEBUG(
            "processes " << s.events() << " events from segment " << s.id());

        std::vector<cow<event>> all_events;
        all_events.reserve(batch_size_);
        std::unordered_map<string, std::vector<cow<event>>> groups;

        segment::reader r{&s};
        while (auto ev = r.read())
        {
          assert(ev);
          cow<event> e{std::move(*ev)};

          all_events.push_back(e);
          if (all_events.size() == batch_size_)
          {
            submit_meta(std::move(all_events));
            all_events = {};
          }

          auto& group = groups[e->name()];
          group.push_back(e);
          if (group.size() == batch_size_)
          {
            submit_data(*e, std::move(group));
            group = {};
          }
        }

        // Send away final events.
        submit_meta(std::move(all_events));
        for (auto& p : groups)
          if (! p.second.empty())
          {
            auto first = p.second[0];
            submit_data(*first, std::move(p.second));
          }

        // Record segment.
        partition_.update(s);

        // Flush partition meta data and data indexes.
        flush();
        send(self, atom("stats"), atom("show"));
      },
      on(atom("stats"), arg_match) >> [=](uint64_t n, uint64_t rate, uint64_t mean)
      {
        auto& s = stats_[last_sender()];
        s.values += n;
        s.rate = rate;
        s.mean = mean;
      },
      on(atom("stats"), atom("show")) >> [=]
      {
        uint64_t total_values = 0;
        uint64_t total_rate = 0;
        uint64_t total_mean = 0;
        for (auto& p : stats_)
        {
          total_values += p.second.values;
          total_rate += p.second.rate;
          total_mean += p.second.mean;
        }

        if (total_rate > 0)
          VAST_LOG_ACTOR_INFO(
              "indexed " << total_values << " values at rate " <<
              total_rate << " values/sec" << " (mean " << total_mean << ')');
      });
}

char const* partition_actor::description() const
{
  return "partition";
}

} // namespace vast
<|endoftext|>


<commit_before>/*
*
* Copyright 2013 Telefonica Investigacion y Desarrollo, S.A.U
*
* This file is part of Orion Context Broker.
*
* Orion Context Broker is free software: you can redistribute it and/or
* modify it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Orion Context Broker is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
* General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.
*
* For those usages not covered by this license please contact with
* iot_support at tid dot es
*
* Author: developer
*/

#include <string.h>             /* strcmp                                    */

#include "parseArgs/paIsSet.h"  /* Own interface                             */

/* ****************************************************************************
*
* paIsSet - is an argument existing in the parse list
*/
bool paIsSet(int argC, char* argV[], const char* option)
{
  int i;

  for (i = 1; i < argC; i++)
  {
    if (strcmp(argV[i], option) == 0)
    {
      return true;
    }
  }

  return false;
}

/* ****************************************************************************
*
* paIsSetSoGet - return value of option 'option'
*/
const char* paIsSetSoGet(int argC, char* argV[], const char* option)
{
  int i;

  for (i = 1; i < argC; i++)
  {
    if (strcmp(argV[i], option) == 0)
    {
      return argV[i + 1];
    }
  }

  return NULL;
}
<commit_msg>further extend logging<commit_after>/*
*
* Copyright 2013 Telefonica Investigacion y Desarrollo, S.A.U
*
* This file is part of Orion Context Broker.
*
* Orion Context Broker is free software: you can redistribute it and/or
* modify it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* Orion Context Broker is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero
* General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with Orion Context Broker. If not, see http://www.gnu.org/licenses/.
*
* For those usages not covered by this license please contact with
* iot_support at tid dot es
*
* Author: developer
*/

#include <string.h>             /* strcmp                                    */

#include "parseArgs/paIsSet.h"  /* Own interface                             */

/* ****************************************************************************
*
* paIsSet - is an argument existing in the parse list
*/
bool paIsSet(int argC, char* argV[], const char* option)
{
  int i;

  for (i = 1; i < argC; i++)
  {
    LM_I(argV[i])
    if (strcmp(argV[i], option) == 0)
    {
      return true;
    }
  }

  return false;
}

/* ****************************************************************************
*
* paIsSetSoGet - return value of option 'option'
*/
const char* paIsSetSoGet(int argC, char* argV[], const char* option)
{
  int i;

  for (i = 1; i < argC; i++)
  {
    if (strcmp(argV[i], option) == 0)
    {
      return argV[i + 1];
    }
  }

  return NULL;
}
<|endoftext|>


<commit_before>/*
 * Launch and manage WAS child processes.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "was_stock.hxx"
#include "was_quark.h"
#include "was_launch.hxx"
#include "stock/MapStock.hxx"
#include "stock/Stock.hxx"
#include "stock/Class.hxx"
#include "stock/Item.hxx"
#include "net/ConnectSocket.hxx"
#include "spawn/ChildOptions.hxx"
#include "spawn/ExitListener.hxx"
#include "spawn/Interface.hxx"
#include "pool.hxx"
#include "event/SocketEvent.hxx"
#include "util/Cancellable.hxx"
#include "util/ConstBuffer.hxx"
#include "util/Cast.hxx"

#include <was/protocol.h>
#include <daemon/log.h>

#include <glib.h>
#include <assert.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <signal.h>
#include <stdlib.h>

static constexpr struct timeval was_idle_timeout = {
    .tv_sec = 300,
    .tv_usec = 0,
};

struct WasChildParams {
    const char *executable_path;

    ConstBuffer<const char *> args;

    const ChildOptions &options;

    WasChildParams(const char *_executable_path,
                   ConstBuffer<const char *> _args,
                   const ChildOptions &_options)
        :executable_path(_executable_path), args(_args),
         options(_options) {}

    const char *GetStockKey(struct pool &pool) const;
};

class WasChild final : public HeapStockItem, ExitListener {
    SpawnService &spawn_service;

    WasProcess process;
    SocketEvent event;

    /**
     * If true, then we're waiting for PREMATURE (after the #WasClient
     * has sent #WAS_COMMAND_STOP).
     */
    bool stopping = false;

    /**
     * The number of bytes received before #WAS_COMMAND_STOP was sent.
     */
    uint64_t input_received;

public:
    explicit WasChild(CreateStockItem c, SpawnService &_spawn_service)
        :HeapStockItem(c), spawn_service(_spawn_service),
         event(c.stock.GetEventLoop(), BIND_THIS_METHOD(EventCallback)) {}

    ~WasChild() override;

    bool Launch(const WasChildParams &params, GError **error_r) {
        process = was_launch(spawn_service,
                             GetStockName(),
                             params.executable_path,
                             params.args,
                             params.options,
                             this,
                             error_r);
        if (!process.IsDefined()) {
            /* explicitly Close() the WasProcess, so our destructor
               doesn't believe we have already initialized the
               Event */
            process.Close();
            return false;
        }

        event.Set(process.control.Get(), EV_READ|EV_TIMEOUT);
        return true;
    }

    const WasProcess &GetProcess() const {
        return process;
    }

    void Stop(uint64_t _received) {
        assert(!is_idle);
        assert(!stopping);

        stopping = true;
        input_received = _received;
    }

private:
    enum class ReceiveResult {
        SUCCESS, ERROR, AGAIN,
    };

    /**
     * Receive data on the control channel.
     *
     * @return true on success
     */
    ReceiveResult ReceiveControl(void *p, size_t size);

    /**
     * Discard the given amount of data from the input pipe.
     *
     * @return true on success
     */
    bool DiscardInput(uint64_t remaining);

    /**
     * Attempt to recover after the WAS client sent STOP to the
     * application.  This method waits for PREMATURE and discards
     * excess data from the pipe.
     */
    void RecoverStop();

    void EventCallback(short events);

public:
    /* virtual methods from class StockItem */
    bool Borrow(gcc_unused void *ctx) override {
        if (stopping)
            /* we havn't yet recovered from #WAS_COMMAND_STOP - give
               up this child process */
            // TODO: improve recovery for this case
            return false;

        event.Delete();
        return true;
    }

    bool Release(gcc_unused void *ctx) override {
        event.Add(was_idle_timeout);
        return true;
    }

private:
    /* virtual methods from class ExitListener */
    void OnChildProcessExit(gcc_unused int status) override {
        process.pid = -1;
    }
};

const char *
WasChildParams::GetStockKey(struct pool &pool) const
{
    const char *key = executable_path;
    for (auto i : args)
        key = p_strcat(&pool, key, " ", i, nullptr);

    for (auto i : options.env)
        key = p_strcat(&pool, key, "$", i, nullptr);

    char options_buffer[4096];
    *options.MakeId(options_buffer) = 0;
    if (*options_buffer != 0)
        key = p_strcat(&pool, key, options_buffer, nullptr);

    return key;
}

WasChild::ReceiveResult
WasChild::ReceiveControl(void *p, size_t size)
{
    ssize_t nbytes = recv(process.control.Get(), p, size, MSG_DONTWAIT);
    if (nbytes == (ssize_t)size)
        return ReceiveResult::SUCCESS;

    if (nbytes < 0 && errno == EAGAIN) {
        /* the WAS application didn't send enough data (yet); don't
           bother waiting for more, just give up on this process */
        return ReceiveResult::AGAIN;
    }

    if (nbytes < 0)
        daemon_log(2, "error on idle WAS control connection '%s': %s\n",
                   GetStockName(), strerror(errno));
    else if (nbytes > 0)
        daemon_log(2, "unexpected data from idle WAS control connection '%s'\n",
                   GetStockName());
    return ReceiveResult::ERROR;
}

inline bool
WasChild::DiscardInput(uint64_t remaining)
{
    while (remaining > 0) {
        uint8_t buffer[16384];
        size_t size = std::min(remaining, uint64_t(sizeof(buffer)));
        ssize_t nbytes = process.input.Read(buffer, size);
        if (nbytes <= 0)
            return false;

        remaining -= nbytes;
    }

    return true;
}

inline void
WasChild::RecoverStop()
{
    uint64_t premature;

    while (true) {
        struct was_header header;
        switch (ReceiveControl(&header, sizeof(header))) {
        case ReceiveResult::SUCCESS:
            break;

        case ReceiveResult::ERROR:
            InvokeIdleDisconnect();
            return;

        case ReceiveResult::AGAIN:
            /* wait for more data */
            event.Add(was_idle_timeout);
            return;
        }

        uint64_t dummy;

        switch ((enum was_command)header.command) {
        case WAS_COMMAND_NOP:
            /* ignore */
            continue;

        case WAS_COMMAND_LENGTH:
        case WAS_COMMAND_STOP:
            /* discard & ignore */
            if (ReceiveControl(&dummy, sizeof(dummy)) != ReceiveResult::SUCCESS) {
                InvokeIdleDisconnect();
                return;
            }
            continue;

        case WAS_COMMAND_REQUEST:
        case WAS_COMMAND_METHOD:
        case WAS_COMMAND_URI:
        case WAS_COMMAND_SCRIPT_NAME:
        case WAS_COMMAND_PATH_INFO:
        case WAS_COMMAND_QUERY_STRING:
        case WAS_COMMAND_HEADER:
        case WAS_COMMAND_PARAMETER:
        case WAS_COMMAND_STATUS:
        case WAS_COMMAND_NO_DATA:
        case WAS_COMMAND_DATA:
            daemon_log(2, "unexpected data from idle WAS control connection '%s'\n",
                       GetStockName());
            InvokeIdleDisconnect();
            return;

        case WAS_COMMAND_PREMATURE:
            /* this is what we're waiting for */
            break;
        }

        if (ReceiveControl(&premature, sizeof(premature)) != ReceiveResult::SUCCESS) {
            InvokeIdleDisconnect();
            return;
        }

        break;
    }

    if (premature < input_received) {
        InvokeIdleDisconnect();
        return;
    }

    if (!DiscardInput(premature - input_received)) {
        InvokeIdleDisconnect();
        return;
    }

    stopping = false;

    event.Add(was_idle_timeout);
}

/*
 * libevent callback
 *
 */

inline void
WasChild::EventCallback(short events)
{
    if ((events & EV_TIMEOUT) == 0) {
        if (stopping) {
            RecoverStop();
            return;
        }

        char buffer;
        ssize_t nbytes = recv(process.control.Get(), &buffer, sizeof(buffer),
                              MSG_DONTWAIT);
        if (nbytes < 0)
            daemon_log(2, "error on idle WAS control connection '%s': %s\n",
                       GetStockName(), strerror(errno));
        else if (nbytes > 0)
            daemon_log(2, "unexpected data from idle WAS control connection '%s'\n",
                       GetStockName());
    }

    InvokeIdleDisconnect();
    pool_commit();
}

/*
 * stock class
 *
 */

static void
was_stock_create(gcc_unused void *ctx,
                 CreateStockItem c,
                 void *info,
                 gcc_unused struct pool &caller_pool,
                 gcc_unused CancellablePointer &cancel_ptr)
{
    auto &spawn_service = *(SpawnService *)ctx;
    WasChildParams *params = (WasChildParams *)info;

    auto *child = new WasChild(c, spawn_service);

    assert(params != nullptr);
    assert(params->executable_path != nullptr);

    GError *error = nullptr;
    if (!child->Launch(*params, &error)) {
        assert(error != nullptr);
        child->InvokeCreateError(error);
        return;
    }

    child->InvokeCreateSuccess();
}

WasChild::~WasChild()
{
    if (process.pid >= 0)
        spawn_service.KillChildProcess(process.pid);

    if (process.control.IsDefined())
        event.Delete();
}

static constexpr StockClass was_stock_class = {
    .create = was_stock_create,
};

/*
 * interface
 *
 */

StockMap *
was_stock_new(unsigned limit, unsigned max_idle,
              EventLoop &event_loop, SpawnService &spawn_service)
{
    return new StockMap(event_loop, was_stock_class, &spawn_service,
                        limit, max_idle);
}

void
was_stock_get(StockMap *hstock, struct pool *pool,
              const ChildOptions &options,
              const char *executable_path,
              ConstBuffer<const char *> args,
              StockGetHandler &handler,
              CancellablePointer &cancel_ptr)
{
    auto params = NewFromPool<WasChildParams>(*pool, executable_path, args,
                                              options);

    hstock->Get(*pool, params->GetStockKey(*pool), params,
                handler, cancel_ptr);
}

const WasProcess &
was_stock_item_get(const StockItem &item)
{
    auto *child = (const WasChild *)&item;

    return child->GetProcess();
}

void
was_stock_item_stop(StockItem &item, uint64_t received)
{
    auto &child = (WasChild &)item;
    child.Stop(received);
}
<commit_msg>was/stock: remove obsolete API documentation<commit_after>/*
 * Launch and manage WAS child processes.
 *
 * author: Max Kellermann <mk@cm4all.com>
 */

#include "was_stock.hxx"
#include "was_quark.h"
#include "was_launch.hxx"
#include "stock/MapStock.hxx"
#include "stock/Stock.hxx"
#include "stock/Class.hxx"
#include "stock/Item.hxx"
#include "net/ConnectSocket.hxx"
#include "spawn/ChildOptions.hxx"
#include "spawn/ExitListener.hxx"
#include "spawn/Interface.hxx"
#include "pool.hxx"
#include "event/SocketEvent.hxx"
#include "util/Cancellable.hxx"
#include "util/ConstBuffer.hxx"
#include "util/Cast.hxx"

#include <was/protocol.h>
#include <daemon/log.h>

#include <glib.h>
#include <assert.h>
#include <stdlib.h>
#include <unistd.h>
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/un.h>
#include <signal.h>
#include <stdlib.h>

static constexpr struct timeval was_idle_timeout = {
    .tv_sec = 300,
    .tv_usec = 0,
};

struct WasChildParams {
    const char *executable_path;

    ConstBuffer<const char *> args;

    const ChildOptions &options;

    WasChildParams(const char *_executable_path,
                   ConstBuffer<const char *> _args,
                   const ChildOptions &_options)
        :executable_path(_executable_path), args(_args),
         options(_options) {}

    const char *GetStockKey(struct pool &pool) const;
};

class WasChild final : public HeapStockItem, ExitListener {
    SpawnService &spawn_service;

    WasProcess process;
    SocketEvent event;

    /**
     * If true, then we're waiting for PREMATURE (after the #WasClient
     * has sent #WAS_COMMAND_STOP).
     */
    bool stopping = false;

    /**
     * The number of bytes received before #WAS_COMMAND_STOP was sent.
     */
    uint64_t input_received;

public:
    explicit WasChild(CreateStockItem c, SpawnService &_spawn_service)
        :HeapStockItem(c), spawn_service(_spawn_service),
         event(c.stock.GetEventLoop(), BIND_THIS_METHOD(EventCallback)) {}

    ~WasChild() override;

    bool Launch(const WasChildParams &params, GError **error_r) {
        process = was_launch(spawn_service,
                             GetStockName(),
                             params.executable_path,
                             params.args,
                             params.options,
                             this,
                             error_r);
        if (!process.IsDefined()) {
            /* explicitly Close() the WasProcess, so our destructor
               doesn't believe we have already initialized the
               Event */
            process.Close();
            return false;
        }

        event.Set(process.control.Get(), EV_READ|EV_TIMEOUT);
        return true;
    }

    const WasProcess &GetProcess() const {
        return process;
    }

    void Stop(uint64_t _received) {
        assert(!is_idle);
        assert(!stopping);

        stopping = true;
        input_received = _received;
    }

private:
    enum class ReceiveResult {
        SUCCESS, ERROR, AGAIN,
    };

    /**
     * Receive data on the control channel.
     */
    ReceiveResult ReceiveControl(void *p, size_t size);

    /**
     * Discard the given amount of data from the input pipe.
     *
     * @return true on success
     */
    bool DiscardInput(uint64_t remaining);

    /**
     * Attempt to recover after the WAS client sent STOP to the
     * application.  This method waits for PREMATURE and discards
     * excess data from the pipe.
     */
    void RecoverStop();

    void EventCallback(short events);

public:
    /* virtual methods from class StockItem */
    bool Borrow(gcc_unused void *ctx) override {
        if (stopping)
            /* we havn't yet recovered from #WAS_COMMAND_STOP - give
               up this child process */
            // TODO: improve recovery for this case
            return false;

        event.Delete();
        return true;
    }

    bool Release(gcc_unused void *ctx) override {
        event.Add(was_idle_timeout);
        return true;
    }

private:
    /* virtual methods from class ExitListener */
    void OnChildProcessExit(gcc_unused int status) override {
        process.pid = -1;
    }
};

const char *
WasChildParams::GetStockKey(struct pool &pool) const
{
    const char *key = executable_path;
    for (auto i : args)
        key = p_strcat(&pool, key, " ", i, nullptr);

    for (auto i : options.env)
        key = p_strcat(&pool, key, "$", i, nullptr);

    char options_buffer[4096];
    *options.MakeId(options_buffer) = 0;
    if (*options_buffer != 0)
        key = p_strcat(&pool, key, options_buffer, nullptr);

    return key;
}

WasChild::ReceiveResult
WasChild::ReceiveControl(void *p, size_t size)
{
    ssize_t nbytes = recv(process.control.Get(), p, size, MSG_DONTWAIT);
    if (nbytes == (ssize_t)size)
        return ReceiveResult::SUCCESS;

    if (nbytes < 0 && errno == EAGAIN) {
        /* the WAS application didn't send enough data (yet); don't
           bother waiting for more, just give up on this process */
        return ReceiveResult::AGAIN;
    }

    if (nbytes < 0)
        daemon_log(2, "error on idle WAS control connection '%s': %s\n",
                   GetStockName(), strerror(errno));
    else if (nbytes > 0)
        daemon_log(2, "unexpected data from idle WAS control connection '%s'\n",
                   GetStockName());
    return ReceiveResult::ERROR;
}

inline bool
WasChild::DiscardInput(uint64_t remaining)
{
    while (remaining > 0) {
        uint8_t buffer[16384];
        size_t size = std::min(remaining, uint64_t(sizeof(buffer)));
        ssize_t nbytes = process.input.Read(buffer, size);
        if (nbytes <= 0)
            return false;

        remaining -= nbytes;
    }

    return true;
}

inline void
WasChild::RecoverStop()
{
    uint64_t premature;

    while (true) {
        struct was_header header;
        switch (ReceiveControl(&header, sizeof(header))) {
        case ReceiveResult::SUCCESS:
            break;

        case ReceiveResult::ERROR:
            InvokeIdleDisconnect();
            return;

        case ReceiveResult::AGAIN:
            /* wait for more data */
            event.Add(was_idle_timeout);
            return;
        }

        uint64_t dummy;

        switch ((enum was_command)header.command) {
        case WAS_COMMAND_NOP:
            /* ignore */
            continue;

        case WAS_COMMAND_LENGTH:
        case WAS_COMMAND_STOP:
            /* discard & ignore */
            if (ReceiveControl(&dummy, sizeof(dummy)) != ReceiveResult::SUCCESS) {
                InvokeIdleDisconnect();
                return;
            }
            continue;

        case WAS_COMMAND_REQUEST:
        case WAS_COMMAND_METHOD:
        case WAS_COMMAND_URI:
        case WAS_COMMAND_SCRIPT_NAME:
        case WAS_COMMAND_PATH_INFO:
        case WAS_COMMAND_QUERY_STRING:
        case WAS_COMMAND_HEADER:
        case WAS_COMMAND_PARAMETER:
        case WAS_COMMAND_STATUS:
        case WAS_COMMAND_NO_DATA:
        case WAS_COMMAND_DATA:
            daemon_log(2, "unexpected data from idle WAS control connection '%s'\n",
                       GetStockName());
            InvokeIdleDisconnect();
            return;

        case WAS_COMMAND_PREMATURE:
            /* this is what we're waiting for */
            break;
        }

        if (ReceiveControl(&premature, sizeof(premature)) != ReceiveResult::SUCCESS) {
            InvokeIdleDisconnect();
            return;
        }

        break;
    }

    if (premature < input_received) {
        InvokeIdleDisconnect();
        return;
    }

    if (!DiscardInput(premature - input_received)) {
        InvokeIdleDisconnect();
        return;
    }

    stopping = false;

    event.Add(was_idle_timeout);
}

/*
 * libevent callback
 *
 */

inline void
WasChild::EventCallback(short events)
{
    if ((events & EV_TIMEOUT) == 0) {
        if (stopping) {
            RecoverStop();
            return;
        }

        char buffer;
        ssize_t nbytes = recv(process.control.Get(), &buffer, sizeof(buffer),
                              MSG_DONTWAIT);
        if (nbytes < 0)
            daemon_log(2, "error on idle WAS control connection '%s': %s\n",
                       GetStockName(), strerror(errno));
        else if (nbytes > 0)
            daemon_log(2, "unexpected data from idle WAS control connection '%s'\n",
                       GetStockName());
    }

    InvokeIdleDisconnect();
    pool_commit();
}

/*
 * stock class
 *
 */

static void
was_stock_create(gcc_unused void *ctx,
                 CreateStockItem c,
                 void *info,
                 gcc_unused struct pool &caller_pool,
                 gcc_unused CancellablePointer &cancel_ptr)
{
    auto &spawn_service = *(SpawnService *)ctx;
    WasChildParams *params = (WasChildParams *)info;

    auto *child = new WasChild(c, spawn_service);

    assert(params != nullptr);
    assert(params->executable_path != nullptr);

    GError *error = nullptr;
    if (!child->Launch(*params, &error)) {
        assert(error != nullptr);
        child->InvokeCreateError(error);
        return;
    }

    child->InvokeCreateSuccess();
}

WasChild::~WasChild()
{
    if (process.pid >= 0)
        spawn_service.KillChildProcess(process.pid);

    if (process.control.IsDefined())
        event.Delete();
}

static constexpr StockClass was_stock_class = {
    .create = was_stock_create,
};

/*
 * interface
 *
 */

StockMap *
was_stock_new(unsigned limit, unsigned max_idle,
              EventLoop &event_loop, SpawnService &spawn_service)
{
    return new StockMap(event_loop, was_stock_class, &spawn_service,
                        limit, max_idle);
}

void
was_stock_get(StockMap *hstock, struct pool *pool,
              const ChildOptions &options,
              const char *executable_path,
              ConstBuffer<const char *> args,
              StockGetHandler &handler,
              CancellablePointer &cancel_ptr)
{
    auto params = NewFromPool<WasChildParams>(*pool, executable_path, args,
                                              options);

    hstock->Get(*pool, params->GetStockKey(*pool), params,
                handler, cancel_ptr);
}

const WasProcess &
was_stock_item_get(const StockItem &item)
{
    auto *child = (const WasChild *)&item;

    return child->GetProcess();
}

void
was_stock_item_stop(StockItem &item, uint64_t received)
{
    auto &child = (WasChild &)item;
    child.Stop(received);
}
<|endoftext|>


<commit_before>#include <animation.hpp>

Animation::Animation(const QString &name)
    :
      QObject(nullptr),
      m_name(name)
{
    m_spritesheet.load("/home/jkbz/test.png");
    m_frames.emplace_back("test2", QRect(64, 0, 64, 64));
    m_frames.emplace_back("test1", QRect(0, 0, 1000, 1000));
    m_frames.emplace_back("test3", QRect(128, 0, 64, 64));
    m_frames.emplace_back("test4", QRect(0, 64, 64, 64));
    m_frames.emplace_back("test5", QRect(64, 64, 64, 64));
    m_frames.emplace_back("test6", QRect(128, 64, 64, 64));
}

Animation::Animation(const Animation &other)
{
   m_name = other.m_name;
   m_frames = std::move(other.m_frames);
   m_spritesheet = other.m_spritesheet;
}

Animation::Animation(Animation &&animation)
    :
      QObject(nullptr),
      m_name(animation.getName()),
      m_frames(std::move(animation.m_frames))
{

}

int Animation::getIndexOf(const QString &name)
{
    auto found = std::find_if(m_frames.begin(), m_frames.end(), [&name](const AnimationFrame& frame)
    {
        return frame.getName() == name;
    });
    if(found != m_frames.end())
        return found - m_frames.begin();
    return -1;
}

int Animation::getIndexOf(const AnimationFrame &ofFrame)
{
    auto found = std::find_if(m_frames.begin(), m_frames.end(), [&ofFrame](const AnimationFrame& frame)
    {
        return frame.getName() == ofFrame.getName();
    });
    if(found != m_frames.end())
        return found - m_frames.begin();
    return -1;
}

void Animation::addFrame(const AnimationFrame &frame)
{
    m_frames.emplace_back(frame);
}

void Animation::addFrame(const QString &name, const QRect &rect)
{
    m_frames.emplace_back(name, rect);
}

void Animation::replaceFrame(const QString& name, const AnimationFrame &frame)
{
    std::replace_if(m_frames.begin(), m_frames.end(), [&name](const AnimationFrame& f)
    {
        return f.getName() == name;
    }, frame);
}

void Animation::removeFrame(int index)
{
    if(index >= 0 && index < m_frames.size())
        m_frames.erase(m_frames.begin() + index);
}

void Animation::removeFrame(const QString &name)
{
    m_frames.erase(std::remove_if(m_frames.begin(), m_frames.end(), [&name](const AnimationFrame& frame){ return frame.getName() == name; }));
}

void Animation::moveFrameTo(int from, int to)
{
    if(to > from)
    {
        auto fromIt = m_frames.begin() + from;
        auto toIt = m_frames.begin() + to;
        while(fromIt != toIt)
            std::iter_swap(fromIt, fromIt++);
    }

    if(to < from)
    {
        auto fromIt = m_frames.begin() + from;
        auto toIt = m_frames.begin() + to + 1;
        while(fromIt != toIt)
            std::iter_swap(fromIt, fromIt--);
    }
}
<commit_msg>Delete hardcoded frames used for testing<commit_after>#include <animation.hpp>

Animation::Animation(const QString &name)
    :
      QObject(nullptr),
      m_name(name)
{

}

Animation::Animation(const Animation &other)
{
   m_name = other.m_name;
   m_frames = std::move(other.m_frames);
   m_spritesheet = other.m_spritesheet;
}

Animation::Animation(Animation &&animation)
    :
      QObject(nullptr),
      m_name(animation.getName()),
      m_frames(std::move(animation.m_frames))
{

}

int Animation::getIndexOf(const QString &name)
{
    auto found = std::find_if(m_frames.begin(), m_frames.end(), [&name](const AnimationFrame& frame)
    {
        return frame.getName() == name;
    });
    if(found != m_frames.end())
        return found - m_frames.begin();
    return -1;
}

int Animation::getIndexOf(const AnimationFrame &ofFrame)
{
    auto found = std::find_if(m_frames.begin(), m_frames.end(), [&ofFrame](const AnimationFrame& frame)
    {
        return frame.getName() == ofFrame.getName();
    });
    if(found != m_frames.end())
        return found - m_frames.begin();
    return -1;
}

void Animation::addFrame(const AnimationFrame &frame)
{
    m_frames.emplace_back(frame);
}

void Animation::addFrame(const QString &name, const QRect &rect)
{
    m_frames.emplace_back(name, rect);
}

void Animation::replaceFrame(const QString& name, const AnimationFrame &frame)
{
    std::replace_if(m_frames.begin(), m_frames.end(), [&name](const AnimationFrame& f)
    {
        return f.getName() == name;
    }, frame);
}

void Animation::removeFrame(int index)
{
    if(index >= 0 && index < m_frames.size())
        m_frames.erase(m_frames.begin() + index);
}

void Animation::removeFrame(const QString &name)
{
    m_frames.erase(std::remove_if(m_frames.begin(), m_frames.end(), [&name](const AnimationFrame& frame){ return frame.getName() == name; }));
}

void Animation::moveFrameTo(int from, int to)
{
    if(to > from)
    {
        auto fromIt = m_frames.begin() + from;
        auto toIt = m_frames.begin() + to;
        while(fromIt != toIt)
            std::iter_swap(fromIt, fromIt++);
    }

    if(to < from)
    {
        auto fromIt = m_frames.begin() + from;
        auto toIt = m_frames.begin() + to + 1;
        while(fromIt != toIt)
            std::iter_swap(fromIt, fromIt--);
    }
}
<|endoftext|>


<commit_before>#include "worker.hpp"

#include <bitcoin/format.hpp>
#include <bitcoin/utility/logger.hpp>
#include "echo.hpp"

namespace obelisk {

using namespace bc;
namespace posix_time = boost::posix_time;
using posix_time::milliseconds;
using posix_time::seconds;
using posix_time::microsec_clock;

const posix_time::time_duration heartbeat_interval = milliseconds(1000);
constexpr size_t interval_init = 4, interval_max = 32;
constexpr long poll_sleep_interval = 50000;

auto now = []() { return microsec_clock::universal_time(); };

bool send_string(zmq::socket_t& socket, const std::string& str)
{
    zmq::message_t message(str.size());
    char* msg_begin = reinterpret_cast<char*>(message.data());
    std::copy(str.begin(), str.end(), msg_begin);
    return socket.send(message);
}

request_worker::request_worker()
  : context_(1)
{
}
bool request_worker::start(config_type& config)
{
    connection_ = config.service;
    name_ = config.name;
    create_new_socket();
    last_heartbeat_ = now();
    heartbeat_at_ = now() + heartbeat_interval;
    interval_ = interval_init;
    // Start thread for processing sends.
    auto process_sends = [this]
    {
        while (!send_stopped_)
        {
            std::unique_lock<std::mutex> lk(send_mutex_);
            send_condition_.wait(lk);
            lk.unlock();
            send_pending();
        }
    };
    send_thread_ = std::thread(process_sends);
}
void request_worker::stop()
{
    send_stopped_ = true;
    send_condition_.notify_one();
    send_thread_.join();
}

void request_worker::create_new_socket()
{
    socket_ = std::make_shared<zmq::socket_t>(context_, ZMQ_DEALER);
    log_debug(LOG_WORKER) << "Connecting: " << connection_;
    // Set the socket identity name.
    if (!name_.empty())
        socket_->setsockopt(ZMQ_IDENTITY, name_.c_str(), name_.size());
    // Connect...
    socket_->connect(connection_.c_str());
    // Configure socket to not wait at close time
    int linger = 0;
    socket_->setsockopt(ZMQ_LINGER, &linger, sizeof (linger));
    // Tell queue we're ready for work
    log_info(LOG_WORKER) << "worker ready";
    send_string(*socket_, "READY");
}

void request_worker::attach(
    const std::string& command, command_handler handler)
{
    handlers_[command] = handler;
}

void request_worker::update()
{
    poll();
}

void request_worker::poll()
{
    // Poll for network updates.
    zmq::pollitem_t items [] = { { *socket_,  0, ZMQ_POLLIN, 0 } };
    int rc = zmq::poll(items, 1, poll_sleep_interval);
    BITCOIN_ASSERT(rc >= 0);

    if (items[0].revents & ZMQ_POLLIN)
    {
        // Get message
        // - 6-part envelope + content -> request
        // - 1-part "HEARTBEAT" -> heartbeat
        incoming_message request;
        request.recv(*socket_);

        if (!request.is_signal())
        {
            last_heartbeat_ = now();
            auto it = handlers_.find(request.command());
            // Perform request if found.
            if (it != handlers_.end())
            {
                // TODO: Slows down queries!
                log_debug(LOG_WORKER)
                    << request.command() << " from " << request.origin();
                auto queue_send = [this](const outgoing_message& message)
                {
                    send_queue_.produce(message);
                    send_condition_.notify_one();
                };
                it->second(request, queue_send);
            }
            else
            {
                log_warning(LOG_WORKER)
                    << "Unhandled request: " << request.command()
                    << " from " << request.origin();
            }
        }
        else if (request.command() == "HEARTBEAT")
        {
            log_debug(LOG_WORKER) << "Received heartbeat";
            last_heartbeat_ = now();
        }
        else
        {
            log_warning(LOG_WORKER) << "invalid message";
        }
        interval_ = interval_init;
    }
    else if (now() - last_heartbeat_ > seconds(interval_))
    {
        log_warning(LOG_WORKER) << "heartbeat failure, can't reach queue";
        log_warning(LOG_WORKER) << "reconnecting in "
            << interval_ << " seconds...";

        if (interval_ < interval_max)
        {
            interval_ *= 2;
        }
        create_new_socket();
        last_heartbeat_ = now();
    }

    // Send heartbeat to queue if it's time
    if (now() > heartbeat_at_)
    {
        heartbeat_at_ = now() + heartbeat_interval;
        log_debug(LOG_WORKER) << "Sending heartbeat";
        send_string(*socket_, "HEARTBEAT");
    }
}

void request_worker::send_pending()
{
    for (const outgoing_message& message: lockless_iterable(send_queue_))
        message.send(*socket_);
}

} // namespace obelisk

<commit_msg>small comment.<commit_after>#include "worker.hpp"

#include <bitcoin/format.hpp>
#include <bitcoin/utility/logger.hpp>
#include "echo.hpp"

namespace obelisk {

using namespace bc;
namespace posix_time = boost::posix_time;
using posix_time::milliseconds;
using posix_time::seconds;
using posix_time::microsec_clock;

const posix_time::time_duration heartbeat_interval = milliseconds(1000);
constexpr size_t interval_init = 4, interval_max = 32;
constexpr long poll_sleep_interval = 50000;

auto now = []() { return microsec_clock::universal_time(); };

bool send_string(zmq::socket_t& socket, const std::string& str)
{
    zmq::message_t message(str.size());
    char* msg_begin = reinterpret_cast<char*>(message.data());
    std::copy(str.begin(), str.end(), msg_begin);
    return socket.send(message);
}

request_worker::request_worker()
  : context_(1)
{
}
bool request_worker::start(config_type& config)
{
    connection_ = config.service;
    name_ = config.name;
    create_new_socket();
    last_heartbeat_ = now();
    heartbeat_at_ = now() + heartbeat_interval;
    interval_ = interval_init;
    // Start thread for processing sends.
    auto process_sends = [this]
    {
        while (!send_stopped_)
        {
            std::unique_lock<std::mutex> lk(send_mutex_);
            send_condition_.wait(lk);
            lk.unlock();
            send_pending();
        }
    };
    send_thread_ = std::thread(process_sends);
}
void request_worker::stop()
{
    // Stop send watching thread.
    send_stopped_ = true;
    send_condition_.notify_one();
    send_thread_.join();
}

void request_worker::create_new_socket()
{
    socket_ = std::make_shared<zmq::socket_t>(context_, ZMQ_DEALER);
    log_debug(LOG_WORKER) << "Connecting: " << connection_;
    // Set the socket identity name.
    if (!name_.empty())
        socket_->setsockopt(ZMQ_IDENTITY, name_.c_str(), name_.size());
    // Connect...
    socket_->connect(connection_.c_str());
    // Configure socket to not wait at close time
    int linger = 0;
    socket_->setsockopt(ZMQ_LINGER, &linger, sizeof (linger));
    // Tell queue we're ready for work
    log_info(LOG_WORKER) << "worker ready";
    send_string(*socket_, "READY");
}

void request_worker::attach(
    const std::string& command, command_handler handler)
{
    handlers_[command] = handler;
}

void request_worker::update()
{
    poll();
}

void request_worker::poll()
{
    // Poll for network updates.
    zmq::pollitem_t items [] = { { *socket_,  0, ZMQ_POLLIN, 0 } };
    int rc = zmq::poll(items, 1, poll_sleep_interval);
    BITCOIN_ASSERT(rc >= 0);

    if (items[0].revents & ZMQ_POLLIN)
    {
        // Get message
        // - 6-part envelope + content -> request
        // - 1-part "HEARTBEAT" -> heartbeat
        incoming_message request;
        request.recv(*socket_);

        if (!request.is_signal())
        {
            last_heartbeat_ = now();
            auto it = handlers_.find(request.command());
            // Perform request if found.
            if (it != handlers_.end())
            {
                // TODO: Slows down queries!
                log_debug(LOG_WORKER)
                    << request.command() << " from " << request.origin();
                auto queue_send = [this](const outgoing_message& message)
                {
                    send_queue_.produce(message);
                    send_condition_.notify_one();
                };
                it->second(request, queue_send);
            }
            else
            {
                log_warning(LOG_WORKER)
                    << "Unhandled request: " << request.command()
                    << " from " << request.origin();
            }
        }
        else if (request.command() == "HEARTBEAT")
        {
            log_debug(LOG_WORKER) << "Received heartbeat";
            last_heartbeat_ = now();
        }
        else
        {
            log_warning(LOG_WORKER) << "invalid message";
        }
        interval_ = interval_init;
    }
    else if (now() - last_heartbeat_ > seconds(interval_))
    {
        log_warning(LOG_WORKER) << "heartbeat failure, can't reach queue";
        log_warning(LOG_WORKER) << "reconnecting in "
            << interval_ << " seconds...";

        if (interval_ < interval_max)
        {
            interval_ *= 2;
        }
        create_new_socket();
        last_heartbeat_ = now();
    }

    // Send heartbeat to queue if it's time
    if (now() > heartbeat_at_)
    {
        heartbeat_at_ = now() + heartbeat_interval;
        log_debug(LOG_WORKER) << "Sending heartbeat";
        send_string(*socket_, "HEARTBEAT");
    }
}

void request_worker::send_pending()
{
    for (const outgoing_message& message: lockless_iterable(send_queue_))
        message.send(*socket_);
}

} // namespace obelisk

<|endoftext|>


<commit_before>Camera::Camera(qreal x, qreal y, qreal z) :
  x(x), y(y), z(z)
{
}

void Camera::setX(qreal x)
{
  this->x = x;
}

void Camera::setY(qreal y)
{
  this->y = y;
}

void Camera::setZ(qreal z)
{
  this->z = z;
}

qreal Camera::getX(void)
{
  return x;
}

qreal Camera::getY(void)
{
  return y;
}

qreal Camera::getZ(void)
{
  return z;
}

IsoCamera::IsoCamera(qreal x, qreal y, qreal z) :
  Camera(x, y, z)
{
}

void IsoCamera::setX0(qreal x0)
{
  this->x0 = x0;
}

void IsoCamera::setY0(qreal y0)
{
  this->y0 = y0;
}

void IsoCamera::setZ0(qreal z0)
{
  this->z0 = z0;
}

void IsoCamera::updateTheta(qreal dTheta)
{
  this->theta += dTheta;
}

void IsoCamera::updatePhi(qreal dPhi)
{
  this->phi += dPhi;
}

void IsoCamera::setDeltaTheta(qreal dTheta)
{
  this->dTheta = dTheta;
}

void IsoCamera::setDeltaPhi(qreal dPhi)
{
  this->dPhi = dPhi;
}

void IsoCamera::setRadius(qreal radius)
{
  this->radius = radius;
}

qreal IsoCamera::getX0(void)
{
  return x0;
}

qreal IsoCamera::getY0(void)
{
  return y0;
}

qreal IsoCamera::getZ0(void)
{
  return z0;
}

qreal IsoCamera::getTheta(void)
{
  return theta;
}

qreal IsoCamera::getPhi(void)
{
  return phi;
}

qreal IsoCamera::getDeltaTheta(void)
{
  return dTheta;
}

qreal IsoCamera::getDeltaPhi(void)
{
  return dPhi;
}

qreal IsoCamera::getRadius(void)
{
  return radius;
}

qreal IsoCamera::calculateRadius(void)
{
  QVector3D currentTarget = targets[target];
  return qSqrt(qPow(x - currentTarget.x(), 2) + qPow(y - currentTarget.y(), 2) + qPow(z - currentTarget.z(), 2));
}

void GLWidget::drawIsoScene(IsoCamera *camera)
{
  gluLookAt(camera->getX(),       camera->getY(),       camera->getZ(),
            camera->getTargetX(), camera->getTargetY(), camera->getTargetZ(),
            0.0,                  1.0,                  0.0);

  glRotatef(xAngle, 1.0, 0.0, 0.0);
  glRotatef(yAngle, 0.0, 1.0, 0.0);
  glRotatef(zAngle, 0.0, 0.0, 1.0);

  glCallList(grid);
  if (cubeVisible) glCallList(cube);
  glCallList(axes);

  spline->drawSpline();
}

void GLWidget::mouseMoveEvent(QMouseEvent *event)
{
  if (event->buttons() & Qt::LeftButton) {
    switch (activeViewport) { // Move cameras in viewports accordingly
      case TOP:   topCamera->setDeltaHorizontal(smooth(event->x() - xClick));
                  topCamera->setDeltaVertical(smooth(event->y() - yClick));
                  topCamera->setX(topCamera->getHorizontal() + topCamera->getDeltaHorizontal());
                  topCamera->setZ(topCamera->getVertical() + topCamera->getDeltaVertical());
                  break;
      case FRONT: frontCamera->setDeltaHorizontal(smooth(event->x() - xClick));
                  frontCamera->setDeltaVertical(smooth(yClick - event->y() + (height / 2)));
                  frontCamera->setX(frontCamera->getHorizontal() + frontCamera->getDeltaHorizontal());
                  frontCamera->setY(frontCamera->getVertical() + frontCamera->getDeltaVertical());
                  break;
      case SIDE:  sideCamera->setDeltaHorizontal(smooth(xClick - event->x() + (width / 2)));
                  sideCamera->setDeltaVertical(smooth(yClick - event->y() + (height / 2)));
                  sideCamera->setZ(sideCamera->getHorizontal() + sideCamera->getDeltaHorizontal());
                  sideCamera->setY(sideCamera->getVertical() + sideCamera->getDeltaVertical());
                  break;
      case ISO:   isoCamera->setDeltaTheta(toRadians(xClick - event->x() + (width / 2)));
                  isoCamera->setDeltaPhi(toRadians(event->y() - yClick));
                  isoCamera->setX(isoCamera->getRadius() * qCos(isoCamera->getTheta() + isoCamera->getDeltaTheta())
                                  * qSin(isoCamera->getPhi() + isoCamera->getDeltaPhi()) + isoCamera->getTargetX());
                  isoCamera->setY(isoCamera->getRadius() * qCos(isoCamera->getPhi() + isoCamera->getDeltaPhi())
                                  + isoCamera->getTargetY());
                  isoCamera->setZ(isoCamera->getRadius() * qSin(isoCamera->getTheta() + isoCamera->getDeltaTheta())
                                  * qSin(isoCamera->getPhi() + isoCamera->getDeltaPhi()) + isoCamera->getTargetZ());
                  break;
    }
  } else if (event->buttons() & Qt::RightButton) {
    switch (activeViewport) { // Zoom in viewports accordingly
      case TOP:   topCamera->setZoom(max(topCamera->getZoom0() + smooth(event->y() - zoomClick), MIN_ZOOM));
                  break;
      case FRONT: frontCamera->setZoom(max(frontCamera->getZoom0() + smooth(event->y() - zoomClick), MIN_ZOOM));
                  break;
      case SIDE:  sideCamera->setZoom(max(sideCamera->getZoom0() + smooth(event->y() - zoomClick), MIN_ZOOM));
                  break;
      case ISO:   qreal t = smooth(zoomClick - event->y());
                  isoCamera->setX(isoCamera->getX0() + t * (isoCamera->getTargetX() - isoCamera->getX0()));
                  isoCamera->setY(isoCamera->getY0() + t * (isoCamera->getTargetY() - isoCamera->getY0()));
                  isoCamera->setZ(isoCamera->getZ0() + t * (isoCamera->getTargetZ() - isoCamera->getZ0()));
                  break;
    }
  }

  updateGL();
}

void GLWidget::mouseReleaseEvent(QMouseEvent *event)
{
  if (event->button() == Qt::LeftButton) {
    switch (activeViewport) {
      case TOP:   topCamera->updateHorizontal(topCamera->getDeltaHorizontal());
                  topCamera->updateVertical(topCamera->getDeltaVertical());
                  break;
      case FRONT: frontCamera->updateHorizontal(frontCamera->getDeltaHorizontal());
                  frontCamera->updateVertical(frontCamera->getDeltaVertical());
                  break;
      case SIDE:  sideCamera->updateHorizontal(sideCamera->getDeltaHorizontal());
                  sideCamera->updateVertical(sideCamera->getDeltaVertical());
                  break;
      case ISO:   isoCamera->updateTheta(isoCamera->getDeltaTheta());
                  isoCamera->updatePhi(isoCamera->getDeltaPhi());
                  isoCamera->setX0(isoCamera->getX());
                  isoCamera->setY0(isoCamera->getY());
                  isoCamera->setZ0(isoCamera->getZ());
                  break;
    }
  } else if (event->button() == Qt::RightButton) {
    switch (activeViewport) {
      case TOP:   topCamera->setZoom0(topCamera->getZoom());
                  break;
      case FRONT: frontCamera->setZoom0(frontCamera->getZoom());
                  break;
      case SIDE:  sideCamera->setZoom0(sideCamera->getZoom());
                  break;
      case ISO:   isoCamera->setX0(isoCamera->getX());
                  isoCamera->setY0(isoCamera->getY());
                  isoCamera->setZ0(isoCamera->getZ());
                  isoCamera->setRadius(isoCamera->calculateRadius());
                  break;
    }
  }
}<commit_msg>Refine notes<commit_after>void GLWidget::mouseMoveEvent(QMouseEvent *event)
{
  if (event->buttons() & Qt::LeftButton) {
    switch (activeViewport) { // Move cameras in viewports accordingly
      case ISO:   isoCamera->setDeltaTheta(toRadians(xClick - event->x() + (width / 2)));
                  isoCamera->setDeltaPhi(toRadians(event->y() - yClick));
                  break;
    }
  }
}

void GLWidget::mouseReleaseEvent(QMouseEvent *event)
{
  if (event->button() == Qt::LeftButton) {
    switch (activeViewport) {
      case ISO:   isoCamera->updateTheta(isoCamera->getDeltaTheta());
                  isoCamera->updatePhi(isoCamera->getDeltaPhi());
                  break;
    }
  }
}<|endoftext|>


<commit_before>/*
    RawSpeed - RAW file decoder.

    Copyright (C) 2009-2014 Klaus Post
    Copyright (C) 2017 Axel Waggershauser

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include "io/Buffer.h"
#include "common/Common.h"  // for uchar8, roundUp
#include "common/Memory.h"  // for alignedFree, alignedMalloc
#include "io/IOException.h" // for IOException (ptr only), ThrowIOE
#include <cassert>          // for assert
#include <memory>           // for unique_ptr

using std::unique_ptr;

namespace RawSpeed {

unique_ptr<uchar8, decltype(&alignedFree)> Buffer::Create(size_type size) {
  if (!size)
    ThrowIOE("Trying to allocate 0 bytes sized buffer.");

  unique_ptr<uchar8, decltype(&alignedFree)> data(
      (uchar8*)alignedMalloc<16>(roundUp(size + BUFFER_PADDING, 16)),
      alignedFree);
  if (!data.get())
    ThrowIOE("Failed to allocate %uz bytes memory buffer.", size);

  return data;
}

Buffer::Buffer(unique_ptr<uchar8, decltype(&alignedFree)> data_,
               size_type size_)
    : size(size_) {
  if (!size)
    ThrowIOE("Buffer has zero size?");

  if (data_.get_deleter() != alignedFree)
    ThrowIOE("Wrong deleter. Expected RawSpeed::alignedFree()");

  data = data_.release();
  if (!data)
    ThrowIOE("Memory buffer is nonexistant");

  isOwner = true;
}

Buffer::Buffer(size_type size_) : Buffer(Create(size_), size_) {}

Buffer::~Buffer() {
  if (isOwner) {
    alignedFree(const_cast<uchar8*>(data));
  }
}

Buffer& Buffer::operator=(Buffer&& rhs) noexcept {
  if (isOwner)
    alignedFree(const_cast<uchar8*>(data));

  data = rhs.data;
  size = rhs.size;
  isOwner = rhs.isOwner;

  rhs.isOwner = false;

  return *this;
}

Buffer& Buffer::operator=(const Buffer& rhs) {
  Buffer unOwningTmp(rhs.data, rhs.size);
  *this = std::move(unOwningTmp);
  assert(!isOwner);
  return *this;
}

#if 0
Buffer* Buffer::clone() {
  Buffer *new_map = new Buffer(size);
  memcpy(new_map->data, data, size);
  return new_map;
}

Buffer* Buffer::cloneRandomSize() {
  uint32 new_size = (rand() | (rand() << 15)) % size;
  Buffer *new_map = new Buffer(new_size);
  memcpy(new_map->data, data, new_size);
  return new_map;
}

void Buffer::corrupt(int errors) {
  for (int i = 0; i < errors; i++) {
    uint32 pos = (rand() | (rand() << 15)) % size;
    data[pos] = rand() & 0xff;
  }
}
#endif

} // namespace RawSpeed
<commit_msg>Buffer: operator=(): check for self-assignment.<commit_after>/*
    RawSpeed - RAW file decoder.

    Copyright (C) 2009-2014 Klaus Post
    Copyright (C) 2017 Axel Waggershauser

    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
    version 2 of the License, or (at your option) any later version.

    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    Lesser General Public License for more details.

    You should have received a copy of the GNU Lesser General Public
    License along with this library; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include "io/Buffer.h"
#include "common/Common.h"  // for uchar8, roundUp
#include "common/Memory.h"  // for alignedFree, alignedMalloc
#include "io/IOException.h" // for IOException (ptr only), ThrowIOE
#include <cassert>          // for assert
#include <memory>           // for unique_ptr

using std::unique_ptr;

namespace RawSpeed {

unique_ptr<uchar8, decltype(&alignedFree)> Buffer::Create(size_type size) {
  if (!size)
    ThrowIOE("Trying to allocate 0 bytes sized buffer.");

  unique_ptr<uchar8, decltype(&alignedFree)> data(
      (uchar8*)alignedMalloc<16>(roundUp(size + BUFFER_PADDING, 16)),
      alignedFree);
  if (!data.get())
    ThrowIOE("Failed to allocate %uz bytes memory buffer.", size);

  return data;
}

Buffer::Buffer(unique_ptr<uchar8, decltype(&alignedFree)> data_,
               size_type size_)
    : size(size_) {
  if (!size)
    ThrowIOE("Buffer has zero size?");

  if (data_.get_deleter() != alignedFree)
    ThrowIOE("Wrong deleter. Expected RawSpeed::alignedFree()");

  data = data_.release();
  if (!data)
    ThrowIOE("Memory buffer is nonexistant");

  isOwner = true;
}

Buffer::Buffer(size_type size_) : Buffer(Create(size_), size_) {}

Buffer::~Buffer() {
  if (isOwner) {
    alignedFree(const_cast<uchar8*>(data));
  }
}

Buffer& Buffer::operator=(Buffer&& rhs) noexcept {
  if (this == &rhs)
    return *this;

  if (isOwner)
    alignedFree(const_cast<uchar8*>(data));

  data = rhs.data;
  size = rhs.size;
  isOwner = rhs.isOwner;

  rhs.isOwner = false;

  return *this;
}

Buffer& Buffer::operator=(const Buffer& rhs) {
  if (this == &rhs)
    return *this;

  Buffer unOwningTmp(rhs.data, rhs.size);
  *this = std::move(unOwningTmp);
  assert(!isOwner);

  return *this;
}

#if 0
Buffer* Buffer::clone() {
  Buffer *new_map = new Buffer(size);
  memcpy(new_map->data, data, size);
  return new_map;
}

Buffer* Buffer::cloneRandomSize() {
  uint32 new_size = (rand() | (rand() << 15)) % size;
  Buffer *new_map = new Buffer(new_size);
  memcpy(new_map->data, data, new_size);
  return new_map;
}

void Buffer::corrupt(int errors) {
  for (int i = 0; i < errors; i++) {
    uint32 pos = (rand() | (rand() << 15)) % size;
    data[pos] = rand() & 0xff;
  }
}
#endif

} // namespace RawSpeed
<|endoftext|>