<commit_before>/*
 * @file adaboost_impl.hpp
 * @author Udit Saxena
 *
 * Implementation of the AdaBoost class.
 *
 * @code
 * @article{schapire1999improved,
 *   author = {Schapire, Robert E. and Singer, Yoram},
 *   title = {Improved Boosting Algorithms Using Confidence-rated Predictions},
 *   journal = {Machine Learning},
 *   volume = {37},
 *   number = {3},
 *   month = dec,
 *   year = {1999},
 *   issn = {0885-6125},
 *   pages = {297--336},
 * }
 * @endcode
 */
#ifndef __MLPACK_METHODS_ADABOOST_ADABOOST_IMPL_HPP
#define __MLPACK_METHODS_ADABOOST_ADABOOST_IMPL_HPP

#include "adaboost.hpp"

namespace mlpack {
namespace adaboost {

/**
 * Constructor. Currently runs the AdaBoost.MH algorithm.
 *
 * @param data Input data
 * @param labels Corresponding labels
 * @param iterations Number of boosting rounds
 * @param tol Tolerance for termination of Adaboost.MH.
 * @param other Weak Learner, which has been initialized already.
 */
template<typename WeakLearnerType, typename MatType>
AdaBoost<WeakLearnerType, MatType>::AdaBoost(
    const MatType& data,
    const arma::Row<size_t>& labels,
    const WeakLearnerType& other,
    const size_t iterations,
    const double tol)
{
  Train(data, labels, other, iterations, tol);
}

// Empty constructor.
template<typename WeakLearnerType, typename MatType>
AdaBoost<WeakLearnerType, MatType>::AdaBoost(const double tolerance) :
    tolerance(tolerance)
{
  // Nothing to do.
}

// Train AdaBoost.
template<typename WeakLearnerType, typename MatType>
void AdaBoost<WeakLearnerType, MatType>::Train(
    const MatType& data,
    const arma::Row<size_t>& labels,
    const WeakLearnerType& other,
    const size_t iterations,
    const double tolerance)
{
  // Clear information from previous runs.
  wl.clear();
  alpha.clear();

  // Count the number of classes.
  classes = (arma::max(labels) - arma::min(labels)) + 1;
  this->tolerance = tolerance;

  // crt is the cumulative rt value for terminating the optimization when rt is
  // changing by less than the tolerance.
  double rt, crt, alphat = 0.0, zt;

  ztProduct = 1.0;

  // To be used for prediction by the weak learner.
  arma::Row<size_t> predictedLabels(labels.n_cols);

  // Use tempData to modify input data for incorporating weights.
  MatType tempData(data);

  // This matrix is a helper matrix used to calculate the final hypothesis.
  arma::mat sumFinalH = arma::zeros<arma::mat>(classes, predictedLabels.n_cols);

  // Load the initial weights into a 2-D matrix.
  const double initWeight = 1.0 / double(data.n_cols * classes);
  arma::mat D(classes, data.n_cols);
  D.fill(initWeight);

  // Weights are stored in this row vector.
  arma::rowvec weights(predictedLabels.n_cols);

  // This is the final hypothesis.
  arma::Row<size_t> finalH(predictedLabels.n_cols);

  // Now, start the boosting rounds.
  for (size_t i = 0; i < iterations; i++)
  {
    // Initialized to zero in every round.  rt is used for calculation of
    // alphat; it is the weighted error.
    // rt = (sum) D(i) y(i) ht(xi)
    rt = 0.0;

    // zt is used for weight normalization.
    zt = 0.0;

    // Build the weight vectors.
    weights = arma::sum(D);

    // Use the existing weak learner to train a new one with new weights.
    WeakLearnerType w(other, tempData, labels, weights);
    w.Classify(tempData, predictedLabels);

    // Now from predictedLabels, build ht, the weak hypothesis
    // buildClassificationMatrix(ht, predictedLabels);

    // Now, calculate alpha(t) using ht.
    for (size_t j = 0; j < D.n_cols; j++) // instead of D, ht
    {
      if (predictedLabels(j) == labels(j))
        rt += arma::accu(D.col(j));
      else
        rt -= arma::accu(D.col(j));
    }

    if ((i > 0) && (std::abs(rt - crt) < tolerance))
      break;

    crt = rt;

    // Our goal is to find alphat which mizimizes or approximately minimizes the
    // value of Z as a function of alpha.
    alphat = 0.5 * log((1 + rt) / (1 - rt));

    alpha.push_back(alphat);
    wl.push_back(w);

    // Now start modifying the weights.
    for (size_t j = 0; j < D.n_cols; j++)
    {
      const double expo = exp(alphat);
      if (predictedLabels(j) == labels(j))
      {
        for (size_t k = 0; k < D.n_rows; k++)
        {
          // We calculate zt, the normalization constant.
          D(k, j) /= expo;
          zt += D(k, j); // * exp(-1 * alphat * yt(j,k) * ht(j,k));

          // Add to the final hypothesis matrix.
          // sumFinalH(k, j) += (alphat * ht(k, j));
          if (k == labels(j))
            sumFinalH(k, j) += (alphat); // * ht(k, j));
          else
            sumFinalH(k, j) -= (alphat);
        }
      }
      else
      {
        for (size_t k = 0; k < D.n_rows; k++)
        {
          // We calculate zt, the normalization constant.
          D(k, j) *= expo;
          zt += D(k, j);

          // Add to the final hypothesis matrix.
          if (k == labels(j))
            sumFinalH(k, j) += alphat; // * ht(k, j));
          else
            sumFinalH(k, j) -= alphat;
        }
      }
    }

    // Normalize D.
    D /= zt;

    // Accumulate the value of zt for the Hamming loss bound.
    ztProduct *= zt;
  }
}

/**
 * Classify the given test points.
 */
template<typename WeakLearnerType, typename MatType>
void AdaBoost<WeakLearnerType, MatType>::Classify(
    const MatType& test,
    arma::Row<size_t>& predictedLabels)
{
  arma::Row<size_t> tempPredictedLabels(test.n_cols);
  arma::mat cMatrix(classes, test.n_cols);

  cMatrix.zeros();
  predictedLabels.set_size(test.n_cols);

  for (size_t i = 0; i < wl.size(); i++)
  {
    wl[i].Classify(test, tempPredictedLabels);

    for (size_t j = 0; j < tempPredictedLabels.n_cols; j++)
      cMatrix(tempPredictedLabels(j), j) += alpha[i];
  }

  arma::colvec cMRow;
  arma::uword maxIndex;

  for (size_t i = 0; i < predictedLabels.n_cols; i++)
  {
    cMRow = cMatrix.unsafe_col(i);
    cMRow.max(maxIndex);
    predictedLabels(i) = maxIndex;
  }
}

/**
 * Serialize the AdaBoost model.
 */
template<typename WeakLearnerType, typename MatType>
template<typename Archive>
void AdaBoost<WeakLearnerType, MatType>::Serialize(Archive& ar,
                                               const unsigned int /* version */)
{
  ar & data::CreateNVP(classes, "classes");
  ar & data::CreateNVP(tolerance, "tolerance");
  ar & data::CreateNVP(ztProduct, "ztProduct");
  ar & data::CreateNVP(alpha, "alpha");

  // Now serialize each weak learner.
  if (Archive::is_loading::value)
  {
    wl.clear();
    wl.resize(alpha.size());
  }
  for (size_t i = 0; i < wl.size(); ++i)
  {
    std::ostringstream oss;
    oss << "weakLearner" << i;
    ar & data::CreateNVP(wl[i], oss.str());
  }
}

} // namespace adaboost
} // namespace mlpack

#endif
<commit_msg>Fix -Wmaybe-uninitialized.<commit_after>/*
 * @file adaboost_impl.hpp
 * @author Udit Saxena
 *
 * Implementation of the AdaBoost class.
 *
 * @code
 * @article{schapire1999improved,
 *   author = {Schapire, Robert E. and Singer, Yoram},
 *   title = {Improved Boosting Algorithms Using Confidence-rated Predictions},
 *   journal = {Machine Learning},
 *   volume = {37},
 *   number = {3},
 *   month = dec,
 *   year = {1999},
 *   issn = {0885-6125},
 *   pages = {297--336},
 * }
 * @endcode
 */
#ifndef __MLPACK_METHODS_ADABOOST_ADABOOST_IMPL_HPP
#define __MLPACK_METHODS_ADABOOST_ADABOOST_IMPL_HPP

#include "adaboost.hpp"

namespace mlpack {
namespace adaboost {

/**
 * Constructor. Currently runs the AdaBoost.MH algorithm.
 *
 * @param data Input data
 * @param labels Corresponding labels
 * @param iterations Number of boosting rounds
 * @param tol Tolerance for termination of Adaboost.MH.
 * @param other Weak Learner, which has been initialized already.
 */
template<typename WeakLearnerType, typename MatType>
AdaBoost<WeakLearnerType, MatType>::AdaBoost(
    const MatType& data,
    const arma::Row<size_t>& labels,
    const WeakLearnerType& other,
    const size_t iterations,
    const double tol)
{
  Train(data, labels, other, iterations, tol);
}

// Empty constructor.
template<typename WeakLearnerType, typename MatType>
AdaBoost<WeakLearnerType, MatType>::AdaBoost(const double tolerance) :
    tolerance(tolerance)
{
  // Nothing to do.
}

// Train AdaBoost.
template<typename WeakLearnerType, typename MatType>
void AdaBoost<WeakLearnerType, MatType>::Train(
    const MatType& data,
    const arma::Row<size_t>& labels,
    const WeakLearnerType& other,
    const size_t iterations,
    const double tolerance)
{
  // Clear information from previous runs.
  wl.clear();
  alpha.clear();

  // Count the number of classes.
  classes = (arma::max(labels) - arma::min(labels)) + 1;
  this->tolerance = tolerance;

  // crt is the cumulative rt value for terminating the optimization when rt is
  // changing by less than the tolerance.
  double rt, crt = 0.0, alphat = 0.0, zt;

  ztProduct = 1.0;

  // To be used for prediction by the weak learner.
  arma::Row<size_t> predictedLabels(labels.n_cols);

  // Use tempData to modify input data for incorporating weights.
  MatType tempData(data);

  // This matrix is a helper matrix used to calculate the final hypothesis.
  arma::mat sumFinalH = arma::zeros<arma::mat>(classes, predictedLabels.n_cols);

  // Load the initial weights into a 2-D matrix.
  const double initWeight = 1.0 / double(data.n_cols * classes);
  arma::mat D(classes, data.n_cols);
  D.fill(initWeight);

  // Weights are stored in this row vector.
  arma::rowvec weights(predictedLabels.n_cols);

  // This is the final hypothesis.
  arma::Row<size_t> finalH(predictedLabels.n_cols);

  // Now, start the boosting rounds.
  for (size_t i = 0; i < iterations; i++)
  {
    // Initialized to zero in every round.  rt is used for calculation of
    // alphat; it is the weighted error.
    // rt = (sum) D(i) y(i) ht(xi)
    rt = 0.0;

    // zt is used for weight normalization.
    zt = 0.0;

    // Build the weight vectors.
    weights = arma::sum(D);

    // Use the existing weak learner to train a new one with new weights.
    WeakLearnerType w(other, tempData, labels, weights);
    w.Classify(tempData, predictedLabels);

    // Now from predictedLabels, build ht, the weak hypothesis
    // buildClassificationMatrix(ht, predictedLabels);

    // Now, calculate alpha(t) using ht.
    for (size_t j = 0; j < D.n_cols; j++) // instead of D, ht
    {
      if (predictedLabels(j) == labels(j))
        rt += arma::accu(D.col(j));
      else
        rt -= arma::accu(D.col(j));
    }

    if ((i > 0) && (std::abs(rt - crt) < tolerance))
      break;

    crt = rt;

    // Our goal is to find alphat which mizimizes or approximately minimizes the
    // value of Z as a function of alpha.
    alphat = 0.5 * log((1 + rt) / (1 - rt));

    alpha.push_back(alphat);
    wl.push_back(w);

    // Now start modifying the weights.
    for (size_t j = 0; j < D.n_cols; j++)
    {
      const double expo = exp(alphat);
      if (predictedLabels(j) == labels(j))
      {
        for (size_t k = 0; k < D.n_rows; k++)
        {
          // We calculate zt, the normalization constant.
          D(k, j) /= expo;
          zt += D(k, j); // * exp(-1 * alphat * yt(j,k) * ht(j,k));

          // Add to the final hypothesis matrix.
          // sumFinalH(k, j) += (alphat * ht(k, j));
          if (k == labels(j))
            sumFinalH(k, j) += (alphat); // * ht(k, j));
          else
            sumFinalH(k, j) -= (alphat);
        }
      }
      else
      {
        for (size_t k = 0; k < D.n_rows; k++)
        {
          // We calculate zt, the normalization constant.
          D(k, j) *= expo;
          zt += D(k, j);

          // Add to the final hypothesis matrix.
          if (k == labels(j))
            sumFinalH(k, j) += alphat; // * ht(k, j));
          else
            sumFinalH(k, j) -= alphat;
        }
      }
    }

    // Normalize D.
    D /= zt;

    // Accumulate the value of zt for the Hamming loss bound.
    ztProduct *= zt;
  }
}

/**
 * Classify the given test points.
 */
template<typename WeakLearnerType, typename MatType>
void AdaBoost<WeakLearnerType, MatType>::Classify(
    const MatType& test,
    arma::Row<size_t>& predictedLabels)
{
  arma::Row<size_t> tempPredictedLabels(test.n_cols);
  arma::mat cMatrix(classes, test.n_cols);

  cMatrix.zeros();
  predictedLabels.set_size(test.n_cols);

  for (size_t i = 0; i < wl.size(); i++)
  {
    wl[i].Classify(test, tempPredictedLabels);

    for (size_t j = 0; j < tempPredictedLabels.n_cols; j++)
      cMatrix(tempPredictedLabels(j), j) += alpha[i];
  }

  arma::colvec cMRow;
  arma::uword maxIndex;

  for (size_t i = 0; i < predictedLabels.n_cols; i++)
  {
    cMRow = cMatrix.unsafe_col(i);
    cMRow.max(maxIndex);
    predictedLabels(i) = maxIndex;
  }
}

/**
 * Serialize the AdaBoost model.
 */
template<typename WeakLearnerType, typename MatType>
template<typename Archive>
void AdaBoost<WeakLearnerType, MatType>::Serialize(Archive& ar,
                                               const unsigned int /* version */)
{
  ar & data::CreateNVP(classes, "classes");
  ar & data::CreateNVP(tolerance, "tolerance");
  ar & data::CreateNVP(ztProduct, "ztProduct");
  ar & data::CreateNVP(alpha, "alpha");

  // Now serialize each weak learner.
  if (Archive::is_loading::value)
  {
    wl.clear();
    wl.resize(alpha.size());
  }
  for (size_t i = 0; i < wl.size(); ++i)
  {
    std::ostringstream oss;
    oss << "weakLearner" << i;
    ar & data::CreateNVP(wl[i], oss.str());
  }
}

} // namespace adaboost
} // namespace mlpack

#endif
<|endoftext|>


<commit_before>// Copyright 2010-2013 RethinkDB, all rights reserved.
#include "rdb_protocol/terms/terms.hpp"

#include <string>
#include <utility>

#include "errors.hpp"
#include <boost/bind.hpp>

#include "rdb_protocol/datum_stream.hpp"
#include "rdb_protocol/error.hpp"
#include "rdb_protocol/func.hpp"
#include "rdb_protocol/op.hpp"
#include "rdb_protocol/term_walker.hpp"

namespace ql {

// NOTE: `asc` and `desc` don't fit into our type system (they're a hack for
// orderby to avoid string parsing), so we instead literally examine the
// protobuf to determine whether they're present.  This is a hack.  (This is
// implemented internally in terms of string concatenation because that's what
// the old string-parsing solution was, and this was a quick way to get the new
// behavior that also avoided dumping already-tested code paths.  I'm probably
// going to hell for it though.)

class asc_term_t : public op_term_t {
public:
    asc_term_t(compile_env_t *env, const protob_t<const Term> &term)
        : op_term_t(env, term, argspec_t(1)) { }
private:
    virtual counted_t<val_t> eval_impl(scope_env_t *env, UNUSED eval_flags_t flags) {
        return arg(env, 0);
    }
    virtual const char *name() const { return "asc"; }
};

class desc_term_t : public op_term_t {
public:
    desc_term_t(compile_env_t *env, const protob_t<const Term> &term)
        : op_term_t(env, term, argspec_t(1)) { }
private:
    virtual counted_t<val_t> eval_impl(scope_env_t *env, UNUSED eval_flags_t flags) {
        return arg(env, 0);
    }
    virtual const char *name() const { return "desc"; }
};

class orderby_term_t : public op_term_t {
public:
    orderby_term_t(compile_env_t *env, const protob_t<const Term> &term)
        : op_term_t(env, term, argspec_t(1, -1),
          optargspec_t({"index"})), src_term(term) { }
private:
    enum order_direction_t { ASC, DESC };
    class lt_cmp_t {
    public:
        typedef bool result_type;
        explicit lt_cmp_t(
            std::vector<std::pair<order_direction_t, counted_t<func_t> > > _comparisons)
            : comparisons(std::move(_comparisons)) { }

        bool operator()(env_t *env,
                        profile::sampler_t *sampler,
                        counted_t<const datum_t> l,
                        counted_t<const datum_t> r) const {
            sampler->new_sample();
            for (auto it = comparisons.begin(); it != comparisons.end(); ++it) {
                counted_t<const datum_t> lval;
                counted_t<const datum_t> rval;
                try {
                    lval = it->second->call(env, l)->as_datum();
                } catch (const base_exc_t &e) {
                    if (e.get_type() != base_exc_t::NON_EXISTENCE) {
                        throw;
                    }
                }

                try {
                    rval = it->second->call(env, r)->as_datum();
                } catch (const base_exc_t &e) {
                    if (e.get_type() != base_exc_t::NON_EXISTENCE) {
                        throw;
                    }
                }

                if (!lval.has() && !rval.has()) {
                    continue;
                }
                if (!lval.has()) {
                    return true != (it->first == DESC);
                }
                if (!rval.has()) {
                    return false != (it->first == DESC);
                }
                // TODO: use datum_t::cmp instead to be faster
                if (*lval == *rval) {
                    continue;
                }
                return (*lval < *rval) != (it->first == DESC);
            }

            return false;
        }

    private:
        const std::vector<std::pair<order_direction_t, counted_t<func_t> > >
            comparisons;
    };

    virtual counted_t<val_t> eval_impl(scope_env_t *env, UNUSED eval_flags_t flags) {
        std::vector<std::pair<order_direction_t, counted_t<func_t> > > comparisons;
        scoped_ptr_t<datum_t> arr(new datum_t(datum_t::R_ARRAY));
        for (size_t i = 1; i < num_args(); ++i) {
            if (get_src()->args(i).type() == Term::DESC) {
                comparisons.push_back(
                        std::make_pair(DESC, arg(env, i)->as_func(GET_FIELD_SHORTCUT)));
            } else {
                comparisons.push_back(
                        std::make_pair(ASC, arg(env, i)->as_func(GET_FIELD_SHORTCUT)));
            }
        }
        lt_cmp_t lt_cmp(comparisons);

        counted_t<table_t> tbl;
        counted_t<datum_stream_t> seq;
        counted_t<val_t> v0 = arg(env, 0);
        if (v0->get_type().is_convertible(val_t::type_t::TABLE)) {
            tbl = v0->as_table();
        } else if (v0->get_type().is_convertible(val_t::type_t::SELECTION)) {
            std::pair<counted_t<table_t>, counted_t<datum_stream_t> > ts
                = v0->as_selection(env->env);
            tbl = ts.first;
            seq = ts.second;
        } else {
            seq = v0->as_seq(env->env);
        }

        /* Add a sorting to the table if we're doing indexed sorting. */
        if (counted_t<val_t> index = optarg(env, "index")) {
            rcheck(tbl.has(), base_exc_t::GENERIC,
                   "Indexed order_by can only be performed on a TABLE.");
            rcheck(!seq.has(), base_exc_t::GENERIC,
                   "Indexed order_by can only be performed on a TABLE.");
            sorting_t sorting = sorting_t::UNORDERED;
            for (int i = 0; i < get_src()->optargs_size(); ++i) {
                if (get_src()->optargs(i).key() == "index") {
                    if (get_src()->optargs(i).val().type() == Term::DESC) {
                        sorting = sorting_t::DESCENDING;
                    } else {
                        sorting = sorting_t::ASCENDING;
                    }
                }
            }
            r_sanity_check(sorting != sorting_t::UNORDERED);
            std::string index_str = index->as_str().to_std();
            tbl->add_sorting(index_str, sorting, this);
            if (index_str != tbl->get_pkey()
                && !comparisons.empty()) {
                seq = make_counted<indexed_sort_datum_stream_t>(
                    tbl->as_datum_stream(env->env, backtrace()), lt_cmp);
            } else {
                seq = tbl->as_datum_stream(env->env, backtrace());
            }
        } else {
            if (!seq.has()) {
                seq = tbl->as_datum_stream(env->env, backtrace());
            }
            rcheck(!comparisons.empty(), base_exc_t::GENERIC,
                   "Must specify something to order by.");
            std::vector<counted_t<const datum_t> > to_sort;
            batchspec_t batchspec = batchspec_t::user(batch_type_t::TERMINAL, env->env);
            for (;;) {
                std::vector<counted_t<const datum_t> > data
                    = seq->next_batch(env->env, batchspec);
                if (data.size() == 0) {
                    break;
                }
                std::move(data.begin(), data.end(), std::back_inserter(to_sort));
                rcheck(to_sort.size() <= array_size_limit(), base_exc_t::GENERIC,
                       strprintf("Array over size limit %zu.",
                                 array_size_limit()).c_str());
            }
            profile::sampler_t sampler("Sorting in-memory.", env->env->trace);
            auto fn = boost::bind(lt_cmp, env->env, &sampler, _1, _2);
            std::sort(to_sort.begin(), to_sort.end(), fn);
            seq = make_counted<array_datum_stream_t>(
                make_counted<const datum_t>(std::move(to_sort)), backtrace());
        }
        return tbl.has() ? new_val(seq, tbl) : new_val(env->env, seq);
    }

    virtual const char *name() const { return "orderby"; }

private:
    protob_t<const Term> src_term;
};

class distinct_term_t : public op_term_t {
public:
    distinct_term_t(compile_env_t *env, const protob_t<const Term> &term)
        : op_term_t(env, term, argspec_t(1)) { }
private:
    static bool lt_cmp(env_t *,
                       counted_t<const datum_t> l,
                       counted_t<const datum_t> r) {
        return *l < *r;
    }
    virtual counted_t<val_t> eval_impl(scope_env_t *env, UNUSED eval_flags_t flags) {
        counted_t<datum_stream_t> s = arg(env, 0)->as_seq(env->env);
        std::vector<counted_t<const datum_t> > arr;
        counted_t<const datum_t> last;
        batchspec_t batchspec = batchspec_t::user(batch_type_t::TERMINAL, env->env);
        {
            profile::sampler_t sampler("Evaluating elements in distinct.",
                                       env->env->trace);
            while (counted_t<const datum_t> d = s->next(env->env, batchspec)) {
                arr.push_back(std::move(d));
                rcheck_array_size(arr, base_exc_t::GENERIC);
                sampler.new_sample();
            }
        }
        std::sort(arr.begin(), arr.end(),
                  std::bind(lt_cmp, env->env,
                            ph::_1, ph::_2));
        std::vector<counted_t<const datum_t> > toret;
        for (auto it = arr.begin(); it != arr.end(); ++it) {
            if (toret.size() == 0 || **it != *toret[toret.size()-1]) {
                toret.push_back(std::move(*it));
            }
        }
        return new_val(make_counted<const datum_t>(std::move(toret)));
    }
    virtual const char *name() const { return "distinct"; }
};

counted_t<term_t> make_orderby_term(compile_env_t *env, const protob_t<const Term> &term) {
    return make_counted<orderby_term_t>(env, term);
}
counted_t<term_t> make_distinct_term(compile_env_t *env, const protob_t<const Term> &term) {
    return make_counted<distinct_term_t>(env, term);
}
counted_t<term_t> make_asc_term(compile_env_t *env, const protob_t<const Term> &term) {
    return make_counted<asc_term_t>(env, term);
}
counted_t<term_t> make_desc_term(compile_env_t *env, const protob_t<const Term> &term) {
    return make_counted<desc_term_t>(env, term);
}

} // namespace ql
<commit_msg>Fix order_by on empty sequences.<commit_after>// Copyright 2010-2013 RethinkDB, all rights reserved.
#include "rdb_protocol/terms/terms.hpp"

#include <string>
#include <utility>

#include "errors.hpp"
#include <boost/bind.hpp>

#include "rdb_protocol/datum_stream.hpp"
#include "rdb_protocol/error.hpp"
#include "rdb_protocol/func.hpp"
#include "rdb_protocol/op.hpp"
#include "rdb_protocol/term_walker.hpp"

namespace ql {

// NOTE: `asc` and `desc` don't fit into our type system (they're a hack for
// orderby to avoid string parsing), so we instead literally examine the
// protobuf to determine whether they're present.  This is a hack.  (This is
// implemented internally in terms of string concatenation because that's what
// the old string-parsing solution was, and this was a quick way to get the new
// behavior that also avoided dumping already-tested code paths.  I'm probably
// going to hell for it though.)

class asc_term_t : public op_term_t {
public:
    asc_term_t(compile_env_t *env, const protob_t<const Term> &term)
        : op_term_t(env, term, argspec_t(1)) { }
private:
    virtual counted_t<val_t> eval_impl(scope_env_t *env, UNUSED eval_flags_t flags) {
        return arg(env, 0);
    }
    virtual const char *name() const { return "asc"; }
};

class desc_term_t : public op_term_t {
public:
    desc_term_t(compile_env_t *env, const protob_t<const Term> &term)
        : op_term_t(env, term, argspec_t(1)) { }
private:
    virtual counted_t<val_t> eval_impl(scope_env_t *env, UNUSED eval_flags_t flags) {
        return arg(env, 0);
    }
    virtual const char *name() const { return "desc"; }
};

class orderby_term_t : public op_term_t {
public:
    orderby_term_t(compile_env_t *env, const protob_t<const Term> &term)
        : op_term_t(env, term, argspec_t(1, -1),
          optargspec_t({"index"})), src_term(term) { }
private:
    enum order_direction_t { ASC, DESC };
    class lt_cmp_t {
    public:
        typedef bool result_type;
        explicit lt_cmp_t(
            std::vector<std::pair<order_direction_t, counted_t<func_t> > > _comparisons)
            : comparisons(std::move(_comparisons)) { }

        bool operator()(env_t *env,
                        profile::sampler_t *sampler,
                        counted_t<const datum_t> l,
                        counted_t<const datum_t> r) const {
            sampler->new_sample();
            for (auto it = comparisons.begin(); it != comparisons.end(); ++it) {
                counted_t<const datum_t> lval;
                counted_t<const datum_t> rval;
                try {
                    lval = it->second->call(env, l)->as_datum();
                } catch (const base_exc_t &e) {
                    if (e.get_type() != base_exc_t::NON_EXISTENCE) {
                        throw;
                    }
                }

                try {
                    rval = it->second->call(env, r)->as_datum();
                } catch (const base_exc_t &e) {
                    if (e.get_type() != base_exc_t::NON_EXISTENCE) {
                        throw;
                    }
                }

                if (!lval.has() && !rval.has()) {
                    continue;
                }
                if (!lval.has()) {
                    return true != (it->first == DESC);
                }
                if (!rval.has()) {
                    return false != (it->first == DESC);
                }
                // TODO: use datum_t::cmp instead to be faster
                if (*lval == *rval) {
                    continue;
                }
                return (*lval < *rval) != (it->first == DESC);
            }

            return false;
        }

    private:
        const std::vector<std::pair<order_direction_t, counted_t<func_t> > >
            comparisons;
    };

    virtual counted_t<val_t> eval_impl(scope_env_t *env, UNUSED eval_flags_t flags) {
        std::vector<std::pair<order_direction_t, counted_t<func_t> > > comparisons;
        scoped_ptr_t<datum_t> arr(new datum_t(datum_t::R_ARRAY));
        for (size_t i = 1; i < num_args(); ++i) {
            if (get_src()->args(i).type() == Term::DESC) {
                comparisons.push_back(
                        std::make_pair(DESC, arg(env, i)->as_func(GET_FIELD_SHORTCUT)));
            } else {
                comparisons.push_back(
                        std::make_pair(ASC, arg(env, i)->as_func(GET_FIELD_SHORTCUT)));
            }
        }
        lt_cmp_t lt_cmp(comparisons);

        counted_t<table_t> tbl;
        counted_t<datum_stream_t> seq;
        counted_t<val_t> v0 = arg(env, 0);
        if (v0->get_type().is_convertible(val_t::type_t::TABLE)) {
            tbl = v0->as_table();
        } else if (v0->get_type().is_convertible(val_t::type_t::SELECTION)) {
            std::pair<counted_t<table_t>, counted_t<datum_stream_t> > ts
                = v0->as_selection(env->env);
            tbl = ts.first;
            seq = ts.second;
        } else {
            seq = v0->as_seq(env->env);
        }

        if (seq.has() && seq->is_exhausted()){
            /* Do nothing for empty sequence */

        /* Add a sorting to the table if we're doing indexed sorting. */
        }else if (counted_t<val_t> index = optarg(env, "index")) {
            rcheck(tbl.has(), base_exc_t::GENERIC,
                   "Indexed order_by can only be performed on a TABLE.");
            rcheck(!seq.has(), base_exc_t::GENERIC,
                   "Indexed order_by can only be performed on a TABLE.");
            sorting_t sorting = sorting_t::UNORDERED;
            for (int i = 0; i < get_src()->optargs_size(); ++i) {
                if (get_src()->optargs(i).key() == "index") {
                    if (get_src()->optargs(i).val().type() == Term::DESC) {
                        sorting = sorting_t::DESCENDING;
                    } else {
                        sorting = sorting_t::ASCENDING;
                    }
                }
            }
            r_sanity_check(sorting != sorting_t::UNORDERED);
            std::string index_str = index->as_str().to_std();
            tbl->add_sorting(index_str, sorting, this);
            if (index_str != tbl->get_pkey()
                && !comparisons.empty()) {
                seq = make_counted<indexed_sort_datum_stream_t>(
                    tbl->as_datum_stream(env->env, backtrace()), lt_cmp);
            } else {
                seq = tbl->as_datum_stream(env->env, backtrace());
            }
        } else {
            if (!seq.has()) {
                seq = tbl->as_datum_stream(env->env, backtrace());
            }
            rcheck(!comparisons.empty(), base_exc_t::GENERIC,
                   "Must specify something to order by.");
            std::vector<counted_t<const datum_t> > to_sort;
            batchspec_t batchspec = batchspec_t::user(batch_type_t::TERMINAL, env->env);
            for (;;) {
                std::vector<counted_t<const datum_t> > data
                    = seq->next_batch(env->env, batchspec);
                if (data.size() == 0) {
                    break;
                }
                std::move(data.begin(), data.end(), std::back_inserter(to_sort));
                rcheck(to_sort.size() <= array_size_limit(), base_exc_t::GENERIC,
                       strprintf("Array over size limit %zu.",
                                 array_size_limit()).c_str());
            }
            profile::sampler_t sampler("Sorting in-memory.", env->env->trace);
            auto fn = boost::bind(lt_cmp, env->env, &sampler, _1, _2);
            std::sort(to_sort.begin(), to_sort.end(), fn);
            seq = make_counted<array_datum_stream_t>(
                make_counted<const datum_t>(std::move(to_sort)), backtrace());
        }
        return tbl.has() ? new_val(seq, tbl) : new_val(env->env, seq);
    }

    virtual const char *name() const { return "orderby"; }

private:
    protob_t<const Term> src_term;
};

class distinct_term_t : public op_term_t {
public:
    distinct_term_t(compile_env_t *env, const protob_t<const Term> &term)
        : op_term_t(env, term, argspec_t(1)) { }
private:
    static bool lt_cmp(env_t *,
                       counted_t<const datum_t> l,
                       counted_t<const datum_t> r) {
        return *l < *r;
    }
    virtual counted_t<val_t> eval_impl(scope_env_t *env, UNUSED eval_flags_t flags) {
        counted_t<datum_stream_t> s = arg(env, 0)->as_seq(env->env);
        std::vector<counted_t<const datum_t> > arr;
        counted_t<const datum_t> last;
        batchspec_t batchspec = batchspec_t::user(batch_type_t::TERMINAL, env->env);
        {
            profile::sampler_t sampler("Evaluating elements in distinct.",
                                       env->env->trace);
            while (counted_t<const datum_t> d = s->next(env->env, batchspec)) {
                arr.push_back(std::move(d));
                rcheck_array_size(arr, base_exc_t::GENERIC);
                sampler.new_sample();
            }
        }
        std::sort(arr.begin(), arr.end(),
                  std::bind(lt_cmp, env->env,
                            ph::_1, ph::_2));
        std::vector<counted_t<const datum_t> > toret;
        for (auto it = arr.begin(); it != arr.end(); ++it) {
            if (toret.size() == 0 || **it != *toret[toret.size()-1]) {
                toret.push_back(std::move(*it));
            }
        }
        return new_val(make_counted<const datum_t>(std::move(toret)));
    }
    virtual const char *name() const { return "distinct"; }
};

counted_t<term_t> make_orderby_term(compile_env_t *env, const protob_t<const Term> &term) {
    return make_counted<orderby_term_t>(env, term);
}
counted_t<term_t> make_distinct_term(compile_env_t *env, const protob_t<const Term> &term) {
    return make_counted<distinct_term_t>(env, term);
}
counted_t<term_t> make_asc_term(compile_env_t *env, const protob_t<const Term> &term) {
    return make_counted<asc_term_t>(env, term);
}
counted_t<term_t> make_desc_term(compile_env_t *env, const protob_t<const Term> &term) {
    return make_counted<desc_term_t>(env, term);
}

} // namespace ql
<|endoftext|>


<commit_before>#include "Bootstrapper.hpp"

#include <functional>

#include <lua-cxx/LuaValue.hpp>
#include <lua-cxx/loaders.hpp>
#include <lua-cxx/userdata.hpp>

#include "LuaPainter.hpp"
#include "LuaFont.hpp"

Bootstrapper::Bootstrapper() :
    _lua(),
    _desktop(_lua),
    _rainback(_lua)
{
    _rainback.setWidget(&_desktop);
    lua::load_file(_lua, "../../demo/init.lua");
}

QWidget& Bootstrapper::mainWidget()
{
    return _desktop;
}

// vim: set ts=4 sw=4 :
<commit_msg>Use Qt to load init.lua<commit_after>#include "Bootstrapper.hpp"

#include <functional>

#include <lua-cxx/LuaValue.hpp>
#include <lua-cxx/loaders.hpp>
#include <lua-cxx/userdata.hpp>

#include "LuaPainter.hpp"
#include "LuaFont.hpp"

Bootstrapper::Bootstrapper() :
    _lua(),
    _desktop(_lua),
    _rainback(_lua)
{
    _rainback.setWidget(&_desktop);
    QFile file("../../demo/init.lua");
    _lua(file);
}

QWidget& Bootstrapper::mainWidget()
{
    return _desktop;
}

// vim: set ts=4 sw=4 :
<|endoftext|>


<commit_before>
/*
 * Copyright 2012 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */
#include "Test.h"
#include "SkChecksum.h"
#include "SkCityHash.h"

// Word size that is large enough to hold results of any checksum type.
typedef uint64_t checksum_result;

namespace skiatest {
    class ChecksumTestClass : public Test {
    public:
        static Test* Factory(void*) {return SkNEW(ChecksumTestClass); }
    protected:
        virtual void onGetName(SkString* name) { name->set("Checksum"); }
        virtual void onRun(Reporter* reporter) {
            this->fReporter = reporter;
            RunTest();
        }
    private:
        enum Algorithm {
            kSkChecksum,
            kSkCityHash32,
            kSkCityHash64
        };

        // Call Compute(data, size) on the appropriate checksum algorithm,
        // depending on this->fWhichAlgorithm.
        checksum_result ComputeChecksum(const char *data, size_t size) {
            switch(fWhichAlgorithm) {
            case kSkChecksum:
                REPORTER_ASSERT_MESSAGE(fReporter,
                                        reinterpret_cast<uintptr_t>(data) % 4 == 0,
                                        "test data pointer is not 32-bit aligned");
                REPORTER_ASSERT_MESSAGE(fReporter, SkIsAlign4(size),
                                        "test data size is not 32-bit aligned");
                return SkChecksum::Compute(reinterpret_cast<const uint32_t *>(data), size);
            case kSkCityHash32:
                return SkCityHash::Compute32(data, size);
            case kSkCityHash64:
                return SkCityHash::Compute64(data, size);
            default:
                SkString message("fWhichAlgorithm has unknown value ");
                message.appendf("%d", fWhichAlgorithm);
                fReporter->reportFailed(message);
            }
            // we never get here
            return 0;
        }

        // Confirm that the checksum algorithm (specified by fWhichAlgorithm)
        // generates the same results if called twice over the same data.
        void TestChecksumSelfConsistency(size_t buf_size) {
            SkAutoMalloc storage(buf_size);
            char* ptr = reinterpret_cast<char *>(storage.get());

            REPORTER_ASSERT(fReporter,
                            GetTestDataChecksum(8, 0) ==
                            GetTestDataChecksum(8, 0));
            REPORTER_ASSERT(fReporter,
                            GetTestDataChecksum(8, 0) !=
                            GetTestDataChecksum(8, 1));

            sk_bzero(ptr, buf_size);
            checksum_result prev = 0;

            // assert that as we change values (from 0 to non-zero) in
            // our buffer, we get a different value
            for (size_t i = 0; i < buf_size; ++i) {
                ptr[i] = (i & 0x7f) + 1; // need some non-zero value here

                // Try checksums of different-sized chunks, but always
                // 32-bit aligned and big enough to contain all the
                // nonzero bytes.  (Remaining bytes will still be zero
                // from the initial sk_bzero() call.)
                size_t checksum_size = (((i/4)+1)*4);
                REPORTER_ASSERT(fReporter, checksum_size <= buf_size);

                checksum_result curr = ComputeChecksum(ptr, checksum_size);
                REPORTER_ASSERT(fReporter, prev != curr);
                checksum_result again = ComputeChecksum(ptr, checksum_size);
                REPORTER_ASSERT(fReporter, again == curr);
                prev = curr;
            }
        }

        // Return the checksum of a buffer of bytes 'len' long.
        // The pattern of values within the buffer will be consistent
        // for every call, based on 'seed'.
        checksum_result GetTestDataChecksum(size_t len, char seed=0) {
            SkAutoMalloc storage(len);
            char* start = reinterpret_cast<char *>(storage.get());
            char* ptr = start;
            for (size_t i = 0; i < len; ++i) {
                *ptr++ = ((seed+i) & 0x7f);
            }
            checksum_result result = ComputeChecksum(start, len);
            return result;
        }

        void RunTest() {
            // Test self-consistency of checksum algorithms.
            fWhichAlgorithm = kSkChecksum;
            TestChecksumSelfConsistency(128);
            fWhichAlgorithm = kSkCityHash32;
            TestChecksumSelfConsistency(128);
            fWhichAlgorithm = kSkCityHash64;
            TestChecksumSelfConsistency(128);

            // Test checksum results that should be consistent across
            // versions and platforms.
            fWhichAlgorithm = kSkChecksum;
            REPORTER_ASSERT(fReporter, ComputeChecksum(NULL, 0) == 0);
            fWhichAlgorithm = kSkCityHash32;
            REPORTER_ASSERT(fReporter, ComputeChecksum(NULL, 0) == 0xdc56d17a);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(4)   == 0x616e1132);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(8)   == 0xeb0fd2d6);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(128) == 0x5321e430);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(132) == 0x924a10e4);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(256) == 0xd4de9dc9);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(260) == 0xecf0325d);
            fWhichAlgorithm = kSkCityHash64;
            REPORTER_ASSERT(fReporter, ComputeChecksum(NULL, 0) == 0x9ae16a3b2f90404f);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(4)   == 0x82bffd898958e540);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(8)   == 0xad5a13e1e8e93b98);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(128) == 0x10b153630af1f395);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(132) == 0x7db71dc4adcc6647);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(256) == 0xeee763519b91b010);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(260) == 0x2fe19e0b2239bc23);

            // TODO: note the weakness exposed by these collisions...
            // We need to improve the SkChecksum algorithm.
            // We would prefer that these asserts FAIL!
            // Filed as https://code.google.com/p/skia/issues/detail?id=981
            // ('SkChecksum algorithm allows for way too many collisions')
            fWhichAlgorithm = kSkChecksum;
            REPORTER_ASSERT(fReporter,
                GetTestDataChecksum(128) == GetTestDataChecksum(256));
            REPORTER_ASSERT(fReporter,
                GetTestDataChecksum(132) == GetTestDataChecksum(260));
        }

        Reporter* fReporter;
        Algorithm fWhichAlgorithm;
    };

    static TestRegistry gReg(ChecksumTestClass::Factory);
}
<commit_msg>Mark 64-bit constants as ULL to fix broken 32-bit Mac 10.6 build TBR=bungeman Review URL: https://codereview.appspot.com/6867079<commit_after>
/*
 * Copyright 2012 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */
#include "Test.h"
#include "SkChecksum.h"
#include "SkCityHash.h"

// Word size that is large enough to hold results of any checksum type.
typedef uint64_t checksum_result;

namespace skiatest {
    class ChecksumTestClass : public Test {
    public:
        static Test* Factory(void*) {return SkNEW(ChecksumTestClass); }
    protected:
        virtual void onGetName(SkString* name) { name->set("Checksum"); }
        virtual void onRun(Reporter* reporter) {
            this->fReporter = reporter;
            RunTest();
        }
    private:
        enum Algorithm {
            kSkChecksum,
            kSkCityHash32,
            kSkCityHash64
        };

        // Call Compute(data, size) on the appropriate checksum algorithm,
        // depending on this->fWhichAlgorithm.
        checksum_result ComputeChecksum(const char *data, size_t size) {
            switch(fWhichAlgorithm) {
            case kSkChecksum:
                REPORTER_ASSERT_MESSAGE(fReporter,
                                        reinterpret_cast<uintptr_t>(data) % 4 == 0,
                                        "test data pointer is not 32-bit aligned");
                REPORTER_ASSERT_MESSAGE(fReporter, SkIsAlign4(size),
                                        "test data size is not 32-bit aligned");
                return SkChecksum::Compute(reinterpret_cast<const uint32_t *>(data), size);
            case kSkCityHash32:
                return SkCityHash::Compute32(data, size);
            case kSkCityHash64:
                return SkCityHash::Compute64(data, size);
            default:
                SkString message("fWhichAlgorithm has unknown value ");
                message.appendf("%d", fWhichAlgorithm);
                fReporter->reportFailed(message);
            }
            // we never get here
            return 0;
        }

        // Confirm that the checksum algorithm (specified by fWhichAlgorithm)
        // generates the same results if called twice over the same data.
        void TestChecksumSelfConsistency(size_t buf_size) {
            SkAutoMalloc storage(buf_size);
            char* ptr = reinterpret_cast<char *>(storage.get());

            REPORTER_ASSERT(fReporter,
                            GetTestDataChecksum(8, 0) ==
                            GetTestDataChecksum(8, 0));
            REPORTER_ASSERT(fReporter,
                            GetTestDataChecksum(8, 0) !=
                            GetTestDataChecksum(8, 1));

            sk_bzero(ptr, buf_size);
            checksum_result prev = 0;

            // assert that as we change values (from 0 to non-zero) in
            // our buffer, we get a different value
            for (size_t i = 0; i < buf_size; ++i) {
                ptr[i] = (i & 0x7f) + 1; // need some non-zero value here

                // Try checksums of different-sized chunks, but always
                // 32-bit aligned and big enough to contain all the
                // nonzero bytes.  (Remaining bytes will still be zero
                // from the initial sk_bzero() call.)
                size_t checksum_size = (((i/4)+1)*4);
                REPORTER_ASSERT(fReporter, checksum_size <= buf_size);

                checksum_result curr = ComputeChecksum(ptr, checksum_size);
                REPORTER_ASSERT(fReporter, prev != curr);
                checksum_result again = ComputeChecksum(ptr, checksum_size);
                REPORTER_ASSERT(fReporter, again == curr);
                prev = curr;
            }
        }

        // Return the checksum of a buffer of bytes 'len' long.
        // The pattern of values within the buffer will be consistent
        // for every call, based on 'seed'.
        checksum_result GetTestDataChecksum(size_t len, char seed=0) {
            SkAutoMalloc storage(len);
            char* start = reinterpret_cast<char *>(storage.get());
            char* ptr = start;
            for (size_t i = 0; i < len; ++i) {
                *ptr++ = ((seed+i) & 0x7f);
            }
            checksum_result result = ComputeChecksum(start, len);
            return result;
        }

        void RunTest() {
            // Test self-consistency of checksum algorithms.
            fWhichAlgorithm = kSkChecksum;
            TestChecksumSelfConsistency(128);
            fWhichAlgorithm = kSkCityHash32;
            TestChecksumSelfConsistency(128);
            fWhichAlgorithm = kSkCityHash64;
            TestChecksumSelfConsistency(128);

            // Test checksum results that should be consistent across
            // versions and platforms.
            fWhichAlgorithm = kSkChecksum;
            REPORTER_ASSERT(fReporter, ComputeChecksum(NULL, 0) == 0);
            fWhichAlgorithm = kSkCityHash32;
            REPORTER_ASSERT(fReporter, ComputeChecksum(NULL, 0) == 0xdc56d17a);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(4)   == 0x616e1132);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(8)   == 0xeb0fd2d6);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(128) == 0x5321e430);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(132) == 0x924a10e4);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(256) == 0xd4de9dc9);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(260) == 0xecf0325d);
            fWhichAlgorithm = kSkCityHash64;
            REPORTER_ASSERT(fReporter, ComputeChecksum(NULL, 0) == 0x9ae16a3b2f90404fULL);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(4)   == 0x82bffd898958e540ULL);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(8)   == 0xad5a13e1e8e93b98ULL);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(128) == 0x10b153630af1f395ULL);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(132) == 0x7db71dc4adcc6647ULL);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(256) == 0xeee763519b91b010ULL);
            REPORTER_ASSERT(fReporter, GetTestDataChecksum(260) == 0x2fe19e0b2239bc23ULL);

            // TODO: note the weakness exposed by these collisions...
            // We need to improve the SkChecksum algorithm.
            // We would prefer that these asserts FAIL!
            // Filed as https://code.google.com/p/skia/issues/detail?id=981
            // ('SkChecksum algorithm allows for way too many collisions')
            fWhichAlgorithm = kSkChecksum;
            REPORTER_ASSERT(fReporter,
                GetTestDataChecksum(128) == GetTestDataChecksum(256));
            REPORTER_ASSERT(fReporter,
                GetTestDataChecksum(132) == GetTestDataChecksum(260));
        }

        Reporter* fReporter;
        Algorithm fWhichAlgorithm;
    };

    static TestRegistry gReg(ChecksumTestClass::Factory);
}
<|endoftext|>


<commit_before>/*********************************************************************************
 *
 * Inviwo - Interactive Visualization Workshop
 *
 * Copyright (c) 2017 Inviwo Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *********************************************************************************/

#include <warn/push>
#include <warn/ignore/all>
#include <gtest/gtest.h>
#include <warn/pop>

#ifdef WIN32
#define NOMINMAX
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#endif

#include <inviwo/core/util/filesystem.h>
#include <inviwo/core/util/exception.h>
#include <modules/cimg/cimgutils.h>
#include <modules/cimg/cimglayerreader.h>

#include <fstream>
#include <array>
#include <cstdio>
#include <algorithm>

namespace inviwo {

#ifdef WIN32
std::wstring get_utf16(const std::string &str, int codepage) {
    if (str.empty()) return std::wstring();
    int sz = MultiByteToWideChar(codepage, 0, &str[0], (int)str.size(), 0, 0);
    std::wstring res(sz, 0);
    MultiByteToWideChar(codepage, 0, &str[0], (int)str.size(), &res[0], sz);
    return res;
}
#endif

class TempFileHandle {
public:
    explicit TempFileHandle(const std::string& prefix = "", const std::string& suffix = "") {

#ifdef WIN32
        // get temp directory
        std::array<wchar_t, MAX_PATH> tempPath;
        std::array<wchar_t, MAX_PATH> tempFile;
        auto retVal = GetTempPath(MAX_PATH, tempPath.data());
        if ((retVal > MAX_PATH) || (retVal == 0)) {
            throw Exception("could not locate temp folder");
        }
        // generate temp file name
        std::wstring prefixW(prefix.begin(), prefix.end());
        auto uRetVal = GetTempFileName(tempPath.data(),   // directory for tmp files
                                       prefixW.c_str(),   // temp file name prefix
                                       0,                 // create unique name
                                       tempFile.data());  // buffer for name
        if (uRetVal == 0) {
            throw Exception("could not create temporary file name");
        }

        filename.assign(tempFile.begin(),
                        tempFile.begin() + std::min<size_t>(wcslen(tempFile.data()), MAX_PATH));
        filename += suffix;

        handle_ = fopen(filename_.c_str(), "w");
        if (!handle_) {
            throw Exception("could not open temporary file");
        }
#else

        static const std::string unqiue = "XXXXXX";

        const int suffixlen = suffix.size();

        std::vector<char> fileTemplate;
        fileTemplate.insert(fileTemplate.end(), prefix.begin(), prefix.end());
        fileTemplate.insert(fileTemplate.end(), unqiue.begin(), unqiue.end());
        fileTemplate.insert(fileTemplate.end(), suffix.begin(), suffix.end());
        fileTemplate.push_back('\0');

        int fd = mkstemps(fileTemplate.data(), suffixlen);
        if (fd == -1) {
            throw Exception("could not create temporary file");
        }
        handle_ = fdopen(fd, "w");
        if (!handle_) {
            throw Exception("could not open temporary file");
        }
        filename_.assign(fileTemplate.begin(), fileTemplate.end() - 1);
#endif
    }

    TempFileHandle(const TempFileHandle&) = delete;
    TempFileHandle& operator=(const TempFileHandle&) = delete;

    TempFileHandle(TempFileHandle&& rhs)
        : handle_{rhs.handle_}, filename_{std::move(rhs.filename_)} {
        rhs.handle_ = nullptr;
        rhs.filename_ = "";
    }
    TempFileHandle& operator=(TempFileHandle&& rhs) {
        if (this != &rhs) {
            cleanup();
            handle_ = rhs.handle_;
            filename_ = std::move(rhs.filename_);
            rhs.handle_ = nullptr;
            rhs.filename_ = "";
        }
        return *this;
    }

    ~TempFileHandle() { cleanup(); }

    const std::string& getFileName() const { return filename_; }

    FILE* getHandle() { return handle_; }
    operator FILE*() { return handle_; };

private:
    void cleanup() {
        if (handle_) fclose(handle_);

        if (!filename_.empty()) {

#ifdef WIN32
            std::wstring fileW(filename_.begin(), filename_.end());
            DeleteFile(fileW.c_str());
#else
            remove(filename_.c_str());
#endif
        }
    }

    FILE* handle_;
    std::string filename_;
};

TEST(CImgUtils, cimgToBuffer) {
    // load source image
    const auto filename = filesystem::getPath(PathType::Tests, "/images/swirl.png");
    CImgLayerReader reader;
    auto layer = reader.readData(filename);

    const std::string testExtension = "png";

    // write layer to a temporary png file
    TempFileHandle tmpFile("cimg", std::string(".") + testExtension);

    cimgutil::saveLayer(tmpFile.getFileName(), layer.get());

    // read file contents
    std::vector<unsigned char> fileContents;
    std::ifstream in(tmpFile.getFileName().c_str(), std::ifstream::binary);
    if (in.is_open()) {
        in.seekg(0, in.end);
        size_t fileLen = in.tellg();
        in.seekg(0, in.beg);

        fileContents.resize(fileLen);
        in.read(reinterpret_cast<char*>(fileContents.data()), fileLen);

        in.close();
    }

    // write layer to buffer
    auto imgBuffer = cimgutil::saveLayerToBuffer(testExtension, layer.get());
    ASSERT_TRUE(imgBuffer != nullptr) << "buffer is empty";

    // compare buffer and file contents

    ASSERT_TRUE(fileContents.size() == imgBuffer->size()) << "buffer and file size does not match";
    EXPECT_EQ(*imgBuffer.get(), fileContents) << "buffer and file contents do not match";
}

}
<commit_msg>Cimg: UnitTest: Windows compile fix<commit_after>/*********************************************************************************
 *
 * Inviwo - Interactive Visualization Workshop
 *
 * Copyright (c) 2017 Inviwo Foundation
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 *********************************************************************************/

#include <warn/push>
#include <warn/ignore/all>
#include <gtest/gtest.h>
#include <warn/pop>

#ifdef WIN32
#define NOMINMAX
#define WIN32_LEAN_AND_MEAN
#include <windows.h>
#endif

#include <inviwo/core/util/filesystem.h>
#include <inviwo/core/util/exception.h>
#include <modules/cimg/cimgutils.h>
#include <modules/cimg/cimglayerreader.h>

#include <fstream>
#include <array>
#include <cstdio>
#include <algorithm>

namespace inviwo {

#ifdef WIN32
std::wstring get_utf16(const std::string &str, int codepage) {
    if (str.empty()) return std::wstring();
    int sz = MultiByteToWideChar(codepage, 0, &str[0], (int)str.size(), 0, 0);
    std::wstring res(sz, 0);
    MultiByteToWideChar(codepage, 0, &str[0], (int)str.size(), &res[0], sz);
    return res;
}
#endif

class TempFileHandle {
public:
    explicit TempFileHandle(const std::string& prefix = "", const std::string& suffix = "") {

#ifdef WIN32
        // get temp directory
        std::array<wchar_t, MAX_PATH> tempPath;
        std::array<wchar_t, MAX_PATH> tempFile;
        auto retVal = GetTempPath(MAX_PATH, tempPath.data());
        if ((retVal > MAX_PATH) || (retVal == 0)) {
            throw Exception("could not locate temp folder");
        }
        // generate temp file name
        std::wstring prefixW(prefix.begin(), prefix.end());
        auto uRetVal = GetTempFileName(tempPath.data(),   // directory for tmp files
                                       prefixW.c_str(),   // temp file name prefix
                                       0,                 // create unique name
                                       tempFile.data());  // buffer for name
        if (uRetVal == 0) {
            throw Exception("could not create temporary file name");
        }

        filename_.assign(tempFile.begin(),
                        tempFile.begin() + std::min<size_t>(wcslen(tempFile.data()), MAX_PATH));
        filename_ += suffix;

        handle_ = fopen(filename_.c_str(), "w");
        if (!handle_) {
            throw Exception("could not open temporary file");
        }
#else

        static const std::string unqiue = "XXXXXX";

        const int suffixlen = suffix.size();

        std::vector<char> fileTemplate;
        fileTemplate.insert(fileTemplate.end(), prefix.begin(), prefix.end());
        fileTemplate.insert(fileTemplate.end(), unqiue.begin(), unqiue.end());
        fileTemplate.insert(fileTemplate.end(), suffix.begin(), suffix.end());
        fileTemplate.push_back('\0');

        int fd = mkstemps(fileTemplate.data(), suffixlen);
        if (fd == -1) {
            throw Exception("could not create temporary file");
        }
        handle_ = fdopen(fd, "w");
        if (!handle_) {
            throw Exception("could not open temporary file");
        }
        filename_.assign(fileTemplate.begin(), fileTemplate.end() - 1);
#endif
    }

    TempFileHandle(const TempFileHandle&) = delete;
    TempFileHandle& operator=(const TempFileHandle&) = delete;

    TempFileHandle(TempFileHandle&& rhs)
        : handle_{rhs.handle_}, filename_{std::move(rhs.filename_)} {
        rhs.handle_ = nullptr;
        rhs.filename_ = "";
    }
    TempFileHandle& operator=(TempFileHandle&& rhs) {
        if (this != &rhs) {
            cleanup();
            handle_ = rhs.handle_;
            filename_ = std::move(rhs.filename_);
            rhs.handle_ = nullptr;
            rhs.filename_ = "";
        }
        return *this;
    }

    ~TempFileHandle() { cleanup(); }

    const std::string& getFileName() const { return filename_; }

    FILE* getHandle() { return handle_; }
    operator FILE*() { return handle_; };

private:
    void cleanup() {
        if (handle_) fclose(handle_);

        if (!filename_.empty()) {

#ifdef WIN32
            std::wstring fileW(filename_.begin(), filename_.end());
            DeleteFile(fileW.c_str());
#else
            remove(filename_.c_str());
#endif
        }
    }

    FILE* handle_;
    std::string filename_;
};

TEST(CImgUtils, cimgToBuffer) {
    // load source image
    const auto filename = filesystem::getPath(PathType::Tests, "/images/swirl.png");
    CImgLayerReader reader;
    auto layer = reader.readData(filename);

    const std::string testExtension = "png";

    // write layer to a temporary png file
    TempFileHandle tmpFile("cimg", std::string(".") + testExtension);

    cimgutil::saveLayer(tmpFile.getFileName(), layer.get());

    // read file contents
    std::vector<unsigned char> fileContents;
    std::ifstream in(tmpFile.getFileName().c_str(), std::ifstream::binary);
    if (in.is_open()) {
        in.seekg(0, in.end);
        size_t fileLen = in.tellg();
        in.seekg(0, in.beg);

        fileContents.resize(fileLen);
        in.read(reinterpret_cast<char*>(fileContents.data()), fileLen);

        in.close();
    }

    // write layer to buffer
    auto imgBuffer = cimgutil::saveLayerToBuffer(testExtension, layer.get());
    ASSERT_TRUE(imgBuffer != nullptr) << "buffer is empty";

    // compare buffer and file contents

    ASSERT_TRUE(fileContents.size() == imgBuffer->size()) << "buffer and file size does not match";
    EXPECT_EQ(*imgBuffer.get(), fileContents) << "buffer and file contents do not match";
}

}
<|endoftext|>


<commit_before>/*
    Copyright 2011 Google Inc.

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

         http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
 */

#include "SkReader32.h"
#include "Test.h"

static void assert_eof(skiatest::Reporter* reporter, const SkReader32& reader) {
    REPORTER_ASSERT(reporter, reader.eof());
    REPORTER_ASSERT(reporter, reader.size() == reader.offset());
    REPORTER_ASSERT(reporter, (const char*)reader.peek() ==
                    (const char*)reader.base() + reader.size());
}

static void assert_start(skiatest::Reporter* reporter, const SkReader32& reader) {
    REPORTER_ASSERT(reporter, 0 == reader.offset());
    REPORTER_ASSERT(reporter, reader.size() == reader.available());
    REPORTER_ASSERT(reporter, reader.isAvailable(reader.size()));
    REPORTER_ASSERT(reporter, !reader.isAvailable(reader.size() + 1));
    REPORTER_ASSERT(reporter, reader.peek() == reader.base());
}

static void assert_empty(skiatest::Reporter* reporter, const SkReader32& reader) {
    REPORTER_ASSERT(reporter, 0 == reader.size());
    REPORTER_ASSERT(reporter, 0 == reader.offset());
    REPORTER_ASSERT(reporter, 0 == reader.available());
    REPORTER_ASSERT(reporter, !reader.isAvailable(1));
    assert_eof(reporter, reader);
    assert_start(reporter, reader);
}

static void Tests(skiatest::Reporter* reporter) {
    SkReader32 reader;
    assert_empty(reporter, reader);
    REPORTER_ASSERT(reporter, NULL == reader.base());
    REPORTER_ASSERT(reporter, NULL == reader.peek());

    size_t i;

    const int32_t data[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
    const SkScalar data2[] = { 0, SK_Scalar1, -SK_Scalar1, SK_Scalar1/2 };
    char buffer[SkMax32(sizeof(data), sizeof(data2))];

    reader.setMemory(data, sizeof(data));
    for (i = 0; i < SK_ARRAY_COUNT(data); ++i) {
        REPORTER_ASSERT(reporter, sizeof(data) == reader.size());
        REPORTER_ASSERT(reporter, i*4 == reader.offset());
        REPORTER_ASSERT(reporter, (const void*)data == reader.base());
        REPORTER_ASSERT(reporter, (const void*)&data[i] == reader.peek());
        REPORTER_ASSERT(reporter, data[i] == reader.readInt());
    }
    assert_eof(reporter, reader);
    reader.rewind();
    assert_start(reporter, reader);
    reader.read(buffer, sizeof(buffer));
    REPORTER_ASSERT(reporter, !memcmp(data, buffer, sizeof(buffer)));

    reader.setMemory(data2, sizeof(data2));
    for (i = 0; i < SK_ARRAY_COUNT(data2); ++i) {
        REPORTER_ASSERT(reporter, sizeof(data2) == reader.size());
        REPORTER_ASSERT(reporter, i*4 == reader.offset());
        REPORTER_ASSERT(reporter, (const void*)data2 == reader.base());
        REPORTER_ASSERT(reporter, (const void*)&data2[i] == reader.peek());
        REPORTER_ASSERT(reporter, data2[i] == reader.readScalar());
    }
    assert_eof(reporter, reader);
    reader.rewind();
    assert_start(reporter, reader);
    reader.read(buffer, sizeof(buffer));
    REPORTER_ASSERT(reporter, !memcmp(data2, buffer, sizeof(buffer)));

    reader.setMemory(NULL, 0);
    assert_empty(reporter, reader);
    REPORTER_ASSERT(reporter, NULL == reader.base());
    REPORTER_ASSERT(reporter, NULL == reader.peek());
}

#include "TestClassDef.h"
DEFINE_TESTCLASS("Reader32", Reader32Class, Tests)

<commit_msg>pass correct size to read(buffer, ...) tests<commit_after>/*
    Copyright 2011 Google Inc.

    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at

         http://www.apache.org/licenses/LICENSE-2.0

    Unless required by applicable law or agreed to in writing, software
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
 */

#include "SkReader32.h"
#include "Test.h"

static void assert_eof(skiatest::Reporter* reporter, const SkReader32& reader) {
    REPORTER_ASSERT(reporter, reader.eof());
    REPORTER_ASSERT(reporter, reader.size() == reader.offset());
    REPORTER_ASSERT(reporter, (const char*)reader.peek() ==
                    (const char*)reader.base() + reader.size());
}

static void assert_start(skiatest::Reporter* reporter, const SkReader32& reader) {
    REPORTER_ASSERT(reporter, 0 == reader.offset());
    REPORTER_ASSERT(reporter, reader.size() == reader.available());
    REPORTER_ASSERT(reporter, reader.isAvailable(reader.size()));
    REPORTER_ASSERT(reporter, !reader.isAvailable(reader.size() + 1));
    REPORTER_ASSERT(reporter, reader.peek() == reader.base());
}

static void assert_empty(skiatest::Reporter* reporter, const SkReader32& reader) {
    REPORTER_ASSERT(reporter, 0 == reader.size());
    REPORTER_ASSERT(reporter, 0 == reader.offset());
    REPORTER_ASSERT(reporter, 0 == reader.available());
    REPORTER_ASSERT(reporter, !reader.isAvailable(1));
    assert_eof(reporter, reader);
    assert_start(reporter, reader);
}

static void Tests(skiatest::Reporter* reporter) {
    SkReader32 reader;
    assert_empty(reporter, reader);
    REPORTER_ASSERT(reporter, NULL == reader.base());
    REPORTER_ASSERT(reporter, NULL == reader.peek());

    size_t i;

    const int32_t data[] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
    const SkScalar data2[] = { 0, SK_Scalar1, -SK_Scalar1, SK_Scalar1/2 };
    char buffer[SkMax32(sizeof(data), sizeof(data2))];

    reader.setMemory(data, sizeof(data));
    for (i = 0; i < SK_ARRAY_COUNT(data); ++i) {
        REPORTER_ASSERT(reporter, sizeof(data) == reader.size());
        REPORTER_ASSERT(reporter, i*4 == reader.offset());
        REPORTER_ASSERT(reporter, (const void*)data == reader.base());
        REPORTER_ASSERT(reporter, (const void*)&data[i] == reader.peek());
        REPORTER_ASSERT(reporter, data[i] == reader.readInt());
    }
    assert_eof(reporter, reader);
    reader.rewind();
    assert_start(reporter, reader);
    reader.read(buffer, sizeof(data));
    REPORTER_ASSERT(reporter, !memcmp(data, buffer, sizeof(data)));

    reader.setMemory(data2, sizeof(data2));
    for (i = 0; i < SK_ARRAY_COUNT(data2); ++i) {
        REPORTER_ASSERT(reporter, sizeof(data2) == reader.size());
        REPORTER_ASSERT(reporter, i*4 == reader.offset());
        REPORTER_ASSERT(reporter, (const void*)data2 == reader.base());
        REPORTER_ASSERT(reporter, (const void*)&data2[i] == reader.peek());
        REPORTER_ASSERT(reporter, data2[i] == reader.readScalar());
    }
    assert_eof(reporter, reader);
    reader.rewind();
    assert_start(reporter, reader);
    reader.read(buffer, sizeof(data2));
    REPORTER_ASSERT(reporter, !memcmp(data2, buffer, sizeof(data2)));

    reader.setMemory(NULL, 0);
    assert_empty(reporter, reader);
    REPORTER_ASSERT(reporter, NULL == reader.base());
    REPORTER_ASSERT(reporter, NULL == reader.peek());
}

#include "TestClassDef.h"
DEFINE_TESTCLASS("Reader32", Reader32Class, Tests)

<|endoftext|>


<commit_before>///
/// @file  LoadBalancerS2.cpp
/// @brief The LoadBalancerS2 assigns work to the individual threads
///        in the computation of the special leaves in the
///        Lagarias-Miller-Odlyzko, Deleglise-Rivat and Gourdon
///        prime counting algorithms. This load balancer is used
///        by the S2_hard(x, y) and D(x, y) functions.
///
///        Simply parallelizing the computation of the special
///        leaves in the Lagarias-Miller-Odlyzko algorithm by
///        subdividing the sieve interval by the number of threads
///        into equally sized subintervals does not scale because
///        the distribution of the special leaves is highly skewed
///        and most special leaves are in the first few segments
///        whereas later on there are very few special leaves.
///
///        This LoadBalancerS2 gradually increases the number of
///        segments to sieve as long the expected runtime of the
///        sieve distance is smaller than the expected finish time
///        of the algorithm. Near the end the LoadBalancerS2 will
///        gradually decrease the number of segments to sieve in
///        order to prevent that 1 thread will run much longer
///        than all the other threads.
///
/// Copyright (C) 2021 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#include <LoadBalancerS2.hpp>
#include <primecount-internal.hpp>
#include <Status.hpp>
#include <Sieve.hpp>
#include <imath.hpp>
#include <int128_t.hpp>
#include <min.hpp>
#include <print.hpp>

#include <stdint.h>

namespace primecount {

LoadBalancerS2::LoadBalancerS2(maxint_t x,
                               int64_t sieve_limit,
                               maxint_t sum_approx,
                               int threads,
                               bool is_print) :
  sieve_limit_(sieve_limit),
  segments_(1),
  sum_approx_(sum_approx),
  time_(get_time()),
  is_print_(is_print),
  status_(x)
{
  // Try to use a segment size that fits exactly
  // into the CPU's L1 data cache. Also the
  // segmented sieve of Eratosthenes requires the
  // segment size to be >= sqrt(sieve_limit).
  int64_t l1_dcache_size = 32 * (1 << 10);
  int64_t numbers_per_byte = 30;
  int64_t sqrt_limit = isqrt(sieve_limit);
  max_size_ = max(sqrt_limit, l1_dcache_size * numbers_per_byte);

  // When a single thread is used (and printing
  // is disabled) we can set segment_size to
  // its maximum size as load balancing is only
  // useful for multi-threading.
  if (threads == 1 && !is_print)
    segment_size_ = max_size_;
  else
  {
    // Start with a tiny segment size of x^(1/4) as
    // most special leaves are in the first few
    // segments and as we need to ensure that all
    // threads are assigned an equal amount of work.
    segment_size_ = isqrt(isqrt(x));
  }

  int64_t min_size = 1 << 9;
  segment_size_ = max(min_size, segment_size_);
  segment_size_ = Sieve::get_segment_size(segment_size_);
}

maxint_t LoadBalancerS2::get_sum() const
{
  return sum_;
}

bool LoadBalancerS2::get_work(ThreadSettings& thread)
{
  LockGuard lockGuard(lock_);
  sum_ += thread.sum;

  if (is_print_)
  {
    uint64_t dist = thread.segments * thread.segment_size;
    uint64_t high = thread.low + dist;
    status_.print(high, sieve_limit_, sum_, sum_approx_);
  }

  update_load_balancing(thread);

  thread.low = low_;
  thread.segments = segments_;
  thread.segment_size = segment_size_;
  thread.sum = 0;
  thread.secs = 0;
  thread.init_secs = 0;

  low_ += segments_ * segment_size_;
  bool is_work = thread.low < sieve_limit_;

  return is_work;
}

void LoadBalancerS2::update_load_balancing(const ThreadSettings& thread)
{
  if (thread.low > max_low_)
  {
    max_low_ = thread.low;
    segments_ = thread.segments;

    // We only start increasing the segment_size and segments
    // per thread once the first special leaves have been
    // found. Near the start there is a very large number of
    // leaves and we don't want a single thread to compute
    // them all by himself (which would cause scaling issues).
    if (sum_ == 0)
      return;

    // Slowly increase the segment size until it reaches
    // sqrt(sieve_limit). Most special leaves are located
    // around y, hence we need to be careful to not assign too
    // much work to a single thread in this region.
    if (segment_size_ < max_size_)
    {
      segment_size_ += segment_size_ / 16;
      segment_size_ = min(segment_size_, max_size_);
      segment_size_ = Sieve::get_segment_size(segment_size_);
    }
    else
      update_segments(thread);
  }
}

/// Increase or decrease the number of segments per thread
/// based on the remaining runtime.
///
void LoadBalancerS2::update_segments(const ThreadSettings& thread)
{
  // Near the end it is important that threads run only for
  // a short amount of time in order to ensure that all
  // threads finish nearly at the same time. Since the
  // remaining time is just a rough estimation we want to be
  // very conservative so we divide the remaining time by 3.
  double rem_secs = remaining_secs() / 3;

  // If the previous thread runtime is larger than the
  // estimated remaining time the factor that we calculate
  // below will be < 1 and we will reduce the number of
  // segments per thread. Otherwise if the factor > 1 we
  // will increase the number of segments per thread.
  double min_secs = 0.001;
  double divider = max(min_secs, thread.secs);
  double factor = rem_secs / divider;

  // For small and medium computations the thread runtime
  // should be about 5000x the thread initialization time.
  // However for very large computations we want to further
  // reduce the thread runtimes in order to increase the
  // backup frequency. If the thread runtime is > 6 hours
  // we reduce the thread runtime to about 50x the thread
  // initialization time.
  double init_secs = max(min_secs, thread.init_secs);
  double init_factor = in_between(50, (3600 * 6) / init_secs, 5000);

  // Reduce the thread runtime if it is much larger than
  // its initialization time. This increases the number of
  // backups without deteriorating performance as we make
  // sure that the thread runtime is still much larger than
  // the thread initialization time.
  if (thread.secs > min_secs &&
      thread.secs > thread.init_secs * init_factor)
  {
    double old = factor;
    double next_runtime = thread.init_secs * init_factor;
    factor = next_runtime / thread.secs;
    factor = min(factor, old);
  }

  // Near the end when the remaining time goes close to 0
  // the load balancer tends to reduce the number of
  // segments per thread also close to 0 which is very bad
  // for performance. The condition below fixes this issue
  // and ensures that the thread runtime is always at
  // least 20x the thread initialization time.
  if (thread.secs > 0 &&
      thread.secs * factor < thread.init_secs * 20)
  {
    double next_runtime = thread.init_secs * 20;
    double current_runtime = thread.secs;
    factor = next_runtime / current_runtime;
  }

  // Since the distribution of the special leaves is highly
  // skewed (at the beginning) we want to increase the
  // number of segments per thread very slowly. Because if
  // the previously sieved interval contained no special
  // leaves but the next interval contains many special
  // leaves then sieving the next interval might take orders
  // of magnitude more time even if the interval size is
  // identical.
  factor = in_between(0.5, factor, 2.0);
  double next_runtime = thread.secs * factor;

  if (next_runtime < min_secs)
    segments_ *= 2;
  else
  {
    double new_segments = round(segments_ * factor);
    segments_ = (int64_t) new_segments;
    segments_ = max(segments_, 1);
  }
}

/// Remaining seconds till finished
double LoadBalancerS2::remaining_secs() const
{
  double percent = status_.getPercent(low_, sieve_limit_, sum_, sum_approx_);
  percent = in_between(10, percent, 100);
  double total_secs = get_time() - time_;
  double secs = total_secs * (100 / percent) - total_secs;
  return secs;
}

} // namespace
<commit_msg>Remove unused header<commit_after>///
/// @file  LoadBalancerS2.cpp
/// @brief The LoadBalancerS2 assigns work to the individual threads
///        in the computation of the special leaves in the
///        Lagarias-Miller-Odlyzko, Deleglise-Rivat and Gourdon
///        prime counting algorithms. This load balancer is used
///        by the S2_hard(x, y) and D(x, y) functions.
///
///        Simply parallelizing the computation of the special
///        leaves in the Lagarias-Miller-Odlyzko algorithm by
///        subdividing the sieve interval by the number of threads
///        into equally sized subintervals does not scale because
///        the distribution of the special leaves is highly skewed
///        and most special leaves are in the first few segments
///        whereas later on there are very few special leaves.
///
///        This LoadBalancerS2 gradually increases the number of
///        segments to sieve as long the expected runtime of the
///        sieve distance is smaller than the expected finish time
///        of the algorithm. Near the end the LoadBalancerS2 will
///        gradually decrease the number of segments to sieve in
///        order to prevent that 1 thread will run much longer
///        than all the other threads.
///
/// Copyright (C) 2021 Kim Walisch, <kim.walisch@gmail.com>
///
/// This file is distributed under the BSD License. See the COPYING
/// file in the top level directory.
///

#include <LoadBalancerS2.hpp>
#include <primecount-internal.hpp>
#include <Status.hpp>
#include <Sieve.hpp>
#include <imath.hpp>
#include <int128_t.hpp>
#include <min.hpp>

#include <stdint.h>

namespace primecount {

LoadBalancerS2::LoadBalancerS2(maxint_t x,
                               int64_t sieve_limit,
                               maxint_t sum_approx,
                               int threads,
                               bool is_print) :
  sieve_limit_(sieve_limit),
  segments_(1),
  sum_approx_(sum_approx),
  time_(get_time()),
  is_print_(is_print),
  status_(x)
{
  // Try to use a segment size that fits exactly
  // into the CPU's L1 data cache. Also the
  // segmented sieve of Eratosthenes requires the
  // segment size to be >= sqrt(sieve_limit).
  int64_t l1_dcache_size = 32 * (1 << 10);
  int64_t numbers_per_byte = 30;
  int64_t sqrt_limit = isqrt(sieve_limit);
  max_size_ = max(sqrt_limit, l1_dcache_size * numbers_per_byte);

  // When a single thread is used (and printing
  // is disabled) we can set segment_size to
  // its maximum size as load balancing is only
  // useful for multi-threading.
  if (threads == 1 && !is_print)
    segment_size_ = max_size_;
  else
  {
    // Start with a tiny segment size of x^(1/4) as
    // most special leaves are in the first few
    // segments and as we need to ensure that all
    // threads are assigned an equal amount of work.
    segment_size_ = isqrt(isqrt(x));
  }

  int64_t min_size = 1 << 9;
  segment_size_ = max(min_size, segment_size_);
  segment_size_ = Sieve::get_segment_size(segment_size_);
}

maxint_t LoadBalancerS2::get_sum() const
{
  return sum_;
}

bool LoadBalancerS2::get_work(ThreadSettings& thread)
{
  LockGuard lockGuard(lock_);
  sum_ += thread.sum;

  if (is_print_)
  {
    uint64_t dist = thread.segments * thread.segment_size;
    uint64_t high = thread.low + dist;
    status_.print(high, sieve_limit_, sum_, sum_approx_);
  }

  update_load_balancing(thread);

  thread.low = low_;
  thread.segments = segments_;
  thread.segment_size = segment_size_;
  thread.sum = 0;
  thread.secs = 0;
  thread.init_secs = 0;

  low_ += segments_ * segment_size_;
  bool is_work = thread.low < sieve_limit_;

  return is_work;
}

void LoadBalancerS2::update_load_balancing(const ThreadSettings& thread)
{
  if (thread.low > max_low_)
  {
    max_low_ = thread.low;
    segments_ = thread.segments;

    // We only start increasing the segment_size and segments
    // per thread once the first special leaves have been
    // found. Near the start there is a very large number of
    // leaves and we don't want a single thread to compute
    // them all by himself (which would cause scaling issues).
    if (sum_ == 0)
      return;

    // Slowly increase the segment size until it reaches
    // sqrt(sieve_limit). Most special leaves are located
    // around y, hence we need to be careful to not assign too
    // much work to a single thread in this region.
    if (segment_size_ < max_size_)
    {
      segment_size_ += segment_size_ / 16;
      segment_size_ = min(segment_size_, max_size_);
      segment_size_ = Sieve::get_segment_size(segment_size_);
    }
    else
      update_segments(thread);
  }
}

/// Increase or decrease the number of segments per thread
/// based on the remaining runtime.
///
void LoadBalancerS2::update_segments(const ThreadSettings& thread)
{
  // Near the end it is important that threads run only for
  // a short amount of time in order to ensure that all
  // threads finish nearly at the same time. Since the
  // remaining time is just a rough estimation we want to be
  // very conservative so we divide the remaining time by 3.
  double rem_secs = remaining_secs() / 3;

  // If the previous thread runtime is larger than the
  // estimated remaining time the factor that we calculate
  // below will be < 1 and we will reduce the number of
  // segments per thread. Otherwise if the factor > 1 we
  // will increase the number of segments per thread.
  double min_secs = 0.001;
  double divider = max(min_secs, thread.secs);
  double factor = rem_secs / divider;

  // For small and medium computations the thread runtime
  // should be about 5000x the thread initialization time.
  // However for very large computations we want to further
  // reduce the thread runtimes in order to increase the
  // backup frequency. If the thread runtime is > 6 hours
  // we reduce the thread runtime to about 50x the thread
  // initialization time.
  double init_secs = max(min_secs, thread.init_secs);
  double init_factor = in_between(50, (3600 * 6) / init_secs, 5000);

  // Reduce the thread runtime if it is much larger than
  // its initialization time. This increases the number of
  // backups without deteriorating performance as we make
  // sure that the thread runtime is still much larger than
  // the thread initialization time.
  if (thread.secs > min_secs &&
      thread.secs > thread.init_secs * init_factor)
  {
    double old = factor;
    double next_runtime = thread.init_secs * init_factor;
    factor = next_runtime / thread.secs;
    factor = min(factor, old);
  }

  // Near the end when the remaining time goes close to 0
  // the load balancer tends to reduce the number of
  // segments per thread also close to 0 which is very bad
  // for performance. The condition below fixes this issue
  // and ensures that the thread runtime is always at
  // least 20x the thread initialization time.
  if (thread.secs > 0 &&
      thread.secs * factor < thread.init_secs * 20)
  {
    double next_runtime = thread.init_secs * 20;
    double current_runtime = thread.secs;
    factor = next_runtime / current_runtime;
  }

  // Since the distribution of the special leaves is highly
  // skewed (at the beginning) we want to increase the
  // number of segments per thread very slowly. Because if
  // the previously sieved interval contained no special
  // leaves but the next interval contains many special
  // leaves then sieving the next interval might take orders
  // of magnitude more time even if the interval size is
  // identical.
  factor = in_between(0.5, factor, 2.0);
  double next_runtime = thread.secs * factor;

  if (next_runtime < min_secs)
    segments_ *= 2;
  else
  {
    double new_segments = round(segments_ * factor);
    segments_ = (int64_t) new_segments;
    segments_ = max(segments_, 1);
  }
}

/// Remaining seconds till finished
double LoadBalancerS2::remaining_secs() const
{
  double percent = status_.getPercent(low_, sieve_limit_, sum_, sum_approx_);
  percent = in_between(10, percent, 100);
  double total_secs = get_time() - time_;
  double secs = total_secs * (100 / percent) - total_secs;
  return secs;
}

} // namespace
<|endoftext|>


<commit_before>#include "MenuStyle.hpp"
#include <Config/Globals.hpp>
#include <algorithm>

MenuStyle::MenuStyle(int h, int w):
	Style(h, w),
	menu(nullptr)
{
	create();
}

void MenuStyle::create()
{
	Style::create();
	destroy();

	title = new Window(main, Globals::title_height + 2, -1, 1, 1);

	int hh = main->getH() - title->getH() - 2;
	int ww = main->getW() / 3;
	int y = title->getH() + 1;
	int x = main->getW() / 3 - 2;

	menu = new Window(main, hh, ww, y, x);

	// windows should be pushed from the background to the foreground
	// otherwise expect the unexpected
	windows.push_back(&main);
	windows.push_back(&title);
	windows.push_back(&menu);
}

MenuStyle::~MenuStyle()
{
	destroy();
}

void MenuStyle::destroy()
{
	if (menu) {
		delete menu;
		menu = nullptr;
	}
}

void MenuStyle::draw(MenuData *data)
{
	clear();
	data->draw(menu);
	title->print(Globals::title, 1, title->getW() / 2 - Globals::title_length/2, COLOR_RED, -1);
	refresh();
}

void MenuStyle::resize(MenuData *data, int h, int w)
{
	_h = h;
	_w = w;
	clearScreen();
	create();
	draw(data);
}

void MenuStyle::clearScreen()
{
	clear();
	refresh();
}

void MenuStyle::clear()
{
	std::for_each(windows.begin(), windows.end(), [this](Window** &w){(**w).clear();});
}

void MenuStyle::refresh()
{
	std::for_each(windows.rbegin(), windows.rend(), [this](Window** &w){(**w).refresh();});
}

void MenuStyle::setBorders()
{
	std::for_each(windows.begin(), windows.end(), [this](Window** &w){(**w).toggleBorders();});
}<commit_msg>Recenter list of menu options<commit_after>#include "MenuStyle.hpp"
#include <Config/Globals.hpp>
#include <algorithm>

MenuStyle::MenuStyle(int h, int w):
	Style(h, w),
	menu(nullptr)
{
	create();
}

void MenuStyle::create()
{
	Style::create();
	destroy();

	title = new Window(main, Globals::title_height + 2, -1, 1, 1);

	int h = main->getH() - title->getH() - 2;
	int w = main->getW() / 3;
	int y = title->getH() + 1;
	int x = main->getW()/2 - w/2;

	menu = new Window(main, h, w, y, x);

	// windows should be pushed from the background to the foreground
	// otherwise expect the unexpected
	windows.push_back(&main);
	windows.push_back(&title);
	windows.push_back(&menu);
}

MenuStyle::~MenuStyle()
{
	destroy();
}

void MenuStyle::destroy()
{
	if (menu) {
		delete menu;
		menu = nullptr;
	}
}

void MenuStyle::draw(MenuData *data)
{
	clear();
	data->draw(menu);
	title->print(Globals::title, 1, title->getW() / 2 - Globals::title_length/2, COLOR_RED, -1);
	refresh();
}

void MenuStyle::resize(MenuData *data, int h, int w)
{
	_h = h;
	_w = w;
	clearScreen();
	create();
	draw(data);
}

void MenuStyle::clearScreen()
{
	clear();
	refresh();
}

void MenuStyle::clear()
{
	std::for_each(windows.begin(), windows.end(), [this](Window** &w){(**w).clear();});
}

void MenuStyle::refresh()
{
	std::for_each(windows.rbegin(), windows.rend(), [this](Window** &w){(**w).refresh();});
}

void MenuStyle::setBorders()
{
	std::for_each(windows.begin(), windows.end(), [this](Window** &w){(**w).toggleBorders();});
}<|endoftext|>