/*
 * This file is part of ELKI:
 * Environment for Developing KDD-Applications Supported by Index-Structures
 *
 * Copyright (C) 2022
 * ELKI Development Team
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */
package elki.application.greedyensemble;

import java.util.ArrayList;
import java.util.Arrays;

import elki.application.AbstractApplication;
import elki.data.DoubleVector;
import elki.data.NumberVector;
import elki.data.type.TypeUtil;
import elki.database.Database;
import elki.database.DatabaseUtil;
import elki.database.datastore.DataStoreFactory;
import elki.database.datastore.DataStoreUtil;
import elki.database.datastore.WritableDataStore;
import elki.database.ids.*;
import elki.database.relation.MaterializedRelation;
import elki.database.relation.Relation;
import elki.database.relation.RelationUtil;
import elki.distance.PrimitiveDistance;
import elki.distance.correlation.WeightedPearsonCorrelationDistance;
import elki.distance.minkowski.WeightedEuclideanDistance;
import elki.distance.minkowski.WeightedManhattanDistance;
import elki.distance.minkowski.WeightedSquaredEuclideanDistance;
import elki.evaluation.scores.ROCEvaluation;
import elki.evaluation.scores.adapter.DecreasingVectorIter;
import elki.logging.Logging;
import elki.math.MeanVariance;
import elki.utilities.datastructures.arraylike.ArrayLikeUtil;
import elki.utilities.documentation.Reference;
import elki.utilities.ensemble.EnsembleVoting;
import elki.utilities.ensemble.EnsembleVotingMean;
import elki.utilities.exceptions.AbortException;
import elki.utilities.io.FormatUtil;
import elki.utilities.optionhandling.OptionID;
import elki.utilities.optionhandling.parameterization.Parameterization;
import elki.utilities.optionhandling.parameters.DoubleParameter;
import elki.utilities.optionhandling.parameters.EnumParameter;
import elki.utilities.optionhandling.parameters.ObjectParameter;
import elki.utilities.scaling.ScalingFunction;
import elki.utilities.scaling.outlier.OutlierScaling;
import elki.workflow.InputStep;

/**
 * Class to load an outlier detection summary file, as produced by
 * {@link ComputeKNNOutlierScores}, and compute a naive ensemble for it. Based
 * on this initial estimation, and optimized ensemble is built using a greedy
 * strategy. Starting with the best candidate only as initial ensemble, the most
 * diverse candidate is investigated at each step. If it improves towards the
 * (estimated) target vector, it is added, otherwise it is discarded.
 * <p>
 * This approach is naive, and it may be surprising that it can improve results.
 * The reason is probably that diversity will result in a comparable ensemble,
 * while the reduced ensemble size is actually responsible for the improvements,
 * by being more decisive and less noisy due to dropping "unhelpful" members.
 * <p>
 * This still leaves quite a bit of room for improvement. If you build upon this
 * basic approach, please acknowledge our proof of concept work.
 * <p>
 * Reference:
 * <p>
 * Erich Schubert, Remigius Wojdanowski, Arthur Zimek, Hans-Peter Kriegel<br>
 * On Evaluation of Outlier Rankings and Outlier Scores<br>
 * Proc. 12th SIAM Int. Conf. on Data Mining (SDM 2012)
 *
 * @author Erich Schubert
 * @since 0.5.0
 */
@Reference(authors = "Erich Schubert, Remigius Wojdanowski, Arthur Zimek, Hans-Peter Kriegel", //
    title = "On Evaluation of Outlier Rankings and Outlier Scores", //
    booktitle = "Proc. 12th SIAM Int. Conf. on Data Mining (SDM 2012)", //
    url = "https://doi.org/10.1137/1.9781611972825.90", //
    bibkey = "DBLP:conf/sdm/SchubertWZK12")
public class GreedyEnsembleExperiment extends AbstractApplication {
  /**
   * Get static logger.
   */
  private static final Logging LOG = Logging.getLogger(GreedyEnsembleExperiment.class);

  /**
   * The data input part.
   */
  private InputStep inputstep;

  /**
   * Variant, where the truth vector is also updated.
   */
  boolean refine_truth = false;

  /**
   * Ensemble voting method.
   */
  EnsembleVoting voting;

  /**
   * Outlier scaling to apply during preprocessing.
   */
  ScalingFunction prescaling;

  /**
   * Outlier scaling to apply to constructed ensembles.
   */
  ScalingFunction scaling;

  /**
   * Expected rate of outliers.
   */
  double rate;

  /**
   * Minimum votes.
   */
  int minvote = 1;

  /**
   * Distance modes.
   */
  public enum Distance {
    PEARSON, //
    SQEUCLIDEAN, //
    EUCLIDEAN, //
    MANHATTAN, //
  }

  /**
   * Distance in use.
   */
  Distance distance = Distance.PEARSON;

  /**
   * Constructor.
   *
   * @param inputstep Input step
   * @param voting Ensemble voting
   * @param distance Distance function
   * @param prescaling Scaling to apply to input data
   * @param scaling Scaling to apply to ensemble members
   * @param rate Expected rate of outliers
   */
  public GreedyEnsembleExperiment(InputStep inputstep, EnsembleVoting voting, Distance distance, ScalingFunction prescaling, ScalingFunction scaling, double rate) {
    super();
    this.inputstep = inputstep;
    this.voting = voting;
    this.distance = distance;
    this.prescaling = prescaling;
    this.scaling = scaling;
    this.rate = rate;
  }

  @Override
  public void run() {
    // Note: the database contains the *result vectors*, not the original data.
    final Database database = inputstep.getDatabase();
    Relation<NumberVector> relation = database.getRelation(TypeUtil.NUMBER_VECTOR_FIELD);
    final Relation<String> labels = DatabaseUtil.guessLabelRepresentation(database);
    final DBID firstid = DBIDUtil.deref(labels.iterDBIDs());
    final String firstlabel = labels.get(firstid);
    if(!firstlabel.matches("bylabel")) {
      throw new AbortException("No 'by label' reference outlier found, which is needed for weighting!");
    }
    relation = applyPrescaling(prescaling, relation, firstid);
    final int numcand = relation.size() - 1;

    // Dimensionality and reference vector
    final int dim = RelationUtil.dimensionality(relation);
    final NumberVector refvec = relation.get(firstid);

    final int desired_outliers = (int) (rate * dim);
    int union_outliers = 0;
    final int[] outliers_seen = new int[dim];
    // Merge the top-k for each ensemble member, until we have enough
    // candidates.
    {
      int k = 0;
      ArrayList<DecreasingVectorIter> iters = new ArrayList<>(numcand);
      if(minvote >= numcand) {
        minvote = Math.max(1, numcand - 1);
      }
      for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) {
        // Skip "by label", obviously
        if(DBIDUtil.equal(firstid, iditer)) {
          continue;
        }
        iters.add(new DecreasingVectorIter(refvec, relation.get(iditer)));
      }
      loop: while(union_outliers < desired_outliers) {
        for(DecreasingVectorIter iter : iters) {
          if(!iter.valid()) {
            LOG.warning("Union_outliers=" + union_outliers + " < desired_outliers=" + desired_outliers + " minvote=" + minvote);
            break loop;
          }
          int cur = iter.dim();
          outliers_seen[cur] += 1;
          if(outliers_seen[cur] == minvote) {
            union_outliers += 1;
          }
          iter.advance();
        }
        k++;
      }
      LOG.verbose("Merged top " + k + " outliers to: " + union_outliers + " outliers (desired: at least " + desired_outliers + ")");
    }
    // Build the final weight vector.
    final double[] estimated_weights = new double[dim];
    final double[] estimated_truth = new double[dim];
    updateEstimations(outliers_seen, union_outliers, estimated_weights, estimated_truth);
    DoubleVector estimated_truth_vec = DoubleVector.wrap(estimated_truth);

    PrimitiveDistance<NumberVector> wdist = getDistance(estimated_weights);
    PrimitiveDistance<NumberVector> tdist = wdist;

    // Build the naive ensemble:
    final double[] naiveensemble = new double[dim];
    {
      double[] buf = new double[numcand];
      for(int d = 0; d < dim; d++) {
        int i = 0;
        for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) {
          if(DBIDUtil.equal(firstid, iditer)) {
            continue;
          }
          buf[i++] = relation.get(iditer).doubleValue(d);
        }
        naiveensemble[d] = voting.combine(buf, i);
        if(Double.isNaN(naiveensemble[d])) {
          LOG.warning("NaN after combining: " + FormatUtil.format(buf) + " i=" + i + " " + voting.toString());
        }
      }
    }
    DoubleVector naivevec = DoubleVector.wrap(naiveensemble);

    // Compute single AUC scores and estimations.
    // Remember the method most similar to the estimation
    double bestauc = 0.0, bestcost = Double.POSITIVE_INFINITY;
    String bestaucstr = "", bestcoststr = "";
    DBID bestid = null;
    double bestest = Double.POSITIVE_INFINITY;
    {
      final double[] greedyensemble = new double[dim];
      // Compute individual scores
      for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) {
        if(DBIDUtil.equal(firstid, iditer)) {
          continue;
        }
        singleEnsemble(greedyensemble, relation.get(iditer));
        double auc = ROCEvaluation.computeAUROC(new DecreasingVectorIter(refvec, DoubleVector.wrap(greedyensemble)));
        double estimated = wdist.distance(DoubleVector.wrap(greedyensemble), estimated_truth_vec);
        double cost = tdist.distance(DoubleVector.wrap(greedyensemble), refvec);
        LOG.verbose("AUROC: " + auc + " estimated " + estimated + " cost " + cost + " " + labels.get(iditer));
        if(auc > bestauc) {
          bestauc = auc;
          bestaucstr = labels.get(iditer);
        }
        if(cost < bestcost) {
          bestcost = cost;
          bestcoststr = labels.get(iditer);
        }
        if(estimated < bestest || bestid == null) {
          bestest = estimated;
          bestid = DBIDUtil.deref(iditer);
        }
      }
    }

    // Initialize ensemble with "best" method
    if(prescaling != null) {
      LOG.verbose("Input prescaling: " + prescaling);
    }
    LOG.verbose("Distance function: " + wdist);
    LOG.verbose("Ensemble voting: " + voting);
    if(scaling != null) {
      LOG.verbose("Ensemble rescaling: " + scaling);
    }
    LOG.verbose("Initial estimation of outliers: " + union_outliers);
    LOG.verbose("Initializing ensemble with: " + labels.get(bestid));
    ModifiableDBIDs ensemble = DBIDUtil.newArray(bestid);
    ModifiableDBIDs enscands = DBIDUtil.newHashSet(relation.getDBIDs());
    ModifiableDBIDs dropped = DBIDUtil.newHashSet(relation.size());
    dropped.add(firstid);
    enscands.remove(bestid);
    enscands.remove(firstid);
    final double[] greedyensemble = new double[dim];
    singleEnsemble(greedyensemble, relation.get(bestid));
    // Greedily grow the ensemble
    final double[] testensemble = new double[dim];
    while(enscands.size() > 0) {
      NumberVector greedyvec = DoubleVector.wrap(greedyensemble);
      final double oldd = wdist.distance(estimated_truth_vec, greedyvec);

      final int heapsize = enscands.size();
      ModifiableDoubleDBIDList heap = DBIDUtil.newDistanceDBIDList(heapsize);
      double[] tmp = new double[dim];
      for(DBIDIter iter = enscands.iter(); iter.valid(); iter.advance()) {
        singleEnsemble(tmp, relation.get(iter));
        heap.add(wdist.distance(DoubleVector.wrap(greedyensemble), greedyvec), iter);
      }
      heap.sort();
      for(DoubleDBIDListMIter it = heap.iter(); heap.size() > 0; it.remove()) {
        enscands.remove(it.seek(heap.size() - 1));
        final NumberVector vec = relation.get(it);
        // Build combined ensemble.
        {
          double[] buf = new double[ensemble.size() + 1];
          for(int i = 0; i < dim; i++) {
            int j = 0;
            for(DBIDIter iter = ensemble.iter(); iter.valid(); iter.advance()) {
              buf[j++] = relation.get(iter).doubleValue(i);
            }
            buf[j] = vec.doubleValue(i);
            testensemble[i] = voting.combine(buf, j + 1);
          }
        }
        applyScaling(testensemble, scaling);
        NumberVector testvec = DoubleVector.wrap(testensemble);
        double newd = wdist.distance(estimated_truth_vec, testvec);
        if(newd < oldd) {
          System.arraycopy(testensemble, 0, greedyensemble, 0, dim);
          ensemble.add(it);
          break; // Recompute heap
        }
        dropped.add(it);
        if(refine_truth) {
          // Update target vectors and weights
          ArrayList<DecreasingVectorIter> iters = new ArrayList<>(numcand);
          for(DBIDIter iditer = relation.iterDBIDs(); iditer.valid(); iditer.advance()) {
            // Skip "by label", obviously
            if(DBIDUtil.equal(firstid, iditer) || dropped.contains(iditer)) {
              continue;
            }
            iters.add(new DecreasingVectorIter(refvec, relation.get(iditer)));
          }
          if(minvote >= iters.size()) {
            minvote = iters.size() - 1;
          }

          union_outliers = 0;
          Arrays.fill(outliers_seen, 0);
          while(union_outliers < desired_outliers) {
            for(DecreasingVectorIter iter : iters) {
              if(!iter.valid()) {
                break;
              }
              if((outliers_seen[iter.dim()] += 1) == minvote) {
                union_outliers += 1;
              }
              iter.advance();
            }
          }
          LOG.warning("New num outliers: " + union_outliers);
          updateEstimations(outliers_seen, union_outliers, estimated_weights, estimated_truth);
          estimated_truth_vec = DoubleVector.wrap(estimated_truth);
        }
      }
    }
    // Build the improved ensemble:
    StringBuilder greedylbl = new StringBuilder();
    for(DBIDIter iter = ensemble.iter(); iter.valid(); iter.advance()) {
      greedylbl.append(labels.get(iter)).append(' ');
    }
    greedylbl.setLength(greedylbl.length() - 1); // last space
    DoubleVector greedyvec = DoubleVector.wrap(greedyensemble);
    if(refine_truth) {
      LOG.verbose("Estimated outliers remaining: " + union_outliers);
    }
    LOG.verbose("Greedy ensemble (" + ensemble.size() + "): " + greedylbl.toString());

    LOG.verbose("Best single AUROC:    " + bestauc + " (" + bestaucstr + ")");
    LOG.verbose("Best single cost:     " + bestcost + " (" + bestcoststr + ")");
    // Evaluate the naive ensemble and the "shrunk" ensemble
    double naiveauc, naivecost;
    {
      naiveauc = ROCEvaluation.computeAUROC(new DecreasingVectorIter(refvec, naivevec));
      naivecost = tdist.distance(naivevec, refvec);
      LOG.verbose("Naive ensemble AUROC:  " + naiveauc + " cost: " + naivecost);
      LOG.verbose("Naive ensemble Gain:   " + gain(naiveauc, bestauc, 1) + " cost gain: " + gain(naivecost, bestcost, 0));
    }
    double greedyauc, greedycost;
    {
      greedyauc = ROCEvaluation.computeAUROC(new DecreasingVectorIter(refvec, greedyvec));
      greedycost = tdist.distance(greedyvec, refvec);
      LOG.verbose("Greedy ensemble AUROC: " + greedyauc + " cost: " + greedycost);
      LOG.verbose("Greedy ensemble Gain to best:  " + gain(greedyauc, bestauc, 1) + " cost gain: " + gain(greedycost, bestcost, 0));
      LOG.verbose("Greedy ensemble Gain to naive: " + gain(greedyauc, naiveauc, 1) + " cost gain: " + gain(greedycost, naivecost, 0));
    }
    {
      MeanVariance meanauc = new MeanVariance(), meancost = new MeanVariance();
      HashSetModifiableDBIDs candidates = DBIDUtil.newHashSet(relation.getDBIDs());
      candidates.remove(firstid);
      for(int i = 0; i < 1000; i++) {
        // Build the improved ensemble:
        final double[] randomensemble = new double[dim];
        {
          DBIDs random = DBIDUtil.randomSample(candidates, ensemble.size(), (long) i);
          double[] buf = new double[random.size()];
          for(int d = 0; d < dim; d++) {
            int j = 0;
            for(DBIDIter iter = random.iter(); iter.valid(); iter.advance()) {
              assert !DBIDUtil.equal(firstid, iter);
              buf[j++] = relation.get(iter).doubleValue(d);
            }
            randomensemble[d] = voting.combine(buf, j);
          }
        }
        applyScaling(randomensemble, scaling);
        NumberVector randomvec = DoubleVector.wrap(randomensemble);
        meanauc.put(ROCEvaluation.computeAUROC(new DecreasingVectorIter(refvec, randomvec)));
        meancost.put(tdist.distance(randomvec, refvec));
      }
      LOG.verbose("Random ensemble AUROC: " + meanauc.getMean() + " + stddev: " + meanauc.getSampleStddev() + " = " + (meanauc.getMean() + meanauc.getSampleStddev()));
      LOG.verbose("Random ensemble Gain:  " + gain(meanauc.getMean(), bestauc, 1));
      LOG.verbose("Greedy improvement:    " + (greedyauc - meanauc.getMean()) / meanauc.getSampleStddev() + " standard deviations.");
      LOG.verbose("Random ensemble Cost:  " + meancost.getMean() + " + stddev: " + meancost.getSampleStddev() + " = " + (meancost.getMean() + meanauc.getSampleStddev()));
      LOG.verbose("Random ensemble Gain:  " + gain(meancost.getMean(), bestcost, 0));
      LOG.verbose("Greedy improvement:    " + (meancost.getMean() - greedycost) / meancost.getSampleStddev() + " standard deviations.");
      LOG.verbose("Naive ensemble Gain to random: " + gain(naiveauc, meanauc.getMean(), 1) + " cost gain: " + gain(naivecost, meancost.getMean(), 0));
      LOG.verbose("Random ensemble Gain to naive: " + gain(meanauc.getMean(), naiveauc, 1) + " cost gain: " + gain(meancost.getMean(), naivecost, 0));
      LOG.verbose("Greedy ensemble Gain to random: " + gain(greedyauc, meanauc.getMean(), 1) + " cost gain: " + gain(greedycost, meancost.getMean(), 0));
    }
  }

  /**
   * Build a single-element "ensemble".
   *
   * @param ensemble
   * @param vec
   */
  protected void singleEnsemble(final double[] ensemble, final NumberVector vec) {
    double[] buf = new double[1];
    for(int i = 0; i < ensemble.length; i++) {
      buf[0] = vec.doubleValue(i);
      ensemble[i] = voting.combine(buf, 1);
      if(Double.isNaN(ensemble[i])) {
        LOG.warning("NaN after combining: " + FormatUtil.format(buf) + " " + voting.toString());
      }
    }
    applyScaling(ensemble, scaling);
  }

  /**
   * Prescale each vector (except when in {@code skip}) with the given scaling
   * function.
   *
   * @param scaling Scaling function
   * @param relation Relation to read
   * @param skip DBIDs to pass unmodified
   * @return New relation
   */
  public static Relation<NumberVector> applyPrescaling(ScalingFunction scaling, Relation<NumberVector> relation, DBIDs skip) {
    if(scaling == null) {
      return relation;
    }
    NumberVector.Factory<NumberVector> factory = RelationUtil.getNumberVectorFactory(relation);
    DBIDs ids = relation.getDBIDs();
    WritableDataStore<NumberVector> contents = DataStoreUtil.makeStorage(ids, DataStoreFactory.HINT_HOT, NumberVector.class);
    for(DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
      double[] raw = relation.get(iter).toArray();
      if(!skip.contains(iter)) {
        applyScaling(raw, scaling);
      }
      contents.put(iter, factory.newNumberVector(raw, ArrayLikeUtil.DOUBLEARRAYADAPTER));
    }
    return new MaterializedRelation<>("rescaled", relation.getDataTypeInformation(), ids, contents);
  }

  private static void applyScaling(double[] raw, ScalingFunction scaling) {
    if(scaling == null) {
      return;
    }
    if(scaling instanceof OutlierScaling) {
      ((OutlierScaling) scaling).prepare(raw, ArrayLikeUtil.DOUBLEARRAYADAPTER);
    }
    for(int i = 0; i < raw.length; i++) {
      final double newval = scaling.getScaled(raw[i]);
      if(Double.isNaN(newval)) {
        LOG.warning("NaN after prescaling: " + raw[i] + " " + scaling.toString() + " -> " + newval);
      }
      raw[i] = newval;
    }
  }

  protected void updateEstimations(final int[] outliers, int numoutliers, final double[] weights, final double[] truth) {
    final double oweight = .5 / numoutliers;
    final double iweight = .5 / (outliers.length - numoutliers);
    // final double orate = union_outliers * 1.0 / (outliers_seen.length);
    final double oscore = 1.; // .5 - .5 * orate;
    final double iscore = 0.; // 1 - .5 * orate;
    for(int i = 0; i < outliers.length; i++) {
      if(outliers[i] >= minvote) {
        weights[i] = oweight;
        truth[i] = oscore;
      }
      else {
        weights[i] = iweight;
        truth[i] = iscore;
      }
    }
  }

  private PrimitiveDistance<NumberVector> getDistance(double[] estimated_weights) {
    switch(distance){
    case SQEUCLIDEAN:
      return new WeightedSquaredEuclideanDistance(estimated_weights);
    case EUCLIDEAN:
      return new WeightedEuclideanDistance(estimated_weights);
    case MANHATTAN:
      return new WeightedManhattanDistance(estimated_weights);
    case PEARSON:
      return new WeightedPearsonCorrelationDistance(estimated_weights);
    default:
      throw new AbortException("Unsupported distance mode: " + distance);
    }
  }

  /**
   * Compute the gain coefficient.
   *
   * @param score New score
   * @param ref Reference score
   * @param optimal Maximum score possible
   * @return Gain
   */
  double gain(double score, double ref, double optimal) {
    return 1 - ((optimal - score) / (optimal - ref));
  }

  /**
   * Parameterization class.
   *
   * @author Erich Schubert
   */
  public static class Par extends AbstractApplication.Par {
    /**
     * Expected rate of outliers
     */
    public static final OptionID RATE_ID = new OptionID("greedy.rate", "Expected rate of outliers.");

    /**
     * Ensemble voting function.
     */
    public static final OptionID VOTING_ID = new OptionID("ensemble.voting", "Ensemble voting function.");

    /**
     * Scaling to apply to input scores.
     */
    public static final OptionID PRESCALING_ID = new OptionID("ensemble.prescaling", "Prescaling to apply to input scores.");

    /**
     * Scaling to apply to ensemble scores.
     */
    public static final OptionID SCALING_ID = new OptionID("ensemble.scaling", "Scaling to apply to ensemble.");

    /**
     * Similarity measure
     */
    public static final OptionID DISTANCE_ID = new OptionID("ensemble.measure", "Similarity measure.");

    /**
     * Data source.
     */
    InputStep inputstep;

    /**
     * Ensemble voting method.
     */
    EnsembleVoting voting;

    /**
     * Distance in use.
     */
    Distance distance = Distance.PEARSON;

    /**
     * Outlier scaling to apply during preprocessing.
     */
    ScalingFunction prescaling;

    /**
     * Outlier scaling to apply to constructed ensembles.
     */
    ScalingFunction scaling;

    /**
     * Expected rate of outliers
     */
    double rate = 0.01;

    @Override
    public void configure(Parameterization config) {
      super.configure(config);
      // Data input
      inputstep = config.tryInstantiate(InputStep.class);
      // Voting method
      new ObjectParameter<EnsembleVoting>(VOTING_ID, EnsembleVoting.class, EnsembleVotingMean.class) //
          .grab(config, x -> voting = x);
      // Similarity measure
      new EnumParameter<Distance>(DISTANCE_ID, Distance.class) //
          .grab(config, x -> distance = x);
      // Prescaling
      new ObjectParameter<ScalingFunction>(PRESCALING_ID, ScalingFunction.class) //
          .setOptional(true) //
          .grab(config, x -> prescaling = x);
      // Ensemble scaling
      new ObjectParameter<ScalingFunction>(SCALING_ID, ScalingFunction.class) //
          .setOptional(true) //
          .grab(config, x -> scaling = x);
      // Expected rate of outliers
      new DoubleParameter(RATE_ID, 0.01) //
          .grab(config, x -> rate = x);
    }

    @Override
    public GreedyEnsembleExperiment make() {
      return new GreedyEnsembleExperiment(inputstep, voting, distance, prescaling, scaling, rate);
    }
  }

  /**
   * Main method.
   *
   * @param args Command line parameters.
   */
  public static void main(String[] args) {
    runCLIApplication(GreedyEnsembleExperiment.class, args);
  }
}
