/* This file is part of the Joshua Machine Translation System.
 * 
 * Joshua is free software; you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1
 * of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free
 * Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
 * MA 02111-1307 USA
 */

package joshua.ZMERT;
import joshua.decoder.*;
import java.math.*;
import java.util.*;
import java.io.*;
import java.text.DecimalFormat;

public class MertCore
{
  private static DecimalFormat f0 = new DecimalFormat("###0");
  private static DecimalFormat f4 = new DecimalFormat("###0.0000");
  private static final Runtime myRuntime = Runtime.getRuntime();

  private static final double NegInf = (-1.0 / 0.0);
  private static final double PosInf = (+1.0 / 0.0);
  private static final double epsilon = 1.0 / 1000000;

  private static int progress;

  private static int verbosity; // anything of priority <= verbosity will be printed
                         // (lower value for priority means more important)

  private static Random randGen;
  private static int generatedRands;

  private static int numSentences;
    // number of sentences in the dev set
    // (aka the "MERT training" set)

  private static int refsPerSen;
    // number of reference translations per sentence

  private static int numParams;
    // number of features for the log-linear model

  private static double[] normalizationOptions;
    // How should a lambda[] vector be normalized (before decoding)?
    //   nO[0] = 0: no normalization
    //   nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
    //   nO[0] = 2: scale so that the maximum absolute value is nO[1]
    //   nO[0] = 3: scale so that the minimum absolute value is nO[1]
    //   nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]

  /* *********************************************************** */
  /*   NOTE: indexing starts at 1 in the following few arrays:   */
  /* *********************************************************** */

  private static String[] paramNames;
    // feature names, needed to read/create config file

  private static double[] lambda;
    // the current weight vector. NOTE: indexing starts at 1.

  private static boolean[] isOptimizable;
    // isOptimizable[c] = true iff lambda[c] should be optimized

  private static double[] minThValue;
  private static double[] maxThValue;
    // when investigating thresholds along the lambda[c] dimension, only values
    // in the [minThValue[c],maxThValue[c]] range will be considered.
    // (*) minThValue and maxThValue can be real values as well as -Infinity and +Infinity
    //     (coded as -Inf and +Inf, respectively, in an input file)

  private static double[] minRandValue;
  private static double[] maxRandValue;
    // when choosing a random value for the lambda[c] parameter, it will be
    // chosen from the [minRandValue[c],maxRandValue[c]] range.
    // (*) minRandValue and maxRandValue must be real values, but not -Inf or +Inf

  private static double[] defaultLambda;
    // "default" parameter values; simply the values read in the parameter file

  /* *********************************************************** */
  /* *********************************************************** */

  private static JoshuaDecoder myDecoder;
    // COMMENT OUT if decoder is not Joshua

  private static String decoderCommand;
    // the command that runs the decoder; read from decoderCommandFileName

  private static int decVerbosity;
    // verbosity level for decoder output.  If 0, decoder output is ignored.
    // If 1, decoder output is printed.

  private static int validDecoderExitValue;
    // return value from running the decoder command that indicates success

  private static int saveInterFiles;
    // 0: nothing, 1: only configs, 2: only n-bests, 3: both configs and n-bests

  private static int sizeOfNBest;
    // size of N-best list generated by decoder at each iteration
    // (aka simply N, but N is a bad variable name)

  private static long seed;
    // seed used to create random number generators

  private static boolean randInit;
    // if true, parameters are initialized randomly.  If false, parameters
    // are initialized using values from parameter file.

  private static int initsPerIt;
    // number of intermediate initial points per iteration

  private static int maxMERTIterations, minMERTIterations, prevMERTIterations;
    // max: maximum number of MERT iterations
    // min: minimum number of MERT iterations before an early MERT exit
    // prev: number of previous MERT iterations from which to consider candidates (in addition to the
    //       candidates from the current iteration)

  private static double stopSigValue;
    // early MERT exit if no weight changes by more than stopSigValue
    // (but see minMERTIterations above and stopMinIts below)

  private static int stopMinIts;
    // some early stopping criterion must be satisfied in stopMinIts *consecutive* iterations before an early exit
    // (but see minMERTIterations above)

  private static boolean oneModificationPerIteration;
    // if true, each MERT iteration performs at most one parameter modification.
    // If false, a new MERT iteration starts (i.e. a new N-best list is
    // generated) only after the previous iteration reaches a local maximum.

  private static String metricName;
    // name of evaluation metric optimized by MERT

  private static String[] metricOptions;
    // options for the evaluation metric (e.g. for BLEU, maxGramLength and effLengthMethod)

  private static EvaluationMetric evalMetric;
    // the evaluation metric used by MERT

  private static int suffStatsCount;
    // number of sufficient statistics for the evaluation metric

  private static String dirPrefix; // where are all these files located?
  private static String paramsFileName, finalLambdaFileName;
  private static String sourceFileName, refFileName, decoderOutFileName;
  private static String decoderConfigFileName, decoderCommandFileName;
  private static String fakeFileNamePrefix;
//  private static int useDisk;

  public static void main(String[] args) throws Exception
  {
    // if bad args[], System.exit(80)

    String configFileName = args[0];
    String stateFileName = args[1];
    int currIteration = Integer.parseInt(args[2]);


    int randsToSkip = 0;
    int earlyStop = 0;
    double FINAL_score = 0.0;
    int[] maxIndex = null;

    if (currIteration == 1) {
      EvaluationMetric.set_knownMetrics();
      processArgsArray(cfgFileToArgsArray(configFileName),true);

      randsToSkip = 0;
      initialize(randsToSkip);

      println("----------------------------------------------------",1);
      println("Z-MERT run started @ " + (new Date()),1);
//      printMemoryUsage();
      println("----------------------------------------------------",1);
      println("",1);

      if (randInit) {
        println("Initializing lambda[] randomly.",1);

        // initialize optimizable parameters randomly (sampling uniformly from that parameter's random value range)
        lambda = randomLambda();
      }

      println("Initial lambda[]: " + lambdaToString(lambda),1);
      println("",1);

      FINAL_score = evalMetric.worstPossibleScore();
      maxIndex = new int[numSentences];
      for (int i = 0; i < numSentences; ++i) { maxIndex[i] = sizeOfNBest - 1; }
      earlyStop = 0;
    } else {

      EvaluationMetric.set_knownMetrics();
      processArgsArray(cfgFileToArgsArray(configFileName),false);

      ObjectInputStream in = new ObjectInputStream(new FileInputStream(stateFileName));
      double[] serA = (double[]) in.readObject();
      in.close();
      // contents of serA[]: last iteration, number of random numbers generated already, earlyStop, FINAL_lambda, lambda[], maxIndex[]
      // => length should be 4+numParams+numSentences

      if (serA.length < 2) {
        println("State file contains an array of length " + serA.length + "; was expecting at least 2");
        System.exit(81);
      }

      if ((int)serA[0] != currIteration-1) {
        println("Iteration in state file is " + (int)serA[0] + "; was expecting " + (currIteration-1));
        System.exit(82);
      }

      randsToSkip = (int)serA[1];
      initialize(randsToSkip); // declares lambda[], sets numParams and numSentences

      if (serA.length != 4+numParams+numSentences) {
        println("State file contains an array of length " + serA.length + "; was expecting " + (4+numParams+numSentences));
        System.exit(83);
      }

      earlyStop = (int)serA[2];
      FINAL_score = serA[3];

      for (int c = 1; c <= numParams; ++c) { lambda[c] = serA[3+c]; }

      maxIndex = new int[numSentences];
      for (int i = 0; i < numSentences; ++i) { maxIndex[i] = (int)serA[3+numParams+1+i]; }
    }


    double[] A = run_single_iteration(currIteration, minMERTIterations, maxMERTIterations, prevMERTIterations, earlyStop, maxIndex);

    if (A != null) {
      FINAL_score = A[0];
      earlyStop = (int)A[1];
      randsToSkip = generatedRands;
    }


    if (A != null && A[2] != 1) {

      double[] serA = new double[4+numParams+numSentences];
      serA[0] = currIteration;
      serA[1] = randsToSkip;
      serA[2] = earlyStop;
      serA[3] = FINAL_score;
      for (int c = 1; c <= numParams; ++c) { serA[3+c] = lambda[c]; }
      for (int i = 0; i < numSentences; ++i) { serA[3+numParams+1+i] = maxIndex[i]; }

      ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(stateFileName));
      out.writeObject(serA);
      out.flush();
      out.close();

      System.exit(91);

    } else {
      // done

      println("",1);

      println("----------------------------------------------------",1);
      println("Z-MERT run ended @ " + (new Date()),1);
//      printMemoryUsage();
      println("----------------------------------------------------",1);
      println("",1);
      println("FINAL lambda: " + lambdaToString(lambda) + " (" + metricName + ": " + FINAL_score + ")",1);
      // check if a lambda is outside its threshold range
      for (int c = 1; c <= numParams; ++c) {
        if (lambda[c] < minThValue[c] || lambda[c] > maxThValue[c]) {
          println("Warning: after normalization, lambda[" + c + "]=" + f4.format(lambda[c]) + " is outside its critical value range.",1);
        }
      }
      println("",1);

      // delete intermediate .temp.*.it* decoder output files
      for (int iteration = 1; iteration <= maxMERTIterations; ++iteration) {
        deleteFile(decoderOutFileName+".temp.sents.it"+iteration);
        deleteFile(decoderOutFileName+".temp.feats.it"+iteration);
        deleteFile(decoderOutFileName+".temp.stats.it"+iteration);
      }

      // delete .temp.stats.merged file
      deleteFile(decoderOutFileName+".temp.stats.merged");


      finish();

      deleteFile(stateFileName);
      System.exit(90);
    }

  }


  public MertCore(String[] args) throws Exception
  {
    EvaluationMetric.set_knownMetrics();
    processArgsArray(args);
    initialize(0);
  }

  public MertCore(String configFileName) throws Exception
  {
    EvaluationMetric.set_knownMetrics();
    processArgsArray(cfgFileToArgsArray(configFileName));
    initialize(0);
  }

  private static void initialize(int randsToSkip) throws Exception
  {
    randGen = new Random(seed);
    generatedRands = randsToSkip;

    if (randsToSkip == 0) {
      println("----------------------------------------------------",1);
      println("Initializing...",1);
      println("----------------------------------------------------",1);
      println("",1);

      println("Random number generator initialized using seed: " + seed,1);
      println("",1);
    } else {
      for (int i = 1; i <= randsToSkip; ++i) {
        double dummy = randGen.nextDouble();
        ++generatedRands;
      }
    }

    numSentences = countLines(refFileName) / refsPerSen;
    numParams = countNonEmptyLines(paramsFileName) - 1;
      // the parameter file contains one line per parameter and one line for the normalization method



    paramNames = new String[1+numParams];
    lambda = new double[1+numParams]; // indexing starts at 1 in these arrays
    isOptimizable = new boolean[1+numParams];
    minThValue = new double[1+numParams];
    maxThValue = new double[1+numParams];
    minRandValue = new double[1+numParams];
    maxRandValue = new double[1+numParams];
//    precision = new double[1+numParams];
    defaultLambda = new double[1+numParams];
    normalizationOptions = new double[3];

    // read paramter names
    BufferedReader inFile_names = new BufferedReader(new FileReader(paramsFileName));

    for (int c = 1; c <= numParams; ++c) {
      String line = "";
      while (line != null && line.length() == 0) { line = inFile_names.readLine(); }
      paramNames[c] = (line.substring(0,line.indexOf("|||"))).trim();
    }

    inFile_names.close();

    processParamFile();
      // sets the arrays declared just above

//    SentenceInfo.createV(); // uncomment ONLY IF using vocabulary implementation of SentenceInfo


    String[][] refSentences = new String[numSentences][refsPerSen];

    // read in reference sentences

    BufferedReader inFile_refs = new BufferedReader(new FileReader(refFileName));
    String line;

    for (int i = 0; i < numSentences; ++i) {
      for (int r = 0; r < refsPerSen; ++r) {
        // read the rth reference translation for the ith sentence
        refSentences[i][r] = inFile_refs.readLine();
      }
    }

    inFile_refs.close();


    // read in decoder command, if any
    decoderCommand = null;
    if (decoderCommandFileName != null) {
      if (fileExists(decoderCommandFileName)) {
        BufferedReader inFile_comm = new BufferedReader(new FileReader(decoderCommandFileName));
        decoderCommand = inFile_comm.readLine();
        inFile_comm.close();
      }
    }

    // set static data members for the EvaluationMetric class
    EvaluationMetric.set_numSentences(numSentences);
    EvaluationMetric.set_refsPerSen(refsPerSen);
    EvaluationMetric.set_refSentences(refSentences);

    // do necessary initialization for the evaluation metric
/*
    if (metricName.equals("BLEU")) {
      evalMetric = new BLEU(4,"closest");
    } else if (metricName.equals("BLEU_SBP")) {
      evalMetric = new BLEU_SBP(4,"closest");
    } else if (metricName.equals("01LOSS")) {
      evalMetric = new ZeroOneLoss();
    } else if (metricName.equals("UserMetric1")) {
      evalMetric = new UserMetric1();
    } else if (metricName.equals("UserMetric2")) {
      evalMetric = new UserMetric2();
    }
*/
    evalMetric = EvaluationMetric.getMetric(metricName,metricOptions);

    suffStatsCount = evalMetric.get_suffStatsCount();

    if (randsToSkip == 0) {
      println("Number of sentences: " + numSentences,1);
      println("Number of features: " + numParams,1);
      print("Feature names: {",1);
      for (int c = 1; c <= numParams; ++c) {
        print("\"" + paramNames[c] + "\"",1);
        if (c < numParams) print(",",1);
      }
      println("}",1);
      println("",1);

      println("c    Default value\tOptimizable?\tCrit. val. range\tRand. val. range",1);

      for (int c = 1; c <= numParams; ++c) {
        print(c + "     " + f4.format(lambda[c]) + "\t\t",1);
        if (!isOptimizable[c]) {
          print(" No",1);
        } else {
          print(" Yes\t\t",1);
  //        print("[" + minThValue[c] + "," + maxThValue[c] + "] @ " + precision[c] + " precision",1);
          print(" [" + minThValue[c] + "," + maxThValue[c] + "]",1);
          print("\t\t",1);
          print(" [" + minRandValue[c] + "," + maxRandValue[c] + "]",1);
          println("",1);
        }
      }

      println("",1);
      print("Normalization method: ",1);
      if (normalizationOptions[0] == 0) {
        println("none.",1);
      } else if (normalizationOptions[0] == 1) {
        println("weights will be scaled so that the \"" + paramNames[(int)normalizationOptions[1]] + "\" weight has an absolute value of " + normalizationOptions[2] + ".",1);
      } else if (normalizationOptions[0] == 2) {
        println("weights will be scaled so that the maximum absolute value is " + normalizationOptions[1] + ".",1);
      } else if (normalizationOptions[0] == 3) {
        println("weights will be scaled so that the minimum absolute value is " + normalizationOptions[1] + ".",1);
      } else if (normalizationOptions[0] == 4) {
        println("weights will be scaled so that the L-" + normalizationOptions[1] + " norm is " + normalizationOptions[2] + ".",1);
      }

      println("",1);

      println("----------------------------------------------------",1);
      println("",1);

      // rename original config file so it doesn't get overwritten (original name will be restored in finish())
      renameFile(decoderConfigFileName,decoderConfigFileName+".ZMERT.orig");

    } // if (randsToSkip == 0)


    if (decoderCommand == null && fakeFileNamePrefix == null) {
      myDecoder = new JoshuaDecoder();
      println("Loading Joshua decoder...",1);
      myDecoder.initializeDecoder(decoderConfigFileName+".ZMERT.orig");
      println("...finished loading @ " + (new Date()),1);
    } else {
      myDecoder = null;
    }

  } // void initialize(...)


  public static void run_MERT() throws Exception
  {
    run_MERT(minMERTIterations,maxMERTIterations,prevMERTIterations);
  }

  public static void run_MERT(int minIts, int maxIts, int prevIts) throws Exception
  {
    println("----------------------------------------------------",1);
    println("Z-MERT run started @ " + (new Date()),1);
//    printMemoryUsage();
    println("----------------------------------------------------",1);
    println("",1);

    if (randInit) {
      println("Initializing lambda[] randomly.",1);

      // initialize optimizable parameters randomly (sampling uniformly from that parameter's random value range)
      lambda = randomLambda();
    }

    println("Initial lambda[]: " + lambdaToString(lambda),1);
    println("",1);

    double FINAL_score = evalMetric.worstPossibleScore();


//    int[] lastUsedIndex = new int[numSentences];
    int[] maxIndex = new int[numSentences];
      // used to grow featVal_array dynamically
//    HashMap<Integer,int[]>[] suffStats_array = new HashMap[numSentences];
      // suffStats_array[i] maps candidates of interest for sentence i to an array
      // storing the sufficient statistics for that candidate
    for (int i = 0; i < numSentences; ++i) {
//      lastUsedIndex[i] = -1;
      maxIndex[i] = sizeOfNBest - 1;
//      suffStats_array[i] = new HashMap<Integer,int[]>();
    }
/*
    double[][][] featVal_array = new double[1+numParams][][]; // indexed by [param][sentence][candidate]
    featVal_array[0] = null; // param indexing starts at 1
    for (int c = 1; c <= numParams; ++c) {
      featVal_array[c] = new double[numSentences][];
      for (int i = 0; i < numSentences; ++i) {
        featVal_array[c][i] = new double[maxIndex[i]];
          // will grow dynamically as needed
      }
    }
*/
    int earlyStop = 0;
      // number of consecutive iteration an early stopping criterion was satisfied

    for (int iteration = 1; ; ++iteration) {

      double[] A = run_single_iteration(iteration, minIts, maxIts, prevIts, earlyStop, maxIndex);
      if (A != null) {
        FINAL_score = A[0];
        earlyStop = (int)A[1];
        if (A[2] == 1) break;
      } else {
        break;
      }

    } // for (iteration)

    println("",1);

    println("----------------------------------------------------",1);
    println("Z-MERT run ended @ " + (new Date()),1);
//    printMemoryUsage();
    println("----------------------------------------------------",1);
    println("",1);
    println("FINAL lambda: " + lambdaToString(lambda) + " (" + metricName + ": " + FINAL_score + ")",1);
    // check if a lambda is outside its threshold range
    for (int c = 1; c <= numParams; ++c) {
      if (lambda[c] < minThValue[c] || lambda[c] > maxThValue[c]) {
        println("Warning: after normalization, lambda[" + c + "]=" + f4.format(lambda[c]) + " is outside its critical value range.",1);
      }
    }
    println("",1);

    // delete intermediate .temp.*.it* decoder output files
    for (int iteration = 1; iteration <= maxIts; ++iteration) {
      deleteFile(decoderOutFileName+".temp.sents.it"+iteration);
      deleteFile(decoderOutFileName+".temp.feats.it"+iteration);
      deleteFile(decoderOutFileName+".temp.stats.it"+iteration);
    }

    // delete .temp.stats.merged file
    deleteFile(decoderOutFileName+".temp.stats.merged");

  } // void run_MERT(int maxIts)


  public static double[] run_single_iteration(int iteration, int minIts, int maxIts, int prevIts, int earlyStop, int[]maxIndex) throws Exception
  {
    double FINAL_score = 0;

    double[] retA = new double[3];
      // retA[0]: FINAL_lambda
      // retA[1]: earlyStop
      // retA[2]: should this be the last iteration?

    boolean done = false;
    retA[2] = 1; // will only made 0 if we don't break from the following loop


    double[][][] featVal_array = new double[1+numParams][][]; // indexed by [param][sentence][candidate]
    featVal_array[0] = null; // param indexing starts at 1
    for (int c = 1; c <= numParams; ++c) {
      featVal_array[c] = new double[numSentences][];
      for (int i = 0; i < numSentences; ++i) {
        featVal_array[c][i] = new double[maxIndex[i]+1];
          // will grow dynamically as needed
      }
    }


    while (!done) { // NOTE: this "loop" will only be carried out once
      println("--- Starting Z-MERT iteration #" + iteration + " @ " + (new Date()) + " ---",1);

//      printMemoryUsage();

      // run the decoder on all the sentences, producing for each sentence a set of
      // sizeOfNBest candidates, with numParams feature values for each candidate

      /******************************/
      // CREATE DECODER CONFIG FILE //
      /******************************/

      createConfigFile(lambda,decoderConfigFileName,decoderConfigFileName+".ZMERT.orig");
        // i.e. use the original config file as a template

      /***************/
      // RUN DECODER //
      /***************/

      if (iteration == 1) {
        println("Decoding using initial weight vector " + lambdaToString(lambda),1);
      } else {
        println("Redecoding using weight vector " + lambdaToString(lambda),1);
      }

      run_decoder(iteration);
        // iteration passed as argument in case fake decoder will be used

      println("...finished decoding @ " + (new Date()),1);

      checkFile(decoderOutFileName);

      println("Producing temp files for iteration "+iteration,3);

      produceTempFiles(iteration);

      if (saveInterFiles == 1 || saveInterFiles == 3) {
        copyFile(decoderConfigFileName,decoderConfigFileName+".ZMERT.it"+iteration);
      }
      if (saveInterFiles == 2 || saveInterFiles == 3) {
        copyFile(decoderOutFileName,decoderOutFileName+".ZMERT.it"+iteration);
      }

      int[] candCount = new int[numSentences];
      int[] lastUsedIndex = new int[numSentences];
      HashMap<Integer,int[]>[] suffStats_array = new HashMap[numSentences];
      for (int i = 0; i < numSentences; ++i) {
        candCount[i] = 0;
        lastUsedIndex[i] = -1;
//        suffStats_array[i].clear();
        suffStats_array[i] = new HashMap<Integer,int[]>();
      }

      double[][] initialLambda = new double[1+initsPerIt][1+numParams]; // the intermediate "initial" lambdas
      double[][] finalLambda = new double[1+initsPerIt][1+numParams]; // the intermediate "final" lambdas

      // set initialLambda[][]
      System.arraycopy(lambda,1,initialLambda[1],1,numParams);
      for (int j = 2; j <= initsPerIt; ++j) { initialLambda[j] = randomLambda(); }

      double[] initialScore = new double[1+initsPerIt];
      double[] finalScore = new double[1+initsPerIt];

      String[][] best1Cand_sen = new String[1+initsPerIt][numSentences];
      double[][] best1Score = new double[1+initsPerIt][numSentences];
        // used to calculate initialScore[]


      int firstIt = Math.max(1,iteration-prevIts);
        // i.e. only process candidates from the current iteration and candidates
        // from up to prevIts previous iterations.
      println("Reading candidate translations from iterations " + firstIt + "-" + iteration,2);
      progress = 0;

//      boolean newCandidatesAdded = false;
      int[] newCandidatesAdded = new int[1+iteration];
      for (int it = 1; it <= iteration; ++it) { newCandidatesAdded[it] = 0; }

      // each inFile corresponds to the output of an iteration
      BufferedReader[] inFile_sents = new BufferedReader[1+iteration];
      BufferedReader[] inFile_feats = new BufferedReader[1+iteration];
      BufferedReader[] inFile_stats = new BufferedReader[1+iteration];
      for (int it = firstIt; it <= iteration; ++it) {
        inFile_sents[it] = new BufferedReader(new FileReader(decoderOutFileName+".temp.sents.it"+it));
        inFile_feats[it] = new BufferedReader(new FileReader(decoderOutFileName+".temp.feats.it"+it));
        inFile_stats[it] = new BufferedReader(new FileReader(decoderOutFileName+".temp.stats.it"+it));
      }

      // to be used to write sufficient statistics from all the sentences from the output files into a single one
      PrintWriter outFile_stats = new PrintWriter(decoderOutFileName+".temp.stats.merged");

      String sents_str, feats_str, stats_str;

      HashSet<String> existingCandidates = new HashSet<String>();
      double[] currFeatVal = new double[1+numParams];
      String[] featVal_str;

      int totalCandidateCount = 0;

      for (int i = 0; i < numSentences; ++i) {

        for (int j = 1; j <= initsPerIt; ++j) {
//          best1Cand[j][i] = null;
          best1Cand_sen[j][i] = null;
          best1Score[j][i] = NegInf;
        }

        for (int it = firstIt; it <= iteration; ++it) {

          for (int n = 0; n < sizeOfNBest; ++n) {

            // for the nth candidate for the ith sentence, read the sentence, feature values,
            // and sufficient statistics from the various temp files

            sents_str = inFile_sents[it].readLine();
            feats_str = inFile_feats[it].readLine();
            stats_str = inFile_stats[it].readLine();

            if (!existingCandidates.contains(sents_str)) {

              outFile_stats.println(stats_str);

              featVal_str = feats_str.split("\\s+");

              for (int c = 1; c <= numParams; ++c) {
                currFeatVal[c] = Double.parseDouble(featVal_str[c-1]);
//                print("fV[" + c + "]=" + featVal[c] + " ",3);
              }
//              println("",3);


              for (int j = 1; j <= initsPerIt; ++j) {
                double score = 0;
                for (int c = 1; c <= numParams; ++c) {
                  score += initialLambda[j][c] * currFeatVal[c];
                }
                if (score > best1Score[j][i]) {
                  best1Score[j][i] = score;
                  best1Cand_sen[j][i] = sents_str;
                }
              }

              existingCandidates.add(sents_str);

              setFeats(featVal_array,i,lastUsedIndex,maxIndex,currFeatVal);
              candCount[i] += 1;
//              ++totalCandidateCount;

              newCandidatesAdded[it] += 1;
//              if (it == iteration) {
//                newCandidatesAdded = true;
//              }

            }

            showProgress();

          } // for (n)

        } // for (it)

        existingCandidates.clear();
        totalCandidateCount += candCount[i];

      } // for (i)

      for (int it = 1; it <= iteration; ++it) {
        inFile_sents[it].close();
        inFile_feats[it].close();
        inFile_stats[it].close();
      }

      outFile_stats.close();

      println("",2); // to finish off progress dot line

//      cleanupMemory();

      println("Processed " + totalCandidateCount + " distinct candidates (about " + totalCandidateCount/numSentences + " per sentence):",1);
      for (int it = firstIt; it <= iteration; ++it) {
        println("newCandidatesAdded[it=" + it + "] = " + newCandidatesAdded[it] + " (about " + newCandidatesAdded[it]/numSentences + " per sentence)",1);
      }

      println("",1);

      if (newCandidatesAdded[iteration] == 0) {
        if (!oneModificationPerIteration) {
          println("No new candidates added in this iteration; exiting Z-MERT.",1);
          println("",1);
          println("---  Z-MERT iteration #" + iteration + " ending @ " + (new Date()) + "  ---",1);
          println("",1);
          return null; // THIS MEANS THAT THE OLD VALUES SHOULD BE KEPT BY THE CALLER
        } else {
          println("Note: No new candidates added in this iteration.",1);
        }
      }

      TreeMap<Double,TreeMap>[] thresholdsAll = new TreeMap[1+numParams];
      thresholdsAll[0] = null;
      for (int c = 1; c <= numParams; ++c) {
        if (isOptimizable[c]) {
          thresholdsAll[c] = new TreeMap<Double,TreeMap>();
        } else {
          thresholdsAll[c] = null;
        }
      }

      for (int j = 1; j <= initsPerIt; ++j) {

//        cleanupMemory();

        println("+++ Optimization of lambda[j=" + j + "] starting @ " + (new Date()) + " +++",1);

        double[] currLambda = new double[1+numParams];
        System.arraycopy(initialLambda[j],1,currLambda,1,numParams);
        initialScore[j] = evalMetric.score(best1Cand_sen[j]);
        println("Initial lambda[j=" + j + "]: " + lambdaToString(initialLambda[j]),1);
        println("(Initial " + metricName + "[j=" + j + "]: " + initialScore[j] + ")",1);
        println("",1);
        finalScore[j] = initialScore[j];

        int c_best = 0; // which param to change?
        double bestLambdaVal = 0; // what value to change to?
        double bestScore = 0; // what score would be achieved?

        while (true) {

          double[] c_best_info = bestParamToChange(j,thresholdsAll,c_best,currLambda,candCount,featVal_array,suffStats_array,1,iteration);
                                                     // we pass in c_best because we don't need to recalculate thresholds for it
          c_best = (int)c_best_info[0]; // which param to change?
          bestLambdaVal = c_best_info[1]; // what value to change to?
          bestScore = c_best_info[2]; // what score would be achieved?

          // now c_best is the parameter giving the most gain

          if (evalMetric.isBetter(bestScore,finalScore[j])) {
            println("*** Changing lambda[j=" + j + "][" + c_best + "] from " + f4.format(currLambda[c_best]) + " (" + metricName + ": " + f4.format(finalScore[j]) + ") to " + f4.format(bestLambdaVal) + " (" + metricName + ": " + f4.format(bestScore) + ") ***",2);
            println("*** Old lambda[j=" + j + "]: " + lambdaToString(currLambda) + " ***",2);
            currLambda[c_best] = bestLambdaVal;
            finalScore[j] = bestScore;
            println("*** New lambda[j=" + j + "]: " + lambdaToString(currLambda) + " ***",2);
            println("",2);
          } else {
            println("*** Not changing any weight in lambda[j=" + j + "] ***",2);
            println("*** lambda[j=" + j + "]: " + lambdaToString(currLambda) + " ***",2);
            println("",2);
            break; // exit while (true) loop
          }

          if (oneModificationPerIteration) { break; } // exit while (true) loop

        } // while (true)

        // now currLambda is the optimized weight vector on the current candidate list (corresponding to initialLambda[j])

        System.arraycopy(currLambda,1,finalLambda[j],1,numParams);
        normalizeLambda(finalLambda[j]);
        // check if a lambda is outside its threshold range
        for (int c = 1; c <= numParams; ++c) {
          if (finalLambda[j][c] < minThValue[c] || finalLambda[j][c] > maxThValue[c]) {
            println("Warning: after normalization, final lambda[j=" + j + "][" + c + "]=" + f4.format(finalLambda[j][c]) + " is outside its critical value range.",1);
          }
        }
        println("Final lambda[j=" + j + "]: " + lambdaToString(finalLambda[j]),1);
        println("(Final " + metricName + "[j=" + j + "]: " + finalScore[j] + ")",1);
        println("",1);

      } // for (j)


      int best_j = 1;
      double bestFinalScore = finalScore[1];
      for (int j = 2; j <= initsPerIt; ++j) {
        if (evalMetric.isBetter(finalScore[j],bestFinalScore)) {
          best_j = j;
          bestFinalScore = finalScore[j];
        }
      }

      if (initsPerIt > 1) {
        println("Best final lambda is lambda[j=" + best_j + "] (" + metricName + ": " + f4.format(bestFinalScore) + ").",1);
        println("",1);
      }

      FINAL_score = bestFinalScore;

      boolean anyParamChanged = false;
      boolean anyParamChangedSignificantly = false;

      for (int c = 1; c <= numParams; ++c) {
        if (finalLambda[best_j][c] != lambda[c]) {
          anyParamChanged = true;
        }
        if (Math.abs(finalLambda[best_j][c] - lambda[c]) > stopSigValue) {
          anyParamChangedSignificantly = true;
        }
      }

      System.arraycopy(finalLambda[best_j],1,lambda,1,numParams);
      println("---  Z-MERT iteration #" + iteration + " ending @ " + (new Date()) + "  ---",1);
      println("",1);

      if (!anyParamChanged) {
        println("No parameter value changed in this iteration; exiting Z-MERT.",1);
        println("",1);
        break; // exit for (iteration) loop preemptively
      }

      // check if a lambda is outside its threshold range
      for (int c = 1; c <= numParams; ++c) {
        if (lambda[c] < minThValue[c] || lambda[c] > maxThValue[c]) {
          println("Warning: after normalization, lambda[" + c + "]=" + f4.format(lambda[c]) + " is outside its critical value range.",1);
        }
      }

      // was an early stopping criterion satisfied?
      boolean critSatisfied = false;
      if (!anyParamChangedSignificantly && stopSigValue >= 0) {
        println("Note: No parameter value changed significantly (i.e. by more than " + stopSigValue + ") in this iteration.",1);
        critSatisfied = true;
      }

      if (critSatisfied) { ++earlyStop; println("",1); }
      else { earlyStop = 0; }

      // if min number of iterations executed, investigate if early exit should happen
      if (iteration >= minIts && earlyStop >= stopMinIts) {
        println("Some early stopping criteria has been ovserved in " + stopMinIts + " consecutive iterations; exiting Z-MERT.",1);
        println("",1);
        break; // exit for (iteration) loop preemptively
      }

      // if max number of iterations executed, exit
      if (iteration >= maxIts) {
        println("Maximum number of MERT iterations reached; exiting Z-MERT.",1);
        println("",1);
        break; // exit for (iteration) loop
      }

      println("Next iteration will decode with lambda: " + lambdaToString(lambda),1);
      println("",1);

//      printMemoryUsage();
      for (int i = 0; i < numSentences; ++i) {
        suffStats_array[i].clear();
      }
//      cleanupMemory();
//      println("",2);


      retA[2] = 0; // i.e. this should NOT be the last iteration
      done = true;

    } // while (true) // NOTE: this "loop" will only be carried out once

    retA[0] = FINAL_score;
    retA[1] = earlyStop;
    return retA;

  } // run_single_iteration







  private static double[] bestParamToChange(int j, TreeMap<Double,TreeMap>[] thresholdsAll, int lastChanged_c, double[] currLambda, int[] candCount, double[][][] featVal_array, HashMap<Integer,int[]>[] suffStats_array, int minIt, int maxIt) throws Exception
  {
    int c_best = 0; // which parameter to change?
    double bestLambdaVal = 0.0;
    double bestScore;
    if (evalMetric.getToBeMinimized()) {
      bestScore = evalMetric.worstPossibleScore() + 1.0;
    } else {
      bestScore = evalMetric.worstPossibleScore() - 1.0;
    }




    // prep for line_opt

    TreeSet<Integer>[] indicesOfInterest = null;
    // indicesOfInterest[i] tells us which candidates for the ith sentence need
    // to be read from the merged decoder output file.

//    if (useDisk == 2) {
      indicesOfInterest = new TreeSet[numSentences];
      for (int i = 0; i < numSentences; ++i) {
        indicesOfInterest[i] = new TreeSet<Integer>();
      }
//    }

    int[][] indexOfCurrBest = new int[1+numParams][numSentences];

    for (int c = 1; c <= numParams; ++c) {
      if (!isOptimizable[c]) {
        println("Not investigating lambda[j=" + j + "][" + c + "].",2);
      } else {
        if (c != lastChanged_c) {
          println("Investigating lambda[j=" + j + "][" + c + "]...",2);
//          thresholdsAll[c] = thresholdsForParam(c,candCount,featVal_array,currLambda,indicesOfInterest);
          set_thresholdsForParam(thresholdsAll[c],c,candCount,featVal_array,currLambda,indicesOfInterest);
        } else {
          println("Keeping thresholds for lambda[j=" + j + "][" + c + "] from previous step.",2);
        }
        // now thresholdsAll has the values for lambda_c at which score changes
        // based on the candidates for *all* the sentences (that satisfy
        // range constraints).
        // Each lambda_c value maps to a Vector of th_info.  An overwhelming majority
        // of these Vectors are of size 1.

        if (thresholdsAll[c].size() != 0) {

          double[] temp_lambda = new double[1+numParams];
          System.arraycopy(currLambda,1,temp_lambda,1,numParams);

          double smallest_th = thresholdsAll[c].firstKey();

          if (minThValue[c] != NegInf) {
            temp_lambda[c] = (minThValue[c] + smallest_th) / 2.0;
          } else {
            temp_lambda[c] = smallest_th - 0.05;
          }

          indexOfCurrBest[c] = initial_indexOfCurrBest(c,candCount,featVal_array,temp_lambda,indicesOfInterest);
        }
      }

      println("",2);

    }



//    if (useDisk == 2) {

      set_suffStats_array(suffStats_array,indicesOfInterest,candCount);

//    } // if (useDisk == 2)



    for (int c = 1; c <= numParams; ++c) {
    // investigate currLambda[j][c]

      if (isOptimizable[c]) {
        double[] bestScoreInfo_c = line_opt(thresholdsAll[c],indexOfCurrBest[c],c,candCount,featVal_array,suffStats_array,currLambda,minIt,maxIt);
          // get best score and its lambda value

        double bestLambdaVal_c = bestScoreInfo_c[0];
        double bestScore_c = bestScoreInfo_c[1];

        if (evalMetric.isBetter(bestScore_c,bestScore)) {
          c_best = c;
          bestLambdaVal = bestLambdaVal_c;
          bestScore = bestScore_c;
        }

      } // if (!isOptimizable[c])

    }




    // delete according to indicesOfInterest

//    printMemoryUsage();

//    if (useDisk == 2) {

      for (int i = 0; i < numSentences; ++i) {

        indicesOfInterest[i].clear();

      }

//    }

//    cleanupMemory();
//    printMemoryUsage();
//    println("",2);








    double[] c_best_info = {c_best,bestLambdaVal,bestScore};
    return c_best_info;

  } // double[] bestParamToChange(int j, double[] currLambda)


  private static String lambdaToString(double[] lambdaA)
  {
    String retStr = "{";
    for (int c = 1; c <= numParams-1; ++c) {
      retStr += "" + lambdaA[c] + ", ";
    }
    retStr += "" + lambdaA[numParams] + "}";

    return retStr;
  }

  private static void run_decoder(int iteration) throws Exception
  {
    if (fakeFileNamePrefix != null && fileExists(fakeFileNamePrefix+iteration)) {
      println("Running fake decoder (making copy of " + fakeFileNamePrefix+iteration + ")...",1);
      copyFile(fakeFileNamePrefix+iteration,decoderOutFileName);
    } else if (decoderCommand == null) {

      if (myDecoder == null) {
        myDecoder = new JoshuaDecoder();
        println("Loading Joshua decoder...",1);
        myDecoder.initializeDecoder(decoderConfigFileName+".ZMERT.orig");
        println("...finished loading @ " + (new Date()),1);
      }

      println("Running Joshua decoder on source file " + sourceFileName + "...",1);
//      myDecoder.initializeDecoder(decoderConfigFileName);
      double[] zeroBased_lambda = new double[numParams];
      System.arraycopy(lambda,1,zeroBased_lambda,0,numParams);
      myDecoder.changeFeatureWeightVector(zeroBased_lambda);
      myDecoder.decodingTestSet(sourceFileName, decoderOutFileName);
    } else {
      println("Running external decoder...",1);

      Runtime rt = Runtime.getRuntime();
      Process p = rt.exec(decoderCommandFileName);
/*
      BufferedReader br_i = new BufferedReader(new InputStreamReader(p.getInputStream()));
      BufferedReader br_e = new BufferedReader(new InputStreamReader(p.getErrorStream()));
      String dummy_line = null;

      while ((dummy_line = br_e.readLine()) != null) { }
	  while ((dummy_line = br_i.readLine()) != null) {
        if (decVerbosity == 1) {
          println(dummy_line);
        }
      }
*/
      StreamGobbler errorGobbler = new StreamGobbler(p.getErrorStream(), decVerbosity);
      StreamGobbler outputGobbler = new StreamGobbler(p.getInputStream(), decVerbosity);

      errorGobbler.start();
      outputGobbler.start();

      int decStatus = p.waitFor();

      if (decStatus != validDecoderExitValue) {
        println("Call to decoder returned " + decStatus + "; was expecting " + validDecoderExitValue + ".");
        System.exit(30);
      }
    }
  }

  private static double[] line_opt(TreeMap<Double,TreeMap> thresholdsAll, int[] indexOfCurrBest, int c, int[] candCount, double[][][] featVal_array, HashMap<Integer,int[]>[] suffStats_array, double[] lambda, int minIt, int maxIt) throws Exception
  {
//    println("Line-optimizing lambda[" + c + "]...",3);

    double[] bestScoreInfo = new double[2];
      // to be returned: [0] will store the best lambda, and [1] will store its score

    if (thresholdsAll.size() == 0) {
      // no thresholds extracted!  Possible in theory...
      // simply return current value for this parameter
      println("No thresholds extracted!  Returning this parameter's current value...",2);

      bestScoreInfo[0] = lambda[c];
      bestScoreInfo[1] = evalMetric.worstPossibleScore();

      return bestScoreInfo;
    }

    double smallest_th = thresholdsAll.firstKey();
    double largest_th = thresholdsAll.lastKey();
//    println("Minimum threshold: " + smallest_th,3);
//    println("Maximum threshold: " + largest_th,3);

    double[] temp_lambda = new double[1+numParams];
    System.arraycopy(lambda,1,temp_lambda,1,numParams);

    double ip_prev = 0.0, ip_curr = 0.0;

    if (minThValue[c] != NegInf) {
      temp_lambda[c] = (minThValue[c] + smallest_th) / 2.0;
      ip_curr = minThValue[c];
    } else {
      temp_lambda[c] = smallest_th - 0.05;
      ip_curr = smallest_th - 0.1;
    }




    int[][] suffStats = new int[numSentences][suffStatsCount];
      // suffStats[i][s] stores the contribution to the sth sufficient
      // statistic from the candidate for the ith sentence (the candidate
      // indicated by indexOfCurrBest[i]).

    int[] suffStats_tot = new int[suffStatsCount];
      // suffStats_tot[s] := SUM_i suffStats[i][s]

    for (int s = 0; s < suffStatsCount; ++s) { suffStats_tot[s] = 0; }

    // Now, set suffStats[][], and increment suffStats_tot[]
    for (int i = 0; i < numSentences; ++i) {
      suffStats[i] = suffStats_array[i].get(indexOfCurrBest[i]);

      for (int s = 0; s < suffStatsCount; ++s) {
        suffStats_tot[s] += suffStats[i][s];
      }
    }



    double bestScore = evalMetric.score(suffStats_tot);
    double bestLambdaVal = temp_lambda[c];
    double nextLambdaVal = bestLambdaVal;
//    println("At lambda[" + c + "] = " + bestLambdaVal + ",\t" + metricName + " = " + bestScore + " (*)",3);

    Iterator<Double> It = (thresholdsAll.keySet()).iterator();
    if (It.hasNext()) { ip_curr = It.next(); }

    while (It.hasNext()) {
      ip_prev = ip_curr;
      ip_curr = It.next();
      nextLambdaVal = (ip_prev + ip_curr)/2.0;

      TreeMap<Integer,int[]> th_info_M = thresholdsAll.get(ip_prev);
      Iterator<Integer> It2 = (th_info_M.keySet()).iterator();
      while (It2.hasNext()) {
        int i = It2.next();
        int[] th_info = th_info_M.get(i);
        int old_k = th_info[0]; // should be equal to indexOfCurrBest[i]
        int new_k = th_info[1];

        for (int s = 0; s < suffStatsCount; ++s) {
          suffStats_tot[s] -= suffStats[i][s]; // subtract stats for candidate old_k
        }

        indexOfCurrBest[i] = new_k;
        suffStats[i] = suffStats_array[i].get(indexOfCurrBest[i]);

        for (int s = 0; s < suffStatsCount; ++s) {
          suffStats_tot[s] += suffStats[i][s]; // add stats for candidate new_k
        }

      }

      double nextTestScore = evalMetric.score(suffStats_tot);
//      print("At lambda[" + c + "] = " + nextLambdaVal + ",\t" + metricName + " = " + nextTestScore,3);

      if (evalMetric.isBetter(nextTestScore,bestScore)) {
        bestScore = nextTestScore;
        bestLambdaVal = nextLambdaVal;
//        print(" (*)",3);
      }

//      println("",3);

    } // while (It.hasNext())

    println("",3);

    // what is the purpose of this block of code ?????????????????????
/*
    if (maxThValue[c] != PosInf) {
      nextLambdaVal = (largest_th + maxThValue[c]) / 2.0;
    } else {
      nextLambdaVal = largest_th + 0.05;
    }
*/
    // ???????????????????????????????????????????????????????????????

    /*************************************************/
    /*************************************************/

    bestScoreInfo[0] = bestLambdaVal;
    bestScoreInfo[1] = bestScore;

    return bestScoreInfo;

  } // double[] line_opt(int c)


//  private static TreeMap<Double,TreeMap> thresholdsForParam(int c, int[] candCount, double[][][] featVal_array, double[] currLambda, TreeSet<Integer>[] indicesOfInterest)
  private static void set_thresholdsForParam(TreeMap<Double,TreeMap> thresholdsAll, int c, int[] candCount, double[][][] featVal_array, double[] currLambda, TreeSet<Integer>[] indicesOfInterest)
  {
/*
    TreeMap[] thresholds = new TreeMap[numSentences];
      // thresholds[i] stores thresholds for the cth parameter obtained by
      // processing the candidates of sentence i.  It not only stores the
      // thresholds themselves, but also a triple of {i,from,to}, where from/to
      // are indices that characterize the 1-best switch at this threshold.

    for (int i = 0; i < numSentences; ++i) {
      thresholds[i] = new TreeMap<Double,int[]>();
    }
*/

//    TreeMap<Double,int[]> thresholds = new TreeMap<Double,int[]>();

    // Find threshold points
//    TreeMap<Double,TreeMap> thresholdsAll = new TreeMap<Double,TreeMap>();
    thresholdsAll.clear();

    int ipCount = 0;
    for (int i = 0; i < numSentences; ++i) {
    // find threshold points contributed by ith sentence

//      println("Processing sentence #" + i,3);

      int numCandidates = candCount[i];
        // aka simply K

      double[] slope = new double[numCandidates];
        // will be h_c from candidatesInfo
        // repeated here for easy access
      double[] offset = new double[numCandidates];
        // SUM_j!=c currLambda_j*h_j(x)

      int minSlopeIndex = -1;          // index of line with steepest descent...
      double minSlope = PosInf;        // ...and its slope...
      double offset_minSlope = NegInf; // ...and its offset (needed to break ties)

      int maxSlopeIndex = -1;          // index of line with steepest ascent...
      double maxSlope = NegInf;        // ...and its slope...
      double offset_maxSlope = NegInf; // ...and its offset (needed to break ties)

      double bestScore_left = NegInf;  // these are used if the min/max values are
      double bestScore_right = NegInf; // not neg/pos infinity

      for (int k = 0; k < numCandidates; ++k) {
        slope[k] = featVal_array[c][i][k];

        offset[k] = 0.0;
        for (int c2 = 1; c2 <= numParams; ++c2) {
          if (c2 != c) { offset[k] += currLambda[c2]*featVal_array[c2][i][k]; }
        }

        // debugging
//        println("@ (i,k)=(" + i + "," + k + "), "
//               + "slope = " + slope[k] + "; offset = " + offset[k],3);

        if (minThValue[c] == NegInf) {
          if (slope[k] < minSlope || (slope[k] == minSlope && offset[k] > offset_minSlope)) {
            minSlopeIndex = k;
            minSlope = slope[k];
            offset_minSlope = offset[k];
          }
        } else {
          double score = offset[k] + ((minThValue[c]-0.1)*slope[k]);
          if (score > bestScore_left || (score == bestScore_left && slope[k] > minSlope)) {
            minSlopeIndex = k;
            minSlope = slope[k];
            bestScore_left = score;
          }
        }

        if (maxThValue[c] == PosInf) {
          if (slope[k] > maxSlope || (slope[k] == maxSlope && offset[k] > offset_maxSlope)) {
            maxSlopeIndex = k;
            maxSlope = slope[k];
            offset_maxSlope = offset[k];
          }
        } else {
          double score = offset[k] + ((maxThValue[c]+0.1)*slope[k]);
          if (score > bestScore_right || (score == bestScore_right && slope[k] < maxSlope)) {
            maxSlopeIndex = k;
            maxSlope = slope[k];
            bestScore_right = score;
          }
        }
      }

      // debugging
//      println("minSlope is @ k = " + minSlopeIndex + ": slope " + minSlope + " (offset " + offset_minSlope + ")",3);
//      println("maxSlope is @ k = " + maxSlopeIndex + ": slope " + maxSlope + " (offset " + offset_maxSlope + ")",3);


      // some lines can be eliminated: the ones that have a lower offset
      // than some other line with the same slope.
      // That is, for any k1 and k2:
      //   if slope[k1] = slope[k2] and offset[k1] > offset[k2],
      //   then k2 can be eliminated.
      // (This is actually important to do as it eliminates a bug.)
//      HashSet<Integer> discardedIndices = indicesToDiscard(slope,offset);


//      println("Extracting thresholds[(i,c)=(" + i + "," + c + ")]",3);

      int currIndex = minSlopeIndex;
        // As we traverse the currLambda_c dimension, the "winner" candidate will
        // change at intersection points.  currIndex tells us which candidate is
        // the winner in the interval currently under investigation.

        // We traverse the lambda_c dimension starting at -Inf.  The line with
        // steepest descent is the winner as lambda_c -> -Inf, so we initialize
        // currIndex to minSlopeIndex to reflect that fact.

        // Similarly, the winner as lambda_c -> +Inf is the line with the
        // steepest *ascent* (i.e. max slope), and so we continue finding
        // intersection points until we hit that line.

        // Notice that we didn't have to investigate the entire space (-Inf,+Inf)
        // if the parameter's range is more restricted than that.  That is why, in
        // the loop above, the "left-most" winner is not necessarily the one with
        // the steepest descent (though it will be if minThValue[c] is -Inf).
        // And similarly, the "right-most" winner is not necessarily the one with
        // the steepest ascent (though it will be if minThValue[c] is +Inf).  The
        // point of doing this is to avoid extracting thresholds that will end up
        // being discarded anyway due to range constraints, this saving us a little
        // bit of time.

      int last_new_k = -1;

      while (currIndex != maxSlopeIndex) {

//        print("cI=" + currIndex + " ",3);

        // find the candidate whose line is the first to intersect the current
        // line.  ("first" meaning with an intersection point that has the
        //         lowest possible lambda_c value.)

        double nearestIntersectionPoint = PosInf;
        int nearestIntersectingLineIndex = -1;

        for (int k = 0; k < numCandidates; ++k) {
//          if (slope[k] > slope[currIndex] && !discardedIndices.contains(k)) {
          if (slope[k] > slope[currIndex]) {
          // only higher-sloped lines will intersect the current line
          // (If we didn't have discardedIndices a bug would creep up here.)

            // find intersection point ip_k
            double ip_k = (offset[k] - offset[currIndex])/(slope[currIndex] - slope[k]);
            if (ip_k < nearestIntersectionPoint) {
              nearestIntersectionPoint = ip_k;
              nearestIntersectingLineIndex = k;
            }
          }
        }

//        print("ip=" + f4.format(nearestIntersectionPoint) + " ",3);
        ++ipCount;

        if (nearestIntersectionPoint > minThValue[c] && nearestIntersectionPoint < maxThValue[c]) {

          int[] th_info = {currIndex,nearestIntersectingLineIndex};
          last_new_k = nearestIntersectingLineIndex;

          indicesOfInterest[i].add(currIndex); // old_k

          if (!thresholdsAll.containsKey(nearestIntersectionPoint)) {
            TreeMap<Integer,int[]> A = new TreeMap<Integer,int[]>();
            A.put(i,th_info);
            thresholdsAll.put(nearestIntersectionPoint,A);
          } else {
            TreeMap<Integer,int[]> A = thresholdsAll.get(nearestIntersectionPoint);
            if (!A.containsKey(i)) {
              A.put(i,th_info);
            } else {
              int[] old_th_info = A.get(i);
              old_th_info[1] = th_info[1]; // replace the existing new_k
              A.put(i,th_info);
            }
            thresholdsAll.put(nearestIntersectionPoint,A);
          }
/*
          if (!thresholds.containsKey(nearestIntersectionPoint)) {
            thresholds.put(nearestIntersectionPoint,th_info);
              // i.e., at lambda_c = nIP, the (index of the) 1-best changes
              // from currIndex to nearestIntersectingLineIndex (which is
              // indicated in th_info)
          } else { // extremely rare, but causes problem if it does occur
            // in essence, just replace the new_k of the existing th_info
            int[] old_th_info = (int[])thresholds.get(nearestIntersectionPoint);
            old_th_info[1] = th_info[1];
            thresholds.put(nearestIntersectionPoint,old_th_info);
            // When does this happen?  If two consecutive intersection points are so close
            // to each other so as to appear as having the same value.  For instance, assume
            // we have two intersection points ip1 and ip2 corresponding to two transitions,
            // one from k_a to k_b, and the other from k_b to k_c.  It might be the case
            // that ip2-ip1 is extremeley small, so that the ip2 entry would actually REPLACE
            // the ip1 entry.  This would be bad.

            // Instead, we pretend that k_b never happened, and just assume there is a single
            // intersection point, ip (which equals whatever value Java calculates for ip1
            // and ip2), with a corresponding transition of k_a to k_c.
          }
*/
        } // if (in-range)

        currIndex = nearestIntersectingLineIndex;

      } // end while (currIndex != maxSlopeIndex)

      if (last_new_k != -1) {
        indicesOfInterest[i].add(last_new_k); // last new_k
      }

//      println("cI=" + currIndex + "(=? " + maxSlopeIndex + " = mxSI)",3);

      // now thresholds has the values for lambda_c at which score changes
      // based on the candidates for the ith sentence

//      println("",3);

/*
      Iterator<Double> It = (thresholds.keySet()).iterator();
      int[] th_info = null;
      while (It.hasNext()) { // process intersection points contributed by this sentence
        double ip = It.next();
        if (ip > minThValue[c] && ip < maxThValue[c]) {
          th_info = thresholds.get(ip);
          if (!thresholdsAll.containsKey(ip)) {
            TreeMap A = new TreeMap();
            A.put(i,th_info);
            thresholdsAll.put(ip,A);
          } else { // not frequent, but does happen (when same intersection point corresponds to a candidate switch for more than one i)
            TreeMap A = thresholdsAll.get(ip);
            A.put(i,th_info);
            thresholdsAll.put(ip,A);
          }

//          if (useDisk == 2) {
            // th_info[0] = old_k, th_info[1] = new_k
            indicesOfInterest[i].add(th_info[0]);
//          }

        } // if (in-range)

      } // while (It.hasNext())
*/

/*
//      if (useDisk == 2 && th_info != null) {
      if (th_info != null) {
        // new_k from the last th_info (previous new_k already appear as the next old_k)
        indicesOfInterest[i].add(th_info[1]);
      }
*/

//      thresholds.clear();

    } // for (i)

    // now thresholdsAll has the values for lambda_c at which score changes
    // based on the candidates for *all* the sentences (that satisfy
    // range constraints).
    // Each lambda_c value maps to a Vector of th_info.  An overwhelming majority
    // of these Vectors are of size 1.

    // indicesOfInterest[i] tells us which candidates for the ith sentence need
    // to be read from the merged decoder output file.

    if (thresholdsAll.size() != 0) {
      double smallest_th = thresholdsAll.firstKey();
      double largest_th = thresholdsAll.lastKey();
      println("Smallest extracted threshold: " + smallest_th,2);
      println("Largest extracted threshold: " + largest_th,2);

      if (maxThValue[c] != PosInf) {
        thresholdsAll.put(maxThValue[c],null);
      } else {
        thresholdsAll.put((thresholdsAll.lastKey() + 0.1),null);
      }
    }

//    return thresholdsAll;

  } // TreeMap<Double,TreeMap> thresholdsForParam (int c)


  private static int[] initial_indexOfCurrBest(int c, int[] candCount, double[][][] featVal_array, double[] temp_lambda, TreeSet<Integer>[] indicesOfInterest)
  {
    int[] indexOfCurrBest = new int[numSentences];
      // As we traverse lambda_c, indexOfCurrBest indicates which is the
      // current best candidate.

    // initialize indexOfCurrBest[]

    for (int i = 0; i < numSentences; ++i) {
      int numCandidates = candCount[i];

      double max = NegInf;
      int indexOfMax = -1;
      for (int k = 0; k < numCandidates; ++k) {
        double score = 0;

        for (int c2 = 1; c2 <= numParams; ++c2) { score += temp_lambda[c2] * featVal_array[c2][i][k]; }
        if (score > max) {
          max = score;
          indexOfMax = k;
        }
      }

      indexOfCurrBest[i] = indexOfMax;

//      if (useDisk == 2) {
        // add indexOfCurrBest[i] to indicesOfInterest
        indicesOfInterest[i].add(indexOfMax);
//      }

    }

    return indexOfCurrBest;

  } // int[] initial_indexOfCurrBest (int c)



  private static void produceTempFiles(int iteration) throws Exception
  {
    String sentsFileName = decoderOutFileName+".temp.sents.it"+iteration;
    String featsFileName = decoderOutFileName+".temp.feats.it"+iteration;
    String statsFileName = decoderOutFileName+".temp.stats.it"+iteration;

    FileOutputStream outStream_sents = new FileOutputStream(sentsFileName, false); // false: don't append
    OutputStreamWriter outStreamWriter_sents = new OutputStreamWriter(outStream_sents, "utf8");
    BufferedWriter outFile_sents = new BufferedWriter(outStreamWriter_sents);

    PrintWriter outFile_feats = new PrintWriter(featsFileName);
    PrintWriter outFile_stats = new PrintWriter(statsFileName);


    InputStream inStream = new FileInputStream(new File(decoderOutFileName));
    BufferedReader inFile = new BufferedReader(new InputStreamReader(inStream, "utf8"));

    String line; //, prevLine;
    String candidate_str = "";
    String feats_str = "";
    int[] stats = new int[suffStatsCount];

    int i = 0; int n = 0;
    line = inFile.readLine();

    while (line != null) {

/*
line format:

i ||| words of candidate translation . ||| feat-1_val feat-2_val ... feat-numParams_val .*

*/

      // in a well formed file, we'd find the nth candidate for the ith sentence

      int read_i = Integer.parseInt((line.substring(0,line.indexOf("|||"))).trim());

      if (read_i != i) { // bad; add dummy copies of last seen candidate
        while (n < sizeOfNBest) {
          writeLine(candidate_str, outFile_sents);
          outFile_feats.println(feats_str);
          for (int s = 0; s < suffStatsCount-1; ++s) { outFile_stats.print(stats[s] + " "); }
          outFile_stats.println(stats[suffStatsCount-1]);
          ++n;
        }
        n = 0; ++i;
      }

      line = (line.substring(line.indexOf("|||")+3)).trim(); // get rid of initial text

      candidate_str = (line.substring(0,line.indexOf("|||"))).trim();
      feats_str = (line.substring(line.indexOf("|||")+3)).trim(); // get rid of candidate string

      int junk_i = feats_str.indexOf("|||");
      if (junk_i >= 0) {
        feats_str = (feats_str.substring(0,junk_i)).trim();
      }

      stats = evalMetric.suffStats(candidate_str,i);

      writeLine(candidate_str, outFile_sents);
      outFile_feats.println(feats_str);
      for (int s = 0; s < suffStatsCount-1; ++s) { outFile_stats.print(stats[s] + " "); }
      outFile_stats.println(stats[suffStatsCount-1]);

      ++n;
      if (n == sizeOfNBest) { n = 0; ++i; }

      line = inFile.readLine();
    }

    if (i != numSentences) { // last sentence had too few candidates
      while (n < sizeOfNBest) {
        writeLine(candidate_str, outFile_sents);
        outFile_feats.println(feats_str);
        for (int s = 0; s < suffStatsCount-1; ++s) { outFile_stats.print(stats[s] + " "); }
        outFile_stats.println(stats[suffStatsCount-1]);
        ++n;
      }
    }

    inFile.close();
    outFile_sents.close();
    outFile_feats.close();
    outFile_stats.close();

  }

  private static void createConfigFile(double[] params, String cfgFileName, String templateFileName) throws Exception
  {
    // i.e. create cfgFileName, which is similar to templateFileName, but with
    // params[] as parameter values

    BufferedReader inFile = new BufferedReader(new FileReader(templateFileName));
    PrintWriter outFile = new PrintWriter(cfgFileName);

    String line = inFile.readLine();

    while (line != null) {
      int c_match = -1;
      for (int c = 1; c <= numParams; ++c) {
        if (line.startsWith(paramNames[c] + " ")) { c_match = c; break; }
      }

      if (c_match == -1) {
        outFile.println(line);
      } else {
        outFile.println(paramNames[c_match] + " " + params[c_match]);
      }

      line = inFile.readLine();
    }

    inFile.close();
    outFile.close();

  }

  private static void processParamFile() throws Exception
  {
    // process parameter file
    Scanner inFile_init = new Scanner(new FileReader(paramsFileName));

    String dummy = "";

    // initialize lambda[] and other related arrays
    for (int c = 1; c <= numParams; ++c) {
      // skip parameter name
      while (!dummy.equals("|||")) { dummy = inFile_init.next(); }

      // read default value
      lambda[c] = inFile_init.nextDouble();
      defaultLambda[c] = lambda[c];

      // read isOptimizable
      dummy = inFile_init.next();
      if (dummy.equals("Opt")) { isOptimizable[c] = true; }
      else if (dummy.equals("Fix")) { isOptimizable[c] = false; }
      else { println("Unknown isOptimizable string " + dummy + " (must be either Opt or Fix)"); System.exit(21); }

      if (!isOptimizable[c]) { // skip next four values
        dummy = inFile_init.next();
        dummy = inFile_init.next();
        dummy = inFile_init.next();
        dummy = inFile_init.next();
      } else {
        // set minThValue[c] and maxThValue[c] (range for thresholds to investigate)
        dummy = inFile_init.next();
        if (dummy.equals("-Inf")) { minThValue[c] = NegInf; }
        else if (dummy.equals("+Inf")) { println("minThValue[" + c + "] cannot be +Inf!"); System.exit(21); }
        else { minThValue[c] = Double.parseDouble(dummy); }

        dummy = inFile_init.next();
        if (dummy.equals("-Inf")) { println("maxThValue[" + c + "] cannot be -Inf!"); System.exit(21); }
        else if (dummy.equals("+Inf")) { maxThValue[c] = PosInf; }
        else { maxThValue[c] = Double.parseDouble(dummy); }

        // set minRandValue[c] and maxRandValue[c] (range for random values)
        dummy = inFile_init.next();
        if (dummy.equals("-Inf") || dummy.equals("+Inf")) { println("minRandValue[" + c + "] cannot be -Inf or +Inf!"); System.exit(21); }
        else { minRandValue[c] = Double.parseDouble(dummy); }

        dummy = inFile_init.next();
        if (dummy.equals("-Inf") || dummy.equals("+Inf")) { println("maxRandValue[" + c + "] cannot be -Inf or +Inf!"); System.exit(21); }
        else { maxRandValue[c] = Double.parseDouble(dummy); }

  
        // check for illogical values
        if (minThValue[c] > maxThValue[c]) {
          println("minThValue[" + c + "]=" + minThValue[c] + " > " + maxThValue[c] + "=maxThValue[" + c + "]!");
          System.exit(21);
        }
        if (minRandValue[c] > maxRandValue[c]) {
          println("minRandValue[" + c + "]=" + minRandValue[c] + " > " + maxRandValue[c] + "=maxRandValue[" + c + "]!");
          System.exit(21);
        }

        // check for odd values
        if (!(minThValue[c] <= lambda[c] && lambda[c] <= maxThValue[c])) {
          println("Warning: lambda[" + c + "] has initial value (" + lambda[c] + ")",1);
          println("         that is outside its critical value range [" + minThValue[c] + "," + maxThValue[c] + "]",1);
        }

        if (minThValue[c] == maxThValue[c]) {
          println("Warning: lambda[" + c + "] has minThValue = maxThValue = " + minThValue[c] + ".",1);
        }

        if (minRandValue[c] == maxRandValue[c]) {
          println("Warning: lambda[" + c + "] has minRandValue = maxRandValue = " + minRandValue[c] + ".",1);
        }

        if (minRandValue[c] < minThValue[c] || minRandValue[c] > maxThValue[c] || maxRandValue[c] < minThValue[c] || maxRandValue[c] > maxThValue[c]) {
          println("Warning: The random value range for lambda[" + c + "] is not contained",1);
          println("         within its critical value range.",1);
        }

      } // if (!isOptimizable[c])

/*
      precision[c] = inFile_init.nextDouble();
      if (precision[c] < 0) {
        println("precision[" + c + "]=" + precision[c] + " < 0!  Must be non-negative.");
        System.exit(21);
      }
*/

    }

    // set normalizationOptions[]
    String origLine = "";
    while (origLine != null && origLine.length() == 0) { origLine = inFile_init.nextLine(); }


    // How should a lambda[] vector be normalized (before decoding)?
    //   nO[0] = 0: no normalization
    //   nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
    //   nO[0] = 2: scale so that the maximum absolute value is nO[1]
    //   nO[0] = 3: scale so that the minimum absolute value is nO[1]
    //   nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]

// normalization = none
// normalization = absval 1 lm
// normalization = maxabsval 1
// normalization = minabsval 1
// normalization = LNorm 2 1

    dummy = (origLine.substring(origLine.indexOf("=")+1)).trim();
    String[] dummyA = dummy.split("\\s+");

    if (dummyA[0].equals("none")) {
      normalizationOptions[0] = 0;
    } else if (dummyA[0].equals("absval")) {
      normalizationOptions[0] = 1;
      normalizationOptions[1] = Double.parseDouble(dummyA[1]);
      String pName = dummyA[2];
      for (int i = 3; i < dummyA.length; ++i) { // in case parameter name has multiple words
        pName = pName + " " + dummyA[i];
      }
      normalizationOptions[2] = c_fromParamName(pName);;

      if (normalizationOptions[1] <= 0) {
        println("Value for the absval normalization method must be positive.",1);
        System.exit(21);
      }
      if (normalizationOptions[2] == 0) {
        println("Unrecognized feature name " + normalizationOptions[2] + " for absval normalization method.",1);
        System.exit(21);
      }
    } else if (dummyA[0].equals("maxabsval")) {
      normalizationOptions[0] = 2;
      normalizationOptions[1] = Double.parseDouble(dummyA[1]);
      if (normalizationOptions[1] <= 0) {
        println("Value for the maxabsval normalization method must be positive.",1);
        System.exit(21);
      }
    } else if (dummyA[0].equals("minabsval")) {
      normalizationOptions[0] = 3;
      normalizationOptions[1] = Double.parseDouble(dummyA[1]);
      if (normalizationOptions[1] <= 0) {
        println("Value for the minabsval normalization method must be positive.",1);
        System.exit(21);
      }
    } else if (dummyA[0].equals("LNorm")) {
      normalizationOptions[0] = 4;
      normalizationOptions[1] = Double.parseDouble(dummyA[1]);
      normalizationOptions[2] = Double.parseDouble(dummyA[2]);
      if (normalizationOptions[1] <= 0 || normalizationOptions[2] <= 0) {
        println("Both values for the LNorm normalization method must be positive.",1);
        System.exit(21);
      }
    } else {
      println("Unrecognized normalization method " + dummyA[0] + "; must be one of none, absval, maxabsval, and LNorm.",1);
      System.exit(21);
    } // if (dummyA[0])

    inFile_init.close();
  }

  private static void copyFile(String origFileName, String newFileName) throws Exception
  {
    InputStream inStream = new FileInputStream(new File(origFileName));
    BufferedReader inFile = new BufferedReader(new InputStreamReader(inStream, "utf8"));

    FileOutputStream outStream = new FileOutputStream(newFileName, false); // false: don't append
    OutputStreamWriter outStreamWriter = new OutputStreamWriter(outStream, "utf8");
    BufferedWriter outFile = new BufferedWriter(outStreamWriter);

    String line;
    while(inFile.ready()) {
      line = inFile.readLine();
      writeLine(line, outFile);
    }

    inFile.close();
    outFile.close();
  }

  private static void renameFile(String origFileName, String newFileName)
  {
    if (fileExists(origFileName)) {
      deleteFile(newFileName);
      File oldFile = new File(origFileName);
      File newFile = new File(newFileName);
      oldFile.renameTo(newFile);
    } else {
      println("Warning: file " + origFileName + " does not exist! (in renameFile)",1);
    }
  }

  private static void deleteFile(String fileName)
  {
    if (fileExists(fileName)) {
      File fd = new File(fileName);
      fd.delete();
    }
  }


  private static void writeLine(String line, BufferedWriter writer) throws IOException
  {
    writer.write(line, 0, line.length());
    writer.newLine();
    writer.flush();
  }

  public static void finish() throws Exception
  {
    if (myDecoder != null) {
      myDecoder.cleanUp();
    }

    if (finalLambdaFileName != null) {
      PrintWriter outFile_lambdas = new PrintWriter(finalLambdaFileName);
      for (int c = 1; c <= numParams; ++c) {
        outFile_lambdas.println(paramNames[c] + " ||| " + lambda[c]);
      }
      outFile_lambdas.close();
    }

    // create config file with final values
    createConfigFile(lambda, decoderConfigFileName+".ZMERT.final",decoderConfigFileName+".ZMERT.orig");

    // delete current decoder config file and decoder output
    deleteFile(decoderConfigFileName);
    deleteFile(decoderOutFileName);

    // restore original name for config file (name was changed in initialize() so it doesn't get overwritten)
    renameFile(decoderConfigFileName+".ZMERT.orig",decoderConfigFileName);

  }

  private static String[] cfgFileToArgsArray(String fileName) throws Exception
  {
    checkFile(fileName);

    Vector<String> argsVector = new Vector<String>();


    BufferedReader inFile = new BufferedReader(new FileReader(fileName));

    String line, origLine;
    do {
      line = inFile.readLine();
      origLine = line; // for error reporting purposes

      if (line != null && line.length() > 0 && line.charAt(0) != '#') {

        if (line.indexOf("#") != -1) { // discard comment
          line = line.substring(0,line.indexOf("#"));
        }

        line = line.trim();

        // now line should look like "-xxx XXX"

        String[] paramA = line.split("\\s+");

        if (paramA.length == 2 && paramA[0].charAt(0) == '-') {
          argsVector.add(paramA[0]);
          argsVector.add(paramA[1]);
        } else if (paramA.length > 2 && paramA[0].equals("-m")) {
          // -m (metricName) is allowed to have extra optinos
          for (int opt = 0; opt < paramA.length; ++opt) { argsVector.add(paramA[opt]); }
        } else {
          println("Malformed line in config file:");
          println(origLine);
          System.exit(70);
        }

      }
    }  while (line != null);

    inFile.close();


    String[] argsArray = new String[argsVector.size()];

    for (int i = 0; i < argsVector.size(); ++i) {
      argsArray[i] = argsVector.elementAt(i);
    }

    return argsArray;
  }

  private static void processArgsArray(String[] args) throws Exception
  {
    processArgsArray(args,true);
  }

  private static void processArgsArray(String[] args, boolean firstTime) throws Exception
  {
    /* set default values */
    // Relevant files
    dirPrefix = null;
    sourceFileName = null;
    refFileName = "reference.txt";
    refsPerSen = 1;
    paramsFileName = "params.txt";
    finalLambdaFileName = null;
    // MERT specs
    metricName = "BLEU";
    metricOptions = new String[2];
    metricOptions[0] = "4";
    metricOptions[1] = "closest";
    maxMERTIterations = 20;
    prevMERTIterations = 20;
    minMERTIterations = 5;
    stopMinIts = 3;
    stopSigValue = -1;
//
//  /* possibly other early stopping criteria here */
//
    saveInterFiles = 3;
    initsPerIt = 20;
    oneModificationPerIteration = false;
    randInit = false;
    seed = System.currentTimeMillis();
//    useDisk = 2;
    // Decoder specs
    decoderCommandFileName = null;
    decoderOutFileName = "output.nbest";
    validDecoderExitValue = 0;
    decoderConfigFileName = "dec_cfg.txt";
    sizeOfNBest = 100;
    fakeFileNamePrefix = null;
    // Output specs
    verbosity = 1;
    decVerbosity = 0;

    int i = 0;

    while (i < args.length) {
      String option = args[i];
      // Relevant files
      if (option.equals("-dir")) { dirPrefix = args[i+1]; }
      else if (option.equals("-s")) { sourceFileName = args[i+1]; }
      else if (option.equals("-r")) { refFileName = args[i+1]; }
      else if (option.equals("-rps")) {
        refsPerSen = Integer.parseInt(args[i+1]);
        if (refsPerSen < 1) { println("refsPerSen must be positive."); System.exit(10); }
      }
      else if (option.equals("-p")) { paramsFileName = args[i+1]; }
      else if (option.equals("-fin")) { finalLambdaFileName = args[i+1]; }
      // MERT specs
      else if (option.equals("-m")) {
        metricName = args[i+1];
        if (EvaluationMetric.knownMetricName(metricName)) {
          int optionCount = EvaluationMetric.metricOptionCount(metricName);
          metricOptions = new String[optionCount];
          for (int opt = 0; opt < optionCount; ++opt) { metricOptions[opt] = args[i+opt+2]; }
          i += optionCount;
        } else {
          println("Unknown metric name " + metricName + "."); System.exit(10);
        }
      }
      else if (option.equals("-maxIt")) {
        maxMERTIterations = Integer.parseInt(args[i+1]);
        if (maxMERTIterations < 1) { println("maxMERTIts must be positive."); System.exit(10); }
      }
      else if (option.equals("-minIt")) {
        minMERTIterations = Integer.parseInt(args[i+1]);
        if (minMERTIterations < 1) { println("minMERTIts must be positive."); System.exit(10); }
      }
      else if (option.equals("-prevIt")) {
        prevMERTIterations = Integer.parseInt(args[i+1]);
        if (prevMERTIterations < 0) { println("prevMERTIts must be non-negative."); System.exit(10); }
      }
      else if (option.equals("-stopIt")) {
        stopMinIts = Integer.parseInt(args[i+1]);
        if (stopMinIts < 1) { println("stopMinIts must be positive."); System.exit(10); }
      }
      else if (option.equals("-stopSig")) {
        stopSigValue = Double.parseDouble(args[i+1]);
      }
//
//  /* possibly other early stopping criteria here */
//
      else if (option.equals("-save")) {
        int saveInterFiles = Integer.parseInt(args[i+1]);
        if (saveInterFiles < 0 || saveInterFiles > 3) { println("save should be between 0 and 3"); System.exit(10); }
      }
      else if (option.equals("-ipi")) {
        initsPerIt = Integer.parseInt(args[i+1]);
        if (initsPerIt < 1) { println("initsPerIt must be positive."); System.exit(10); }
      }
      else if (option.equals("-opi")) {
        int opi = Integer.parseInt(args[i+1]);
        if (opi == 1) oneModificationPerIteration = true;
        else if (opi == 0) oneModificationPerIteration = false;
        else { println("oncePerIt must be either 0 or 1."); System.exit(10); }
      }
      else if (option.equals("-rand")) {
        int rand = Integer.parseInt(args[i+1]);
        if (rand == 1) randInit = true;
        else if (rand == 0) randInit = false;
        else { println("randInit must be either 0 or 1."); System.exit(10); }
      }
      else if (option.equals("-seed")) {
        if (args[i+1].equals("time")) {
          seed = System.currentTimeMillis();
        } else {
          seed = Long.parseLong(args[i+1]);
        }
      }
/*
      else if (option.equals("-ud")) {
        useDisk = Integer.parseInt(args[i+1]);
        if (useDisk < 0 || useDisk > 2) { println("useDisk should be between 0 and 2"); System.exit(10); }
      }
*/
      // Decoder specs
      else if (option.equals("-cmd")) { decoderCommandFileName = args[i+1]; }
      else if (option.equals("-decOut")) { decoderOutFileName = args[i+1]; }
      else if (option.equals("-decExit")) {
        validDecoderExitValue = Integer.parseInt(args[i+1]);
      }
      else if (option.equals("-dcfg")) { decoderConfigFileName = args[i+1]; }
      else if (option.equals("-N")) {
        sizeOfNBest = Integer.parseInt(args[i+1]);
        if (sizeOfNBest < 1) { println("N must be positive."); System.exit(10); }
      }
      // Output specs
      else if (option.equals("-v")) {
        verbosity = Integer.parseInt(args[i+1]);
        if (verbosity < 0 || verbosity > 4) { println("verbosity should be between 0 and 4"); System.exit(10); }
      }
      else if (option.equals("-decV")) {
        decVerbosity = Integer.parseInt(args[i+1]);
        if (decVerbosity < 0 || decVerbosity > 1) { println("decVerbosity should be either 0 or 1"); System.exit(10); }
      }
      else if (option.equals("-fake")) { fakeFileNamePrefix = args[i+1]; }
      else {
        println("Unknown option " + option); System.exit(10);
      }

      i += 2;

    } // while (i)

    if (maxMERTIterations < minMERTIterations) {

      if (firstTime)
        println("Warning: maxMERTIts is smaller than minMERTIts; decreasing minMERTIts from " + minMERTIterations + " to maxMERTIts (i.e. " + maxMERTIterations + ").",1);

      minMERTIterations = maxMERTIterations;
    }

    if (dirPrefix != null) { // append dirPrefix to file names
      refFileName = fullPath(dirPrefix,refFileName);
      decoderOutFileName = fullPath(dirPrefix,decoderOutFileName);
      paramsFileName = fullPath(dirPrefix,paramsFileName);
      decoderConfigFileName = fullPath(dirPrefix,decoderConfigFileName);

      if (sourceFileName != null) { sourceFileName = fullPath(dirPrefix,sourceFileName); }
      if (finalLambdaFileName != null) { finalLambdaFileName = fullPath(dirPrefix,finalLambdaFileName); }
      if (decoderCommandFileName != null) { decoderCommandFileName = fullPath(dirPrefix,decoderCommandFileName); }
      if (fakeFileNamePrefix != null) { fakeFileNamePrefix = fullPath(dirPrefix,fakeFileNamePrefix); }
    }

    checkFile(paramsFileName);
    checkFile(decoderConfigFileName);

    boolean canRunCommand = fileExists(decoderCommandFileName);
    if (decoderCommandFileName != null && !canRunCommand) {
      // i.e. a decoder command file was specified, but it was not found
      if (firstTime)
        println("Warning: specified decoder command file " + decoderCommandFileName + " was not found.",1);
    }
    boolean canRunJoshua = fileExists(sourceFileName);
    if (sourceFileName != null && !canRunJoshua) {
      // i.e. a source file was specified, but it was not found
      if (firstTime)
        println("Warning: specified source file " + sourceFileName + " was not found.",1);
    }
    boolean canRunFake = (fakeFileNamePrefix != null);

    if (!canRunCommand && !canRunJoshua) { // can only run fake decoder

      if (!canRunFake) {
        println("Z-MERT cannot decode; must provide one of: command file (for external decoder),");
        println("                                           source file (for Joshua decoder),");
        println("                                        or prefix for existing output files (for fake decoder).");
        System.exit(12);
      }

      int lastGoodIt = 0;
      for (int it = 1; it <= maxMERTIterations; ++it) {
        if (fileExists(fakeFileNamePrefix+it)) {
          lastGoodIt = it;
        } else {
          break; // from for (it) loop
        }
      }

      if (lastGoodIt == 0) {
        println("Fake decoder cannot find first output file " + (fakeFileNamePrefix+1));
        System.exit(13);
      } else if (lastGoodIt < maxMERTIterations) {
        if (firstTime)
          println("Warning: can only run fake decoder; existing output files are only available for the first " + lastGoodIt + " iteration(s).",1);
      }

    }



    if (refsPerSen > 1) {
      // the provided refFileName might be a prefix
      File dummy = new File(refFileName);
      if (!dummy.exists()) {
        refFileName = createUnifiedRefFile(refFileName,refsPerSen);
      }
    } else {
      checkFile(refFileName);
    }


    if (firstTime) {
      println("Processed the following args array:",1);
      print("  ",1);
      for (i = 0; i < args.length; ++i) {
        print(args[i] + " ",1);
      }
      println("",1);
      println("",1);
    }

  } // processArgs(String[] args)

  private static void checkFile(String fileName)
  {
    if (!fileExists(fileName)) {
      println("The file " + fileName + " was not found!");
      System.exit(40);
    }
  }

  private static boolean fileExists(String fileName)
  {
    if (fileName == null) return false;
    File checker = new File(fileName);
    return checker.exists();
  }

  private static String createUnifiedRefFile(String prefix, int numFiles) throws Exception
  {
    if (numFiles < 2) {
      println("Warning: createUnifiedRefFile called with numFiles = " + numFiles + "; doing nothing.",1);
      return prefix;
    } else {
      File checker;
      checker = new File(prefix+"1");

      if (!checker.exists()) {
        checker = new File(prefix+".1");
        if (!checker.exists()) {
          println("Can't find reference files.");
          System.exit(50);
        } else {
          prefix = prefix + ".";
        }
      }

      String outFileName;
      if (prefix.endsWith(".")) { outFileName = prefix+"all"; }
      else { outFileName = prefix+".all"; }

      PrintWriter outFile = new PrintWriter(outFileName);

      BufferedReader[] inFile = new BufferedReader[numFiles];

      int nextIndex;
      checker = new File(prefix+"0");
      if (checker.exists()) { nextIndex = 0; }
      else { nextIndex = 1; }
      int lineCount = countLines(prefix+nextIndex);

      for (int r = 0; r < numFiles; ++r) {
        if (countLines(prefix+nextIndex) != lineCount) {
          println("Line count mismatch in " + (prefix+nextIndex) + ".");
          System.exit(60);
        }
        inFile[r] = new BufferedReader(new FileReader(prefix+nextIndex));
        ++nextIndex;
      }

      String line;

      for (int i = 0; i < lineCount; ++i) {
        for (int r = 0; r < numFiles; ++r) {
          line = inFile[r].readLine();
          outFile.println(line);
        }
      }

      outFile.close();

      for (int r = 0; r < numFiles; ++r) { inFile[r].close(); }

      return outFileName;

    }

  } // createUnifiedRefFile(String prefix, int numFiles)

  private static int countLines(String fileName) throws Exception
  {
    BufferedReader inFile = new BufferedReader(new FileReader(fileName));

    String line;
    int count = 0;
    do {
      line = inFile.readLine();
      if (line != null) ++count;
    }  while (line != null);

    inFile.close();

    return count;
  }

  private static int countNonEmptyLines(String fileName) throws Exception
  {
    BufferedReader inFile = new BufferedReader(new FileReader(fileName));

    String line;
    int count = 0;
    do {
      line = inFile.readLine();
      if (line != null && line.length() > 0) ++count;
    }  while (line != null);

    inFile.close();

    return count;
  }

  private static int countWords(String fileName) throws Exception
  {
    Scanner inFile = new Scanner(new FileReader(fileName));

    String word;
    int count = 0;
    while (inFile.hasNext()) {
      word = inFile.next();
      ++count;
    }

    inFile.close();

    return count;
  }

  private static String fullPath(String dir, String fileName)
  {
    File dummyFile = new File(dir,fileName);
    return dummyFile.getAbsolutePath();
  }

  private static void cleanupMemory()
  {
    cleanupMemory(100,false);
  }

  private static void cleanupMemorySilently()
  {
    cleanupMemory(100,true);
  }

  private static void cleanupMemory(int reps, boolean silent)
  {
    int bytesPerMB = 1024 * 1024;

    long totalMemBefore = myRuntime.totalMemory();
    long freeMemBefore = myRuntime.freeMemory();
    long usedMemBefore = totalMemBefore - freeMemBefore;


    long usedCurr = usedMemBefore; long usedPrev = usedCurr;

    // perform garbage collection repeatedly, until there is no decrease in
    // the amount of used memory
    for (int i = 1; i <= reps; ++i) {
      myRuntime.runFinalization();
      myRuntime.gc();
      (Thread.currentThread()).yield();

      usedPrev = usedCurr;
      usedCurr = myRuntime.totalMemory() - myRuntime.freeMemory();

      if (usedCurr == usedPrev) break;
    }


    if (!silent) {
      long totalMemAfter = myRuntime.totalMemory();
      long freeMemAfter = myRuntime.freeMemory();
      long usedMemAfter = totalMemAfter - freeMemAfter;

      println("GC: d_used = " + ((usedMemAfter - usedMemBefore) / bytesPerMB) + " MB (d_tot = " + ((totalMemAfter - totalMemBefore) / bytesPerMB) + " MB).",2);
    }
  }

  private static void printMemoryUsage()
  {
    int bytesPerMB = 1024 * 1024;
    long totalMem = myRuntime.totalMemory();
    long freeMem = myRuntime.freeMemory();
    long usedMem = totalMem - freeMem;

    println("Allocated memory: " + (totalMem / bytesPerMB) + " MB (of which " + (usedMem / bytesPerMB) + " MB is being used).",2);
  }

  private static void println(Object obj, int priority) { if (priority <= verbosity) println(obj); }
  private static void print(Object obj, int priority) { if (priority <= verbosity) print(obj); }

  private static void println(Object obj) { System.out.println(obj); }
  private static void print(Object obj) { System.out.print(obj); }

  private static void showProgress()
  {
    ++progress;
    if (progress % 100000 == 0) print(".",2);
  }



  private static double[] randomLambda()
  {
    double[] retVal = new double[1+numParams];

    for (int c = 1; c <= numParams; ++c) {
      if (isOptimizable[c]) {
        double randVal = randGen.nextDouble(); // number in [0.0,1.0]
        ++generatedRands;
        randVal = randVal * (maxRandValue[c] - minRandValue[c]); // number in [0.0,max-min]
        randVal = minRandValue[c] + randVal; // number in [min,max]
        retVal[c] = randVal;
      } else {
        retVal[c] = defaultLambda[c];
      }
    }

    return retVal;
  }

  private static void normalizeLambda(double[] origLambda)
  {
    // private static String[] normalizationOptions;
      // How should a lambda[] vector be normalized (before decoding)?
      //   nO[0] = 0: no normalization
      //   nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
      //   nO[0] = 2: scale so that the maximum absolute value is nO[1]
      //   nO[0] = 3: scale so that the minimum absolute value is nO[1]
      //   nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]

    int normalizationMethod = (int)normalizationOptions[0];
    double scalingFactor = 1.0;
    if (normalizationMethod == 0) {

      scalingFactor = 1.0;

    } else if (normalizationMethod == 1) {

      int c = (int)normalizationOptions[2];
      scalingFactor = normalizationOptions[1]/Math.abs(origLambda[c]);

    } else if (normalizationMethod == 2) {

      double maxAbsVal = -1;
      int maxAbsVal_c = 0;
      for (int c = 1; c <= numParams; ++c) {
        if (Math.abs(origLambda[c]) > maxAbsVal) {
          maxAbsVal = Math.abs(origLambda[c]);
          maxAbsVal_c = c;
        }
      }
      scalingFactor = normalizationOptions[1]/Math.abs(origLambda[maxAbsVal_c]);

    } else if (normalizationMethod == 3) {

      double minAbsVal = PosInf;
      int minAbsVal_c = 0;
      for (int c = 1; c <= numParams; ++c) {
        if (Math.abs(origLambda[c]) < minAbsVal) {
          minAbsVal = Math.abs(origLambda[c]);
          minAbsVal_c = c;
        }
      }
      scalingFactor = normalizationOptions[1]/Math.abs(origLambda[minAbsVal_c]);

    } else if (normalizationMethod == 4) {

      double pow = normalizationOptions[1];
      double norm = L_norm(origLambda,pow);
      scalingFactor = normalizationOptions[2]/norm;

    }

    for (int c = 1; c <= numParams; ++c) {
      origLambda[c] *= scalingFactor;
    }

  }

  private static int c_fromParamName (String pName)
  {
    for (int c = 1; c <= numParams; ++c) {
      if (paramNames[c].equals(pName)) return c;
    }
    return 0; // no parameter with that name!
  }

  private static double L_norm(double[] A, double pow)
  {
    // calculates the L-pow norm of A[]
    // NOTE: this calculation ignores A[0]
    double sum = 0.0;
    for (int i = 1; i < A.length; ++i) {
      sum += Math.pow(Math.abs(A[i]),pow);
    }
    return Math.pow(sum,1/pow);
  }

  private static void setFeats(double[][][] featVal_array, int i, int[] lastUsedIndex, int[] maxIndex, double[] featVal)
  {
    int k = lastUsedIndex[i] + 1;

    if (k > maxIndex[i]) {
      for (int c = 1; c <= numParams; ++c) {
        double[] temp = featVal_array[c][i];
        featVal_array[c][i] = new double[1+maxIndex[i]+sizeOfNBest];

        for (int k2 = 0; k2 <= maxIndex[i]; ++k2) {
          featVal_array[c][i][k2] = temp[k2];
        }
      }
      maxIndex[i] += sizeOfNBest;
//      cleanupMemorySilently(); // UNCOMMENT THIS if cleaning up memory
    }

    for (int c = 1; c <= numParams; ++c) {
      featVal_array[c][i][k] = featVal[c];
    }
    lastUsedIndex[i] += 1;
  }


  private static HashSet<Integer> indicesToDiscard(double[] slope, double[] offset)
  {
    // some lines can be eliminated: the ones that have a lower offset
    // than some other line with the same slope.
    // That is, for any k1 and k2:
    //   if slope[k1] = slope[k2] and offset[k1] > offset[k2],
    //   then k2 can be eliminated.
    // (This is actually important to do as it eliminates a bug.)
//    print("discarding: ",3);

    int numCandidates = slope.length;
    HashSet<Integer> discardedIndices = new HashSet<Integer>();
    HashMap<Double,Integer> indicesOfSlopes = new HashMap<Double,Integer>();
    // maps slope to index of best candidate that has that slope.
    // ("best" as in the one with the highest offset)

    for (int k1 = 0; k1 < numCandidates; ++k1) {
      double currSlope = slope[k1];
      if (!indicesOfSlopes.containsKey(currSlope)) {
        indicesOfSlopes.put(currSlope,k1);
      } else {
        int existingIndex = indicesOfSlopes.get(currSlope);
        if (offset[existingIndex] > offset[k1]) {
          discardedIndices.add(k1);
//          print(k1 + " ",3);
        } else if (offset[k1] > offset[existingIndex]) {
          indicesOfSlopes.put(currSlope,k1);
          discardedIndices.add(existingIndex);
//          print(existingIndex + " ",3);
        }
      }
    }


    // old way of doing it; takes quadratic time (vs. linear time above)
/*
    for (int k1 = 0; k1 < numCandidates; ++k1) {
      for (int k2 = 0; k2 < numCandidates; ++k2) {
        if (k1 != k2 && slope[k1] == slope[k2] && offset[k1] > offset[k2]) {
          discardedIndices.add(k2);
//          print(k2 + " ",3);
        }
      }
    }
*/

//    println("",3);
    return discardedIndices;
  } // indicesToDiscard(double[] slope, double[] offset)

  private static void set_suffStats_array(HashMap<Integer,int[]>[] suffStats_array, TreeSet<Integer>[] indicesOfInterest, int[] candCount) throws Exception
  {
    // process the merged sufficient statistics file, and read (and store) the
    // stats for candidates of interest
    BufferedReader inFile = new BufferedReader(new FileReader(decoderOutFileName+".temp.stats.merged"));
    String line, candidate_suffStats;

    for (int i = 0; i < numSentences; ++i) {
      int numCandidates = candCount[i];

      int currCand = 0;
      Iterator<Integer> It = indicesOfInterest[i].iterator();

      while (It.hasNext()) {
        int nextIndex = It.next();

        // skip candidates until you get to the nextIndex'th candidate
        while (currCand < nextIndex) {
          line = inFile.readLine();
          ++currCand;
        }

        // now currCand == nextIndex, and the next line in inFile contains the sufficient statistics we want

        candidate_suffStats = inFile.readLine();
        ++currCand;

        String[] suffStats_str = candidate_suffStats.split("\\s+");

        int[] suffStats = new int[suffStatsCount];

        for (int s = 0; s < suffStatsCount; ++s) {
          suffStats[s] = Integer.parseInt(suffStats_str[s]);
        }

        suffStats_array[i].put(nextIndex,suffStats);

      }

      // skip the rest of ith sentence's candidates
      while (currCand < numCandidates) {
        line = inFile.readLine();
        ++currCand;
      }

    }

    inFile.close();

  } // set_suffStats_array(HashMap[] suffStats_array, TreeSet[] indicesOfInterest, Vector[] candidates)


  public static void printZMERTUsage(int argsLen, boolean detailed)
  {
    if (!detailed) {
      println("Oops, you provided " + argsLen + " args!");
      println("");
      println("Usage:");
      println("           ZMERT -maxMem maxMemoryInMB MERT_configFile");
      println("");
      println("Where -maxMem specifies the maximum amount of memory (in MB) Z-MERT is");
      println("allowed to use when performing its calculations (no memroy is needed while");
      println("the decoder is running),");
      println("and the config file contains any subset of Z-MERT's 20-some parameters,");
      println("one per line.  Run   ZMERT -h   for more details on those parameters.");
    } else {
      println("Usage:");
      println("           ZMERT -maxMem maxMemoryInMB MERT_configFile");
      println("");
      println("Where -maxMem specifies the maximum amount of memory (in MB) Z-MERT is");
      println("allowed to use when performing its calculations (no memroy is needed while");
      println("the decoder is running),");
      println("and the config file contains any subset of Z-MERT's 20-some parameters,");
      println("one per line.  Those parameters, and their default values, are:");
      println("");
      println("Relevant files:");
      println("  -dir dirPrefix: working directory\n    [[default: null string (i.e. they are in the current directory)]]");
      println("  -s sourceFile: source sentences (foreign sentences) of the MERT dataset\n    [[default: null string (i.e. file name is not needed by MERT)]]");
      println("  -r refFile: target sentences (reference translations) of the MERT dataset\n    [[default: reference.txt]]");
      println("  -rps refsPerSen: number of reference translations per sentence\n    [[default: 1]]");
      println("  -p paramsFile: file containing parameter names, initial values, and ranges\n    [[default: params.txt]]");
      println("  -fin finalLambda: file name for final lambda[] values\n    [[default: null string (i.e. no such file will be created)]]");
      println("");
      println("MERT specs:");
      println("  -m metricName metric options: name of evaluation metric and its options\n    [[default: BLEU 4 closest]]");
      println("  -maxIt maxMERTIts: maximum number of MERT iterations\n    [[default: 20]]");
      println("  -prevIt prevMERTIts: maximum number of previous MERT iterations to\n    construct candidate sets from\n    [[default: 20]]");
      println("  -minIt minMERTIts: number of iterations before considering an early exit\n    [[default: 5]]");
      println("  -stopIt stopMinIts: some early stopping criterion must be satisfied in\n    stopMinIts *consecutive* iterations before an early exit\n    [[default: 3]]");
      println("  -stopSig sigValue: early MERT exit if no weight changes by more than sigValue\n    [[default: -1 (i.e. this criterion is never investigated)]]");
      println("  -save saveInter: save intermediate cfg files (1) or decoder outputs (2)\n    or both (3) or neither (0)\n    [[default: 3]]");
      println("  -ipi initsPerIt: number of intermediate initial points per iteration\n    [[default: 20]]");
      println("  -opi oncePerIt: modify a parameter only once per iteration (1) or not (0)\n    [[default: 0]]");
      println("  -rand randInit: choose initial point randomly (1) or from paramsFile (0)\n    [[default: 0]]");
      println("  -seed seed: seed used to initialize random number generator\n    [[default: time (i.e. value returned by System.currentTimeMillis()]]");
      println("  -ud useDisk: reliance on disk (0-2; higher value => more reliance)\n    [[default: 2]]");
      println("");
      println("Decoder specs:");
      println("  -cmd commandFile: name of file containing commands to run the decoder\n    [[default: null string (i.e. decoder is a JoshuaDecoder object)]]");
      println("  -decOut decoderOutFile: name of the output file produced by the decoder\n    [[default: output.nbest]]");
      println("  -decExit validExit: value returned by decoder to indicate success\n    [[default: 0]]");
      println("  -dcfg decConfigFile: name of decoder config file\n    [[default: dec_cfg.txt]]");
      println("  -N N: size of N-best list (per sentence) generated in each MERT iteration\n    [[default: 100]]");
      println("");
      println("Output specs:");
      println("  -v verbosity: Z-MERT verbosity level (0-2; higher value => more verbose)\n    [[default: 1]]");
      println("  -decV decVerbosity: should decoder output be printed (1) or ignored (0)\n    [[default: 0]]");
      println("");
    }
  }


}

// based on:
// http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html?page=4
class StreamGobbler extends Thread
{
  InputStream istream;
  boolean verbose;

  StreamGobbler(InputStream is, int p)
  {
    istream = is;
    if (p == 0) verbose = false;
    else verbose = true;
  }

  public void run()
  {
    try
    {
      InputStreamReader isreader = new InputStreamReader(istream);
      BufferedReader br = new BufferedReader(isreader);
      String line = null;
      while ((line = br.readLine()) != null) {
        if (verbose) System.out.println(line);    
      }
    } catch (IOException ioe)
      {
        ioe.printStackTrace();  
      }
  }
}


/*

fake:
-----
ex2_N300:
java -javaagent:shiftone-jrat.jar -Xmx300m -cp bin joshua.ZMERT.ZMERT -dir MERT_example -s src.txt -r ref.all -rps 4 -cmd decoder_command_ex2.txt -dcfg config_ex2.txt -decOut nbest_ex2.out -N 300 -p params.txt -maxIt 25 -opi 0 -ipi 20 -v 2 -rand 0 -seed 1226091488390 -save 1 -fake nbest_ex2.out.N300.it > ex2_N300ipi20opi0_300max+defratios.it10.noMemRep.bugFixes.monitored.txt

ex2_N500:
java -javaagent:shiftone-jrat.jar -Xmx300m -cp bin joshua.ZMERT.ZMERT -dir MERT_example -s src.txt -r ref.all -rps 4 -cmd decoder_command_ex2.txt -dcfg config_ex2.txt -decOut nbest_ex2.out -N 500 -p params.txt -maxIt 25 -opi 0 -ipi 20 -v 2 -rand 0 -seed 1226091488390 -save 1 -fake nbest_ex2.out.N500.it > ex2_N500ipi20opi0_300max+defratios.it05.noMemRep.bugFixes.monitored.txt

exL_N300__600max:
java -javaagent:shiftone-jrat.jar -Xmx600m -cp bin joshua.ZMERT.ZMERT -dir MERT_example -s mt06_source.txt -r mt06_ref.all -rps 4 -cmd decoder_command_ex2.txt -dcfg config_ex2.txt -decOut nbest_exL.out -N 300 -p params.txt -maxIt 5 -opi 0 -ipi 20 -v 2 -rand 0 -seed 1226091488390 -save 1 -fake nbest_exL.out.it > exL_N300ipi20opi0_600max+defratios.it05.noMemRep.bugFixes.monitored.txt

exL_N300__300max:
java -javaagent:shiftone-jrat.jar -Xmx300m -cp bin joshua.ZMERT.ZMERT -dir MERT_example -s mt06_source.txt -r mt06_ref.all -rps 4 -cmd decoder_command_ex2.txt -dcfg config_ex2.txt -decOut nbest_exL.out -N 300 -p params.txt -maxIt 5 -opi 0 -ipi 20 -v 2 -rand 0 -seed 1226091488390 -save 1 -fake nbest_exL.out.it > exL_N300ipi20opi0_300max+defratios.it05.noMemRep.bugFixes.monitored.txt

gen:
----
ex2_N300:
make sure top_n=300 in MERT_example\config_ex2.txt
java -javaagent:shiftone-jrat.jar -Xmx300m -cp bin joshua.ZMERT.ZMERT -dir MERT_example -s src.txt -r ref.all -rps 4 -cmd decoder_command_ex2.txt -dcfg config_ex2.txt -decOut nbest_ex2.out -N 300 -p params.txt -maxIt 25 -opi 0 -ipi 20 -v 2 -rand 0 -seed 1226091488390 -save 1 > ex2_N300ipi20opi0_300max+defratios.itxx.monitored.txt.gen

ex2_N500:
make sure top_n=500 in MERT_example\config_ex2.txt
java -javaagent:shiftone-jrat.jar -Xmx300m -cp bin joshua.ZMERT.ZMERT -dir MERT_example -s src.txt -r ref.all -rps 4 -cmd decoder_command_ex2.txt -dcfg config_ex2.txt -decOut nbest_ex2.out -N 500 -p params.txt -maxIt 25 -opi 0 -ipi 20 -v 2 -rand 0 -seed 1226091488390 -save 1 > ex2_N500ipi20opi0_300max+defratios.itxx.monitored.txt.gen

exL_N300__600max:
run on CLSP machines only! (e.g. z12)
$JAVA_bin/java -javaagent:shiftone-jrat.jar -Xmx600m -cp bin joshua.ZMERT.ZMERT -dir YOURDIR -s mt06_source.txt -r mt06_ref.all -rps 4 -cmd decoder_command.txt -dcfg config_exL.txt -decOut nbest_exL.out -N 300 -p params.txt -maxIt 25 -opi 0 -ipi 20 -v 2 -rand 0 -seed 1226091488390 -save 1 > exL_N300ipi20opi0_600max+defratios.itxx.monitored.txt.gen

exL_N300__300max:
run on CLSP machines only! (e.g. z12)
$JAVA_bin/java -javaagent:shiftone-jrat.jar -Xmx300m -cp bin joshua.ZMERT.ZMERT -dir YOURDIR -s mt06_source.txt -r mt06_ref.all -rps 4 -cmd decoder_command.txt -dcfg config_exL.txt -decOut nbest_exL.out -N 300 -p params.txt -maxIt 25 -opi 0 -ipi 20 -v 2 -rand 0 -seed 1226091488390 -save 1 > exL_N300ipi20opi0_600max+defratios.itxx.monitored.txt.gen

*/
