XFoldLearner extends Learner { @Override public RetrievalModelInstance learn() throws Exception { if (execute) { final List<RetrievalModelInstance> learntParams = new ArrayList<>(); HashMap<String, List<EvalDoc>> allTestResMap = new HashMap<>(); for (int foldId : foldLearners.keySet()) { RetrievalModelInstance result = foldLearners.get(foldId).learn(); Pair<Double, HashMap<String, List<EvalDoc>>> testResult = evaluateSpecificQueries(result, testQueryFolds.get(foldId)); result.setAnnotation("testScore", Double.toString(testResult.getFirst())); allTestResMap.putAll(testResult.getSecond()); double allScore = evaluateSpecificQueries(result, queryNumbers).getFirst(); result.setAnnotation("allScore", Double.toString(allScore)); this.outputPrintStream.println(result.toPrettyString()); learntParams.add(result); } QuerySetResults allTestResults = new QuerySetResults(allTestResMap); allTestResults.ensureQuerySet(queries.getQueryParameters()); if (outputFolder != null) { PrintWriter out = new PrintWriter(new File(outputFolder, name + "-test-fold-all.run"), "UTF-8"); for (String queryNumber : allTestResults.getQueryIterator()) { QueryResults results = allTestResults.get(queryNumber); results.outputTrecrun(out, "galago"); } out.close(); } double allTestScore = evalFunction.evaluate(allTestResults, qrels); outputTraceStream.println(String.format("Score on all test sets combined: %f", allTestScore)); Parameters settings = Parameters.create(); for (String param : this.learnableParameters.getParams()) { double setting = 0.0; for (RetrievalModelInstance foldOpt : learntParams) { setting += foldOpt.get(param); } setting /= learntParams.size(); settings.set(param, setting); } RetrievalModelInstance averageParams = new RetrievalModelInstance(this.learnableParameters, settings); double score = evaluateSpecificQueries(averageParams, queryNumbers).getFirst(); averageParams.setAnnotation("score", Double.toString(score)); averageParams.setAnnotation("name", name + "-xfold-avg"); outputPrintStream.println(averageParams.toPrettyString()); return averageParams; } else { outputPrintStream.println("NOT OPTIMIZING, returning random parameters."); return this.generateRandomInitalValues(); } } XFoldLearner(Parameters p, Retrieval r); void close(); @Override RetrievalModelInstance learn();  }
@Test public void testCoordinateSearch() throws Exception { File index = null; File qrels = null; try { File[] files = TestingUtils.make10DocIndex(); Assert.assertTrue(files[0].delete()); FSUtil.deleteDirectory(files[1]); index = files[2]; qrels = FileUtility.createTemporary(); Retrieval ret = RetrievalFactory.instance(index.getAbsolutePath(), Parameters.create()); String qrelData = "q1 x 2 1\n" + "q1 x 5 1\n" + "q1 x 8 1\n" + "q2 x 3 1\n" + "q2 x 7 1\n"; StreamUtil.copyStringToFile(qrelData, qrels); Parameters learnParams = Parameters.parseString("{\"queries\": [{\"number\":\"q1\",\"text\":\"#combine:0=0.5:1=0.5( jump moon )\"}, {\"number\":\"q2\",\"text\":\"#combine:0=0.5:1=0.5( everything shoe )\"}]}"); learnParams.set("learner", "xfold"); learnParams.set("xfolds", 2); learnParams.set("xfoldLearner", "coord"); learnParams.set("qrels", qrels.getAbsolutePath()); List<Parameters> learnableParams = new ArrayList<Parameters>(); learnableParams.add(Parameters.parseString("{\"name\":\"0\"}")); learnableParams.add(Parameters.parseString("{\"name\":\"1\"}")); learnParams.set("learnableParameters", learnableParams); Parameters normalRule = Parameters.create(); normalRule.set("mode", "sum"); normalRule.set("params", Arrays.asList(new String[]{"0", "1"})); normalRule.set("value", 1.0); learnParams.set("normalization", new ArrayList()); learnParams.getList("normalization").add(normalRule); learnParams.set("restarts", 1); learnParams.set("initialParameters", new ArrayList()); learnParams.getList("initialParameters").add(Parameters.parseString("{\"0\":0.9,\"1\":-0.2}")); Learner learner = LearnerFactory.instance(learnParams, ret); RetrievalModelInstance res = learner.learn(); } finally { if (index != null) { FSUtil.deleteDirectory(index); } if (qrels != null) { Assert.assertTrue(qrels.delete()); } } }