Evaluation_ImplBase { public List<STATS_TYPE> crossValidation(List<ITEM_TYPE> items, int nFolds) throws Exception { if (nFolds > items.size()) { String message = "Cannot have %d folds with only %d items"; throw new IllegalArgumentException(String.format(message, nFolds, items.size())); } List<STATS_TYPE> stats = new ArrayList<STATS_TYPE>(); for (int fold = 0; fold < nFolds; ++fold) { File subDirectory = new File(this.baseDirectory, "fold_" + fold); subDirectory.mkdirs(); List<ITEM_TYPE> trainItems = this.selectFoldTrainItems(items, nFolds, fold); List<ITEM_TYPE> testItems = this.selectFoldTestItems(items, nFolds, fold); this.train(this.getCollectionReader(trainItems), subDirectory); stats.add(this.test(this.getCollectionReader(testItems), subDirectory)); } return stats; } Evaluation_ImplBase(File baseDirectory); STATS_TYPE trainAndTest(List<ITEM_TYPE> trainItems, List<ITEM_TYPE> testItems); List<STATS_TYPE> crossValidation(List<ITEM_TYPE> items, int nFolds);  }
@Test public void testCrossValidation() throws Exception { TestableEvaluation evaluation = new TestableEvaluation(this.outputDirectory); List<String> trainItems = Arrays.asList("a", "b", "c", "d", "e"); List<List<String>> trainFolds = new ArrayList<List<String>>(); trainFolds.add(Arrays.asList("b", "c", "e")); trainFolds.add(Arrays.asList("a", "c", "d")); trainFolds.add(Arrays.asList("a", "b", "d", "e")); List<List<String>> testFolds = new ArrayList<List<String>>(); testFolds.add(Arrays.asList("a", "d")); testFolds.add(Arrays.asList("b", "e")); testFolds.add(Arrays.asList("c")); evaluation.crossValidation(trainItems, 3); Assert.assertEquals(3, evaluation.trainDirectories.size()); Assert.assertEquals(3, evaluation.testDirectories.size()); Assert.assertEquals(3, evaluation.trainDirectoryItems.size()); Assert.assertEquals(3, evaluation.testDirectoryItems.size()); Assert.assertEquals(trainFolds, evaluation.trainDirectoryItems); Assert.assertEquals(testFolds, evaluation.testDirectoryItems); for (File dir : evaluation.trainDirectories) { Assert.assertTrue(dir.getPath().startsWith(this.outputDirectory.getPath())); } for (File dir : evaluation.testDirectories) { Assert.assertTrue(dir.getPath().startsWith(this.outputDirectory.getPath())); } try { evaluation.crossValidation(trainItems, 6); Assert.fail("evaluation should fail with more folds than items"); } catch (IllegalArgumentException e) { } }