@Test public void createRandomSplitsForKFoldCrossValidationTest() { Random rand = new Random(0); for (int j=0; j<10; j++) { List<Example> examples = new LinkedList<>(); int EXAMPLES_NUM = 100; int FOLDS_NUM = 4; assert EXAMPLES_NUM%FOLDS_NUM == 0; int WORKERS_NUM = 30; SafeMap<String, Integer> workerIdToExampleNum = new SafeHashMap<>(); for (int i = 0; i<EXAMPLES_NUM; i++) { String workerId = Integer.toString(rand.nextInt(WORKERS_NUM)); Example ex = new Example(Integer.toString(i), null, null, null, null); if (!workerIdToExampleNum.safeContainsKey(workerId)) workerIdToExampleNum.put(workerId,0); workerIdToExampleNum.put(workerId, workerIdToExampleNum.safeGet(workerId)+1); examples.add(ex); } int largestExampleNumDoneBySameWorker = Collections.max(workerIdToExampleNum.values()); ArrayList<ExampleSplit> splits = ExampleSplit.createRandomSplitsForKFoldCrossValidation( examples, FOLDS_NUM, rand, false); aux_verifyKFoldSplitsAreValid(EXAMPLES_NUM, FOLDS_NUM, 0, splits); int maxDifferenceBetweenActualAndDesiredExamplesNum = (FOLDS_NUM-1) * (largestExampleNumDoneBySameWorker-1); aux_verifyKFoldSplitsAreValid(EXAMPLES_NUM, FOLDS_NUM, maxDifferenceBetweenActualAndDesiredExamplesNum, splits); int K = 10; assert K < EXAMPLES_NUM / FOLDS_NUM; splits = ExampleSplit.createRandomSplitsForKFoldCrossValidation( examples, FOLDS_NUM, rand, false); SafeSet<List<Example>> k_sequences = new SafeHashSet<>(); for (ExampleSplit split : splits) { aux_addAllKSequencesVerifyingNotAlreadyIn(split.getTrainExamples(), K, k_sequences); aux_addAllKSequencesVerifyingNotAlreadyIn(split.getTestExamples(), K, k_sequences); } } }
static ArrayList<ExampleSplit> createRandomSplitsForKFoldCrossValidation( List<Example> examples, int foldsNum, Random rand, boolean sameWorkerExamplesStayTogether) { List<Double> proportionSubsetSizes = Collections.nCopies(foldsNum, 1.0/foldsNum); List<List<Example>> partition = randomlyShuffleAndCreateExamplePartition(examples, proportionSubsetSizes, rand, sameWorkerExamplesStayTogether); ArrayList<ExampleSplit> result = new ArrayList<>(foldsNum); for (int subsetUsedAsTestInd=0; subsetUsedAsTestInd<foldsNum; subsetUsedAsTestInd++) { List<Example> foldTestExamples = partition.get(subsetUsedAsTestInd); List<Example> foldTrainExamples = new LinkedList<>(); for (int i=0; i<foldsNum; i++) if (i != subsetUsedAsTestInd) foldTrainExamples.addAll(partition.get(i)); ExampleSplit split = new ExampleSplit(foldTrainExamples, foldTestExamples); split.shuffleOrders(rand); Verify.verify(split.getAllExamples().count() == examples.size()); result.add(split); } return result; }
ExampleSplit implements Serializable { static ArrayList<ExampleSplit> createRandomSplitsForKFoldCrossValidation( List<Example> examples, int foldsNum, Random rand, boolean sameWorkerExamplesStayTogether) { List<Double> proportionSubsetSizes = Collections.nCopies(foldsNum, 1.0/foldsNum); List<List<Example>> partition = randomlyShuffleAndCreateExamplePartition(examples, proportionSubsetSizes, rand, sameWorkerExamplesStayTogether); ArrayList<ExampleSplit> result = new ArrayList<>(foldsNum); for (int subsetUsedAsTestInd=0; subsetUsedAsTestInd<foldsNum; subsetUsedAsTestInd++) { List<Example> foldTestExamples = partition.get(subsetUsedAsTestInd); List<Example> foldTrainExamples = new LinkedList<>(); for (int i=0; i<foldsNum; i++) if (i != subsetUsedAsTestInd) foldTrainExamples.addAll(partition.get(i)); ExampleSplit split = new ExampleSplit(foldTrainExamples, foldTestExamples); split.shuffleOrders(rand); Verify.verify(split.getAllExamples().count() == examples.size()); result.add(split); } return result; } }
ExampleSplit implements Serializable { static ArrayList<ExampleSplit> createRandomSplitsForKFoldCrossValidation( List<Example> examples, int foldsNum, Random rand, boolean sameWorkerExamplesStayTogether) { List<Double> proportionSubsetSizes = Collections.nCopies(foldsNum, 1.0/foldsNum); List<List<Example>> partition = randomlyShuffleAndCreateExamplePartition(examples, proportionSubsetSizes, rand, sameWorkerExamplesStayTogether); ArrayList<ExampleSplit> result = new ArrayList<>(foldsNum); for (int subsetUsedAsTestInd=0; subsetUsedAsTestInd<foldsNum; subsetUsedAsTestInd++) { List<Example> foldTestExamples = partition.get(subsetUsedAsTestInd); List<Example> foldTrainExamples = new LinkedList<>(); for (int i=0; i<foldsNum; i++) if (i != subsetUsedAsTestInd) foldTrainExamples.addAll(partition.get(i)); ExampleSplit split = new ExampleSplit(foldTrainExamples, foldTestExamples); split.shuffleOrders(rand); Verify.verify(split.getAllExamples().count() == examples.size()); result.add(split); } return result; } ExampleSplit();  ExampleSplit(List<Example> trainExamples, List<Example> testExamples); }
ExampleSplit implements Serializable { static ArrayList<ExampleSplit> createRandomSplitsForKFoldCrossValidation( List<Example> examples, int foldsNum, Random rand, boolean sameWorkerExamplesStayTogether) { List<Double> proportionSubsetSizes = Collections.nCopies(foldsNum, 1.0/foldsNum); List<List<Example>> partition = randomlyShuffleAndCreateExamplePartition(examples, proportionSubsetSizes, rand, sameWorkerExamplesStayTogether); ArrayList<ExampleSplit> result = new ArrayList<>(foldsNum); for (int subsetUsedAsTestInd=0; subsetUsedAsTestInd<foldsNum; subsetUsedAsTestInd++) { List<Example> foldTestExamples = partition.get(subsetUsedAsTestInd); List<Example> foldTrainExamples = new LinkedList<>(); for (int i=0; i<foldsNum; i++) if (i != subsetUsedAsTestInd) foldTrainExamples.addAll(partition.get(i)); ExampleSplit split = new ExampleSplit(foldTrainExamples, foldTestExamples); split.shuffleOrders(rand); Verify.verify(split.getAllExamples().count() == examples.size()); result.add(split); } return result; } ExampleSplit();  ExampleSplit(List<Example> trainExamples, List<Example> testExamples); static List<ExampleSplit> getRandomSplitsForInDomainTrainingExperiment(
            List<Example> examples, Function<Example, ExampleCategory> exampleToCategory, Domain domain,
            Random rand, int foldsNum); static ExampleSplit getRandomSplitForZeroShotExperiment(List<Example> examples, Domain testDomain,
                                                                   Random rand); Stream<Example> getAllExamples(); void shuffleOrders(Random rand); static ExampleSplit createRandomly(List<Example> examples, double trainRatio, Random rand,
                                               boolean sameWorkerExamplesStayTogether); ExampleSplit getSubsetByFilteringExamples(@Nullable SplitPart splitPartToFilter,
                                                     Predicate<Example> predicate); ExampleSplit getSubsetContainedInExampleCollection(Collection<Example> c); ExampleSplit getSubsetNotContainedInExampleCollection(Collection<Example> c); ExampleSplit getSubsetNotContainingMultiSentenceExamples(); ExampleSplit getSubsetContainingSingleExample(String exId); static ExampleSplit addSplits(ExampleSplit split1, ExampleSplit split2, @Nullable Random rand); boolean contains(Example ex); ArrayList<Example> getTrainExamples(); ArrayList<Example> getTestExamples(); @Nullable SplitPart getSplitPart(Example ex); @Override String toString(); static ArrayList<ExampleSplit> createRandomTrainDevSplitsForKFoldCrossValidationStratifiedOnExampleCategory(
            List<Example> examples, int foldsNum, Function<Example, ExampleCategory> exampleToCategory,
            Random rand, boolean sameWorkerExamplesStayTogether); }
ExampleSplit implements Serializable { static ArrayList<ExampleSplit> createRandomSplitsForKFoldCrossValidation( List<Example> examples, int foldsNum, Random rand, boolean sameWorkerExamplesStayTogether) { List<Double> proportionSubsetSizes = Collections.nCopies(foldsNum, 1.0/foldsNum); List<List<Example>> partition = randomlyShuffleAndCreateExamplePartition(examples, proportionSubsetSizes, rand, sameWorkerExamplesStayTogether); ArrayList<ExampleSplit> result = new ArrayList<>(foldsNum); for (int subsetUsedAsTestInd=0; subsetUsedAsTestInd<foldsNum; subsetUsedAsTestInd++) { List<Example> foldTestExamples = partition.get(subsetUsedAsTestInd); List<Example> foldTrainExamples = new LinkedList<>(); for (int i=0; i<foldsNum; i++) if (i != subsetUsedAsTestInd) foldTrainExamples.addAll(partition.get(i)); ExampleSplit split = new ExampleSplit(foldTrainExamples, foldTestExamples); split.shuffleOrders(rand); Verify.verify(split.getAllExamples().count() == examples.size()); result.add(split); } return result; } ExampleSplit();  ExampleSplit(List<Example> trainExamples, List<Example> testExamples); static List<ExampleSplit> getRandomSplitsForInDomainTrainingExperiment(
            List<Example> examples, Function<Example, ExampleCategory> exampleToCategory, Domain domain,
            Random rand, int foldsNum); static ExampleSplit getRandomSplitForZeroShotExperiment(List<Example> examples, Domain testDomain,
                                                                   Random rand); Stream<Example> getAllExamples(); void shuffleOrders(Random rand); static ExampleSplit createRandomly(List<Example> examples, double trainRatio, Random rand,
                                               boolean sameWorkerExamplesStayTogether); ExampleSplit getSubsetByFilteringExamples(@Nullable SplitPart splitPartToFilter,
                                                     Predicate<Example> predicate); ExampleSplit getSubsetContainedInExampleCollection(Collection<Example> c); ExampleSplit getSubsetNotContainedInExampleCollection(Collection<Example> c); ExampleSplit getSubsetNotContainingMultiSentenceExamples(); ExampleSplit getSubsetContainingSingleExample(String exId); static ExampleSplit addSplits(ExampleSplit split1, ExampleSplit split2, @Nullable Random rand); boolean contains(Example ex); ArrayList<Example> getTrainExamples(); ArrayList<Example> getTestExamples(); @Nullable SplitPart getSplitPart(Example ex); @Override String toString(); static ArrayList<ExampleSplit> createRandomTrainDevSplitsForKFoldCrossValidationStratifiedOnExampleCategory(
            List<Example> examples, int foldsNum, Function<Example, ExampleCategory> exampleToCategory,
            Random rand, boolean sameWorkerExamplesStayTogether);  }