@Test public void createRandomTrainDevSplitsForKFoldCrossValidationStratifiedOnExampleCategoryTest(){ { Random rand = new Random(0); final ArrayList<ExampleCategory> CATEGORIES = new ArrayList<>(Arrays.asList( new ExampleCategory(new MethodId(Object.class, "a", Object.class), InstructionQueryability.OTHER), new ExampleCategory(new MethodId(Object.class, "a", List.class), InstructionQueryability.OTHER), new ExampleCategory(new MethodId(List.class, "a", Object.class), InstructionQueryability.OTHER), new ExampleCategory(new MethodId(Object.class, "b", Object.class), InstructionQueryability.OTHER), new ExampleCategory(new MethodId(Object.class, "a", Object.class), InstructionQueryability.QUERYABLE_BY_PRIMITIVE_REL_AND_ARG_QUERYABLE_BY_SUPERLATIVE) )); Function<Example, ExampleCategory> exampleToCategory = ex -> CATEGORIES.get(Integer.parseInt(ex.getId()) % CATEGORIES.size()); for (int j=0; j<10; j++) { List<Example> examples = new LinkedList<>(); final int FOLDS_NUM = 4; final int EXAMPLES_NUM = CATEGORIES.size() * FOLDS_NUM * 10; final int EXAMPLES_NUM_PER_CATEGORY = EXAMPLES_NUM / CATEGORIES.size(); for (int i = 0; i < EXAMPLES_NUM; i++) examples.add(new Example(Integer.toString(i), null, null, null, null)); ArrayList<ExampleSplit> splits = ExampleSplit.createRandomTrainDevSplitsForKFoldCrossValidationStratifiedOnExampleCategory( examples, FOLDS_NUM, exampleToCategory, rand, false); aux_verifyKFoldSplitsAreValid(EXAMPLES_NUM, FOLDS_NUM, 0, splits); for (ExampleCategory category : CATEGORIES) { for (int foldInd = 0; foldInd < FOLDS_NUM; foldInd++) { ExampleSplit s = splits.get(foldInd).getSubsetByFilteringExamples(null, ex -> exampleToCategory.apply(ex).equals(category)); int examplesPerCategoryPerFold = EXAMPLES_NUM_PER_CATEGORY / FOLDS_NUM; assertEquals(examplesPerCategoryPerFold, s.getTestExamples().size()); assertEquals(EXAMPLES_NUM_PER_CATEGORY-examplesPerCategoryPerFold, s.getTrainExamples().size()); } } } } }
public static ArrayList<ExampleSplit> createRandomTrainDevSplitsForKFoldCrossValidationStratifiedOnExampleCategory( List<Example> examples, int foldsNum, Function<Example, ExampleCategory> exampleToCategory, Random rand, boolean sameWorkerExamplesStayTogether) { SafeMap<ExampleCategory, List<Example>> categoryToExamples = new SafeHashMap<>(); Stack<ExampleCategory> deterministicCategoryOrder = new Stack<>(); examples.forEach(ex-> { ExampleCategory category = exampleToCategory.apply(ex); Maps.addToMapOfCollections(categoryToExamples, category, ex, LinkedList::new); deterministicCategoryOrder.add(category); }); ArrayList<ExampleSplit> result = new ArrayList<>(); for (int i=0;i<foldsNum;i++) result.add(new ExampleSplit()); while (!categoryToExamples.isEmpty()) { ExampleCategory category = deterministicCategoryOrder.pop(); if (!categoryToExamples.safeContainsKey(category)) continue; List<Example> examplesOfCategory = categoryToExamples.safeGet(category); categoryToExamples.safeRemove(category); ArrayList<ExampleSplit> splitsForCurrentCategory = createRandomSplitsForKFoldCrossValidation( examplesOfCategory, foldsNum, rand, sameWorkerExamplesStayTogether); for (int i=0;i<foldsNum;i++) result.set(i, addSplits(result.get(i), splitsForCurrentCategory.get(i), null)); } for (int i=0;i<foldsNum;i++) result.get(i).shuffleOrders(rand); return result; }
ExampleSplit implements Serializable { public static ArrayList<ExampleSplit> createRandomTrainDevSplitsForKFoldCrossValidationStratifiedOnExampleCategory( List<Example> examples, int foldsNum, Function<Example, ExampleCategory> exampleToCategory, Random rand, boolean sameWorkerExamplesStayTogether) { SafeMap<ExampleCategory, List<Example>> categoryToExamples = new SafeHashMap<>(); Stack<ExampleCategory> deterministicCategoryOrder = new Stack<>(); examples.forEach(ex-> { ExampleCategory category = exampleToCategory.apply(ex); Maps.addToMapOfCollections(categoryToExamples, category, ex, LinkedList::new); deterministicCategoryOrder.add(category); }); ArrayList<ExampleSplit> result = new ArrayList<>(); for (int i=0;i<foldsNum;i++) result.add(new ExampleSplit()); while (!categoryToExamples.isEmpty()) { ExampleCategory category = deterministicCategoryOrder.pop(); if (!categoryToExamples.safeContainsKey(category)) continue; List<Example> examplesOfCategory = categoryToExamples.safeGet(category); categoryToExamples.safeRemove(category); ArrayList<ExampleSplit> splitsForCurrentCategory = createRandomSplitsForKFoldCrossValidation( examplesOfCategory, foldsNum, rand, sameWorkerExamplesStayTogether); for (int i=0;i<foldsNum;i++) result.set(i, addSplits(result.get(i), splitsForCurrentCategory.get(i), null)); } for (int i=0;i<foldsNum;i++) result.get(i).shuffleOrders(rand); return result; } }
ExampleSplit implements Serializable { public static ArrayList<ExampleSplit> createRandomTrainDevSplitsForKFoldCrossValidationStratifiedOnExampleCategory( List<Example> examples, int foldsNum, Function<Example, ExampleCategory> exampleToCategory, Random rand, boolean sameWorkerExamplesStayTogether) { SafeMap<ExampleCategory, List<Example>> categoryToExamples = new SafeHashMap<>(); Stack<ExampleCategory> deterministicCategoryOrder = new Stack<>(); examples.forEach(ex-> { ExampleCategory category = exampleToCategory.apply(ex); Maps.addToMapOfCollections(categoryToExamples, category, ex, LinkedList::new); deterministicCategoryOrder.add(category); }); ArrayList<ExampleSplit> result = new ArrayList<>(); for (int i=0;i<foldsNum;i++) result.add(new ExampleSplit()); while (!categoryToExamples.isEmpty()) { ExampleCategory category = deterministicCategoryOrder.pop(); if (!categoryToExamples.safeContainsKey(category)) continue; List<Example> examplesOfCategory = categoryToExamples.safeGet(category); categoryToExamples.safeRemove(category); ArrayList<ExampleSplit> splitsForCurrentCategory = createRandomSplitsForKFoldCrossValidation( examplesOfCategory, foldsNum, rand, sameWorkerExamplesStayTogether); for (int i=0;i<foldsNum;i++) result.set(i, addSplits(result.get(i), splitsForCurrentCategory.get(i), null)); } for (int i=0;i<foldsNum;i++) result.get(i).shuffleOrders(rand); return result; } ExampleSplit();  ExampleSplit(List<Example> trainExamples, List<Example> testExamples); }
ExampleSplit implements Serializable { public static ArrayList<ExampleSplit> createRandomTrainDevSplitsForKFoldCrossValidationStratifiedOnExampleCategory( List<Example> examples, int foldsNum, Function<Example, ExampleCategory> exampleToCategory, Random rand, boolean sameWorkerExamplesStayTogether) { SafeMap<ExampleCategory, List<Example>> categoryToExamples = new SafeHashMap<>(); Stack<ExampleCategory> deterministicCategoryOrder = new Stack<>(); examples.forEach(ex-> { ExampleCategory category = exampleToCategory.apply(ex); Maps.addToMapOfCollections(categoryToExamples, category, ex, LinkedList::new); deterministicCategoryOrder.add(category); }); ArrayList<ExampleSplit> result = new ArrayList<>(); for (int i=0;i<foldsNum;i++) result.add(new ExampleSplit()); while (!categoryToExamples.isEmpty()) { ExampleCategory category = deterministicCategoryOrder.pop(); if (!categoryToExamples.safeContainsKey(category)) continue; List<Example> examplesOfCategory = categoryToExamples.safeGet(category); categoryToExamples.safeRemove(category); ArrayList<ExampleSplit> splitsForCurrentCategory = createRandomSplitsForKFoldCrossValidation( examplesOfCategory, foldsNum, rand, sameWorkerExamplesStayTogether); for (int i=0;i<foldsNum;i++) result.set(i, addSplits(result.get(i), splitsForCurrentCategory.get(i), null)); } for (int i=0;i<foldsNum;i++) result.get(i).shuffleOrders(rand); return result; } ExampleSplit();  ExampleSplit(List<Example> trainExamples, List<Example> testExamples); static List<ExampleSplit> getRandomSplitsForInDomainTrainingExperiment(
            List<Example> examples, Function<Example, ExampleCategory> exampleToCategory, Domain domain,
            Random rand, int foldsNum); static ExampleSplit getRandomSplitForZeroShotExperiment(List<Example> examples, Domain testDomain,
                                                                   Random rand); Stream<Example> getAllExamples(); void shuffleOrders(Random rand); static ExampleSplit createRandomly(List<Example> examples, double trainRatio, Random rand,
                                               boolean sameWorkerExamplesStayTogether); ExampleSplit getSubsetByFilteringExamples(@Nullable SplitPart splitPartToFilter,
                                                     Predicate<Example> predicate); ExampleSplit getSubsetContainedInExampleCollection(Collection<Example> c); ExampleSplit getSubsetNotContainedInExampleCollection(Collection<Example> c); ExampleSplit getSubsetNotContainingMultiSentenceExamples(); ExampleSplit getSubsetContainingSingleExample(String exId); static ExampleSplit addSplits(ExampleSplit split1, ExampleSplit split2, @Nullable Random rand); boolean contains(Example ex); ArrayList<Example> getTrainExamples(); ArrayList<Example> getTestExamples(); @Nullable SplitPart getSplitPart(Example ex); @Override String toString(); static ArrayList<ExampleSplit> createRandomTrainDevSplitsForKFoldCrossValidationStratifiedOnExampleCategory(
            List<Example> examples, int foldsNum, Function<Example, ExampleCategory> exampleToCategory,
            Random rand, boolean sameWorkerExamplesStayTogether); }
ExampleSplit implements Serializable { public static ArrayList<ExampleSplit> createRandomTrainDevSplitsForKFoldCrossValidationStratifiedOnExampleCategory( List<Example> examples, int foldsNum, Function<Example, ExampleCategory> exampleToCategory, Random rand, boolean sameWorkerExamplesStayTogether) { SafeMap<ExampleCategory, List<Example>> categoryToExamples = new SafeHashMap<>(); Stack<ExampleCategory> deterministicCategoryOrder = new Stack<>(); examples.forEach(ex-> { ExampleCategory category = exampleToCategory.apply(ex); Maps.addToMapOfCollections(categoryToExamples, category, ex, LinkedList::new); deterministicCategoryOrder.add(category); }); ArrayList<ExampleSplit> result = new ArrayList<>(); for (int i=0;i<foldsNum;i++) result.add(new ExampleSplit()); while (!categoryToExamples.isEmpty()) { ExampleCategory category = deterministicCategoryOrder.pop(); if (!categoryToExamples.safeContainsKey(category)) continue; List<Example> examplesOfCategory = categoryToExamples.safeGet(category); categoryToExamples.safeRemove(category); ArrayList<ExampleSplit> splitsForCurrentCategory = createRandomSplitsForKFoldCrossValidation( examplesOfCategory, foldsNum, rand, sameWorkerExamplesStayTogether); for (int i=0;i<foldsNum;i++) result.set(i, addSplits(result.get(i), splitsForCurrentCategory.get(i), null)); } for (int i=0;i<foldsNum;i++) result.get(i).shuffleOrders(rand); return result; } ExampleSplit();  ExampleSplit(List<Example> trainExamples, List<Example> testExamples); static List<ExampleSplit> getRandomSplitsForInDomainTrainingExperiment(
            List<Example> examples, Function<Example, ExampleCategory> exampleToCategory, Domain domain,
            Random rand, int foldsNum); static ExampleSplit getRandomSplitForZeroShotExperiment(List<Example> examples, Domain testDomain,
                                                                   Random rand); Stream<Example> getAllExamples(); void shuffleOrders(Random rand); static ExampleSplit createRandomly(List<Example> examples, double trainRatio, Random rand,
                                               boolean sameWorkerExamplesStayTogether); ExampleSplit getSubsetByFilteringExamples(@Nullable SplitPart splitPartToFilter,
                                                     Predicate<Example> predicate); ExampleSplit getSubsetContainedInExampleCollection(Collection<Example> c); ExampleSplit getSubsetNotContainedInExampleCollection(Collection<Example> c); ExampleSplit getSubsetNotContainingMultiSentenceExamples(); ExampleSplit getSubsetContainingSingleExample(String exId); static ExampleSplit addSplits(ExampleSplit split1, ExampleSplit split2, @Nullable Random rand); boolean contains(Example ex); ArrayList<Example> getTrainExamples(); ArrayList<Example> getTestExamples(); @Nullable SplitPart getSplitPart(Example ex); @Override String toString(); static ArrayList<ExampleSplit> createRandomTrainDevSplitsForKFoldCrossValidationStratifiedOnExampleCategory(
            List<Example> examples, int foldsNum, Function<Example, ExampleCategory> exampleToCategory,
            Random rand, boolean sameWorkerExamplesStayTogether);  }