@Test public void testEvaluate() throws Exception { PredicateArgumentEvaluator evaluator = new PredicateArgumentEvaluator(); evaluator.evaluate(argLabelTester, gold, predicted); assertEquals(1.0, argLabelTester.getMicroF1(), 0); Constituent predicate = predicted.getPredicates().get(0); Constituent argument = new Constituent("test", ViewNames.SRL_VERB, predicate.getTextAnnotation(), 0, 1); predicted.addRelation(new Relation("test", predicate, argument, 0)); evaluator = new PredicateArgumentEvaluator(); evaluator.evaluate(argLabelTester, gold, predicted); assertEquals(0.92, argLabelTester.getMicroF1(), 0.1); }
public void evaluate(ClassificationTester tester, View goldView, View predictionView) { gold = (PredicateArgumentView) goldView; prediction = (PredicateArgumentView) predictionView; goldToPredictionPredicateMapping = getGoldToPredictionPredicateMapping(); for (Constituent gp : gold.getPredicates()) { if (!goldToPredictionPredicateMapping.containsKey(gp)) { tester.recordGoldOnly("V"); continue; } Constituent pp = goldToPredictionPredicateMapping.get(gp); Map<IntPair, Record> goldLabels = getArgumentMap(gold, gp); Map<IntPair, Record> predictedLabels = getArgumentMap(prediction, pp); Set<IntPair> goldDone = new HashSet<>(); for (IntPair predictedSpan : predictedLabels.keySet()) { Record p = predictedLabels.get(predictedSpan); Record g = goldLabels.get(predictedSpan); if (g == null) { tester.recordPredictionOnly(p.baseLabel); continue; } Map<IntPair, String> gComponents = g.components; Map<IntPair, String> pComponents = p.components; assert gComponents != null; assert pComponents != null; if (gComponents.size() == 1 && pComponents.size() == 1) { tester.record(g.baseLabel, p.baseLabel); goldDone.add(predictedSpan); } else if (gComponents.size() > 1 && pComponents.size() == 1) { tester.recordPredictionOnly(p.baseLabel); } else if (gComponents.size() == 1 && pComponents.size() > 1) { tester.recordPredictionOnly(p.baseLabel); } else { if (p.baseLabel.startsWith("AM")) { Set<IntPair> set = new HashSet<>(); set.addAll(gComponents.keySet()); set.addAll(pComponents.keySet()); for (IntPair s : set) { String gLabel = gComponents.get(s); String pLabel = pComponents.get(s); if (gLabel != null && pLabel != null) tester.record(gLabel, pLabel); else if (gLabel == null) tester.recordPredictionOnly(pLabel); else tester.recordGoldOnly(gLabel); } goldDone.add(predictedSpan); } else { boolean allOK = p.baseLabel.equals(g.baseLabel); Set<IntPair> goldSpansLeft = new HashSet<>(gComponents.keySet()); for (IntPair pSpan : pComponents.keySet()) { if (gComponents.containsKey(pSpan)) goldSpansLeft.remove(pSpan); else { allOK = false; break; } } if (allOK) { tester.record(g.baseLabel, p.baseLabel); goldDone.add(predictedSpan); } else { tester.recordPredictionOnly(p.baseLabel); } } } } for (IntPair gSpan : goldLabels.keySet()) { if (!goldDone.contains(gSpan)) tester.recordGoldOnly(goldLabels.get(gSpan).baseLabel); } } }
PredicateArgumentEvaluator extends Evaluator { public void evaluate(ClassificationTester tester, View goldView, View predictionView) { gold = (PredicateArgumentView) goldView; prediction = (PredicateArgumentView) predictionView; goldToPredictionPredicateMapping = getGoldToPredictionPredicateMapping(); for (Constituent gp : gold.getPredicates()) { if (!goldToPredictionPredicateMapping.containsKey(gp)) { tester.recordGoldOnly("V"); continue; } Constituent pp = goldToPredictionPredicateMapping.get(gp); Map<IntPair, Record> goldLabels = getArgumentMap(gold, gp); Map<IntPair, Record> predictedLabels = getArgumentMap(prediction, pp); Set<IntPair> goldDone = new HashSet<>(); for (IntPair predictedSpan : predictedLabels.keySet()) { Record p = predictedLabels.get(predictedSpan); Record g = goldLabels.get(predictedSpan); if (g == null) { tester.recordPredictionOnly(p.baseLabel); continue; } Map<IntPair, String> gComponents = g.components; Map<IntPair, String> pComponents = p.components; assert gComponents != null; assert pComponents != null; if (gComponents.size() == 1 && pComponents.size() == 1) { tester.record(g.baseLabel, p.baseLabel); goldDone.add(predictedSpan); } else if (gComponents.size() > 1 && pComponents.size() == 1) { tester.recordPredictionOnly(p.baseLabel); } else if (gComponents.size() == 1 && pComponents.size() > 1) { tester.recordPredictionOnly(p.baseLabel); } else { if (p.baseLabel.startsWith("AM")) { Set<IntPair> set = new HashSet<>(); set.addAll(gComponents.keySet()); set.addAll(pComponents.keySet()); for (IntPair s : set) { String gLabel = gComponents.get(s); String pLabel = pComponents.get(s); if (gLabel != null && pLabel != null) tester.record(gLabel, pLabel); else if (gLabel == null) tester.recordPredictionOnly(pLabel); else tester.recordGoldOnly(gLabel); } goldDone.add(predictedSpan); } else { boolean allOK = p.baseLabel.equals(g.baseLabel); Set<IntPair> goldSpansLeft = new HashSet<>(gComponents.keySet()); for (IntPair pSpan : pComponents.keySet()) { if (gComponents.containsKey(pSpan)) goldSpansLeft.remove(pSpan); else { allOK = false; break; } } if (allOK) { tester.record(g.baseLabel, p.baseLabel); goldDone.add(predictedSpan); } else { tester.recordPredictionOnly(p.baseLabel); } } } } for (IntPair gSpan : goldLabels.keySet()) { if (!goldDone.contains(gSpan)) tester.recordGoldOnly(goldLabels.get(gSpan).baseLabel); } } } }
PredicateArgumentEvaluator extends Evaluator { public void evaluate(ClassificationTester tester, View goldView, View predictionView) { gold = (PredicateArgumentView) goldView; prediction = (PredicateArgumentView) predictionView; goldToPredictionPredicateMapping = getGoldToPredictionPredicateMapping(); for (Constituent gp : gold.getPredicates()) { if (!goldToPredictionPredicateMapping.containsKey(gp)) { tester.recordGoldOnly("V"); continue; } Constituent pp = goldToPredictionPredicateMapping.get(gp); Map<IntPair, Record> goldLabels = getArgumentMap(gold, gp); Map<IntPair, Record> predictedLabels = getArgumentMap(prediction, pp); Set<IntPair> goldDone = new HashSet<>(); for (IntPair predictedSpan : predictedLabels.keySet()) { Record p = predictedLabels.get(predictedSpan); Record g = goldLabels.get(predictedSpan); if (g == null) { tester.recordPredictionOnly(p.baseLabel); continue; } Map<IntPair, String> gComponents = g.components; Map<IntPair, String> pComponents = p.components; assert gComponents != null; assert pComponents != null; if (gComponents.size() == 1 && pComponents.size() == 1) { tester.record(g.baseLabel, p.baseLabel); goldDone.add(predictedSpan); } else if (gComponents.size() > 1 && pComponents.size() == 1) { tester.recordPredictionOnly(p.baseLabel); } else if (gComponents.size() == 1 && pComponents.size() > 1) { tester.recordPredictionOnly(p.baseLabel); } else { if (p.baseLabel.startsWith("AM")) { Set<IntPair> set = new HashSet<>(); set.addAll(gComponents.keySet()); set.addAll(pComponents.keySet()); for (IntPair s : set) { String gLabel = gComponents.get(s); String pLabel = pComponents.get(s); if (gLabel != null && pLabel != null) tester.record(gLabel, pLabel); else if (gLabel == null) tester.recordPredictionOnly(pLabel); else tester.recordGoldOnly(gLabel); } goldDone.add(predictedSpan); } else { boolean allOK = p.baseLabel.equals(g.baseLabel); Set<IntPair> goldSpansLeft = new HashSet<>(gComponents.keySet()); for (IntPair pSpan : pComponents.keySet()) { if (gComponents.containsKey(pSpan)) goldSpansLeft.remove(pSpan); else { allOK = false; break; } } if (allOK) { tester.record(g.baseLabel, p.baseLabel); goldDone.add(predictedSpan); } else { tester.recordPredictionOnly(p.baseLabel); } } } } for (IntPair gSpan : goldLabels.keySet()) { if (!goldDone.contains(gSpan)) tester.recordGoldOnly(goldLabels.get(gSpan).baseLabel); } } }  }
PredicateArgumentEvaluator extends Evaluator { public void evaluate(ClassificationTester tester, View goldView, View predictionView) { gold = (PredicateArgumentView) goldView; prediction = (PredicateArgumentView) predictionView; goldToPredictionPredicateMapping = getGoldToPredictionPredicateMapping(); for (Constituent gp : gold.getPredicates()) { if (!goldToPredictionPredicateMapping.containsKey(gp)) { tester.recordGoldOnly("V"); continue; } Constituent pp = goldToPredictionPredicateMapping.get(gp); Map<IntPair, Record> goldLabels = getArgumentMap(gold, gp); Map<IntPair, Record> predictedLabels = getArgumentMap(prediction, pp); Set<IntPair> goldDone = new HashSet<>(); for (IntPair predictedSpan : predictedLabels.keySet()) { Record p = predictedLabels.get(predictedSpan); Record g = goldLabels.get(predictedSpan); if (g == null) { tester.recordPredictionOnly(p.baseLabel); continue; } Map<IntPair, String> gComponents = g.components; Map<IntPair, String> pComponents = p.components; assert gComponents != null; assert pComponents != null; if (gComponents.size() == 1 && pComponents.size() == 1) { tester.record(g.baseLabel, p.baseLabel); goldDone.add(predictedSpan); } else if (gComponents.size() > 1 && pComponents.size() == 1) { tester.recordPredictionOnly(p.baseLabel); } else if (gComponents.size() == 1 && pComponents.size() > 1) { tester.recordPredictionOnly(p.baseLabel); } else { if (p.baseLabel.startsWith("AM")) { Set<IntPair> set = new HashSet<>(); set.addAll(gComponents.keySet()); set.addAll(pComponents.keySet()); for (IntPair s : set) { String gLabel = gComponents.get(s); String pLabel = pComponents.get(s); if (gLabel != null && pLabel != null) tester.record(gLabel, pLabel); else if (gLabel == null) tester.recordPredictionOnly(pLabel); else tester.recordGoldOnly(gLabel); } goldDone.add(predictedSpan); } else { boolean allOK = p.baseLabel.equals(g.baseLabel); Set<IntPair> goldSpansLeft = new HashSet<>(gComponents.keySet()); for (IntPair pSpan : pComponents.keySet()) { if (gComponents.containsKey(pSpan)) goldSpansLeft.remove(pSpan); else { allOK = false; break; } } if (allOK) { tester.record(g.baseLabel, p.baseLabel); goldDone.add(predictedSpan); } else { tester.recordPredictionOnly(p.baseLabel); } } } } for (IntPair gSpan : goldLabels.keySet()) { if (!goldDone.contains(gSpan)) tester.recordGoldOnly(goldLabels.get(gSpan).baseLabel); } } }  void evaluateSense(ClassificationTester senseTester, View goldView, View predictionView); void evaluate(ClassificationTester tester, View goldView, View predictionView); }
PredicateArgumentEvaluator extends Evaluator { public void evaluate(ClassificationTester tester, View goldView, View predictionView) { gold = (PredicateArgumentView) goldView; prediction = (PredicateArgumentView) predictionView; goldToPredictionPredicateMapping = getGoldToPredictionPredicateMapping(); for (Constituent gp : gold.getPredicates()) { if (!goldToPredictionPredicateMapping.containsKey(gp)) { tester.recordGoldOnly("V"); continue; } Constituent pp = goldToPredictionPredicateMapping.get(gp); Map<IntPair, Record> goldLabels = getArgumentMap(gold, gp); Map<IntPair, Record> predictedLabels = getArgumentMap(prediction, pp); Set<IntPair> goldDone = new HashSet<>(); for (IntPair predictedSpan : predictedLabels.keySet()) { Record p = predictedLabels.get(predictedSpan); Record g = goldLabels.get(predictedSpan); if (g == null) { tester.recordPredictionOnly(p.baseLabel); continue; } Map<IntPair, String> gComponents = g.components; Map<IntPair, String> pComponents = p.components; assert gComponents != null; assert pComponents != null; if (gComponents.size() == 1 && pComponents.size() == 1) { tester.record(g.baseLabel, p.baseLabel); goldDone.add(predictedSpan); } else if (gComponents.size() > 1 && pComponents.size() == 1) { tester.recordPredictionOnly(p.baseLabel); } else if (gComponents.size() == 1 && pComponents.size() > 1) { tester.recordPredictionOnly(p.baseLabel); } else { if (p.baseLabel.startsWith("AM")) { Set<IntPair> set = new HashSet<>(); set.addAll(gComponents.keySet()); set.addAll(pComponents.keySet()); for (IntPair s : set) { String gLabel = gComponents.get(s); String pLabel = pComponents.get(s); if (gLabel != null && pLabel != null) tester.record(gLabel, pLabel); else if (gLabel == null) tester.recordPredictionOnly(pLabel); else tester.recordGoldOnly(gLabel); } goldDone.add(predictedSpan); } else { boolean allOK = p.baseLabel.equals(g.baseLabel); Set<IntPair> goldSpansLeft = new HashSet<>(gComponents.keySet()); for (IntPair pSpan : pComponents.keySet()) { if (gComponents.containsKey(pSpan)) goldSpansLeft.remove(pSpan); else { allOK = false; break; } } if (allOK) { tester.record(g.baseLabel, p.baseLabel); goldDone.add(predictedSpan); } else { tester.recordPredictionOnly(p.baseLabel); } } } } for (IntPair gSpan : goldLabels.keySet()) { if (!goldDone.contains(gSpan)) tester.recordGoldOnly(goldLabels.get(gSpan).baseLabel); } } }  void evaluateSense(ClassificationTester senseTester, View goldView, View predictionView); void evaluate(ClassificationTester tester, View goldView, View predictionView);  }