ParseOutputRewriter implements ParseConfigurationProcessor { List<CorpusLine> getCorpusLines(ParseConfiguration parseConfiguration) throws TalismaneException { List<CorpusLine> corpusLines = new ArrayList<>(); for (PosTaggedToken posTaggedToken : parseConfiguration.getPosTagSequence()) { if (!posTaggedToken.isRoot()) { DependencyArc arc = parseConfiguration.getGoverningDependency(posTaggedToken); DependencyArc nonProjArc = parseConfiguration.getGoverningDependency(posTaggedToken, false); String line = posTaggedToken.getIndex() + "\t" + posTaggedToken.getToken().getOriginalText() + "\t" + posTaggedToken.getLemmaForCoNLL() + "\t" + posTaggedToken.getTag().getCode() + "\t" + posTaggedToken.getTag().getCode() + "\t" + posTaggedToken.getMorphologyForCoNLL() + "\t" + (arc != null ? arc.getHead().getIndex() : 0) + "\t" + (arc != null ? arc.getLabel() : "_"); CorpusLine corpusLine = new CorpusLine(line, posTaggedToken.getToken().getLineNumber()); corpusLine.setIndex(posTaggedToken.getIndex()); corpusLine.setToken(posTaggedToken.getToken().getOriginalText()); corpusLine.setLemma(posTaggedToken.getLemmaForCoNLL()); corpusLine.setPosTag(posTaggedToken.getTag().getCode()); String morphology = posTaggedToken.getMorphologyForCoNLL(); corpusLine.setMorphology(morphology.length() == 0 ? "_" : morphology); corpusLine.setGovernorIndex(arc != null ? arc.getHead().getIndex() : 0); corpusLine.setLabel(arc != null ? arc.getLabel() : "_"); corpusLine.setNonProjGovernorIndex(nonProjArc != null ? nonProjArc.getHead().getIndex() : 0); corpusLine.setNonProjLabel(nonProjArc != null ? nonProjArc.getLabel() : "_"); if (posTaggedToken.getToken().getPrecedingRawOutput() != null) corpusLine.setElement(CorpusElement.PRECEDING_RAW_OUTPUT, posTaggedToken.getToken().getPrecedingRawOutput()); if (posTaggedToken.getToken().getTrailingRawOutput() != null) corpusLine.setElement(CorpusElement.TRAILING_RAW_OUTPUT, posTaggedToken.getToken().getTrailingRawOutput()); corpusLine.setTokenProbability(posTaggedToken.getToken().getProbability()); corpusLine.setPosTagProbability(posTaggedToken.getProbability()); if (arc != null) corpusLine.setParseProbability(arc.getProbability()); corpusLines.add(corpusLine); } } Map<CorpusLine, SplitAction> splitActions = new HashMap<>(); for (CorpusLine corpusLine : corpusLines) { if (LOG.isDebugEnabled()) LOG.debug(corpusLine.toString()); for (RewriteRule rewriteRule : rewriteRules) { boolean matches = true; conditionLoop: for (CorpusElement corpusElement : rewriteRule.conditions.keySet()) { Pattern pattern = rewriteRule.conditions.get(corpusElement); if (LOG.isTraceEnabled()) LOG.trace("For " + corpusElement.name() + ", matching " + pattern.pattern()); switch (corpusElement) { case POSTAG: if (!pattern.matcher(corpusLine.getPosTag()).matches()) { if (LOG.isTraceEnabled()) LOG.trace("Match failed for " + corpusLine.getPosTag()); matches = false; break conditionLoop; } break; case TOKEN: if (!pattern.matcher(corpusLine.getToken()).matches()) { matches = false; break conditionLoop; } break; case LEMMA: if (!pattern.matcher(corpusLine.getLemma()).matches()) { matches = false; break conditionLoop; } break; case LABEL: if (!pattern.matcher(corpusLine.getLabel()).matches()) { matches = false; break conditionLoop; } break; default: throw new TalismaneException(ParseOutputRewriter.class.getSimpleName() + " cannot match on " + corpusElement.name()); } } if (matches) { if (rewriteRule.action instanceof SplitAction) { SplitAction splitAction = (SplitAction) rewriteRule.action; splitActions.put(corpusLine, splitAction); } } } } if (splitActions.size() > 0) { List<CorpusLine> newCorpusLines = new ArrayList<>(); Map<Integer, Integer> oldToNewIndexMap = new HashMap<>(); oldToNewIndexMap.put(0, 0); int currentIndex = 1; for (int i = 0; i < corpusLines.size(); i++) { CorpusLine corpusLine = corpusLines.get(i); oldToNewIndexMap.put(i + 1, currentIndex); if (splitActions.containsKey(corpusLine)) { SplitAction splitAction = splitActions.get(corpusLine); currentIndex += splitAction.elementValues.size(); } else { currentIndex++; } } for (int i = 0; i < corpusLines.size(); i++) { CorpusLine corpusLine = corpusLines.get(i); CorpusLine newCorpusLine = corpusLine.cloneCorpusLine(); newCorpusLine.setIndex(oldToNewIndexMap.get(corpusLine.getIndex())); newCorpusLine.setGovernorIndex(oldToNewIndexMap.get(corpusLine.getGovernorIndex())); newCorpusLine.setNonProjGovernorIndex(oldToNewIndexMap.get(corpusLine.getNonProjGovernorIndex())); if (splitActions.containsKey(corpusLine)) { SplitAction splitAction = splitActions.get(corpusLine); for (int j = 0; j < splitAction.elementValues.size(); j++) { CorpusLine splitCorpusLine = new CorpusLine(corpusLine.getLine(), corpusLine.getLineNumber()); splitCorpusLine.setIndex(oldToNewIndexMap.get(corpusLine.getIndex()) + j); Map<CorpusElement, String> elementValues = splitAction.elementValues.get(j); this.setElementValues(elementValues, oldToNewIndexMap, newCorpusLine, splitCorpusLine); List<ConditionalAction> conditionalActions = splitAction.conditionalValues.get(j); boolean groupHasMatch = false; for (ConditionalAction conditionalAction : conditionalActions) { CorpusLine baseLine = corpusLines.get(i + conditionalAction.relativeIndex); if (conditionalAction.isDefault) { if (!groupHasMatch) { Map<CorpusElement, String> conditionalElementValues = conditionalAction.elementValues; this.setElementValues(conditionalElementValues, oldToNewIndexMap, newCorpusLine, splitCorpusLine); } groupHasMatch = false; } else { boolean match = true; for (CorpusElement corpusElement : conditionalAction.conditions.keySet()) { String origValue = baseLine.getElement(corpusElement); Pattern pattern = conditionalAction.conditions.get(corpusElement); if (!pattern.matcher(origValue).matches()) { match = false; break; } } if (match) { Map<CorpusElement, String> conditionalElementValues = conditionalAction.elementValues; this.setElementValues(conditionalElementValues, oldToNewIndexMap, newCorpusLine, splitCorpusLine); groupHasMatch = true; } } } newCorpusLines.add(splitCorpusLine); } } else { newCorpusLines.add(newCorpusLine); } } corpusLines = newCorpusLines; } return corpusLines; } ParseOutputRewriter(File outDir, String sessionId);  ParseOutputRewriter(Writer writer, String sessionId); @Override void close(); @Override void onNextParseConfiguration(ParseConfiguration parseConfiguration); @Override void onCompleteParse();  }
@Test public void testGetCorpusLines() throws Exception { TalismaneSession.clearSessions(); System.setProperty("config.file", "src/test/resources/testWithOutputRules.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; String input = ""; input += "1\tAu\tau\tADP+DET\t0\troot\n"; input += "2\tsein\tsein\tNOUN\t1\tfixed\n"; input += "3\tmême\tmême\tADV\t1\tadvmod\n"; input += "4\tdu\tdu\tADP+DET\t5\tcase\n"; input += "5\tParti\tParti\tPROPN\t1\tnmod\n"; input += "6\tsocialiste\tsocialiste\tADJ\t5\tfixed\n"; input += "7\tauquel\tauquel\tADP+PRON\t8\tobl\n"; input += "8\tappartient\tappartenir\tVERB\t5\tacl:relcl\n"; input += "9\tM.\tmonsieur\tNOUN\t8\tnsubj\n"; input += "10\tDupont\tDupont\tPROPN\t9\tflat:name\n"; StringReader stringReader = new StringReader(input); ParserRegexBasedCorpusReader reader = new ParserRegexBasedCorpusReader(stringReader, config.getConfig("talismane.core.test.parser.input"), sessionId); ParseConfiguration parseConfiguration = reader.nextConfiguration(); final StringWriter writer = new StringWriter(); try (ParseOutputRewriter rewriter = new ParseOutputRewriter(writer, sessionId)) { List<CorpusLine> corpusLines = rewriter.getCorpusLines(parseConfiguration); int i = 1; for (CorpusLine corpusLine : corpusLines) { LOG.debug("line " + corpusLine.getIndex() + ": " + corpusLine.getElements()); if (i == 1) { assertEquals(1, corpusLine.getIndex()); assertEquals("à", corpusLine.getToken()); assertEquals("à", corpusLine.getLemma()); assertEquals("ADP", corpusLine.getPosTag()); assertEquals(0, corpusLine.getGovernorIndex()); assertEquals("root", corpusLine.getLabel()); } else if (i == 2) { assertEquals(2, corpusLine.getIndex()); assertEquals("le", corpusLine.getToken()); assertEquals("le", corpusLine.getLemma()); assertEquals("DET", corpusLine.getPosTag()); assertEquals(1, corpusLine.getGovernorIndex()); assertEquals("fixed", corpusLine.getLabel()); } else if (i == 3) { assertEquals(3, corpusLine.getIndex()); assertEquals("sein", corpusLine.getToken()); assertEquals(1, corpusLine.getGovernorIndex()); assertEquals("fixed", corpusLine.getLabel()); } else if (i == 4) { assertEquals(4, corpusLine.getIndex()); assertEquals("même", corpusLine.getToken()); assertEquals(1, corpusLine.getGovernorIndex()); assertEquals("advmod", corpusLine.getLabel()); } else if (i == 5) { assertEquals(5, corpusLine.getIndex()); assertEquals("de", corpusLine.getToken()); assertEquals("de", corpusLine.getLemma()); assertEquals("ADP", corpusLine.getPosTag()); assertEquals(7, corpusLine.getGovernorIndex()); assertEquals("case", corpusLine.getLabel()); } else if (i == 6) { assertEquals(6, corpusLine.getIndex()); assertEquals("le", corpusLine.getToken()); assertEquals("le", corpusLine.getLemma()); assertEquals("DET", corpusLine.getPosTag()); assertEquals(7, corpusLine.getGovernorIndex()); assertEquals("det", corpusLine.getLabel()); } else if (i == 7) { assertEquals(7, corpusLine.getIndex()); assertEquals("Parti", corpusLine.getToken()); assertEquals(1, corpusLine.getGovernorIndex()); assertEquals("nmod", corpusLine.getLabel()); } else if (i == 8) { assertEquals(8, corpusLine.getIndex()); assertEquals("socialiste", corpusLine.getToken()); assertEquals(7, corpusLine.getGovernorIndex()); assertEquals("fixed", corpusLine.getLabel()); } else if (i == 9) { assertEquals(9, corpusLine.getIndex()); assertEquals("à", corpusLine.getToken()); assertEquals("à", corpusLine.getLemma()); assertEquals("ADP", corpusLine.getPosTag()); assertEquals(10, corpusLine.getGovernorIndex()); assertEquals("case", corpusLine.getLabel()); } else if (i == 10) { assertEquals(10, corpusLine.getIndex()); assertEquals("lequel", corpusLine.getToken()); assertEquals("lequel", corpusLine.getLemma()); assertEquals("PRON", corpusLine.getPosTag()); assertEquals(11, corpusLine.getGovernorIndex()); assertEquals("obl", corpusLine.getLabel()); } else if (i == 11) { assertEquals(11, corpusLine.getIndex()); assertEquals("appartient", corpusLine.getToken()); assertEquals("VERB", corpusLine.getPosTag()); assertEquals(7, corpusLine.getGovernorIndex()); assertEquals("acl:relcl", corpusLine.getLabel()); } else if (i == 12) { assertEquals(12, corpusLine.getIndex()); assertEquals("M.", corpusLine.getToken()); assertEquals("NOUN", corpusLine.getPosTag()); assertEquals(11, corpusLine.getGovernorIndex()); assertEquals("nsubj", corpusLine.getLabel()); } else if (i == 13) { assertEquals(13, corpusLine.getIndex()); assertEquals("Dupont", corpusLine.getToken()); assertEquals("PROPN", corpusLine.getPosTag()); assertEquals(12, corpusLine.getGovernorIndex()); assertEquals("flat:name", corpusLine.getLabel()); } i++; } assertEquals(13, corpusLines.size()); } }