PennTreebankCombinedReader extends JCasResourceCollectionReader_ImplBase { @Override public void getNext(JCas aJCas) throws IOException, CollectionException { Resource res = nextFile(); initCas(aJCas.getCas(), res); try { posMappingProvider.configure(aJCas.getCas()); constituentMappingProvider.configure(aJCas.getCas()); } catch (AnalysisEngineProcessException e) { throw new IOException(e); } StringBuilder text = new StringBuilder(); try (InputStream is = res.getInputStream()) { lineNumber = 0; LineIterator li = IOUtils.lineIterator(is, sourceEncoding); while (li.hasNext()) { PennTreeNode tree = readTree(li); if (removeTraces) { doRemoveTraces(tree); } Constituent root = converter.convertPennTree(aJCas, text, tree); Sentence sentence = new Sentence(aJCas, root.getBegin(), root.getEnd()); sentence.addToIndexes(); text.append('\n'); } } aJCas.setDocumentText(text.toString()); }  @Override void initialize(UimaContext aContext); @Override void getNext(JCas aJCas); static final String PARAM_SOURCE_ENCODING; static final String PARAM_POS_TAG_SET; static final String PARAM_READ_POS; static final String PARAM_CONSTITUENT_TAG_SET; static final String PARAM_MAPPING_ENABLED; static final String PARAM_POS_MAPPING_LOCATION; static final String PARAM_CONSTITUENT_MAPPING_LOCATION; static final String PARAM_REMOVE_TRACES; static final String PARAM_WRITE_TRACES_TO_TEXT; }
@Test public void test() throws Exception { CollectionReader reader = createReader(PennTreebankCombinedReader.class, PennTreebankCombinedReader.PARAM_SOURCE_LOCATION, "src/test/resources/stanfordPennTrees/stanford-english-trees-first2.mrg"); JCas jcas = JCasFactory.createJCas(); reader.getNext(jcas.getCas()); String text = "Al Qaida Endorses George W. Bush for President\n" + "Al-Qaeda tries to incite more violence in Iraq\n"; String[] sentences = { "Al Qaida Endorses George W. Bush for President", "Al-Qaeda tries to incite more violence in Iraq" }; String[] tokens1 = { "Al", "Qaida", "Endorses", "George", "W.", "Bush", "for", "President" }; String[] constituentMapped1 = { "Constituent 0,46", "Constituent 0,8", "Constituent 18,32", "Constituent 33,46", "Constituent 37,46", "Constituent 9,46", "ROOT 0,46" }; String[] constituentOriginal1 = { "NP 0,8", "NP 18,32", "NP 37,46", "PP 33,46", "ROOT 0,46", "S 0,46", "VP 9,46" }; String[] tokens2 = { "Al-Qaeda", "tries", "to", "incite", "more", "violence", "in", "Iraq" }; String[] constituentMapped2 = { "Constituent 47,55", "Constituent 47,93", "Constituent 56,93", "Constituent 62,93", "Constituent 65,93", "Constituent 72,85", "Constituent 86,93", "Constituent 89,93", "ROOT 47,93" }; String[] constituentOriginal2 = { "NP 47,55", "NP 72,85", "NP 89,93", "PP 86,93", "ROOT 47,93", "S 47,93", "S 62,93", "VP 56,93", "VP 62,93", "VP 65,93" }; assertEquals(text, jcas.getDocumentText()); assertSentence(sentences, select(jcas, Sentence.class)); Sentence[] actualSentences = select(jcas, Sentence.class).toArray(new Sentence[0]); assertToken(tokens1, selectCovered(Token.class, actualSentences[0])); assertConstituents(constituentMapped1, constituentOriginal1, selectCovered(Constituent.class, actualSentences[0])); assertToken(tokens2, selectCovered(Token.class, actualSentences[1])); assertConstituents(constituentMapped2, constituentOriginal2, selectCovered(Constituent.class, actualSentences[1])); }
@Test public void testWithDirectSpeech() throws Exception { CollectionReader reader = createReader(PennTreebankCombinedReader.class, PennTreebankCombinedReader.PARAM_LANGUAGE, "en", PennTreebankCombinedReader.PARAM_SOURCE_LOCATION, "src/test/resources/stanfordPennTrees/tree_with_direct_speech.mrg"); JCas jcas = JCasFactory.createJCas(); reader.getNext(jcas.getCas()); String[] sentences = { "`` And what do you know ? ''" }; String[] tokens = { "``", "And", "what", "do", "you", "know", "?", "''" }; String[] posMapped = { "POS_PUNCT", "POS_CONJ", "POS_PRON", "POS_VERB", "POS_PRON", "POS_VERB", "POS_PUNCT", "POS_PUNCT" }; String[] posOriginal = { "``", "CC", "WP", "VBP", "PRP", "VB", ".", "''" }; String[] constituentMapped = { "NP 15,18", "ROOT 0,28", "SBARQ 0,28", "SQ 12,23", "VP 19,23", "WHNP 7,11" }; String[] constituentOriginal = { "NP 15,18", "ROOT 0,28", "SBARQ 0,28", "SQ 12,23", "VP 19,23", "WHNP 7,11" }; assertSentence(sentences, select(jcas, Sentence.class)); assertToken(tokens, select(jcas, Token.class)); assertPOS(posMapped, posOriginal, select(jcas, POS.class)); assertConstituents(constituentMapped, constituentOriginal, select(jcas, Constituent.class)); }
@Test public void testWithParentheses() throws Exception { CollectionReader reader = createReader(PennTreebankCombinedReader.class, PennTreebankCombinedReader.PARAM_LANGUAGE, "en", PennTreebankCombinedReader.PARAM_SOURCE_LOCATION, "src/test/resources/stanfordPennTrees/tree_with_parentheses.mrg"); JCas jcas = JCasFactory.createJCas(); reader.getNext(jcas.getCas()); String[] sentences = { "( CNN ) ." }; String[] tokens = { "(", "CNN", ")", "." }; String[] posMapped = { "POS_PUNCT", "POS_PROPN", "POS_PUNCT", "POS_PUNCT" }; String[] posOriginal = { "-LRB-", "NNP", "-RRB-", "." }; String[] constituentMapped = { "FRAG 0,9", "NP 2,5", "ROOT 0,9" }; String[] constituentOriginal = { "FRAG 0,9", "NP 2,5", "ROOT 0,9" }; assertSentence(sentences, select(jcas, Sentence.class)); assertToken(tokens, select(jcas, Token.class)); assertPOS(posMapped, posOriginal, select(jcas, POS.class)); assertConstituents(constituentMapped, constituentOriginal, select(jcas, Constituent.class)); }