SentenceTokenConsumer extends JCasAnnotator_ImplBase { @Override public void process(JCas jcas) throws AnalysisEngineProcessException { LOGGER.trace("Processing next document ... "); try { String fileId = getDocID(jcas); if (fileId == null) fileId = new Integer(docs++).toString(); if (mode == Mode.TOKEN) { FSIterator sentenceIterator = jcas.getAnnotationIndex(Sentence.type).iterator(); AnnotationIndex tokenIndex = jcas.getAnnotationIndex(Token.type); ArrayList<String> sentences = new ArrayList<>(); while (sentenceIterator.hasNext()) { Sentence sentence = (Sentence) sentenceIterator.next(); FSIterator tokIterator = tokenIndex.subiterator(sentence); String sentenceText = ""; while (tokIterator.hasNext()) { if (addPOSTAG) { sentenceText = returnWithPOSTAG(tokIterator, sentenceText); } else { sentenceText = returnWithoutPOSTAG(tokIterator, sentenceText); } } sentences.add(sentenceText); } writeSentences2File(fileId, sentences); } else if (mode == Mode.DOCUMENT) { File outputFile = new File(directory.getCanonicalPath() + File.separator + fileId + ".txt" + (gzip ? ".gz" : "")); LOGGER.trace("Writing the verbatim CAS document text to {}", outputFile); writeSentences2File(fileId, Arrays.asList(jcas.getDocumentText())); } } catch (CASRuntimeException | CASException | IOException e) { LOGGER.error("Error while writing: ", e); throw new AnalysisEngineProcessException(e); } }  @Override void initialize(UimaContext aContext); @Override void process(JCas jcas); String getDocID(JCas jcas); @Override void collectionProcessComplete(); static final String PARAM_OUTPUT_DIR; static final String PARAM_DELIMITER; static final String PARAM_LOWERCASE; static final String PARAM_MODE; static final String PARAM_GZIP; static final String PARAM_ZIP_ARCHIVE; static final String PARAM_ZIP_MAX_SIZE; static final String PARAM_ZIP_PREFIX; }
@Test public void testProcessEmptyCAS() throws Exception { JCas cas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-all-types"); AnalysisEngine consumer = AnalysisEngineFactory.createEngine(SentenceTokenConsumer.class, PARAM_OUTPUT_DIR, "src/test/resources/data"); consumer.process(cas); }
@Test public void testProcessWithoutPOSTags() throws Exception { JCas cas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-all-types"); AnalysisEngine consumer = AnalysisEngineFactory.createEngine(SentenceTokenConsumer.class, PARAM_OUTPUT_DIR, "src/test/resources/data"); cas.setDocumentText("I love food. I like sleeping."); Sentence s1 = new Sentence(cas, 0, 12); s1.addToIndexes(); Sentence s2 = new Sentence(cas, 13, 29); s2.addToIndexes(); Token t1 = new Token(cas, 0, 1); t1.addToIndexes(); Token t2 = new Token(cas, 2, 6); t2.addToIndexes(); Token t3 = new Token(cas, 7, 11); t3.addToIndexes(); Token t4 = new Token(cas, 11, 12); t4.addToIndexes(); Token t5 = new Token(cas, 13, 14); t5.addToIndexes(); Token t6 = new Token(cas, 15, 19); t6.addToIndexes(); Token t7 = new Token(cas, 20, 28); t7.addToIndexes(); Token t8 = new Token(cas, 28, 29); t8.addToIndexes(); Header header = new Header(cas); header.setDocId("withoutPOS"); header.addToIndexes(); consumer.process(cas); assertTrue(new File("src/test/resources/data/withoutPOS.txt").exists()); List<String> file = readFile("src/test/resources/data/withoutPOS.txt"); assertTrue(file.contains("I love food .")); assertTrue(file.contains("I like sleeping .")); }
@Test public void testProcessWithPOSTags() throws Exception { JCas cas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-all-types"); AnalysisEngine consumer = AnalysisEngineFactory.createEngine(SentenceTokenConsumer.class, PARAM_OUTPUT_DIR, "src/test/resources/data", SentenceTokenConsumer.PARAM_DELIMITER, "$"); cas.setDocumentText("I love food. I like sleeping."); Sentence s1 = new Sentence(cas, 0, 12); s1.addToIndexes(); Sentence s2 = new Sentence(cas, 13, 29); s2.addToIndexes(); Token t1 = new Token(cas, 0, 1); t1.addToIndexes(); FSArray array1 = new FSArray(cas, 1); PennBioIEPOSTag pos1 = new PennBioIEPOSTag(cas); pos1.setValue("PRP"); array1.set(0, pos1); t1.setPosTag(array1); Token t2 = new Token(cas, 2, 6); t2.addToIndexes(); FSArray array2 = new FSArray(cas, 1); PennBioIEPOSTag pos2 = new PennBioIEPOSTag(cas); pos2.setValue("VBP"); array2.set(0, pos2); t2.setPosTag(array2); Token t3 = new Token(cas, 7, 11); t3.addToIndexes(); FSArray array3 = new FSArray(cas, 1); PennBioIEPOSTag pos3 = new PennBioIEPOSTag(cas); pos3.setValue("NN"); array3.set(0, pos3); t3.setPosTag(array3); Token t4 = new Token(cas, 11, 12); t4.addToIndexes(); FSArray array4 = new FSArray(cas, 1); PennBioIEPOSTag pos4 = new PennBioIEPOSTag(cas); pos4.setValue("."); array4.set(0, pos4); t4.setPosTag(array4); Token t5 = new Token(cas, 13, 14); t5.addToIndexes(); FSArray array5 = new FSArray(cas, 1); PennBioIEPOSTag pos5 = new PennBioIEPOSTag(cas); pos5.setValue("PRP"); array5.set(0, pos5); t5.setPosTag(array5); Token t6 = new Token(cas, 15, 19); t6.addToIndexes(); FSArray array6 = new FSArray(cas, 1); PennBioIEPOSTag pos6 = new PennBioIEPOSTag(cas); pos6.setValue("VBP"); array6.set(0, pos6); t6.setPosTag(array6); Token t7 = new Token(cas, 20, 28); t7.addToIndexes(); FSArray array7 = new FSArray(cas, 1); PennBioIEPOSTag pos7 = new PennBioIEPOSTag(cas); pos7.setValue("VBG"); array7.set(0, pos7); t7.setPosTag(array7); Token t8 = new Token(cas, 28, 29); t8.addToIndexes(); FSArray array8 = new FSArray(cas, 1); PennBioIEPOSTag pos8 = new PennBioIEPOSTag(cas); pos8.setValue("."); array8.set(0, pos8); t8.setPosTag(array8); Header header = new Header(cas); header.setDocId("withPOS"); header.addToIndexes(); consumer.process(cas); assertTrue(new File("src/test/resources/data/withPOS.txt").exists()); assertFalse(new File("src/test/resources/data/nemo.txt").exists()); List<String> file = readFile("src/test/resources/data/withPOS.txt"); assertFalse(file.contains("I_PRP love_VBP food_NN ._.")); assertTrue(file.contains("I$PRP like$VBP sleeping$VBG .$.")); System.out.println(file.get(0) + "\n" + file.get(1)); }
@Test public void testWriteDocumentText() throws Exception { JCas jcas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types", "de.julielab.jcore.types.jcore-document-meta-pubmed-types"); jcas.setDocumentText("This is line 1\nthis is line two."); new Token(jcas, 0, 4).addToIndexes(); new Token(jcas, 5, 7).addToIndexes(); new Token(jcas, 8, 12).addToIndexes(); new Sentence(jcas, 0, 7).addToIndexes(); Header header = new Header(jcas, 0, 0); header.setDocId("documentTest"); header.addToIndexes(); AnalysisEngine consumer = AnalysisEngineFactory.createEngine(SentenceTokenConsumer.class, PARAM_OUTPUT_DIR, "src/test/resources/data", SentenceTokenConsumer.PARAM_MODE, "DOCUMENT"); consumer.process(jcas); File outputFile = new File("src/test/resources/data/documentTest.txt"); assertTrue(outputFile.exists()); List<String> lines = readFile(outputFile.getAbsolutePath()); assertEquals(2, lines.size()); assertEquals("This is line 1", lines.get(0)); assertEquals("this is line two.", lines.get(1)); }
@Test public void testFromDescriptor1() throws Exception { JCas jcas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types", "de.julielab.jcore.types.jcore-document-meta-pubmed-types"); jcas.setDocumentText("This is line 1\nthis is line two."); new Token(jcas, 0, 4).addToIndexes(); new Token(jcas, 5, 7).addToIndexes(); new Token(jcas, 8, 12).addToIndexes(); new Sentence(jcas, 0, 7).addToIndexes(); Header header = new Header(jcas, 0, 0); header.setDocId("documentTest"); header.addToIndexes(); AnalysisEngine consumer = AnalysisEngineFactory.createEngine("de.julielab.jcore.consumer.txt.desc.jcore-txt-consumer", PARAM_OUTPUT_DIR, "src/test/resources/data", SentenceTokenConsumer.PARAM_MODE, "DOCUMENT"); consumer.process(jcas); File outputFile = new File("src/test/resources/data/documentTest.txt"); assertTrue(outputFile.exists()); List<String> lines = readFile(outputFile.getAbsolutePath()); assertEquals(2, lines.size()); assertEquals("This is line 1", lines.get(0)); assertEquals("this is line two.", lines.get(1)); }
@Test public void testFromDescriptor2() throws Exception { JCas jcas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types", "de.julielab.jcore.types.jcore-document-meta-pubmed-types"); jcas.setDocumentText("This is line 1\nthis is line two."); new Token(jcas, 0, 4).addToIndexes(); new Token(jcas, 5, 7).addToIndexes(); new Token(jcas, 8, 12).addToIndexes(); new Sentence(jcas, 0, 7).addToIndexes(); Header header = new Header(jcas, 0, 0); header.setDocId("tokenTest"); header.addToIndexes(); AnalysisEngine consumer = AnalysisEngineFactory.createEngine("de.julielab.jcore.consumer.txt.desc.jcore-txt-consumer", PARAM_OUTPUT_DIR, "src/test/resources/data", SentenceTokenConsumer.PARAM_MODE, "TOKEN"); consumer.process(jcas); File outputFile = new File("src/test/resources/data/tokenTest.txt"); assertTrue(outputFile.exists()); List<String> lines = readFile(outputFile.getAbsolutePath()); assertEquals(1, lines.size()); assertEquals("This is", lines.get(0)); }
@Test public void testLowercasing() throws Exception { final AnalysisEngine consumer = AnalysisEngineFactory.createEngine(SentenceTokenConsumer.class, PARAM_OUTPUT_DIR, "src/test/resources/data", PARAM_LOWERCASE, true, PARAM_MODE, "DOCUMENT"); JCas jcas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-morpho-syntax-types", "de.julielab.jcore.types.jcore-document-meta-pubmed-types"); jcas.setDocumentText("DoCUmeNt 1"); consumer.process(jcas); final String line = FileUtilities.getReaderFromFile(new File("src/test/resources/data/0.txt")).readLine(); assertThat(line).isEqualTo("document 1"); }