@Test public void testProcessWithoutPOSTags() throws Exception { JCas cas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-all-types"); AnalysisEngine consumer = AnalysisEngineFactory.createEngine(SentenceTokenConsumer.class, PARAM_OUTPUT_DIR, "src/test/resources/data"); cas.setDocumentText("I love food. I like sleeping."); Sentence s1 = new Sentence(cas, 0, 12); s1.addToIndexes(); Sentence s2 = new Sentence(cas, 13, 29); s2.addToIndexes(); Token t1 = new Token(cas, 0, 1); t1.addToIndexes(); Token t2 = new Token(cas, 2, 6); t2.addToIndexes(); Token t3 = new Token(cas, 7, 11); t3.addToIndexes(); Token t4 = new Token(cas, 11, 12); t4.addToIndexes(); Token t5 = new Token(cas, 13, 14); t5.addToIndexes(); Token t6 = new Token(cas, 15, 19); t6.addToIndexes(); Token t7 = new Token(cas, 20, 28); t7.addToIndexes(); Token t8 = new Token(cas, 28, 29); t8.addToIndexes(); Header header = new Header(cas); header.setDocId("withoutPOS"); header.addToIndexes(); consumer.process(cas); assertTrue(new File("src/test/resources/data/withoutPOS.txt").exists()); List<String> file = readFile("src/test/resources/data/withoutPOS.txt"); assertTrue(file.contains("I love food .")); assertTrue(file.contains("I like sleeping .")); }
@Override public void process(JCas jcas) throws AnalysisEngineProcessException { LOGGER.trace("Processing next document ... "); try { String fileId = getDocID(jcas); if (fileId == null) fileId = new Integer(docs++).toString(); if (mode == Mode.TOKEN) { FSIterator sentenceIterator = jcas.getAnnotationIndex(Sentence.type).iterator(); AnnotationIndex tokenIndex = jcas.getAnnotationIndex(Token.type); ArrayList<String> sentences = new ArrayList<>(); while (sentenceIterator.hasNext()) { Sentence sentence = (Sentence) sentenceIterator.next(); FSIterator tokIterator = tokenIndex.subiterator(sentence); String sentenceText = ""; while (tokIterator.hasNext()) { if (addPOSTAG) { sentenceText = returnWithPOSTAG(tokIterator, sentenceText); } else { sentenceText = returnWithoutPOSTAG(tokIterator, sentenceText); } } sentences.add(sentenceText); } writeSentences2File(fileId, sentences); } else if (mode == Mode.DOCUMENT) { File outputFile = new File(directory.getCanonicalPath() + File.separator + fileId + ".txt" + (gzip ? ".gz" : "")); LOGGER.trace("Writing the verbatim CAS document text to {}", outputFile); writeSentences2File(fileId, Arrays.asList(jcas.getDocumentText())); } } catch (CASRuntimeException | CASException | IOException e) { LOGGER.error("Error while writing: ", e); throw new AnalysisEngineProcessException(e); } }
SentenceTokenConsumer extends JCasAnnotator_ImplBase { @Override public void process(JCas jcas) throws AnalysisEngineProcessException { LOGGER.trace("Processing next document ... "); try { String fileId = getDocID(jcas); if (fileId == null) fileId = new Integer(docs++).toString(); if (mode == Mode.TOKEN) { FSIterator sentenceIterator = jcas.getAnnotationIndex(Sentence.type).iterator(); AnnotationIndex tokenIndex = jcas.getAnnotationIndex(Token.type); ArrayList<String> sentences = new ArrayList<>(); while (sentenceIterator.hasNext()) { Sentence sentence = (Sentence) sentenceIterator.next(); FSIterator tokIterator = tokenIndex.subiterator(sentence); String sentenceText = ""; while (tokIterator.hasNext()) { if (addPOSTAG) { sentenceText = returnWithPOSTAG(tokIterator, sentenceText); } else { sentenceText = returnWithoutPOSTAG(tokIterator, sentenceText); } } sentences.add(sentenceText); } writeSentences2File(fileId, sentences); } else if (mode == Mode.DOCUMENT) { File outputFile = new File(directory.getCanonicalPath() + File.separator + fileId + ".txt" + (gzip ? ".gz" : "")); LOGGER.trace("Writing the verbatim CAS document text to {}", outputFile); writeSentences2File(fileId, Arrays.asList(jcas.getDocumentText())); } } catch (CASRuntimeException | CASException | IOException e) { LOGGER.error("Error while writing: ", e); throw new AnalysisEngineProcessException(e); } } }
SentenceTokenConsumer extends JCasAnnotator_ImplBase { @Override public void process(JCas jcas) throws AnalysisEngineProcessException { LOGGER.trace("Processing next document ... "); try { String fileId = getDocID(jcas); if (fileId == null) fileId = new Integer(docs++).toString(); if (mode == Mode.TOKEN) { FSIterator sentenceIterator = jcas.getAnnotationIndex(Sentence.type).iterator(); AnnotationIndex tokenIndex = jcas.getAnnotationIndex(Token.type); ArrayList<String> sentences = new ArrayList<>(); while (sentenceIterator.hasNext()) { Sentence sentence = (Sentence) sentenceIterator.next(); FSIterator tokIterator = tokenIndex.subiterator(sentence); String sentenceText = ""; while (tokIterator.hasNext()) { if (addPOSTAG) { sentenceText = returnWithPOSTAG(tokIterator, sentenceText); } else { sentenceText = returnWithoutPOSTAG(tokIterator, sentenceText); } } sentences.add(sentenceText); } writeSentences2File(fileId, sentences); } else if (mode == Mode.DOCUMENT) { File outputFile = new File(directory.getCanonicalPath() + File.separator + fileId + ".txt" + (gzip ? ".gz" : "")); LOGGER.trace("Writing the verbatim CAS document text to {}", outputFile); writeSentences2File(fileId, Arrays.asList(jcas.getDocumentText())); } } catch (CASRuntimeException | CASException | IOException e) { LOGGER.error("Error while writing: ", e); throw new AnalysisEngineProcessException(e); } }  }
SentenceTokenConsumer extends JCasAnnotator_ImplBase { @Override public void process(JCas jcas) throws AnalysisEngineProcessException { LOGGER.trace("Processing next document ... "); try { String fileId = getDocID(jcas); if (fileId == null) fileId = new Integer(docs++).toString(); if (mode == Mode.TOKEN) { FSIterator sentenceIterator = jcas.getAnnotationIndex(Sentence.type).iterator(); AnnotationIndex tokenIndex = jcas.getAnnotationIndex(Token.type); ArrayList<String> sentences = new ArrayList<>(); while (sentenceIterator.hasNext()) { Sentence sentence = (Sentence) sentenceIterator.next(); FSIterator tokIterator = tokenIndex.subiterator(sentence); String sentenceText = ""; while (tokIterator.hasNext()) { if (addPOSTAG) { sentenceText = returnWithPOSTAG(tokIterator, sentenceText); } else { sentenceText = returnWithoutPOSTAG(tokIterator, sentenceText); } } sentences.add(sentenceText); } writeSentences2File(fileId, sentences); } else if (mode == Mode.DOCUMENT) { File outputFile = new File(directory.getCanonicalPath() + File.separator + fileId + ".txt" + (gzip ? ".gz" : "")); LOGGER.trace("Writing the verbatim CAS document text to {}", outputFile); writeSentences2File(fileId, Arrays.asList(jcas.getDocumentText())); } } catch (CASRuntimeException | CASException | IOException e) { LOGGER.error("Error while writing: ", e); throw new AnalysisEngineProcessException(e); } }  @Override void initialize(UimaContext aContext); @Override void process(JCas jcas); String getDocID(JCas jcas); @Override void collectionProcessComplete(); }
SentenceTokenConsumer extends JCasAnnotator_ImplBase { @Override public void process(JCas jcas) throws AnalysisEngineProcessException { LOGGER.trace("Processing next document ... "); try { String fileId = getDocID(jcas); if (fileId == null) fileId = new Integer(docs++).toString(); if (mode == Mode.TOKEN) { FSIterator sentenceIterator = jcas.getAnnotationIndex(Sentence.type).iterator(); AnnotationIndex tokenIndex = jcas.getAnnotationIndex(Token.type); ArrayList<String> sentences = new ArrayList<>(); while (sentenceIterator.hasNext()) { Sentence sentence = (Sentence) sentenceIterator.next(); FSIterator tokIterator = tokenIndex.subiterator(sentence); String sentenceText = ""; while (tokIterator.hasNext()) { if (addPOSTAG) { sentenceText = returnWithPOSTAG(tokIterator, sentenceText); } else { sentenceText = returnWithoutPOSTAG(tokIterator, sentenceText); } } sentences.add(sentenceText); } writeSentences2File(fileId, sentences); } else if (mode == Mode.DOCUMENT) { File outputFile = new File(directory.getCanonicalPath() + File.separator + fileId + ".txt" + (gzip ? ".gz" : "")); LOGGER.trace("Writing the verbatim CAS document text to {}", outputFile); writeSentences2File(fileId, Arrays.asList(jcas.getDocumentText())); } } catch (CASRuntimeException | CASException | IOException e) { LOGGER.error("Error while writing: ", e); throw new AnalysisEngineProcessException(e); } }  @Override void initialize(UimaContext aContext); @Override void process(JCas jcas); String getDocID(JCas jcas); @Override void collectionProcessComplete(); static final String PARAM_OUTPUT_DIR; static final String PARAM_DELIMITER; static final String PARAM_LOWERCASE; static final String PARAM_MODE; static final String PARAM_GZIP; static final String PARAM_ZIP_ARCHIVE; static final String PARAM_ZIP_MAX_SIZE; static final String PARAM_ZIP_PREFIX; }