@Test public void testProcess() { XMLInputSource tokenXML = null; ResourceSpecifier tokenSpec = null; AnalysisEngine tokenAnnotator = null; try { tokenXML = new XMLInputSource(DESCRIPTOR); tokenSpec = UIMAFramework.getXMLParser().parseResourceSpecifier( tokenXML); tokenAnnotator = UIMAFramework.produceAnalysisEngine(tokenSpec); tokenAnnotator.setConfigParameterValue("UseDocText", false); tokenAnnotator.reconfigure(); } catch (final Exception e) { LOGGER.error("testProcess()", e); } JCas jcas = null; try { jcas = tokenAnnotator.newJCas(); } catch (final ResourceInitializationException e) { LOGGER.error("testProcess()", e); } initSentenceCas(jcas); try { tokenAnnotator.process(jcas); } catch (final Exception e) { e.printStackTrace(); } final JFSIndexRepository indexes = jcas.getJFSIndexRepository(); Iterator<?> tokIter = indexes.getAnnotationIndex(Token.type).iterator(); final String predictedOffsets = getPredictedOffsets(tokIter); LOGGER.debug("testProcess() - predicted: " + predictedOffsets); LOGGER.debug("testProcess() - wanted: " + TEST_SENTENCES_OFFSETS); System.out.println("predicted: " + predictedOffsets); System.out.println("wanted: " + TEST_SENTENCES_OFFSETS); assertEquals(TEST_SENTENCES_OFFSETS, predictedOffsets); tokIter = indexes.getAnnotationIndex(Token.type).iterator(); final String tokenNumbers = getTokenNumbers(tokIter); LOGGER.debug("testProcess() - predicted: " + tokenNumbers); LOGGER.debug("testProcess() - wanted: " + TEST_SENTENCES_TOKEN_NUMBERS); System.out.println("predicted: "+tokenNumbers); System.out.println("wanted: " +TEST_SENTENCES_TOKEN_NUMBERS); assertEquals(TEST_SENTENCES_TOKEN_NUMBERS, tokenNumbers); }
@Override public void process(final JCas aJCas) throws AnalysisEngineProcessException { LOGGER.debug("starting processing document"); tokenNumber = 1; if (useCompleteDocText) { LOGGER.debug("process() - tokenizing whole document text!"); final String text = aJCas.getDocumentText(); writeTokensToCAS(text, 0, aJCas); } else { final JFSIndexRepository indexes = aJCas.getJFSIndexRepository(); final Iterator<Annotation> sentenceIter = indexes.getAnnotationIndex(Sentence.type).iterator(); while (sentenceIter.hasNext()) { final Sentence sentence = (Sentence) sentenceIter.next(); int length = sentence.getEnd() - sentence .getBegin(); LOGGER.debug("going to next sentence having length: " + length); if (length > 1000) LOGGER.warn("Current sentence has length {}.", length); final String text = sentence.getCoveredText(); writeTokensToCAS(text, sentence.getBegin(), aJCas); } } }
TokenAnnotator extends JCasAnnotator_ImplBase { @Override public void process(final JCas aJCas) throws AnalysisEngineProcessException { LOGGER.debug("starting processing document"); tokenNumber = 1; if (useCompleteDocText) { LOGGER.debug("process() - tokenizing whole document text!"); final String text = aJCas.getDocumentText(); writeTokensToCAS(text, 0, aJCas); } else { final JFSIndexRepository indexes = aJCas.getJFSIndexRepository(); final Iterator<Annotation> sentenceIter = indexes.getAnnotationIndex(Sentence.type).iterator(); while (sentenceIter.hasNext()) { final Sentence sentence = (Sentence) sentenceIter.next(); int length = sentence.getEnd() - sentence .getBegin(); LOGGER.debug("going to next sentence having length: " + length); if (length > 1000) LOGGER.warn("Current sentence has length {}.", length); final String text = sentence.getCoveredText(); writeTokensToCAS(text, sentence.getBegin(), aJCas); } } } }
TokenAnnotator extends JCasAnnotator_ImplBase { @Override public void process(final JCas aJCas) throws AnalysisEngineProcessException { LOGGER.debug("starting processing document"); tokenNumber = 1; if (useCompleteDocText) { LOGGER.debug("process() - tokenizing whole document text!"); final String text = aJCas.getDocumentText(); writeTokensToCAS(text, 0, aJCas); } else { final JFSIndexRepository indexes = aJCas.getJFSIndexRepository(); final Iterator<Annotation> sentenceIter = indexes.getAnnotationIndex(Sentence.type).iterator(); while (sentenceIter.hasNext()) { final Sentence sentence = (Sentence) sentenceIter.next(); int length = sentence.getEnd() - sentence .getBegin(); LOGGER.debug("going to next sentence having length: " + length); if (length > 1000) LOGGER.warn("Current sentence has length {}.", length); final String text = sentence.getCoveredText(); writeTokensToCAS(text, sentence.getBegin(), aJCas); } } }  }
TokenAnnotator extends JCasAnnotator_ImplBase { @Override public void process(final JCas aJCas) throws AnalysisEngineProcessException { LOGGER.debug("starting processing document"); tokenNumber = 1; if (useCompleteDocText) { LOGGER.debug("process() - tokenizing whole document text!"); final String text = aJCas.getDocumentText(); writeTokensToCAS(text, 0, aJCas); } else { final JFSIndexRepository indexes = aJCas.getJFSIndexRepository(); final Iterator<Annotation> sentenceIter = indexes.getAnnotationIndex(Sentence.type).iterator(); while (sentenceIter.hasNext()) { final Sentence sentence = (Sentence) sentenceIter.next(); int length = sentence.getEnd() - sentence .getBegin(); LOGGER.debug("going to next sentence having length: " + length); if (length > 1000) LOGGER.warn("Current sentence has length {}.", length); final String text = sentence.getCoveredText(); writeTokensToCAS(text, sentence.getBegin(), aJCas); } } }  @Override void initialize(final UimaContext aContext); @Override void process(final JCas aJCas); }
TokenAnnotator extends JCasAnnotator_ImplBase { @Override public void process(final JCas aJCas) throws AnalysisEngineProcessException { LOGGER.debug("starting processing document"); tokenNumber = 1; if (useCompleteDocText) { LOGGER.debug("process() - tokenizing whole document text!"); final String text = aJCas.getDocumentText(); writeTokensToCAS(text, 0, aJCas); } else { final JFSIndexRepository indexes = aJCas.getJFSIndexRepository(); final Iterator<Annotation> sentenceIter = indexes.getAnnotationIndex(Sentence.type).iterator(); while (sentenceIter.hasNext()) { final Sentence sentence = (Sentence) sentenceIter.next(); int length = sentence.getEnd() - sentence .getBegin(); LOGGER.debug("going to next sentence having length: " + length); if (length > 1000) LOGGER.warn("Current sentence has length {}.", length); final String text = sentence.getCoveredText(); writeTokensToCAS(text, sentence.getBegin(), aJCas); } } }  @Override void initialize(final UimaContext aContext); @Override void process(final JCas aJCas); static final String PARAM_MODEL; static final String USE_DOC_TEXT_PARAM; }