Tokenizer implements Analyzer { public void analyze(Document document) { List<Sentence> sentences = document.getSentences(); for (Sentence sentence : sentences) { String sentenceString = sentence.getText(); Span[] tokensSpan; synchronized (this.tokenizer) { tokensSpan = tokenizer.tokenizePos(preprocess(sentenceString)); } List<Token> tokens = new ArrayList<Token>(tokensSpan.length); for (int i = 0; i < tokensSpan.length; i++) { Token token = new TokenImpl(tokensSpan[i].getStart(), tokensSpan[i].getEnd() , tokensSpan[i] .getCoveredText(sentenceString).toString()); tokens.add(token); } sentence.setTokens(tokens); } } Tokenizer(TokenizerME tokenizer); void analyze(Document document);  }
@Test public void testAnalyze() throws FileNotFoundException { DocumentImpl document = new DocumentImpl(); String text = "A menina pequena andava para lá."; document.setText(text); List<Sentence> sentences = new ArrayList<Sentence>(); Sentence sentence = new SentenceImpl(0, 32, document); sentences.add(sentence); document.setSentences(sentences); Span[] spansTokens = { new Span(0,1), new Span(2,8), new Span(9,16), new Span (17, 23), new Span (24,28), new Span (29, 31), new Span (31,32)}; when(mockedTokenizer.tokenizePos(text)).thenReturn(spansTokens); tokenizer.analyze(document); assertEquals(7, document.getSentences().get(0).getTokens().size()); assertEquals("A", document.getSentences().get(0).getTokens().get(0) .getLexeme()); assertEquals("menina", document.getSentences().get(0).getTokens().get(1).getLexeme()); assertEquals("pequena", document.getSentences().get(0).getTokens().get(2).getLexeme()); assertEquals("andava", document.getSentences().get(0).getTokens().get(3).getLexeme()); assertEquals("para", document.getSentences().get(0).getTokens().get(4).getLexeme()); assertEquals("lá", document.getSentences().get(0).getTokens().get(5).getLexeme()); assertEquals(".", document.getSentences().get(0).getTokens().get(6).getLexeme()); }