TokenisedAtomicTokenSequence extends TaggedTokenSequence<TokeniserOutcome> implements ClassificationSolution, Comparable<TokenisedAtomicTokenSequence>, Serializable { public TokenSequence inferTokenSequence() { if (tokenSequence == null) { Map<Integer, TaggedToken<TokeniserOutcome>> indexTokenMap = new HashMap<Integer, TaggedToken<TokeniserOutcome>>(); tokenSequence = new TokenSequence(sentence, this, sessionId); int currentStart = 0; int currentEnd = 0; List<TaggedToken<TokeniserOutcome>> currentAtomicParts = new ArrayList<TaggedToken<TokeniserOutcome>>(); boolean isWhiteSpace = true; for (TaggedToken<TokeniserOutcome> decisionTag : this) { currentAtomicParts.add(decisionTag); indexTokenMap.put(decisionTag.getToken().getStartIndex(), decisionTag); Token token = decisionTag.getToken(); if (decisionTag.getTag().equals(TokeniserOutcome.SEPARATE)) { if (!isWhiteSpace) { this.addToken(tokenSequence, currentStart, currentEnd, currentAtomicParts); currentAtomicParts = new ArrayList<TaggedToken<TokeniserOutcome>>(); } else { this.addToken(tokenSequence, currentStart, currentEnd, null); } currentStart = token.getStartIndex(); isWhiteSpace = true; } isWhiteSpace = isWhiteSpace && token.isWhiteSpace(); currentEnd = token.getEndIndex(); } if (!isWhiteSpace) { this.addToken(tokenSequence, currentStart, currentEnd, currentAtomicParts); } else { this.addToken(tokenSequence, currentStart, currentEnd, null); } } return tokenSequence; } TokenisedAtomicTokenSequence(Sentence sentence, String sessionId);  TokenisedAtomicTokenSequence(Sentence sentence, int initialCapacity, String sessionId);  TokenisedAtomicTokenSequence(TokenisedAtomicTokenSequence history); TokenSequence inferTokenSequence(); @Override List<Decision> getDecisions(); @Override List<Solution> getUnderlyingSolutions(); @Override void addDecision(Decision decision); @Override @SuppressWarnings("rawtypes") ScoringStrategy getScoringStrategy(); @Override void setScoringStrategy(@SuppressWarnings("rawtypes") ScoringStrategy scoringStrategy); @Override @SuppressWarnings("unchecked") double getScore(); @Override int compareTo(TokenisedAtomicTokenSequence o); Sentence getSentence(); TokenisedAtomicTokenSequence cloneSequence();  }
@Test public void testGetTokenSequence() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; final Sentence sentence = new Sentence("Je n'ai pas encore l'ourang-outan.", sessionId); TokeniserOutcome[] tokeniserOutcomeArray = new TokeniserOutcome[] { TokeniserOutcome.SEPARATE, TokeniserOutcome.SEPARATE, TokeniserOutcome.SEPARATE, TokeniserOutcome.JOIN, TokeniserOutcome.SEPARATE, TokeniserOutcome.SEPARATE, TokeniserOutcome.SEPARATE, TokeniserOutcome.JOIN, TokeniserOutcome.JOIN, TokeniserOutcome.SEPARATE, TokeniserOutcome.SEPARATE, TokeniserOutcome.JOIN, TokeniserOutcome.SEPARATE, TokeniserOutcome.JOIN, TokeniserOutcome.JOIN, TokeniserOutcome.SEPARATE }; TokenisedAtomicTokenSequence atomicTokenSequence = new TokenisedAtomicTokenSequence(sentence, sessionId); TokenSequence tokenSequence = new TokenSequence(sentence, sessionId); tokenSequence.findDefaultTokens(); int i = 0; for (Token token : tokenSequence.listWithWhiteSpace()) { Decision decision = new Decision(tokeniserOutcomeArray[i++].name()); TaggedToken<TokeniserOutcome> taggedToken = new TaggedToken<>(token, decision, TokeniserOutcome.valueOf(decision.getOutcome())); atomicTokenSequence.add(taggedToken); } TokenSequence newTokenSequence = atomicTokenSequence.inferTokenSequence(); LOG.debug(newTokenSequence.toString()); i = 0; for (Token token : newTokenSequence) { if (i == 0) { assertEquals("Je", token.getAnalyisText()); } else if (i == 1) { assertEquals("n'", token.getAnalyisText()); } else if (i == 2) { assertEquals("ai", token.getAnalyisText()); } else if (i == 3) { assertEquals("pas encore", token.getAnalyisText()); } else if (i == 4) { assertEquals("l'", token.getAnalyisText()); } else if (i == 5) { assertEquals("ourang-outan", token.getAnalyisText()); } else if (i == 6) { assertEquals(".", token.getAnalyisText()); } i++; } assertEquals(7, newTokenSequence.size()); }