PretokenisedSequence extends TokenSequence { public Token addToken(String string) throws TalismaneException { CharSequence text = this.getSentence().getText(); int start = 0; if (this.size() > 0) start = this.get(this.size() - 1).getEndIndex(); for (; start < text.length(); start++) { char c = text.charAt(start); if (!Character.isWhitespace(c)) break; } for (int i = 0; i < string.length(); i++) { char s = string.charAt(i); if (Character.isWhitespace(s)) { start--; char t = text.charAt(start); if (!Character.isWhitespace(t)) break; } else { break; } } int end = start + string.length(); if (end > text.length()) throw new TalismaneException("Add token failed: Expected |" + string + "| at positions " + start + ", " + end + ", but only remaining text (length " + text.length() + ") is |" + text.subSequence(start, text.length()) + "| in sentence: |" + text + "|"); if (!string.equals(text.subSequence(start, end).toString())) throw new TalismaneException("Add token failed: Expected |" + string + "| but was |" + text.subSequence(start, end) + "| in sentence: |" + text + "|"); return this.addToken(start, end); } PretokenisedSequence(PretokenisedSequence sequenceToClone);  PretokenisedSequence(Sentence sentence, String sessionId); Token addToken(String string); @Override TokenSequence cloneTokenSequence();  }
@Test public void testAddTokenString() throws Exception { System.setProperty("config.file", "src/test/resources/test.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; final Sentence sentence = new Sentence("« Il est là. »", sessionId); PretokenisedSequence sequence = new PretokenisedSequence(sentence, sessionId); sequence.addToken("« "); sequence.addToken("Il"); sequence.addToken("est"); sequence.addToken("là"); sequence.addToken("."); sequence.addToken(" »"); System.out.println(sequence.toString()); assertEquals(6, sequence.size()); }