IllinoisTokenizer implements Tokenizer { @Override public Tokenization tokenizeTextSpan(String text) { String[] splitterInput = new String[1]; splitterInput[0] = text; SentenceSplitter splitter = new SentenceSplitter(splitterInput); Sentence[] sentences = splitter.splitAll(); List<IntPair> characterOffsets = new LinkedList<>(); int[] sentenceEndTokenOffsets = new int[sentences.length]; int sentenceEndTokenIndex = 0; int sentIndex = 0; List<String> tokens = new LinkedList<>(); for (Sentence s : splitter.splitAll()) { LinkedVector words = s.wordSplit(); if (s.end >= text.length()) { throw new IllegalArgumentException("Error in tokenizer, sentence end ( " + s.end + ") is greater than rawtext length (" + text.length() + ")."); } for (int i = 0; i < words.size(); i++) { Word word = (Word) words.get(i); IntPair wordOffsets = new IntPair(word.start, word.end + 1); characterOffsets.add(wordOffsets); tokens.add(text.substring(wordOffsets.getFirst(), wordOffsets.getSecond())); } sentenceEndTokenIndex += words.size(); sentenceEndTokenOffsets[sentIndex++] = sentenceEndTokenIndex; } String[] tokenArray = tokens.toArray(new String[tokens.size()]); IntPair[] charOffsetArray = characterOffsets.toArray(new IntPair[characterOffsets.size()]); return new Tokenization(tokenArray, charOffsetArray, sentenceEndTokenOffsets); }  @Override Pair<String[], IntPair[]> tokenizeSentence(String sentence); @Override Tokenization tokenizeTextSpan(String text);  }
@Test public void testIllinoisTokenizerMultiline() { Tokenizer tkr = new IllinoisTokenizer(); String text = "Mr. Dawkins -- a liberal professor -- doesn't like fundamentalists. " + System.lineSeparator() + "He is intolerant of intolerance!"; Tokenizer.Tokenization tknzn = tkr.tokenizeTextSpan(text); int[] sentEndOffsets = tknzn.getSentenceEndTokenIndexes(); assertEquals(2, sentEndOffsets.length); assertEquals(12, sentEndOffsets[0]); assertEquals(18, sentEndOffsets[1]); String[] tokens = tknzn.getTokens(); assertEquals("--", tokens[6]); assertEquals("of", tokens[15]); IntPair[] tokenOffsets = tknzn.getCharacterOffsets(); int notIndex = 8; IntPair notOffsets = new IntPair(42, 45); assertEquals(notOffsets, tokenOffsets[notIndex]); int intolerantIndex = 14; IntPair intolerantOffsets = new IntPair(77, 87); assertEquals(intolerantOffsets, tokenOffsets[intolerantIndex]); }
@Test public void testIllinoisTokenizerEmptyString() { Tokenizer tkr = new IllinoisTokenizer(); String text = ""; Tokenizer.Tokenization tknzn = tkr.tokenizeTextSpan(text); assertEquals(tknzn.getTokens().length, 0); }
@Test public void testIllinoisTokenizerStringWithNewline() { Tokenizer tkr = new IllinoisTokenizer(); String text = "this\nsentence"; Tokenizer.Tokenization tknzn = tkr.tokenizeTextSpan(text); assertEquals(tknzn.getTokens().length, 2); }