WordChunkUtils { public static List<WordChunk> getWordChunks(String string) { string = string.replaceAll("\\s+", " "); for (char c : SENTENCE_ENDING_INTERPUNCTIONS.toCharArray()) { string = string.replace(" " + c, "" + c); } ArrayList<WordChunk> res = new ArrayList<>(); String[] parts = string.split("\\s+"); for (int i = 0; i < parts.length; i++) { String part = parts[i]; String word = getWord(part); if (res.size() > 0 && StringUtils.isEmpty(word)) { res.get(res.size() - 1).chunk += " " + part; } else { res.add(new WordChunk(part, word)); } } WordChunk firstChunk = res.get(0); if (res.size() > 1 && StringUtils.isEmpty(firstChunk.word)) { res.remove(0); res.get(0).chunk = firstChunk.chunk + " " + res.get(0).chunk; } return res; } private  WordChunkUtils(); static List<WordChunk> getWordChunks(String string); static String getWord(String str); static void main(String[] args);  }
@Test public void testRomanianWithDash() { String sentence = "L-a trezit."; List<WordChunk> chunks = WordChunkUtils.getWordChunks(sentence); Assert.assertEquals(2, chunks.size()); Assert.assertEquals("L-a", chunks.get(0).word); Assert.assertEquals("trezit", chunks.get(1).word); }
@Test public void testSentenceWithNumbers() { String sentence = "L-a 123 trezit."; List<WordChunk> chunks = WordChunkUtils.getWordChunks(sentence); Assert.assertEquals(3, chunks.size()); Assert.assertEquals("L-a", chunks.get(0).word); Assert.assertEquals("123", chunks.get(1).word); Assert.assertEquals("trezit", chunks.get(2).word); }
@Test public void testSpaceBeforeQuestionMark() { String sentence = "Je li ?"; List<WordChunk> chunks = WordChunkUtils.getWordChunks(sentence); Assert.assertEquals(2, chunks.size()); Assert.assertEquals("Je", chunks.get(0).word); Assert.assertEquals("Je", chunks.get(0).chunk); Assert.assertEquals("li", chunks.get(1).word); Assert.assertEquals("li?", chunks.get(1).chunk); }
@Test public void testSpaceAroundNonLetter() { String sentence = "Je - li ?"; List<WordChunk> chunks = WordChunkUtils.getWordChunks(sentence); Assert.assertEquals(2, chunks.size()); Assert.assertEquals("Je", chunks.get(0).word); Assert.assertEquals("Je -", chunks.get(0).chunk); Assert.assertEquals("li", chunks.get(1).word); Assert.assertEquals("li?", chunks.get(1).chunk); }
@Test public void testSpaceAroundNonLetter2() { String sentence = " - Je - li ?"; List<WordChunk> chunks = WordChunkUtils.getWordChunks(sentence); Assert.assertEquals(2, chunks.size()); Assert.assertEquals("Je", chunks.get(0).word); Assert.assertEquals(" - Je -", chunks.get(0).chunk); Assert.assertEquals("li", chunks.get(1).word); Assert.assertEquals("li?", chunks.get(1).chunk); }
@Test public void testSpaceAroundNonLetter3() { String sentence = " - Je - li ? - "; List<WordChunk> chunks = WordChunkUtils.getWordChunks(sentence); Assert.assertEquals(2, chunks.size()); Assert.assertEquals("Je", chunks.get(0).word); Assert.assertEquals(" - Je -", chunks.get(0).chunk); Assert.assertEquals("li", chunks.get(1).word); Assert.assertEquals("li? -", chunks.get(1).chunk); }