@Test public void testTokenize() { logger.info("tokenize"); String text = "In publishing and graphic design, lorem ipsum[1] is a placeholder text (filler text) commonly used to demonstrate the graphic elements of a document or visual presentation, such as font, typography, and layout, by removing the distraction of meaningful content. The lorem ipsum text is typically a section of a Latin text by Cicero with words altered, added, and removed that make it nonsensical and not proper Latin.[1]"; WhitespaceTokenizer instance = new WhitespaceTokenizer(); List<String> expResult = new ArrayList<>(Arrays.asList("In", "publishing", "and", "graphic", "design,", "lorem", "ipsum[1]", "is", "a", "placeholder", "text", "(filler", "text)", "commonly", "used", "to", "demonstrate", "the", "graphic", "elements", "of", "a", "document", "or", "visual", "presentation,", "such", "as", "font,", "typography,", "and", "layout,", "by", "removing", "the", "distraction", "of", "meaningful", "content.", "The", "lorem", "ipsum", "text", "is", "typically", "a", "section", "of", "a", "Latin", "text", "by", "Cicero", "with", "words", "altered,", "added,", "and", "removed", "that", "make", "it", "nonsensical", "and", "not", "proper", "Latin.[1]")); List<String> result = instance.tokenize(text); assertEquals(expResult, result); }
@Override public List<String> tokenize(String text) { List<String> tokens = new ArrayList<>(Arrays.asList(text.split("[\\p{Z}\\p{C}]+"))); return tokens; }
WhitespaceTokenizer extends AbstractTokenizer { @Override public List<String> tokenize(String text) { List<String> tokens = new ArrayList<>(Arrays.asList(text.split("[\\p{Z}\\p{C}]+"))); return tokens; } }
WhitespaceTokenizer extends AbstractTokenizer { @Override public List<String> tokenize(String text) { List<String> tokens = new ArrayList<>(Arrays.asList(text.split("[\\p{Z}\\p{C}]+"))); return tokens; }  }
WhitespaceTokenizer extends AbstractTokenizer { @Override public List<String> tokenize(String text) { List<String> tokens = new ArrayList<>(Arrays.asList(text.split("[\\p{Z}\\p{C}]+"))); return tokens; }  @Override List<String> tokenize(String text); }
WhitespaceTokenizer extends AbstractTokenizer { @Override public List<String> tokenize(String text) { List<String> tokens = new ArrayList<>(Arrays.asList(text.split("[\\p{Z}\\p{C}]+"))); return tokens; }  @Override List<String> tokenize(String text);  }