RuleBasedTokenizer implements Tokenizer { @Override public String[] tokenize(String text) { char[] chars = text.trim().toCharArray(); ArrayList<String> tokens = new ArrayList<String>(); StringBuilder sb = new StringBuilder(); for (int index = 0; index < chars.length; index++) { char previousChar = (index - 1) > 0 ? chars[index - 1] : ' '; char currentChar = chars[index]; char nextChar = (index + 1) < chars.length ? chars[index + 1] : ' '; if (currentChar == '\'') { if (Character.isLetter(previousChar) && Character.isLetter(nextChar)) { sb.append(currentChar); continue; } } if (currentChar == '.' || currentChar == ',') { if (Character.isDigit(previousChar) && Character.isDigit(nextChar)) { sb.append(currentChar); continue; } } if (currentChar == ' ') { if (sb.length() > 0) { tokens.add(sb.toString()); sb.setLength(0); } continue; } if (!Character.isLetterOrDigit(currentChar)) { tokens.add(sb.toString()); sb.setLength(0); if (!ignorePunctuation) { tokens.add(String.valueOf(currentChar)); } continue; } sb.append(currentChar); } if (sb.length() > 0) { tokens.add(sb.toString()); } return tokens.toArray(new String[tokens.size()]); } RuleBasedTokenizer();  RuleBasedTokenizer(boolean ignorePunctuation); @Override String[] tokenize(String text);  }
@Test public void testTokenize() { String testString = "Today is the first day of the rest of your life."; Tokenizer tokenizer = new RuleBasedTokenizer(); String[] tokens = tokenizer.tokenize(testString); assertEquals("Today", tokens[0]); assertEquals("rest", tokens[7]); assertEquals("life", tokens[10]); assertEquals(".", tokens[11]); }
@Test public void testTokenizeWithoutPunctuation() { String testString = "George, an athlete, ran a marathon yesterday."; Tokenizer tokenizer = new RuleBasedTokenizer(true); String[] tokens = tokenizer.tokenize(testString); assertEquals(7, tokens.length); assertEquals("an", tokens[1]); assertEquals("ran", tokens[3]); }
@Test public void testTokenizeWithContraction() { String testString = "I can't believe this"; Tokenizer tokenizer = new RuleBasedTokenizer(); String[] tokens = tokenizer.tokenize(testString); assertEquals("I", tokens[0]); assertEquals("can't", tokens[1]); assertEquals("believe", tokens[2]); assertEquals("this", tokens[3]); }
@Test public void testTokenizeWithDecimalNumbers() { String testString = "Add 2.3 to 4,305.7"; Tokenizer tokenizer = new RuleBasedTokenizer(); String[] tokens = tokenizer.tokenize(testString); assertEquals("Add", tokens[0]); assertEquals("2.3", tokens[1]); assertEquals("to", tokens[2]); assertEquals("4,305.7", tokens[3]); }