@Test void tokenize() { Tokenizer tokenizer = new BasicTokenizer(); String example = "$100, 10:45 1/2/3 4-5-6 1st 2nd 3RD 4th 10pm 3May"; List<String> expected = Arrays.asList( "$100", "1045", "1", "2", "3", "4", "5", "6", "1", "2", "3", "4", "10", "pm", "3", "May" ); assertEquals(expected, tokenizer.tokenize(example)); }
@Override public List<String> tokenize(String input) { String cleaned = input.trim() .replaceAll("[:,]", "") .replaceAll("[/\\-]", " ") .replaceAll("(\\d)(st|nd|rd|th|ST|ND|RD|TH)?", "$1") .replaceAll("(\\d)([a-zA-Z])", "$1 $2"); return Arrays.asList(cleaned.split(" ")); }
BasicTokenizer implements Tokenizer { @Override public List<String> tokenize(String input) { String cleaned = input.trim() .replaceAll("[:,]", "") .replaceAll("[/\\-]", " ") .replaceAll("(\\d)(st|nd|rd|th|ST|ND|RD|TH)?", "$1") .replaceAll("(\\d)([a-zA-Z])", "$1 $2"); return Arrays.asList(cleaned.split(" ")); } }
BasicTokenizer implements Tokenizer { @Override public List<String> tokenize(String input) { String cleaned = input.trim() .replaceAll("[:,]", "") .replaceAll("[/\\-]", " ") .replaceAll("(\\d)(st|nd|rd|th|ST|ND|RD|TH)?", "$1") .replaceAll("(\\d)([a-zA-Z])", "$1 $2"); return Arrays.asList(cleaned.split(" ")); }  }
BasicTokenizer implements Tokenizer { @Override public List<String> tokenize(String input) { String cleaned = input.trim() .replaceAll("[:,]", "") .replaceAll("[/\\-]", " ") .replaceAll("(\\d)(st|nd|rd|th|ST|ND|RD|TH)?", "$1") .replaceAll("(\\d)([a-zA-Z])", "$1 $2"); return Arrays.asList(cleaned.split(" ")); }  @Override List<String> tokenize(String input); }
BasicTokenizer implements Tokenizer { @Override public List<String> tokenize(String input) { String cleaned = input.trim() .replaceAll("[:,]", "") .replaceAll("[/\\-]", " ") .replaceAll("(\\d)(st|nd|rd|th|ST|ND|RD|TH)?", "$1") .replaceAll("(\\d)([a-zA-Z])", "$1 $2"); return Arrays.asList(cleaned.split(" ")); }  @Override List<String> tokenize(String input);  }