@Test public void testSplit() { System.out.println("tokenize"); String text = "Good muffins cost $3.88\nin New York. Please buy " + "me\ntwo of them.\n\nYou cannot eat them. I gonna eat them. " + "Thanks."; String[] expResult = {"Good", "muffins", "cost", "$", "3.88", "in", "New", "York.", "Please", "buy", "me", "two", "of", "them", ".", "You", "can", "not", "eat", "them.", "I", "gon", "na", "eat", "them.", "Thanks", "."}; PennTreebankTokenizer instance = PennTreebankTokenizer.getInstance(); String[] result = instance.split(text); assertEquals(expResult.length, result.length); for (int i = 0; i < result.length; i++) { assertEquals(expResult[i], result[i]); } }
@Override public String[] split(String text) { for (Pattern regexp : CONTRACTIONS2) text = regexp.matcher(text).replaceAll("$1 $2"); for (Pattern regexp : CONTRACTIONS3) text = regexp.matcher(text).replaceAll("$1 $2 $3"); text = DELIMITERS[0].matcher(text).replaceAll(" $1 "); text = DELIMITERS[1].matcher(text).replaceAll(" $1"); text = DELIMITERS[2].matcher(text).replaceAll(" $1"); text = DELIMITERS[3].matcher(text).replaceAll(" . "); String[] words = WHITESPACE.split(text); if (words.length > 1 && words[words.length-1].equals(".")) { if (EnglishAbbreviations.contains(words[words.length-2])) { words[words.length-2] = words[words.length-2] + "."; } } return words; }
PennTreebankTokenizer implements Tokenizer { @Override public String[] split(String text) { for (Pattern regexp : CONTRACTIONS2) text = regexp.matcher(text).replaceAll("$1 $2"); for (Pattern regexp : CONTRACTIONS3) text = regexp.matcher(text).replaceAll("$1 $2 $3"); text = DELIMITERS[0].matcher(text).replaceAll(" $1 "); text = DELIMITERS[1].matcher(text).replaceAll(" $1"); text = DELIMITERS[2].matcher(text).replaceAll(" $1"); text = DELIMITERS[3].matcher(text).replaceAll(" . "); String[] words = WHITESPACE.split(text); if (words.length > 1 && words[words.length-1].equals(".")) { if (EnglishAbbreviations.contains(words[words.length-2])) { words[words.length-2] = words[words.length-2] + "."; } } return words; } }
PennTreebankTokenizer implements Tokenizer { @Override public String[] split(String text) { for (Pattern regexp : CONTRACTIONS2) text = regexp.matcher(text).replaceAll("$1 $2"); for (Pattern regexp : CONTRACTIONS3) text = regexp.matcher(text).replaceAll("$1 $2 $3"); text = DELIMITERS[0].matcher(text).replaceAll(" $1 "); text = DELIMITERS[1].matcher(text).replaceAll(" $1"); text = DELIMITERS[2].matcher(text).replaceAll(" $1"); text = DELIMITERS[3].matcher(text).replaceAll(" . "); String[] words = WHITESPACE.split(text); if (words.length > 1 && words[words.length-1].equals(".")) { if (EnglishAbbreviations.contains(words[words.length-2])) { words[words.length-2] = words[words.length-2] + "."; } } return words; } private  PennTreebankTokenizer(); }
PennTreebankTokenizer implements Tokenizer { @Override public String[] split(String text) { for (Pattern regexp : CONTRACTIONS2) text = regexp.matcher(text).replaceAll("$1 $2"); for (Pattern regexp : CONTRACTIONS3) text = regexp.matcher(text).replaceAll("$1 $2 $3"); text = DELIMITERS[0].matcher(text).replaceAll(" $1 "); text = DELIMITERS[1].matcher(text).replaceAll(" $1"); text = DELIMITERS[2].matcher(text).replaceAll(" $1"); text = DELIMITERS[3].matcher(text).replaceAll(" . "); String[] words = WHITESPACE.split(text); if (words.length > 1 && words[words.length-1].equals(".")) { if (EnglishAbbreviations.contains(words[words.length-2])) { words[words.length-2] = words[words.length-2] + "."; } } return words; } private  PennTreebankTokenizer(); static PennTreebankTokenizer getInstance(); @Override String[] split(String text); }
PennTreebankTokenizer implements Tokenizer { @Override public String[] split(String text) { for (Pattern regexp : CONTRACTIONS2) text = regexp.matcher(text).replaceAll("$1 $2"); for (Pattern regexp : CONTRACTIONS3) text = regexp.matcher(text).replaceAll("$1 $2 $3"); text = DELIMITERS[0].matcher(text).replaceAll(" $1 "); text = DELIMITERS[1].matcher(text).replaceAll(" $1"); text = DELIMITERS[2].matcher(text).replaceAll(" $1"); text = DELIMITERS[3].matcher(text).replaceAll(" . "); String[] words = WHITESPACE.split(text); if (words.length > 1 && words[words.length-1].equals(".")) { if (EnglishAbbreviations.contains(words[words.length-2])) { words[words.length-2] = words[words.length-2] + "."; } } return words; } private  PennTreebankTokenizer(); static PennTreebankTokenizer getInstance(); @Override String[] split(String text);  }