@Test public void testSplitContraction() { System.out.println("tokenize contraction"); String text = "Here are some examples of contractions: 'tis, " + "'twas, ain't, aren't, Can't, could've, couldn't, didn't, doesn't, " + "don't, hasn't, he'd, he'll, he's, how'd, how'll, how's, i'd, i'll, i'm, " + "i've, isn't, it's, might've, mightn't, must've, mustn't, Shan't, " + "she'd, she'll, she's, should've, shouldn't, that'll, that's, " + "there's, they'd, they'll, they're, they've, wasn't, we'd, we'll, " + "we're, weren't, what'd, what's, when'd, when'll, when's, " + "where'd, where'll, where's, who'd, who'll, who's, why'd, why'll, " + "why's, Won't, would've, wouldn't, you'd, you'll, you're, you've"; String[] expResult = {"Here", "are", "some", "examples", "of", "contractions", ":", "'", "tis", ",", "'", "twas", ",", "ain't", ",", "aren't", ",", "Can't", ",", "could've", ",", "couldn't", ",", "didn't", ",", "doesn't", ",", "don't", ",", "hasn't", ",", "he'd", ",", "he'll", ",", "he's", ",", "how'd", ",", "how'll", ",", "how's", ",", "i'd", ",", "i'll", ",", "i'm", ",", "i've", ",", "isn't", ",", "it's", ",", "might've", ",", "mightn't", ",", "must've", ",", "mustn't", ",", "Shan't", ",", "she'd", ",", "she'll", ",", "she's", ",", "should've", ",", "shouldn't", ",", "that'll", ",", "that's", ",", "there's", ",", "they'd", ",", "they'll", ",", "they're", ",", "they've", ",", "wasn't", ",", "we'd", ",", "we'll", ",", "we're", ",", "weren't", ",", "what'd", ",", "what's", ",", "when'd", ",", "when'll", ",", "when's", ",", "where'd", ",", "where'll", ",", "where's", ",", "who'd", ",", "who'll", ",", "who's", ",", "why'd", ",", "why'll", ",", "why's", ",", "Won't", ",", "would've", ",", "wouldn't", ",", "you'd", ",", "you'll", ",", "you're", ",", "you've"}; BreakIteratorTokenizer instance = new BreakIteratorTokenizer(); String[] result = instance.split(text); assertEquals(expResult.length, result.length); for (int i = 0; i < result.length; i++) { assertEquals(expResult[i], result[i]); } }
@Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; }
BreakIteratorTokenizer implements Tokenizer { @Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; } }
BreakIteratorTokenizer implements Tokenizer { @Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; } BreakIteratorTokenizer();  BreakIteratorTokenizer(Locale locale); }
BreakIteratorTokenizer implements Tokenizer { @Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; } BreakIteratorTokenizer();  BreakIteratorTokenizer(Locale locale); @Override String[] split(String text); }
BreakIteratorTokenizer implements Tokenizer { @Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; } BreakIteratorTokenizer();  BreakIteratorTokenizer(Locale locale); @Override String[] split(String text);  }