@Test public void testSplitSingleQuote() { System.out.println("tokenize single quote"); String text = "String literals can be enclosed in matching single " + "quotes ('). But it's also appearing in contractions such as can't."; String[] expResult = {"String", "literals", "can", "be", "enclosed", "in", "matching", "single", "quotes", "(", "'", ")", ".", "But", "it's", "also", "appearing", "in", "contractions", "such", "as", "can't", "."}; BreakIteratorTokenizer instance = new BreakIteratorTokenizer(); String[] result = instance.split(text); assertEquals(expResult.length, result.length); for (int i = 0; i < result.length; i++) { assertEquals(expResult[i], result[i]); } }
@Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; }
BreakIteratorTokenizer implements Tokenizer { @Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; } }
BreakIteratorTokenizer implements Tokenizer { @Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; } BreakIteratorTokenizer();  BreakIteratorTokenizer(Locale locale); }
BreakIteratorTokenizer implements Tokenizer { @Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; } BreakIteratorTokenizer();  BreakIteratorTokenizer(Locale locale); @Override String[] split(String text); }
BreakIteratorTokenizer implements Tokenizer { @Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; } BreakIteratorTokenizer();  BreakIteratorTokenizer(Locale locale); @Override String[] split(String text);  }