@Test public void testSplitHyphen() { System.out.println("tokenize hyphen"); String text = "On a noncash basis for the quarter, the bank reported a " + "loss of $7.3 billion because of a $10.4 billion write-down " + "in the value of its credit card unit, attributed to federal " + "regulations that limit debit fees and other charges."; String[] expResult = {"On", "a", "noncash", "basis", "for", "the", "quarter", ",", "the", "bank", "reported", "a", "loss", "of", "$7.3", "billion", "because", "of", "a", "$10.4", "billion", "write-down", "in", "the", "value", "of", "its", "credit", "card", "unit", ",", "attributed", "to", "federal", "regulations", "that", "limit", "debit", "fees", "and", "other", "charges", "."}; BreakIteratorTokenizer instance = new BreakIteratorTokenizer(); String[] result = instance.split(text); assertEquals(expResult.length, result.length); for (int i = 0; i < result.length; i++) { assertEquals(expResult[i], result[i]); } }
@Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; }
BreakIteratorTokenizer implements Tokenizer { @Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; } }
BreakIteratorTokenizer implements Tokenizer { @Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; } BreakIteratorTokenizer();  BreakIteratorTokenizer(Locale locale); }
BreakIteratorTokenizer implements Tokenizer { @Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; } BreakIteratorTokenizer();  BreakIteratorTokenizer(Locale locale); @Override String[] split(String text); }
BreakIteratorTokenizer implements Tokenizer { @Override public String[] split(String text) { boundary.setText(text); ArrayList<String> words = new ArrayList<>(); int start = boundary.first(); int end = boundary.next(); while (end != BreakIterator.DONE) { String word = text.substring(start, end).trim(); if (!word.isEmpty()) { words.add(word); } start = end; end = boundary.next(); } String[] array = new String[words.size()]; for (int i = 0; i < array.length; i++) { array[i] = words.get(i); } return array; } BreakIteratorTokenizer();  BreakIteratorTokenizer(Locale locale); @Override String[] split(String text);  }