@Test(dataProvider = "data") public void textToWords(String text, String expanded) { List<String> tokens = expander.expand(text); assertThat(Utilities.join(tokens), equalTo(expanded)); }
public List<String> expand(String text) { String simplifiedText = simplifyChars(text); CharTokenizer tokenizer = new CharTokenizer(); tokenizer.setWhitespaceSymbols(UsEnglish.WHITESPACE_SYMBOLS); tokenizer.setSingleCharSymbols(UsEnglish.SINGLE_CHAR_SYMBOLS); tokenizer.setPrepunctuationSymbols(UsEnglish.PREPUNCTUATION_SYMBOLS); tokenizer.setPostpunctuationSymbols(UsEnglish.PUNCTUATION_SYMBOLS); tokenizer.setInputText(simplifiedText); Utterance utterance = new Utterance(tokenizer); Relation tokenRelation; if ((tokenRelation = utterance.getRelation(Relation.TOKEN)) == null) { throw new IllegalStateException("token relation does not exist"); } wordRelation = WordRelation.createWordRelation(utterance, this); for (tokenItem = tokenRelation.getHead(); tokenItem != null; tokenItem = tokenItem.getNext()) { FeatureSet featureSet = tokenItem.getFeatures(); String tokenVal = featureSet.getString("name"); tokenToWords(tokenVal); } List<String> words = new ArrayList<String>(); for (Item item = utterance.getRelation(Relation.WORD).getHead(); item != null; item = item.getNext()) { if (!item.toString().isEmpty() && !item.toString().contains("#")) { words.add(item.toString()); } } return words; }
UsEnglishWordExpander implements WordExpander { public List<String> expand(String text) { String simplifiedText = simplifyChars(text); CharTokenizer tokenizer = new CharTokenizer(); tokenizer.setWhitespaceSymbols(UsEnglish.WHITESPACE_SYMBOLS); tokenizer.setSingleCharSymbols(UsEnglish.SINGLE_CHAR_SYMBOLS); tokenizer.setPrepunctuationSymbols(UsEnglish.PREPUNCTUATION_SYMBOLS); tokenizer.setPostpunctuationSymbols(UsEnglish.PUNCTUATION_SYMBOLS); tokenizer.setInputText(simplifiedText); Utterance utterance = new Utterance(tokenizer); Relation tokenRelation; if ((tokenRelation = utterance.getRelation(Relation.TOKEN)) == null) { throw new IllegalStateException("token relation does not exist"); } wordRelation = WordRelation.createWordRelation(utterance, this); for (tokenItem = tokenRelation.getHead(); tokenItem != null; tokenItem = tokenItem.getNext()) { FeatureSet featureSet = tokenItem.getFeatures(); String tokenVal = featureSet.getString("name"); tokenToWords(tokenVal); } List<String> words = new ArrayList<String>(); for (Item item = utterance.getRelation(Relation.WORD).getHead(); item != null; item = item.getNext()) { if (!item.toString().isEmpty() && !item.toString().contains("#")) { words.add(item.toString()); } } return words; } }
UsEnglishWordExpander implements WordExpander { public List<String> expand(String text) { String simplifiedText = simplifyChars(text); CharTokenizer tokenizer = new CharTokenizer(); tokenizer.setWhitespaceSymbols(UsEnglish.WHITESPACE_SYMBOLS); tokenizer.setSingleCharSymbols(UsEnglish.SINGLE_CHAR_SYMBOLS); tokenizer.setPrepunctuationSymbols(UsEnglish.PREPUNCTUATION_SYMBOLS); tokenizer.setPostpunctuationSymbols(UsEnglish.PUNCTUATION_SYMBOLS); tokenizer.setInputText(simplifiedText); Utterance utterance = new Utterance(tokenizer); Relation tokenRelation; if ((tokenRelation = utterance.getRelation(Relation.TOKEN)) == null) { throw new IllegalStateException("token relation does not exist"); } wordRelation = WordRelation.createWordRelation(utterance, this); for (tokenItem = tokenRelation.getHead(); tokenItem != null; tokenItem = tokenItem.getNext()) { FeatureSet featureSet = tokenItem.getFeatures(); String tokenVal = featureSet.getString("name"); tokenToWords(tokenVal); } List<String> words = new ArrayList<String>(); for (Item item = utterance.getRelation(Relation.WORD).getHead(); item != null; item = item.getNext()) { if (!item.toString().isEmpty() && !item.toString().contains("#")) { words.add(item.toString()); } } return words; } UsEnglishWordExpander(); }
UsEnglishWordExpander implements WordExpander { public List<String> expand(String text) { String simplifiedText = simplifyChars(text); CharTokenizer tokenizer = new CharTokenizer(); tokenizer.setWhitespaceSymbols(UsEnglish.WHITESPACE_SYMBOLS); tokenizer.setSingleCharSymbols(UsEnglish.SINGLE_CHAR_SYMBOLS); tokenizer.setPrepunctuationSymbols(UsEnglish.PREPUNCTUATION_SYMBOLS); tokenizer.setPostpunctuationSymbols(UsEnglish.PUNCTUATION_SYMBOLS); tokenizer.setInputText(simplifiedText); Utterance utterance = new Utterance(tokenizer); Relation tokenRelation; if ((tokenRelation = utterance.getRelation(Relation.TOKEN)) == null) { throw new IllegalStateException("token relation does not exist"); } wordRelation = WordRelation.createWordRelation(utterance, this); for (tokenItem = tokenRelation.getHead(); tokenItem != null; tokenItem = tokenItem.getNext()) { FeatureSet featureSet = tokenItem.getFeatures(); String tokenVal = featureSet.getString("name"); tokenToWords(tokenVal); } List<String> words = new ArrayList<String>(); for (Item item = utterance.getRelation(Relation.WORD).getHead(); item != null; item = item.getNext()) { if (!item.toString().isEmpty() && !item.toString().contains("#")) { words.add(item.toString()); } } return words; } UsEnglishWordExpander(); Item getTokenItem(); List<String> expand(String text); static boolean kingLike(Item tokenItem); static boolean sectionLike(Item tokenItem); boolean isPronounceable(String word); }
UsEnglishWordExpander implements WordExpander { public List<String> expand(String text) { String simplifiedText = simplifyChars(text); CharTokenizer tokenizer = new CharTokenizer(); tokenizer.setWhitespaceSymbols(UsEnglish.WHITESPACE_SYMBOLS); tokenizer.setSingleCharSymbols(UsEnglish.SINGLE_CHAR_SYMBOLS); tokenizer.setPrepunctuationSymbols(UsEnglish.PREPUNCTUATION_SYMBOLS); tokenizer.setPostpunctuationSymbols(UsEnglish.PUNCTUATION_SYMBOLS); tokenizer.setInputText(simplifiedText); Utterance utterance = new Utterance(tokenizer); Relation tokenRelation; if ((tokenRelation = utterance.getRelation(Relation.TOKEN)) == null) { throw new IllegalStateException("token relation does not exist"); } wordRelation = WordRelation.createWordRelation(utterance, this); for (tokenItem = tokenRelation.getHead(); tokenItem != null; tokenItem = tokenItem.getNext()) { FeatureSet featureSet = tokenItem.getFeatures(); String tokenVal = featureSet.getString("name"); tokenToWords(tokenVal); } List<String> words = new ArrayList<String>(); for (Item item = utterance.getRelation(Relation.WORD).getHead(); item != null; item = item.getNext()) { if (!item.toString().isEmpty() && !item.toString().contains("#")) { words.add(item.toString()); } } return words; } UsEnglishWordExpander(); Item getTokenItem(); List<String> expand(String text); static boolean kingLike(Item tokenItem); static boolean sectionLike(Item tokenItem); boolean isPronounceable(String word);  }