@Test public void testParseFullText() throws Exception { checkParseFullText("brown dog fail fox jump lazy over quick", "The quick brown fox jumps over the lazy dog -- and fails!"); checkParseFullText("aime cafe jure pas", "J'aime PAS le caf\u00e9, je te jure."); checkParseFullText("007 bond jame thx1138", "James Bond 007 && THX1138"); }
public static Set<String> parseFullText(String string, boolean removeDiacritics) { if (string == null) { return Collections.emptySet(); } Set<String> set = new LinkedHashSet<>(); for (String word : wordPattern.split(string)) { String w = parseWord(word, removeDiacritics); if (w != null) { set.add(w); } } return set; }
FullTextUtils { public static Set<String> parseFullText(String string, boolean removeDiacritics) { if (string == null) { return Collections.emptySet(); } Set<String> set = new LinkedHashSet<>(); for (String word : wordPattern.split(string)) { String w = parseWord(word, removeDiacritics); if (w != null) { set.add(w); } } return set; } }
FullTextUtils { public static Set<String> parseFullText(String string, boolean removeDiacritics) { if (string == null) { return Collections.emptySet(); } Set<String> set = new LinkedHashSet<>(); for (String word : wordPattern.split(string)) { String w = parseWord(word, removeDiacritics); if (w != null) { set.add(w); } } return set; } private  FullTextUtils(); }
FullTextUtils { public static Set<String> parseFullText(String string, boolean removeDiacritics) { if (string == null) { return Collections.emptySet(); } Set<String> set = new LinkedHashSet<>(); for (String word : wordPattern.split(string)) { String w = parseWord(word, removeDiacritics); if (w != null) { set.add(w); } } return set; } private  FullTextUtils(); static Set<String> parseFullText(String string, boolean removeDiacritics); static String parseWord(String string, boolean removeDiacritics); }
FullTextUtils { public static Set<String> parseFullText(String string, boolean removeDiacritics) { if (string == null) { return Collections.emptySet(); } Set<String> set = new LinkedHashSet<>(); for (String word : wordPattern.split(string)) { String w = parseWord(word, removeDiacritics); if (w != null) { set.add(w); } } return set; } private  FullTextUtils(); static Set<String> parseFullText(String string, boolean removeDiacritics); static String parseWord(String string, boolean removeDiacritics); static final Pattern wordPattern; static final int MIN_SIZE; static final String STOP_WORDS; static final Set<String> stopWords; static final String UNACCENTED; }