@Test public void testParse() { assertNull(FullTextUtils.parseWord("gr", false)); assertNull(FullTextUtils.parseWord("are", false)); assertNull(FullTextUtils.parseWord("THE", false)); String str = "\u0153ufcaf\u00e9"; assertEquals("oeufcafe", FullTextUtils.parseWord(str, true)); assertEquals(str, FullTextUtils.parseWord(str, false)); assertEquals("foo", FullTextUtils.parseWord("foo", false)); assertEquals("foo", FullTextUtils.parseWord("fOoS", false)); }
public static String parseWord(String string, boolean removeDiacritics) { int len = string.length(); if (len < MIN_SIZE) { return null; } StringBuilder sb = new StringBuilder(len); for (int i = 0; i < len; i++) { char c = Character.toLowerCase(string.charAt(i)); if (removeDiacritics) { if (c == '\u00e6') { sb.append("ae"); } else if (c >= '\u00e0' && c <= '\u00ff') { sb.append(UNACCENTED.charAt((c) - 0xe0)); } else if (c == '\u0153') { sb.append("oe"); } else { sb.append(c); } } else { sb.append(c); } } int l = sb.length(); if (l > 3 && sb.charAt(l - 1) == 's') { sb.setLength(l - 1); } String word = sb.toString(); if (stopWords.contains(word)) { return null; } return word; }
FullTextUtils { public static String parseWord(String string, boolean removeDiacritics) { int len = string.length(); if (len < MIN_SIZE) { return null; } StringBuilder sb = new StringBuilder(len); for (int i = 0; i < len; i++) { char c = Character.toLowerCase(string.charAt(i)); if (removeDiacritics) { if (c == '\u00e6') { sb.append("ae"); } else if (c >= '\u00e0' && c <= '\u00ff') { sb.append(UNACCENTED.charAt((c) - 0xe0)); } else if (c == '\u0153') { sb.append("oe"); } else { sb.append(c); } } else { sb.append(c); } } int l = sb.length(); if (l > 3 && sb.charAt(l - 1) == 's') { sb.setLength(l - 1); } String word = sb.toString(); if (stopWords.contains(word)) { return null; } return word; } }
FullTextUtils { public static String parseWord(String string, boolean removeDiacritics) { int len = string.length(); if (len < MIN_SIZE) { return null; } StringBuilder sb = new StringBuilder(len); for (int i = 0; i < len; i++) { char c = Character.toLowerCase(string.charAt(i)); if (removeDiacritics) { if (c == '\u00e6') { sb.append("ae"); } else if (c >= '\u00e0' && c <= '\u00ff') { sb.append(UNACCENTED.charAt((c) - 0xe0)); } else if (c == '\u0153') { sb.append("oe"); } else { sb.append(c); } } else { sb.append(c); } } int l = sb.length(); if (l > 3 && sb.charAt(l - 1) == 's') { sb.setLength(l - 1); } String word = sb.toString(); if (stopWords.contains(word)) { return null; } return word; } private  FullTextUtils(); }
FullTextUtils { public static String parseWord(String string, boolean removeDiacritics) { int len = string.length(); if (len < MIN_SIZE) { return null; } StringBuilder sb = new StringBuilder(len); for (int i = 0; i < len; i++) { char c = Character.toLowerCase(string.charAt(i)); if (removeDiacritics) { if (c == '\u00e6') { sb.append("ae"); } else if (c >= '\u00e0' && c <= '\u00ff') { sb.append(UNACCENTED.charAt((c) - 0xe0)); } else if (c == '\u0153') { sb.append("oe"); } else { sb.append(c); } } else { sb.append(c); } } int l = sb.length(); if (l > 3 && sb.charAt(l - 1) == 's') { sb.setLength(l - 1); } String word = sb.toString(); if (stopWords.contains(word)) { return null; } return word; } private  FullTextUtils(); static Set<String> parseFullText(String string, boolean removeDiacritics); static String parseWord(String string, boolean removeDiacritics); }
FullTextUtils { public static String parseWord(String string, boolean removeDiacritics) { int len = string.length(); if (len < MIN_SIZE) { return null; } StringBuilder sb = new StringBuilder(len); for (int i = 0; i < len; i++) { char c = Character.toLowerCase(string.charAt(i)); if (removeDiacritics) { if (c == '\u00e6') { sb.append("ae"); } else if (c >= '\u00e0' && c <= '\u00ff') { sb.append(UNACCENTED.charAt((c) - 0xe0)); } else if (c == '\u0153') { sb.append("oe"); } else { sb.append(c); } } else { sb.append(c); } } int l = sb.length(); if (l > 3 && sb.charAt(l - 1) == 's') { sb.setLength(l - 1); } String word = sb.toString(); if (stopWords.contains(word)) { return null; } return word; } private  FullTextUtils(); static Set<String> parseFullText(String string, boolean removeDiacritics); static String parseWord(String string, boolean removeDiacritics); static final Pattern wordPattern; static final int MIN_SIZE; static final String STOP_WORDS; static final Set<String> stopWords; static final String UNACCENTED; }