TextUtil { static public String normalize(TextIndex textIndex, String string){ if(textIndex.hasTextIndexNormalizations()){ List<TextIndexNormalization> textIndexNormalizations = textIndex.getTextIndexNormalizations(); for(TextIndexNormalization textIndexNormalization : textIndexNormalizations){ string = TextUtil.normalize(textIndex, textIndexNormalization, string); } } return string; } private  TextUtil(); static String normalize(TextIndex textIndex, String string); static String normalize(TextIndex textIndex, TextIndexNormalization textIndexNormalization, String string); static List<String> tokenize(TextIndex textIndex, String text); static int termFrequency(TextIndex textIndex, List<String> textTokens, List<String> termTokens); static Map<List<String>, Integer> termFrequencyTable(TextIndex textIndex, List<String> textTokens, Set<List<String>> termTokenSet, int maxLength); static boolean matches(TextIndex textIndex, List<String> leftTokens, List<String> rightTokens);  }
@Test public void normalize(){ List<List<String>> regexRows = Arrays.asList( Arrays.asList("[\u00c0|\u00c1|\u00c2|\u00c3|\u00c4|\u00c5]", "A", "true") ); List<List<String>> noRegexRows = Arrays.asList( Arrays.asList("\u00c0", "A", null), Arrays.asList("\u00c1", "A", null), Arrays.asList("\u00c2", "A", null), Arrays.asList("\u00c3", "A", "false"), Arrays.asList("\u00c4", "A", "false"), Arrays.asList("\u00c5", "A", "false") ); for(List<List<String>> rows : Arrays.asList(regexRows, noRegexRows)){ String text = "\u00c0BC"; assertEquals("ABC", normalize(rows, text, null, true, 0)); text = text.toLowerCase(); assertEquals(text, normalize(rows, text, null, true, 0)); assertEquals("Abc", normalize(rows, text, null, false, 0)); text = "\u00c0\u00c2\u00c4"; assertEquals("AAA", normalize(rows, text, null, true, 0)); text = text.toLowerCase(); assertEquals(text, normalize(rows, text, null, true, 0)); assertEquals("AAA", normalize(rows, text, null, false, 0)); } }