UnicodeUtil { public static String normaliseText(String text) { if (text == null) return null; text = text.replaceAll(my_whitespace_chars, " "); text = text.replace("\r\n", "\n").replaceAll(new_line_chars, "\n"); text = text.replaceAll("\\p{Pd}", "-"); text = text.replaceAll(horizontal_low_lines_chars, "_"); text = text.replaceAll(vertical_lines_chars, "|"); text = text.replaceAll(bullet_chars, "•"); text = text.replaceAll(open_parenthesis, "("); text = text.replaceAll(close_parenthesis, ")"); return text; }  static String normaliseText(String text); static String normaliseTextAndRemoveSpaces(String text); static String whitespace_chars; static String my_whitespace_chars; static String horizontal_low_lines_chars; static String vertical_lines_chars; static String new_line_chars; static String bullet_chars; static String open_parenthesis; static String close_parenthesis; }
@Test public void testNormaliseToken() throws Exception { String test = "´\rÓÑÔÙØØØ"; String result = UnicodeUtil.normaliseText(test); assertThat("´\nÓÑÔÙØØØ", is(result)); ArrayList<String> tokens = Lists.newArrayList( "½ºº´\r", "½ºº´\n", "½ºº´\t", "½ºº´\f", "½ºº´ ", "½ºº´\f\n", "½ºº´\r\t"); for (String token : tokens) { assertEquals("½ºº´", UnicodeUtil.normaliseText(token.replace(" ", "").replace("\n", ""))); } }