TextUtilities { @Deprecated public static String dehyphenizeHard(String text) { if (text == null) return null; String res = ""; text.replaceAll("\n", SPACE); StringTokenizer st = new StringTokenizer(text, "-"); boolean hyphen = false; boolean failure = false; String lastToken = null; while (st.hasMoreTokens()) { String section = st.nextToken().trim(); if (hyphen) { StringTokenizer st2 = new StringTokenizer(section, " ,.);!"); if (st2.countTokens() > 0) { String firstToken = st2.nextToken(); String hyphenToken = lastToken + firstToken; Lexicon lex = Lexicon.getInstance(); if (lex.inDictionary(hyphenToken.toLowerCase()) & !(test_digit(hyphenToken))) { res += firstToken; section = section.substring(firstToken.length(), section.length()); } else { res += "-"; failure = true; } } else { res += "-"; } hyphen = false; } hyphen = true; lastToken = getLastToken(section); if (failure) { res += section; failure = false; } else res += SPACE + section; } res = res.replace(" . ", ". "); res = res.replace(" ", SPACE); return res.trim(); }  static String shadowNumbers(String string); @Deprecated static List<LayoutToken> dehyphenize(List<LayoutToken> tokens); static String dehyphenize(String text); static String getLastToken(String section); static String getFirstToken(String section); @Deprecated static String dehyphenizeHard(String text); static int getLevenshteinDistance(String s, String t); final static void appendN(StringBuffer buffer, char c, int nb); final static String removeAccents(String input); final static String cleanField(String input0, boolean applyStopwordsFilter); final static List<String> segment(String input, String segments); static String HTMLEncode(String string); static String HTMLEncode(String string, boolean fullHTML); static String normalizeRegex(String string); static String convertStreamToString(InputStream is); static int countDigit(String text); static String clean(String token); static String formatTwoDecimals(double d); static String formatFourDecimals(double d); static boolean isAllUpperCase(String text); static boolean isAllLowerCase(String text); static List<String> generateEmailVariants(String firstName, String lastName); static String capitalizeFully(String input, String delimiters); static String wordShape(String word); static String wordShapeTrimmed(String word); static String punctuationProfile(String line); static int getNbTokens(String line, int currentLinePos, List<String> tokenization); static String trimEncodedCharaters(String string); static boolean filterLine(String line); static StringBuilder replaceAll(StringBuilder sb, String regex, String replacement); static String prefix(String s, int count); static String suffix(String s, int count); static String JSONEncode(String json); static String strrep(char c, int times); static int getOccCount(String term, String string); static boolean test_digit(String tok); static boolean isAllUpperCaseOrDigitOrDot(String text); static final String punctuations; static final String fullPunctuations; static final String restrictedPunctuations; static String delimiters; static final String OR; static final String NEW_LINE; static final String SPACE; static final String COMMA; static final String QUOTE; static final String END_BRACKET; static final String START_BRACKET; static final String SHARP; static final String COLON; static final String DOUBLE_QUOTE; static final String ESC_DOUBLE_QUOTE; static final String LESS_THAN; static final String ESC_LESS_THAN; static final String GREATER_THAN; static final String ESC_GREATER_THAN; static final String AND; static final String ESC_AND; static final String SLASH; static final Pattern ORCIDPattern; static final Pattern DOIPattern; static final Pattern arXivPattern; static final Pattern pmidPattern; static final Pattern pmcidPattern; static final Pattern urlPattern; static final Pattern emailPattern; final static List<String> stopwords; }
@Ignore @Test public void testDephynizationHard_withoutSpaces() { assertThat(TextUtilities.dehyphenizeHard("This is hype-\nnized.We are here."), is("This is hypenized.We are here.")); assertThat(TextUtilities.dehyphenizeHard("This is hype-\nnized. We are here."), is("This is hypenized. We are here.")); }
@Ignore @Test public void testDephynizationHard_withSpaces() { assertThat(TextUtilities.dehyphenizeHard("This is hype- \n nized. We are here."), is("This is hypenyzed. We are here.")); assertThat(TextUtilities.dehyphenizeHard("This is hype- \nnized. We are here."), is("This is hypenyzed. We are here.")); assertThat(TextUtilities.dehyphenizeHard("This is hype - \n nized. We are here."), is("This is hypenyzed. We are here.")); }
@Ignore @Test public void testDephynizationHard_withDigits_shouldNotDephypenize() { assertThat(TextUtilities.dehyphenizeHard("This is 1234-\n44A. Patent."), is("This is 1234-44A. Patent.")); assertThat(TextUtilities.dehyphenizeHard("This is 1234 - \n44A. Patent."), is("This is 1234 - 44A.Patent.")); }
@Ignore @Test public void testDephynizationHard_citation() { assertThat(TextUtilities.dehyphenizeHard("Anonymous. Runtime process infection. Phrack, 11(59):ar-\n+ " + " ticle 8 of 18, December 2002."), is("Anonymous. Runtime process infection. Phrack, 11(59):article 8 of 18, December 2002.")); }