TextUtilities { @Deprecated public static List<LayoutToken> dehyphenize(List<LayoutToken> tokens) { return LayoutTokensUtil.dehyphenize(tokens); }  static String shadowNumbers(String string); @Deprecated static List<LayoutToken> dehyphenize(List<LayoutToken> tokens); static String dehyphenize(String text); static String getLastToken(String section); static String getFirstToken(String section); @Deprecated static String dehyphenizeHard(String text); static int getLevenshteinDistance(String s, String t); final static void appendN(StringBuffer buffer, char c, int nb); final static String removeAccents(String input); final static String cleanField(String input0, boolean applyStopwordsFilter); final static List<String> segment(String input, String segments); static String HTMLEncode(String string); static String HTMLEncode(String string, boolean fullHTML); static String normalizeRegex(String string); static String convertStreamToString(InputStream is); static int countDigit(String text); static String clean(String token); static String formatTwoDecimals(double d); static String formatFourDecimals(double d); static boolean isAllUpperCase(String text); static boolean isAllLowerCase(String text); static List<String> generateEmailVariants(String firstName, String lastName); static String capitalizeFully(String input, String delimiters); static String wordShape(String word); static String wordShapeTrimmed(String word); static String punctuationProfile(String line); static int getNbTokens(String line, int currentLinePos, List<String> tokenization); static String trimEncodedCharaters(String string); static boolean filterLine(String line); static StringBuilder replaceAll(StringBuilder sb, String regex, String replacement); static String prefix(String s, int count); static String suffix(String s, int count); static String JSONEncode(String json); static String strrep(char c, int times); static int getOccCount(String term, String string); static boolean test_digit(String tok); static boolean isAllUpperCaseOrDigitOrDot(String text); static final String punctuations; static final String fullPunctuations; static final String restrictedPunctuations; static String delimiters; static final String OR; static final String NEW_LINE; static final String SPACE; static final String COMMA; static final String QUOTE; static final String END_BRACKET; static final String START_BRACKET; static final String SHARP; static final String COLON; static final String DOUBLE_QUOTE; static final String ESC_DOUBLE_QUOTE; static final String LESS_THAN; static final String ESC_LESS_THAN; static final String GREATER_THAN; static final String ESC_GREATER_THAN; static final String AND; static final String ESC_AND; static final String SLASH; static final Pattern ORCIDPattern; static final Pattern DOIPattern; static final Pattern arXivPattern; static final Pattern pmidPattern; static final Pattern pmcidPattern; static final Pattern urlPattern; static final Pattern emailPattern; final static List<String> stopwords; }
@Test public void testDephynization_withoutSpaces() { assertThat(TextUtilities.dehyphenize("This is hype-\nnized.We are here."), is("This is hypenized.We are here.")); assertThat(TextUtilities.dehyphenize("This is hype-\nnized. We are here."), is("This is hypenized. We are here.")); }
@Test public void testDephynization_withSpaces() { assertThat(TextUtilities.dehyphenize("This is hype- \n nized. We are here."), is("This is hypenized. We are here.")); assertThat(TextUtilities.dehyphenize("This is hype- \nnized. We are here."), is("This is hypenized. We are here.")); assertThat(TextUtilities.dehyphenize("This is hype - \n nized. We are here."), is("This is hypenized. We are here.")); }
@Test public void testDephynization_withDigits_shouldNotDephypenize() { assertThat(TextUtilities.dehyphenize("This is 1234-44A. Patent."), is("This is 1234-44A. Patent.")); assertThat(TextUtilities.dehyphenize("This is 1234 - 44A. Patent."), is("This is 1234 - 44A. Patent.")); }
@Test public void testDephynization_citation() { assertThat(TextUtilities.dehyphenize("Anonymous. Runtime process infection. Phrack, 11(59):ar-\n" + " ticle 8 of 18, December 2002."), is("Anonymous. Runtime process infection. Phrack, 11(59):article 8 of 18, December 2002.")); }
@Test public void testDephynization_falseTruncation_shouldReturnSameString() { assertThat(TextUtilities.dehyphenize("sd. Linux on-the-fly kernel patching without lkm. Phrack, 11(58):article 7 of 15, December 2001."), is("sd. Linux on-the-fly kernel patching without lkm. Phrack, 11(58):article 7 of 15, December 2001.")); }
@Test public void testDephynization_FalseTruncation_shouldReturnSameString() { assertThat(TextUtilities.dehyphenize("Nettop also relies on VMware Workstation for its VMM. Ultimately, since VMware is a closed-source product, it is impossible to verify this claim through open review."), is("Nettop also relies on VMware Workstation for its VMM. Ultimately, since VMware is a closed-source product, it is impossible to verify this claim through open review.")); }
@Test public void testDephynization_NormalCase() { assertThat(TextUtilities.dehyphenize("Implementation bugs in the VMM can compromise its ability to provide secure isolation, and modify-\n ing the VMM presents the risk of introducing bugs."), is("Implementation bugs in the VMM can compromise its ability to provide secure isolation, and modifying the VMM presents the risk of introducing bugs.")); }
@Test public void testDehyphenizationWithLayoutTokens() throws Exception { List<String> tokens = GrobidAnalyzer.getInstance().tokenize("This is hype-\n nized."); List<LayoutToken> layoutTokens = new ArrayList<>(); for (String token : tokens) { if (token.equals("\n")) { layoutTokens.get(layoutTokens.size() - 1).setNewLineAfter(true); } layoutTokens.add(new LayoutToken(token)); } List<LayoutToken> output = TextUtilities.dehyphenize(layoutTokens); assertNotNull(output); assertThat(LayoutTokensUtil.toText(output), is("This is hypenized.")); }