TextUtilities { @Deprecated protected static boolean doesRequireDehypenisation(List<LayoutToken> tokens, int i) { return LayoutTokensUtil.doesRequireDehypenisation(tokens, i); }  static String shadowNumbers(String string); @Deprecated static List<LayoutToken> dehyphenize(List<LayoutToken> tokens); static String dehyphenize(String text); static String getLastToken(String section); static String getFirstToken(String section); @Deprecated static String dehyphenizeHard(String text); static int getLevenshteinDistance(String s, String t); final static void appendN(StringBuffer buffer, char c, int nb); final static String removeAccents(String input); final static String cleanField(String input0, boolean applyStopwordsFilter); final static List<String> segment(String input, String segments); static String HTMLEncode(String string); static String HTMLEncode(String string, boolean fullHTML); static String normalizeRegex(String string); static String convertStreamToString(InputStream is); static int countDigit(String text); static String clean(String token); static String formatTwoDecimals(double d); static String formatFourDecimals(double d); static boolean isAllUpperCase(String text); static boolean isAllLowerCase(String text); static List<String> generateEmailVariants(String firstName, String lastName); static String capitalizeFully(String input, String delimiters); static String wordShape(String word); static String wordShapeTrimmed(String word); static String punctuationProfile(String line); static int getNbTokens(String line, int currentLinePos, List<String> tokenization); static String trimEncodedCharaters(String string); static boolean filterLine(String line); static StringBuilder replaceAll(StringBuilder sb, String regex, String replacement); static String prefix(String s, int count); static String suffix(String s, int count); static String JSONEncode(String json); static String strrep(char c, int times); static int getOccCount(String term, String string); static boolean test_digit(String tok); static boolean isAllUpperCaseOrDigitOrDot(String text); static final String punctuations; static final String fullPunctuations; static final String restrictedPunctuations; static String delimiters; static final String OR; static final String NEW_LINE; static final String SPACE; static final String COMMA; static final String QUOTE; static final String END_BRACKET; static final String START_BRACKET; static final String SHARP; static final String COLON; static final String DOUBLE_QUOTE; static final String ESC_DOUBLE_QUOTE; static final String LESS_THAN; static final String ESC_LESS_THAN; static final String GREATER_THAN; static final String ESC_GREATER_THAN; static final String AND; static final String ESC_AND; static final String SLASH; static final Pattern ORCIDPattern; static final Pattern DOIPattern; static final Pattern arXivPattern; static final Pattern pmidPattern; static final Pattern pmcidPattern; static final Pattern urlPattern; static final Pattern emailPattern; final static List<String> stopwords; }
@Test public void testDoesRequireDehypenisation_trickySequence7_shouldReturnTrue() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("-\ncore is a bad sample."); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 0), is(true)); }
@Test public void testDoesRequireDehypenisation_trickySequence8_shouldReturnTrue() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("- \n\n core is a bad sample."); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 0), is(true)); }
@Test public void testDoesRequireDehypenisation_falseFriend1_shouldReturnTrue() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("which was mediated through the inhibition of expression of α2-\n integrin (1,2). "); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 19), is(false)); }
@Test public void testDoesRequireDehypenisation_falseFriend2_shouldReturnTrue() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("which was mediated through the inhibition of expression of α2 -\n integrin (1,2). "); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 19), is(false)); }
@Test public void testDoesRequireDehypenisation_falseFriend3_shouldReturnTrue() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("which was mediated through the inhibition of expression of α 2 - \n integrin (1,2). "); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 19), is(false)); }
@Test public void testDoesRequireDehypenisation_standard_shouldReturnTrue() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a sample dehypen-\nyzation text"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 9), is(true)); }
@Test public void testDoesRequireDehypenisation_withSpaceAfter_shouldReturnTrue() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a sample dehypen- \nyzation text"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 9), is(true)); }
@Test public void testDoesRequireDehypenisation_withSpacesAfter_shouldReturnTrue() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a sample dehypen- \n yzation text"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 9), is(true)); }
@Test public void testDoesRequireDehypenisation_withSpaceBefore_shouldReturnTrue() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a sample dehypen -\nyzation text"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 10), is(true)); }
@Test public void testDoesRequireDehypenisation_withSpacesBefore_shouldReturnTrue() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a sample dehypen -\nyzation text"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 13), is(true)); }
@Test public void testDoesRequireDehypenisation_usualWord_shouldReturnFalse() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a sample open-source text"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 9), is(false)); }
@Test public void testDoesRequireDehypenisation_usualWordWithSpace_shouldReturnFalse() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a sample open- source text"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 9), is(false)); }
@Test public void testDoesRequireDehypenisation_usualWordWith2Space_shouldReturnFalse() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a sample open - source text"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 9), is(false)); }
@Test public void testDoesRequireDehypenisation_sequence_shouldReturnFalse() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a sample ABC123-3434 text"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 9), is(false)); }
@Test public void testDoesRequireDehypenisation_sequence2_shouldReturnFalse() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a sample ABC123 - 3434 text"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 9), is(false)); }
@Test public void testDoesRequireDehypenisation_sequence3_shouldReturnFalse() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a sample ABC123 - 3434 text"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 9), is(false)); }
@Test public void testDoesRequireDehypenisation_trickySequence1_shouldReturnFalse() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a bad sample -\n\n"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 10), is(false)); }
@Test public void testDoesRequireDehypenisation_trickySequence2_shouldReturnFalse() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a bad sample -"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 10), is(false)); }
@Test public void testDoesRequireDehypenisation_trickySequence3_shouldReturnFalse() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a bad sample - \n\n "); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 10), is(false)); }
@Test public void testDoesRequireDehypenisation_trickySequence4_shouldReturnFalse() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a bad sample-\n\n"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 9), is(false)); }
@Test public void testDoesRequireDehypenisation_trickySequence5_shouldReturnFalse() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a bad sample-"); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 9), is(false)); }
@Test public void testDoesRequireDehypenisation_trickySequence6_shouldReturnFalse() { List<LayoutToken> tokens = GrobidDefaultAnalyzer.getInstance().tokenizeWithLayoutToken("This is a bad sample- \n\n "); assertThat(TextUtilities.doesRequireDehypenisation(tokens, 9), is(false)); }