LayoutTokensUtil { protected static boolean doesRequireDehypenisation(List<LayoutToken> tokens, int i) { boolean forward = false; boolean backward = false; int j = i + 1; int breakLine = 0; int spacesAfter = 0; double coordinateY = tokens.get(i).getY(); while (j < tokens.size() && (tokens.get(j).getText().equals(" ") || tokens.get(j).getText().equals("\n"))) { if (tokens.get(j).getText().equals("\n")) { breakLine++; } else if (tokens.get(j).getText().equals(" ")) { spacesAfter++; } else if (tokens.get(j).getY() > coordinateY) { breakLine++; } j++; } if (breakLine == 0) { if (j < tokens.size() && tokens.get(j).getY() == coordinateY) { return false; } } if (j < tokens.size()) { forward = StringUtils.isAllLowerCase(tokens.get(j).getText()); if (forward) { if (i < 1) { return forward; } if (tokens.get(j).getY() > coordinateY) { return forward; } int z = i - 1; while (z > 0 && (tokens.get(z).getText().equals(" ") || tokens.get(z).getText().equals("\n"))) { z--; } if (StringUtils.isAlpha(tokens.get(z).getText())) { if (tokens.get(z).getY() < coordinateY) { backward = true; } else if(coordinateY == -1 && breakLine > 0) { backward = true; } } } } return backward; }  static List<LayoutToken> enrichWithNewLineInfo(List<LayoutToken> toks); static String normalizeText(String text); static String normalizeText(List<LayoutToken> tokens); static String normalizeDehyphenizeText(List<LayoutToken> tokens); static String toText(List<LayoutToken> tokens); static boolean noCoords(LayoutToken t); static boolean spaceyToken(String tok); static boolean newLineToken(String tok); static boolean containsToken(List<LayoutToken> toks, String text); static int tokenPos(List<LayoutToken> toks, String text); static int tokenPos(List<LayoutToken> toks, Pattern p); static List<List<LayoutToken>> split(List<LayoutToken> toks, Pattern p, boolean preserveSeparator); static List<List<LayoutToken>> split(List<LayoutToken> toks, Pattern p, boolean preserveSeparator, boolean preserveLeftOvers); static boolean tooFarAwayVertically(List<BoundingBox> boxes, double distance); static String getCoordsString(List<LayoutToken> toks); static String getCoordsStringForOneBox(List<LayoutToken> toks); static List<LayoutToken> dehyphenize(List<LayoutToken> tokens); static List<LayoutToken> subListByOffset(List<LayoutToken> token, int startIncluded); static List<LayoutToken> subListByOffset(List<LayoutToken> token, int startIncluded, int endExcluded); static List<LayoutToken> getLayoutTokensForTokenizedText(List<String> tokens); static final Function<LayoutToken, String> TO_TEXT_FUNCTION; }
@Test public void testDoesRequireDehyphenization_shouldReturnTrue() throws Exception { String input = "The study of iron-based supercondu- \n" + "ctors superconductivity in the iron-pnictide LaFeAsO 1-x F x has been expanding and has \n"; List<LayoutToken> layoutTokens = GrobidAnalyzer.getInstance().tokenizeWithLayoutToken(input); assertThat(LayoutTokensUtil.doesRequireDehypenisation(layoutTokens, 11), is(true)); }
@Test public void testDoesRequireDehyphenization2_shouldReturnTrue() throws Exception { String input = "The study of iron-based supercondu - \n" + "ctors superconductivity in the iron-pnictide LaFeAsO 1-x F x has been expanding and has \n"; List<LayoutToken> layoutTokens = GrobidAnalyzer.getInstance().tokenizeWithLayoutToken(input); assertThat(LayoutTokensUtil.doesRequireDehypenisation(layoutTokens, 12), is(true)); }
@Test public void testDoesRequireDehyphenization_composedWords_shouldReturnFalse() throws Exception { String input = "The study of iron-based supercondu - \n" + "ctors superconductivity in the iron-pnictide LaFeAsO 1-x F x has been expanding and has \n"; List<LayoutToken> layoutTokens = GrobidAnalyzer.getInstance().tokenizeWithLayoutToken(input); assertThat(LayoutTokensUtil.doesRequireDehypenisation(layoutTokens, 7), is(false)); assertThat(LayoutTokensUtil.doesRequireDehypenisation(layoutTokens, 24), is(false)); }
@Test public void testDoesRequireDehyphenization2_composedWords_shouldReturnFalse() throws Exception { String input = "The study of iron- based supercondu - \n" + "ctors superconductivity in the iron-pnictide LaFeAsO 1-x F x has been expanding and has \n"; List<LayoutToken> layoutTokens = GrobidAnalyzer.getInstance().tokenizeWithLayoutToken(input); assertThat(LayoutTokensUtil.doesRequireDehypenisation(layoutTokens, 7), is(false)); }
@Test public void testDoesRequireDehyphenization3_composedWords_shouldReturnFalse() throws Exception { String input = "The study of iron - based supercondu - \n" + "ctors superconductivity in the iron-pnictide LaFeAsO 1-x F x has been expanding and has \n"; List<LayoutToken> layoutTokens = GrobidAnalyzer.getInstance().tokenizeWithLayoutToken(input); assertThat(LayoutTokensUtil.doesRequireDehypenisation(layoutTokens, 8), is(false)); }
@Test public void testDoesRequireDehyphenization_usingCoordinates_shouldReturnTrue() throws Exception { String input = "The study of iron-based supercondu - " + "ctors superconductivity in the iron-pnictide LaFeAsO 1-x F x has been expanding and has \n"; List<LayoutToken> layoutTokens = GrobidAnalyzer.getInstance().tokenizeWithLayoutToken(input); IntStream.range(0, 15).forEach(i -> layoutTokens.get(i).setY(10)); IntStream.range(15, layoutTokens.size()).forEach(i -> layoutTokens.get(i).setY(30)); assertThat(LayoutTokensUtil.doesRequireDehypenisation(layoutTokens, 12), is(true)); }
@Test public void testDoesRequireDehyphenization_hypenAtEndOfString_shouldReturnFalse() throws Exception { String input = "The study of iron-based supercondu-"; List<LayoutToken> layoutTokens = GrobidAnalyzer.getInstance().tokenizeWithLayoutToken(input); assertThat(LayoutTokensUtil.doesRequireDehypenisation(layoutTokens, 11), is(false)); }