GrobidAnalyzer implements Analyzer { public List<LayoutToken> tokenizeWithLayoutToken(String text) { return tokenizeWithLayoutToken(text, null); } private  GrobidAnalyzer(); static GrobidAnalyzer getInstance(); String getName(); List<String> tokenize(String text); List<String> tokenize(String text, Language lang); List<String> retokenize(List<String> textTokenized); List<String> retokenize(List<String> textTokenized, Language lang); List<LayoutToken> tokenizeWithLayoutToken(String text); List<LayoutToken> tokenizeWithLayoutToken(String text, Language lang);  }
@Test public void testTokenizeWithLayoutToken() { final List<LayoutToken> layoutTokens = target.tokenizeWithLayoutToken("This is a normal \ntext,\n\n\n on several lines.\n"); assertThat(layoutTokens, hasSize(22)); assertThat(layoutTokens.get(0).getText(), is("This")); assertThat(layoutTokens.get(1).getText(), is(" ")); assertThat(layoutTokens.get(6).getText(), is("normal")); assertThat(layoutTokens.get(7).getText(), is(" ")); assertThat(layoutTokens.get(7).isNewLineAfter(), is(true)); assertThat(layoutTokens.get(8).getText(), is("\n")); assertThat(layoutTokens.get(8).isNewLineAfter(), is(false)); assertThat(layoutTokens.get(10).getText(), is(",")); assertThat(layoutTokens.get(10).isNewLineAfter(), is(true)); assertThat(layoutTokens.get(11).getText(), is("\n")); assertThat(layoutTokens.get(11).isNewLineAfter(), is(true)); assertThat(layoutTokens.get(12).getText(), is("\n")); assertThat(layoutTokens.get(12).isNewLineAfter(), is(true)); assertThat(layoutTokens.get(13).getText(), is("\n")); assertThat(layoutTokens.get(13).isNewLineAfter(), is(false)); }
@Test public void testTokenizeWithLayoutToken_emptyText() { assertThat(target.tokenizeWithLayoutToken(""), hasSize(0)); }