@Test public void testTokenizeWithDocumentAffiliation() { String text = "Cóż ro123bić?"; List<Token<AffiliationLabel>> expected = Arrays.asList( new Token<AffiliationLabel>("Coz", 0, 5), new Token<AffiliationLabel>("ro", 6, 8), new Token<AffiliationLabel>("123", 8, 11), new Token<AffiliationLabel>("bic", 11, 15), new Token<AffiliationLabel>("?", 15, 16) ); DocumentAffiliation instance = new DocumentAffiliation(text); List<Token<AffiliationLabel>> actual = TOKENIZER.tokenize(instance.getRawText()); assertEquals(expected, actual); }
@Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); }
AffiliationTokenizer implements TextTokenizer<Token<AffiliationLabel>> { @Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); } }
AffiliationTokenizer implements TextTokenizer<Token<AffiliationLabel>> { @Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); }  }
AffiliationTokenizer implements TextTokenizer<Token<AffiliationLabel>> { @Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); }  @Override List<Token<AffiliationLabel>> tokenize(String text); }
AffiliationTokenizer implements TextTokenizer<Token<AffiliationLabel>> { @Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); }  @Override List<Token<AffiliationLabel>> tokenize(String text);  }