@Test public void testTokenizeAscii() { String input = "ko pi_es123_@@123Kot"; List<Token<AffiliationLabel>> expected = Arrays.asList( new Token<AffiliationLabel>("ko", 0, 2), new Token<AffiliationLabel>("pi", 4, 6), new Token<AffiliationLabel>("_", 6, 7), new Token<AffiliationLabel>("es", 7, 9), new Token<AffiliationLabel>("123", 9, 12), new Token<AffiliationLabel>("_", 12, 13), new Token<AffiliationLabel>("@", 13, 14), new Token<AffiliationLabel>("@", 14, 15), new Token<AffiliationLabel>("123", 15, 18), new Token<AffiliationLabel>("Kot", 18, 21) ); List<Token<AffiliationLabel>> actual = TOKENIZER.tokenize(input); assertEquals(expected, actual); }
@Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); }
AffiliationTokenizer implements TextTokenizer<Token<AffiliationLabel>> { @Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); } }
AffiliationTokenizer implements TextTokenizer<Token<AffiliationLabel>> { @Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); }  }
AffiliationTokenizer implements TextTokenizer<Token<AffiliationLabel>> { @Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); }  @Override List<Token<AffiliationLabel>> tokenize(String text); }
AffiliationTokenizer implements TextTokenizer<Token<AffiliationLabel>> { @Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); }  @Override List<Token<AffiliationLabel>> tokenize(String text);  }