@Test public void testTokenizeNonAscii() { String input = "śćdź óó"; input = MetadataTools.cleanAndNormalize(input); List<Token<AffiliationLabel>> expected = Arrays.asList( new Token<AffiliationLabel>("scdz", 0, 7), new Token<AffiliationLabel>("oo", 8, 12) ); List<Token<AffiliationLabel>> actual = TOKENIZER.tokenize(input); assertEquals(expected, actual); }
@Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); }
AffiliationTokenizer implements TextTokenizer<Token<AffiliationLabel>> { @Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); } }
AffiliationTokenizer implements TextTokenizer<Token<AffiliationLabel>> { @Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); }  }
AffiliationTokenizer implements TextTokenizer<Token<AffiliationLabel>> { @Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); }  @Override List<Token<AffiliationLabel>> tokenize(String text); }
AffiliationTokenizer implements TextTokenizer<Token<AffiliationLabel>> { @Override public List<Token<AffiliationLabel>> tokenize(String text) { List<Integer> asciiIndices = getAsciiSubstringIndices(text); String asciiText = getSubstring(text, asciiIndices); asciiIndices.add(text.length()); return asciiTextToTokens(asciiText, asciiIndices); }  @Override List<Token<AffiliationLabel>> tokenize(String text);  }