@Test public void testSimpleSegmentation() { String input = "スペースステーションに行きます。うたがわしい。"; String[] surfaces = {"スペース", "ステーション", "に", "行き", "ます", "。", "うたがわしい", "。"}; List<Token> tokens = tokenizer.tokenize(input); assertTrue(tokens.size() == surfaces.length); for (int i = 0; i < tokens.size(); i++) { assertEquals(surfaces[i], tokens.get(i).getSurface()); } }
@Override public List<Token> tokenize(String text) { return createTokenList(text); }
Tokenizer extends TokenizerBase { @Override public List<Token> tokenize(String text) { return createTokenList(text); } }
Tokenizer extends TokenizerBase { @Override public List<Token> tokenize(String text) { return createTokenList(text); } Tokenizer(); private  Tokenizer(Builder builder); }
Tokenizer extends TokenizerBase { @Override public List<Token> tokenize(String text) { return createTokenList(text); } Tokenizer(); private  Tokenizer(Builder builder); @Override List<Token> tokenize(String text); }
Tokenizer extends TokenizerBase { @Override public List<Token> tokenize(String text) { return createTokenList(text); } Tokenizer(); private  Tokenizer(Builder builder); @Override List<Token> tokenize(String text);  }