@Test public void testNakaguroSplit() { Tokenizer defaultTokenizer = new Tokenizer(); Tokenizer nakakuroSplittingTokenizer = new Tokenizer.Builder().isSplitOnNakaguro(true).build(); String input = "ラレ・プールカリムの音楽が好き。"; assertTokenSurfacesEquals(Arrays.asList("ラレ・プールカリム", "の", "音楽", "が", "好き", "。"), defaultTokenizer.tokenize(input)); assertTokenSurfacesEquals(Arrays.asList("ラレ", "・", "プールカリム", "の", "音楽", "が", "好き", "。"), nakakuroSplittingTokenizer.tokenize(input)); }
@Override public List<Token> tokenize(String text) { return createTokenList(text); }
Tokenizer extends TokenizerBase { @Override public List<Token> tokenize(String text) { return createTokenList(text); } }
Tokenizer extends TokenizerBase { @Override public List<Token> tokenize(String text) { return createTokenList(text); } Tokenizer(); private  Tokenizer(Builder builder); }
Tokenizer extends TokenizerBase { @Override public List<Token> tokenize(String text) { return createTokenList(text); } Tokenizer(); private  Tokenizer(Builder builder); @Override List<Token> tokenize(String text); }
Tokenizer extends TokenizerBase { @Override public List<Token> tokenize(String text) { return createTokenList(text); } Tokenizer(); private  Tokenizer(Builder builder); @Override List<Token> tokenize(String text);  }