@Test public void testCustomPenalties() { String input = "シニアソフトウェアエンジニアを探しています"; Tokenizer customTokenizer = new Tokenizer.Builder().mode(Tokenizer.Mode.SEARCH).kanjiPenalty(3, 10000) .otherPenalty(Integer.MAX_VALUE, 0).build(); String[] expected1 = {"シニアソフトウェアエンジニア", "を", "探し", "て", "い", "ます"}; assertTokenSurfacesEquals(Arrays.asList(expected1), customTokenizer.tokenize(input)); Tokenizer searchTokenizer = new Tokenizer.Builder().mode(Tokenizer.Mode.SEARCH).build(); String[] expected2 = {"シニア", "ソフトウェア", "エンジニア", "を", "探し", "て", "い", "ます"}; assertTokenSurfacesEquals(Arrays.asList(expected2), searchTokenizer.tokenize(input)); }
@Override public List<Token> tokenize(String text) { return createTokenList(text); }
Tokenizer extends TokenizerBase { @Override public List<Token> tokenize(String text) { return createTokenList(text); } }
Tokenizer extends TokenizerBase { @Override public List<Token> tokenize(String text) { return createTokenList(text); } Tokenizer(); private  Tokenizer(Builder builder); }
Tokenizer extends TokenizerBase { @Override public List<Token> tokenize(String text) { return createTokenList(text); } Tokenizer(); private  Tokenizer(Builder builder); @Override List<Token> tokenize(String text); }
Tokenizer extends TokenizerBase { @Override public List<Token> tokenize(String text) { return createTokenList(text); } Tokenizer(); private  Tokenizer(Builder builder); @Override List<Token> tokenize(String text);  }