@Test public void testNGramTokenizer() throws Exception { String toTokenize = "Mary had a little lamb."; TokenizerFactory factory = new NGramTokenizerFactory(new DefaultTokenizerFactory(), 1, 2); Tokenizer tokenizer = factory.create(toTokenize); Tokenizer tokenizer2 = factory.create(toTokenize); while (tokenizer.hasMoreTokens()) { assertEquals(tokenizer.nextToken(), tokenizer2.nextToken()); } int stringCount = factory.create(toTokenize).countTokens(); List<String> tokens = factory.create(toTokenize).getTokens(); assertEquals(9, stringCount); assertTrue(tokens.contains("Mary")); assertTrue(tokens.contains("had")); assertTrue(tokens.contains("a")); assertTrue(tokens.contains("little")); assertTrue(tokens.contains("lamb.")); assertTrue(tokens.contains("Mary had")); assertTrue(tokens.contains("had a")); assertTrue(tokens.contains("a little")); assertTrue(tokens.contains("little lamb.")); factory = new NGramTokenizerFactory(new DefaultTokenizerFactory(), 2, 2); tokens = factory.create(toTokenize).getTokens(); assertEquals(4, tokens.size()); assertTrue(tokens.contains("Mary had")); assertTrue(tokens.contains("had a")); assertTrue(tokens.contains("a little")); assertTrue(tokens.contains("little lamb.")); }
public NGramTokenizer(Tokenizer tokenizer, Integer minN, Integer maxN) { this.tokens = new ArrayList<>(); while (tokenizer.hasMoreTokens()) { String nextToken = tokenizer.nextToken(); this.tokens.add(nextToken); } if (maxN != 1) { this.originalTokens = this.tokens; this.tokens = new ArrayList<>(); Integer nOriginalTokens = this.originalTokens.size(); Integer min = Math.min(maxN + 1, nOriginalTokens + 1); for (int i = minN; i < min; i++) { for (int j = 0; j < nOriginalTokens - i + 1; j++) { List<String> originalTokensSlice = this.originalTokens.subList(j, j + i); this.tokens.add(StringUtils.join(originalTokensSlice, " ")); } } } }
NGramTokenizer implements Tokenizer { public NGramTokenizer(Tokenizer tokenizer, Integer minN, Integer maxN) { this.tokens = new ArrayList<>(); while (tokenizer.hasMoreTokens()) { String nextToken = tokenizer.nextToken(); this.tokens.add(nextToken); } if (maxN != 1) { this.originalTokens = this.tokens; this.tokens = new ArrayList<>(); Integer nOriginalTokens = this.originalTokens.size(); Integer min = Math.min(maxN + 1, nOriginalTokens + 1); for (int i = minN; i < min; i++) { for (int j = 0; j < nOriginalTokens - i + 1; j++) { List<String> originalTokensSlice = this.originalTokens.subList(j, j + i); this.tokens.add(StringUtils.join(originalTokensSlice, " ")); } } } } }
NGramTokenizer implements Tokenizer { public NGramTokenizer(Tokenizer tokenizer, Integer minN, Integer maxN) { this.tokens = new ArrayList<>(); while (tokenizer.hasMoreTokens()) { String nextToken = tokenizer.nextToken(); this.tokens.add(nextToken); } if (maxN != 1) { this.originalTokens = this.tokens; this.tokens = new ArrayList<>(); Integer nOriginalTokens = this.originalTokens.size(); Integer min = Math.min(maxN + 1, nOriginalTokens + 1); for (int i = minN; i < min; i++) { for (int j = 0; j < nOriginalTokens - i + 1; j++) { List<String> originalTokensSlice = this.originalTokens.subList(j, j + i); this.tokens.add(StringUtils.join(originalTokensSlice, " ")); } } } } NGramTokenizer(Tokenizer tokenizer, Integer minN, Integer maxN); }
NGramTokenizer implements Tokenizer { public NGramTokenizer(Tokenizer tokenizer, Integer minN, Integer maxN) { this.tokens = new ArrayList<>(); while (tokenizer.hasMoreTokens()) { String nextToken = tokenizer.nextToken(); this.tokens.add(nextToken); } if (maxN != 1) { this.originalTokens = this.tokens; this.tokens = new ArrayList<>(); Integer nOriginalTokens = this.originalTokens.size(); Integer min = Math.min(maxN + 1, nOriginalTokens + 1); for (int i = minN; i < min; i++) { for (int j = 0; j < nOriginalTokens - i + 1; j++) { List<String> originalTokensSlice = this.originalTokens.subList(j, j + i); this.tokens.add(StringUtils.join(originalTokensSlice, " ")); } } } } NGramTokenizer(Tokenizer tokenizer, Integer minN, Integer maxN); @Override boolean hasMoreTokens(); @Override int countTokens(); @Override String nextToken(); @Override List<String> getTokens(); @Override void setTokenPreProcessor(TokenPreProcess tokenPreProcessor); }
NGramTokenizer implements Tokenizer { public NGramTokenizer(Tokenizer tokenizer, Integer minN, Integer maxN) { this.tokens = new ArrayList<>(); while (tokenizer.hasMoreTokens()) { String nextToken = tokenizer.nextToken(); this.tokens.add(nextToken); } if (maxN != 1) { this.originalTokens = this.tokens; this.tokens = new ArrayList<>(); Integer nOriginalTokens = this.originalTokens.size(); Integer min = Math.min(maxN + 1, nOriginalTokens + 1); for (int i = minN; i < min; i++) { for (int j = 0; j < nOriginalTokens - i + 1; j++) { List<String> originalTokensSlice = this.originalTokens.subList(j, j + i); this.tokens.add(StringUtils.join(originalTokensSlice, " ")); } } } } NGramTokenizer(Tokenizer tokenizer, Integer minN, Integer maxN); @Override boolean hasMoreTokens(); @Override int countTokens(); @Override String nextToken(); @Override List<String> getTokens(); @Override void setTokenPreProcessor(TokenPreProcess tokenPreProcessor);  }