@Test public void testNGramTermGenerator() throws Exception { JCas cas = JCasFactory.createJCas("de.julielab.jcore.types.jcore-all-types"); cas.setDocumentText("1234567"); Annotation a = new Annotation(cas, 0, 7); IndexTermGenerator<String> unigramGenerator = TermGenerators.nGramTermGenerator(1); assertEquals("1 2 3 4 5 6 7", unigramGenerator.asStream(a).reduce(concatentor).get()); IndexTermGenerator<String> bigramGenerator = TermGenerators.nGramTermGenerator(2); assertEquals("12 23 34 45 56 67", bigramGenerator.asStream(a).reduce(concatentor).get()); IndexTermGenerator<String> trigramGenerator = TermGenerators.nGramTermGenerator(3); assertEquals("123 234 345 456 567", trigramGenerator.asStream(a).reduce(concatentor).get()); }
public static IndexTermGenerator<String> nGramTermGenerator(int n) { return a -> { String text = a.getCoveredText(); return IntStream.range(0, text.length()).mapToObj(i -> { if (i + n <= text.length()) return text.substring(i, i + n); return null; }).filter(s -> s != null); }; }
TermGenerators { public static IndexTermGenerator<String> nGramTermGenerator(int n) { return a -> { String text = a.getCoveredText(); return IntStream.range(0, text.length()).mapToObj(i -> { if (i + n <= text.length()) return text.substring(i, i + n); return null; }).filter(s -> s != null); }; } }
TermGenerators { public static IndexTermGenerator<String> nGramTermGenerator(int n) { return a -> { String text = a.getCoveredText(); return IntStream.range(0, text.length()).mapToObj(i -> { if (i + n <= text.length()) return text.substring(i, i + n); return null; }).filter(s -> s != null); }; }  }
TermGenerators { public static IndexTermGenerator<String> nGramTermGenerator(int n) { return a -> { String text = a.getCoveredText(); return IntStream.range(0, text.length()).mapToObj(i -> { if (i + n <= text.length()) return text.substring(i, i + n); return null; }).filter(s -> s != null); }; }  static IndexTermGenerator<String> nGramTermGenerator(int n); static IndexTermGenerator<String> edgeNGramTermGenerator(int n); static IndexTermGenerator<String> prefixTermGenerator(int maxLength); static IndexTermGenerator<String> suffixTermGenerator(int maxLength); static IndexTermGenerator<String> exactPrefixTermGenerator(int length); static IndexTermGenerator<String> exactSuffixTermGenerator(int length); static LongOffsetIndexTermGenerator longOffsetTermGenerator(); }
TermGenerators { public static IndexTermGenerator<String> nGramTermGenerator(int n) { return a -> { String text = a.getCoveredText(); return IntStream.range(0, text.length()).mapToObj(i -> { if (i + n <= text.length()) return text.substring(i, i + n); return null; }).filter(s -> s != null); }; }  static IndexTermGenerator<String> nGramTermGenerator(int n); static IndexTermGenerator<String> edgeNGramTermGenerator(int n); static IndexTermGenerator<String> prefixTermGenerator(int maxLength); static IndexTermGenerator<String> suffixTermGenerator(int maxLength); static IndexTermGenerator<String> exactPrefixTermGenerator(int length); static IndexTermGenerator<String> exactSuffixTermGenerator(int length); static LongOffsetIndexTermGenerator longOffsetTermGenerator();  }