@Test public void testGetNGrams() throws Exception { Collection<StringList> nGrams = NGramUtils.getNGrams(new StringList("I", "saw", "brown", "fox"), 2); Assert.assertEquals(3, nGrams.size()); nGrams = NGramUtils.getNGrams(new StringList("I", "saw", "brown", "fox"), 3); Assert.assertEquals(2, nGrams.size()); }
public static Collection<StringList> getNGrams(StringList sequence, int size) { Collection<StringList> ngrams = new LinkedList<>(); if (size == -1 || size >= sequence.size()) { ngrams.add(sequence); } else { String[] ngram = new String[size]; for (int i = 0; i < sequence.size() - size + 1; i++) { ngram[0] = sequence.getToken(i); for (int j = 1; j < size; j++) { ngram[j] = sequence.getToken(i + j); } ngrams.add(new StringList(ngram)); } } return ngrams; }
NGramUtils { public static Collection<StringList> getNGrams(StringList sequence, int size) { Collection<StringList> ngrams = new LinkedList<>(); if (size == -1 || size >= sequence.size()) { ngrams.add(sequence); } else { String[] ngram = new String[size]; for (int i = 0; i < sequence.size() - size + 1; i++) { ngram[0] = sequence.getToken(i); for (int j = 1; j < size; j++) { ngram[j] = sequence.getToken(i + j); } ngrams.add(new StringList(ngram)); } } return ngrams; } }
NGramUtils { public static Collection<StringList> getNGrams(StringList sequence, int size) { Collection<StringList> ngrams = new LinkedList<>(); if (size == -1 || size >= sequence.size()) { ngrams.add(sequence); } else { String[] ngram = new String[size]; for (int i = 0; i < sequence.size() - size + 1; i++) { ngram[0] = sequence.getToken(i); for (int j = 1; j < size; j++) { ngram[j] = sequence.getToken(i + j); } ngrams.add(new StringList(ngram)); } } return ngrams; }  }
NGramUtils { public static Collection<StringList> getNGrams(StringList sequence, int size) { Collection<StringList> ngrams = new LinkedList<>(); if (size == -1 || size >= sequence.size()) { ngrams.add(sequence); } else { String[] ngram = new String[size]; for (int i = 0; i < sequence.size() - size + 1; i++) { ngram[0] = sequence.getToken(i); for (int j = 1; j < size; j++) { ngram[j] = sequence.getToken(i + j); } ngrams.add(new StringList(ngram)); } } return ngrams; }  static double calculateLaplaceSmoothingProbability(StringList ngram,
      Iterable<StringList> set, Double k); static double calculateUnigramMLProbability(String word, Collection<StringList> set); static double calculateBigramMLProbability(String x0, String x1, Collection<StringList> set); static double calculateTrigramMLProbability(String x0, String x1, String x2,
                                                     Iterable<StringList> set); static double calculateNgramMLProbability(StringList ngram, Iterable<StringList> set); static double calculateBigramPriorSmoothingProbability(String x0, String x1,
                                                                Collection<StringList> set, Double k); static double calculateTrigramLinearInterpolationProbability(String x0, String x1,
                                                                      String x2, Collection<StringList> set,
      Double lambda1, Double lambda2, Double lambda3); static double calculateMissingNgramProbabilityMass(StringList ngram, Double discount,
                                                            Iterable<StringList> set); static StringList getNMinusOneTokenFirst(StringList ngram); static StringList getNMinusOneTokenLast(StringList ngram); static Collection<StringList> getNGrams(StringList sequence, int size); static Collection<String[]> getNGrams(String[] sequence, int size); }
NGramUtils { public static Collection<StringList> getNGrams(StringList sequence, int size) { Collection<StringList> ngrams = new LinkedList<>(); if (size == -1 || size >= sequence.size()) { ngrams.add(sequence); } else { String[] ngram = new String[size]; for (int i = 0; i < sequence.size() - size + 1; i++) { ngram[0] = sequence.getToken(i); for (int j = 1; j < size; j++) { ngram[j] = sequence.getToken(i + j); } ngrams.add(new StringList(ngram)); } } return ngrams; }  static double calculateLaplaceSmoothingProbability(StringList ngram,
      Iterable<StringList> set, Double k); static double calculateUnigramMLProbability(String word, Collection<StringList> set); static double calculateBigramMLProbability(String x0, String x1, Collection<StringList> set); static double calculateTrigramMLProbability(String x0, String x1, String x2,
                                                     Iterable<StringList> set); static double calculateNgramMLProbability(StringList ngram, Iterable<StringList> set); static double calculateBigramPriorSmoothingProbability(String x0, String x1,
                                                                Collection<StringList> set, Double k); static double calculateTrigramLinearInterpolationProbability(String x0, String x1,
                                                                      String x2, Collection<StringList> set,
      Double lambda1, Double lambda2, Double lambda3); static double calculateMissingNgramProbabilityMass(StringList ngram, Double discount,
                                                            Iterable<StringList> set); static StringList getNMinusOneTokenFirst(StringList ngram); static StringList getNMinusOneTokenLast(StringList ngram); static Collection<StringList> getNGrams(StringList sequence, int size); static Collection<String[]> getNGrams(String[] sequence, int size);  }