@Test public void testLinearInterpolation() throws Exception { Collection<StringList> set = new LinkedList<>(); set.add(new StringList("the", "green", "book", "STOP")); set.add(new StringList("my", "blue", "book", "STOP")); set.add(new StringList("his", "green", "house", "STOP")); set.add(new StringList("book", "STOP")); Double lambda = 1d / 3d; Double d = NGramUtils.calculateTrigramLinearInterpolationProbability("the", "green", "book", set, lambda, lambda, lambda); Assert.assertNotNull(d); Assert.assertEquals("wrong result", Double.valueOf(0.5714285714285714d), d); }
public static double calculateTrigramLinearInterpolationProbability(String x0, String x1, String x2, Collection<StringList> set, Double lambda1, Double lambda2, Double lambda3) { assert lambda1 + lambda2 + lambda3 == 1 : "lambdas sum should be equals to 1"; assert lambda1 > 0 && lambda2 > 0 && lambda3 > 0 : "lambdas should all be greater than 0"; return lambda1 * calculateTrigramMLProbability(x0, x1, x2, set) + lambda2 * calculateBigramMLProbability(x1, x2, set) + lambda3 * calculateUnigramMLProbability(x2, set); }
NGramUtils { public static double calculateTrigramLinearInterpolationProbability(String x0, String x1, String x2, Collection<StringList> set, Double lambda1, Double lambda2, Double lambda3) { assert lambda1 + lambda2 + lambda3 == 1 : "lambdas sum should be equals to 1"; assert lambda1 > 0 && lambda2 > 0 && lambda3 > 0 : "lambdas should all be greater than 0"; return lambda1 * calculateTrigramMLProbability(x0, x1, x2, set) + lambda2 * calculateBigramMLProbability(x1, x2, set) + lambda3 * calculateUnigramMLProbability(x2, set); } }
NGramUtils { public static double calculateTrigramLinearInterpolationProbability(String x0, String x1, String x2, Collection<StringList> set, Double lambda1, Double lambda2, Double lambda3) { assert lambda1 + lambda2 + lambda3 == 1 : "lambdas sum should be equals to 1"; assert lambda1 > 0 && lambda2 > 0 && lambda3 > 0 : "lambdas should all be greater than 0"; return lambda1 * calculateTrigramMLProbability(x0, x1, x2, set) + lambda2 * calculateBigramMLProbability(x1, x2, set) + lambda3 * calculateUnigramMLProbability(x2, set); }  }
NGramUtils { public static double calculateTrigramLinearInterpolationProbability(String x0, String x1, String x2, Collection<StringList> set, Double lambda1, Double lambda2, Double lambda3) { assert lambda1 + lambda2 + lambda3 == 1 : "lambdas sum should be equals to 1"; assert lambda1 > 0 && lambda2 > 0 && lambda3 > 0 : "lambdas should all be greater than 0"; return lambda1 * calculateTrigramMLProbability(x0, x1, x2, set) + lambda2 * calculateBigramMLProbability(x1, x2, set) + lambda3 * calculateUnigramMLProbability(x2, set); }  static double calculateLaplaceSmoothingProbability(StringList ngram,
      Iterable<StringList> set, Double k); static double calculateUnigramMLProbability(String word, Collection<StringList> set); static double calculateBigramMLProbability(String x0, String x1, Collection<StringList> set); static double calculateTrigramMLProbability(String x0, String x1, String x2,
                                                     Iterable<StringList> set); static double calculateNgramMLProbability(StringList ngram, Iterable<StringList> set); static double calculateBigramPriorSmoothingProbability(String x0, String x1,
                                                                Collection<StringList> set, Double k); static double calculateTrigramLinearInterpolationProbability(String x0, String x1,
                                                                      String x2, Collection<StringList> set,
      Double lambda1, Double lambda2, Double lambda3); static double calculateMissingNgramProbabilityMass(StringList ngram, Double discount,
                                                            Iterable<StringList> set); static StringList getNMinusOneTokenFirst(StringList ngram); static StringList getNMinusOneTokenLast(StringList ngram); static Collection<StringList> getNGrams(StringList sequence, int size); static Collection<String[]> getNGrams(String[] sequence, int size); }
NGramUtils { public static double calculateTrigramLinearInterpolationProbability(String x0, String x1, String x2, Collection<StringList> set, Double lambda1, Double lambda2, Double lambda3) { assert lambda1 + lambda2 + lambda3 == 1 : "lambdas sum should be equals to 1"; assert lambda1 > 0 && lambda2 > 0 && lambda3 > 0 : "lambdas should all be greater than 0"; return lambda1 * calculateTrigramMLProbability(x0, x1, x2, set) + lambda2 * calculateBigramMLProbability(x1, x2, set) + lambda3 * calculateUnigramMLProbability(x2, set); }  static double calculateLaplaceSmoothingProbability(StringList ngram,
      Iterable<StringList> set, Double k); static double calculateUnigramMLProbability(String word, Collection<StringList> set); static double calculateBigramMLProbability(String x0, String x1, Collection<StringList> set); static double calculateTrigramMLProbability(String x0, String x1, String x2,
                                                     Iterable<StringList> set); static double calculateNgramMLProbability(StringList ngram, Iterable<StringList> set); static double calculateBigramPriorSmoothingProbability(String x0, String x1,
                                                                Collection<StringList> set, Double k); static double calculateTrigramLinearInterpolationProbability(String x0, String x1,
                                                                      String x2, Collection<StringList> set,
      Double lambda1, Double lambda2, Double lambda3); static double calculateMissingNgramProbabilityMass(StringList ngram, Double discount,
                                                            Iterable<StringList> set); static StringList getNMinusOneTokenFirst(StringList ngram); static StringList getNMinusOneTokenLast(StringList ngram); static Collection<StringList> getNGrams(StringList sequence, int size); static Collection<String[]> getNGrams(String[] sequence, int size);  }