@Test public void testEmptyVocabularyProbability() { NGramLanguageModel model = new NGramLanguageModel(); Assert.assertEquals("probability with an empty vocabulary is always 0", 0d, model.calculateProbability(""), 0d); Assert.assertEquals("probability with an empty vocabulary is always 0", 0d, model.calculateProbability("1", "2", "3"), 0d); }
@Override public double calculateProbability(StringList tokens) { double probability = 0d; if (size() > 0) { for (StringList ngram : NGramUtils.getNGrams(tokens, n)) { double score = stupidBackoff(ngram); probability += StrictMath.log(score); if (Double.isNaN(probability)) { probability = 0d; break; } } probability = StrictMath.exp(probability); } return probability; }
NGramLanguageModel extends NGramModel implements LanguageModel { @Override public double calculateProbability(StringList tokens) { double probability = 0d; if (size() > 0) { for (StringList ngram : NGramUtils.getNGrams(tokens, n)) { double score = stupidBackoff(ngram); probability += StrictMath.log(score); if (Double.isNaN(probability)) { probability = 0d; break; } } probability = StrictMath.exp(probability); } return probability; } }
NGramLanguageModel extends NGramModel implements LanguageModel { @Override public double calculateProbability(StringList tokens) { double probability = 0d; if (size() > 0) { for (StringList ngram : NGramUtils.getNGrams(tokens, n)) { double score = stupidBackoff(ngram); probability += StrictMath.log(score); if (Double.isNaN(probability)) { probability = 0d; break; } } probability = StrictMath.exp(probability); } return probability; } NGramLanguageModel();  NGramLanguageModel(int n);  NGramLanguageModel(InputStream in);  NGramLanguageModel(InputStream in, int n); }
NGramLanguageModel extends NGramModel implements LanguageModel { @Override public double calculateProbability(StringList tokens) { double probability = 0d; if (size() > 0) { for (StringList ngram : NGramUtils.getNGrams(tokens, n)) { double score = stupidBackoff(ngram); probability += StrictMath.log(score); if (Double.isNaN(probability)) { probability = 0d; break; } } probability = StrictMath.exp(probability); } return probability; } NGramLanguageModel();  NGramLanguageModel(int n);  NGramLanguageModel(InputStream in);  NGramLanguageModel(InputStream in, int n); void add(String... tokens); @Override double calculateProbability(StringList tokens); @Override double calculateProbability(String... tokens); @Override StringList predictNextTokens(StringList tokens); @Override String[] predictNextTokens(String... tokens); }
NGramLanguageModel extends NGramModel implements LanguageModel { @Override public double calculateProbability(StringList tokens) { double probability = 0d; if (size() > 0) { for (StringList ngram : NGramUtils.getNGrams(tokens, n)) { double score = stupidBackoff(ngram); probability += StrictMath.log(score); if (Double.isNaN(probability)) { probability = 0d; break; } } probability = StrictMath.exp(probability); } return probability; } NGramLanguageModel();  NGramLanguageModel(int n);  NGramLanguageModel(InputStream in);  NGramLanguageModel(InputStream in, int n); void add(String... tokens); @Override double calculateProbability(StringList tokens); @Override double calculateProbability(String... tokens); @Override StringList predictNextTokens(StringList tokens); @Override String[] predictNextTokens(String... tokens);  }