@Test public void getLogLikelihood(){ double expected = -0.002387; double actual = target.evaluateCollocation(nGram, new ArrayList<TreeMap<String, Object>>()); Assert.assertEquals(expected, actual, 1e-4); }
private double getLogLikelihood(Long total, Long tokenOne, Long tokenTwo, Long colloc, List<TreeMap<String,Object>> output) { total = total == null ? 0 : total; tokenOne = tokenOne == null || tokenOne == 0 ? 1 : tokenOne; tokenTwo = tokenTwo == null ? 0 : tokenTwo; colloc = colloc == null ? 0 : Math.round(colloc * parameterSet.collocationBoost); long nonCollocTokenTwo = Math.max(tokenOne - colloc, 0); double pColloc = colloc / (double) tokenOne; double pNotColloc= (tokenTwo - colloc) / (double)((total - tokenOne)); pColloc = pColloc >= 1 ? 1 - Double.MIN_VALUE : pColloc; pNotColloc = pNotColloc <= 0 ? Double.MIN_VALUE : pNotColloc; pNotColloc = pNotColloc >= 1 ? 1 - Double.MIN_VALUE : pNotColloc; double pCollocUnderCollocHypothesis = Math.max(pColloc, pNotColloc); double pCollocUnderNotCollocHypothesis = Math.max(Math.min(pColloc, pNotColloc), Double.MIN_VALUE); double likelihoodUnderCollocHypothesis = colloc*Math.log(pCollocUnderCollocHypothesis) + nonCollocTokenTwo*Math.log(Math.max(1-pCollocUnderCollocHypothesis, Double.MIN_VALUE)) + nonCollocTokenTwo*Math.log(pNotColloc) + (total - tokenOne - nonCollocTokenTwo)*Math.log(1-pNotColloc); double likelihoodUnderNotCollocHypothesis = colloc*Math.log(pCollocUnderNotCollocHypothesis) + nonCollocTokenTwo*Math.log(1-pCollocUnderNotCollocHypothesis) + nonCollocTokenTwo*Math.log(pNotColloc) + (total - tokenOne -nonCollocTokenTwo)*Math.log(1-pNotColloc); double logLikelihoodRatio = likelihoodUnderCollocHypothesis - likelihoodUnderNotCollocHypothesis; double scaledLikelihood = scaleLikelihood(logLikelihoodRatio); if(parameterSet.returnScoringOutput) { TreeMap<String,Object> scoringElement = new TreeMap<String,Object>(); scoringElement.put("p_colloc", pColloc); scoringElement.put("p_not_colloc", pNotColloc); scoringElement.put("lr_numerator", likelihoodUnderCollocHypothesis); scoringElement.put("lr_denominator", likelihoodUnderNotCollocHypothesis); scoringElement.put("log_likelihood_ratio", logLikelihoodRatio); scoringElement.put("scaled_llr", scaledLikelihood); output.add(scoringElement); } return scaledLikelihood; }
LikelihoodCollocationScorer implements ICollocationScorer { private double getLogLikelihood(Long total, Long tokenOne, Long tokenTwo, Long colloc, List<TreeMap<String,Object>> output) { total = total == null ? 0 : total; tokenOne = tokenOne == null || tokenOne == 0 ? 1 : tokenOne; tokenTwo = tokenTwo == null ? 0 : tokenTwo; colloc = colloc == null ? 0 : Math.round(colloc * parameterSet.collocationBoost); long nonCollocTokenTwo = Math.max(tokenOne - colloc, 0); double pColloc = colloc / (double) tokenOne; double pNotColloc= (tokenTwo - colloc) / (double)((total - tokenOne)); pColloc = pColloc >= 1 ? 1 - Double.MIN_VALUE : pColloc; pNotColloc = pNotColloc <= 0 ? Double.MIN_VALUE : pNotColloc; pNotColloc = pNotColloc >= 1 ? 1 - Double.MIN_VALUE : pNotColloc; double pCollocUnderCollocHypothesis = Math.max(pColloc, pNotColloc); double pCollocUnderNotCollocHypothesis = Math.max(Math.min(pColloc, pNotColloc), Double.MIN_VALUE); double likelihoodUnderCollocHypothesis = colloc*Math.log(pCollocUnderCollocHypothesis) + nonCollocTokenTwo*Math.log(Math.max(1-pCollocUnderCollocHypothesis, Double.MIN_VALUE)) + nonCollocTokenTwo*Math.log(pNotColloc) + (total - tokenOne - nonCollocTokenTwo)*Math.log(1-pNotColloc); double likelihoodUnderNotCollocHypothesis = colloc*Math.log(pCollocUnderNotCollocHypothesis) + nonCollocTokenTwo*Math.log(1-pCollocUnderNotCollocHypothesis) + nonCollocTokenTwo*Math.log(pNotColloc) + (total - tokenOne -nonCollocTokenTwo)*Math.log(1-pNotColloc); double logLikelihoodRatio = likelihoodUnderCollocHypothesis - likelihoodUnderNotCollocHypothesis; double scaledLikelihood = scaleLikelihood(logLikelihoodRatio); if(parameterSet.returnScoringOutput) { TreeMap<String,Object> scoringElement = new TreeMap<String,Object>(); scoringElement.put("p_colloc", pColloc); scoringElement.put("p_not_colloc", pNotColloc); scoringElement.put("lr_numerator", likelihoodUnderCollocHypothesis); scoringElement.put("lr_denominator", likelihoodUnderNotCollocHypothesis); scoringElement.put("log_likelihood_ratio", logLikelihoodRatio); scoringElement.put("scaled_llr", scaledLikelihood); output.add(scoringElement); } return scaledLikelihood; } }
LikelihoodCollocationScorer implements ICollocationScorer { private double getLogLikelihood(Long total, Long tokenOne, Long tokenTwo, Long colloc, List<TreeMap<String,Object>> output) { total = total == null ? 0 : total; tokenOne = tokenOne == null || tokenOne == 0 ? 1 : tokenOne; tokenTwo = tokenTwo == null ? 0 : tokenTwo; colloc = colloc == null ? 0 : Math.round(colloc * parameterSet.collocationBoost); long nonCollocTokenTwo = Math.max(tokenOne - colloc, 0); double pColloc = colloc / (double) tokenOne; double pNotColloc= (tokenTwo - colloc) / (double)((total - tokenOne)); pColloc = pColloc >= 1 ? 1 - Double.MIN_VALUE : pColloc; pNotColloc = pNotColloc <= 0 ? Double.MIN_VALUE : pNotColloc; pNotColloc = pNotColloc >= 1 ? 1 - Double.MIN_VALUE : pNotColloc; double pCollocUnderCollocHypothesis = Math.max(pColloc, pNotColloc); double pCollocUnderNotCollocHypothesis = Math.max(Math.min(pColloc, pNotColloc), Double.MIN_VALUE); double likelihoodUnderCollocHypothesis = colloc*Math.log(pCollocUnderCollocHypothesis) + nonCollocTokenTwo*Math.log(Math.max(1-pCollocUnderCollocHypothesis, Double.MIN_VALUE)) + nonCollocTokenTwo*Math.log(pNotColloc) + (total - tokenOne - nonCollocTokenTwo)*Math.log(1-pNotColloc); double likelihoodUnderNotCollocHypothesis = colloc*Math.log(pCollocUnderNotCollocHypothesis) + nonCollocTokenTwo*Math.log(1-pCollocUnderNotCollocHypothesis) + nonCollocTokenTwo*Math.log(pNotColloc) + (total - tokenOne -nonCollocTokenTwo)*Math.log(1-pNotColloc); double logLikelihoodRatio = likelihoodUnderCollocHypothesis - likelihoodUnderNotCollocHypothesis; double scaledLikelihood = scaleLikelihood(logLikelihoodRatio); if(parameterSet.returnScoringOutput) { TreeMap<String,Object> scoringElement = new TreeMap<String,Object>(); scoringElement.put("p_colloc", pColloc); scoringElement.put("p_not_colloc", pNotColloc); scoringElement.put("lr_numerator", likelihoodUnderCollocHypothesis); scoringElement.put("lr_denominator", likelihoodUnderNotCollocHypothesis); scoringElement.put("log_likelihood_ratio", logLikelihoodRatio); scoringElement.put("scaled_llr", scaledLikelihood); output.add(scoringElement); } return scaledLikelihood; } LikelihoodCollocationScorer(HashMap<String, Long> frequencyMap,
                                       HashMap<NGramField, Long> totalFrequencies,
                                       ScorerParameterSet parameterSet); }
LikelihoodCollocationScorer implements ICollocationScorer { private double getLogLikelihood(Long total, Long tokenOne, Long tokenTwo, Long colloc, List<TreeMap<String,Object>> output) { total = total == null ? 0 : total; tokenOne = tokenOne == null || tokenOne == 0 ? 1 : tokenOne; tokenTwo = tokenTwo == null ? 0 : tokenTwo; colloc = colloc == null ? 0 : Math.round(colloc * parameterSet.collocationBoost); long nonCollocTokenTwo = Math.max(tokenOne - colloc, 0); double pColloc = colloc / (double) tokenOne; double pNotColloc= (tokenTwo - colloc) / (double)((total - tokenOne)); pColloc = pColloc >= 1 ? 1 - Double.MIN_VALUE : pColloc; pNotColloc = pNotColloc <= 0 ? Double.MIN_VALUE : pNotColloc; pNotColloc = pNotColloc >= 1 ? 1 - Double.MIN_VALUE : pNotColloc; double pCollocUnderCollocHypothesis = Math.max(pColloc, pNotColloc); double pCollocUnderNotCollocHypothesis = Math.max(Math.min(pColloc, pNotColloc), Double.MIN_VALUE); double likelihoodUnderCollocHypothesis = colloc*Math.log(pCollocUnderCollocHypothesis) + nonCollocTokenTwo*Math.log(Math.max(1-pCollocUnderCollocHypothesis, Double.MIN_VALUE)) + nonCollocTokenTwo*Math.log(pNotColloc) + (total - tokenOne - nonCollocTokenTwo)*Math.log(1-pNotColloc); double likelihoodUnderNotCollocHypothesis = colloc*Math.log(pCollocUnderNotCollocHypothesis) + nonCollocTokenTwo*Math.log(1-pCollocUnderNotCollocHypothesis) + nonCollocTokenTwo*Math.log(pNotColloc) + (total - tokenOne -nonCollocTokenTwo)*Math.log(1-pNotColloc); double logLikelihoodRatio = likelihoodUnderCollocHypothesis - likelihoodUnderNotCollocHypothesis; double scaledLikelihood = scaleLikelihood(logLikelihoodRatio); if(parameterSet.returnScoringOutput) { TreeMap<String,Object> scoringElement = new TreeMap<String,Object>(); scoringElement.put("p_colloc", pColloc); scoringElement.put("p_not_colloc", pNotColloc); scoringElement.put("lr_numerator", likelihoodUnderCollocHypothesis); scoringElement.put("lr_denominator", likelihoodUnderNotCollocHypothesis); scoringElement.put("log_likelihood_ratio", logLikelihoodRatio); scoringElement.put("scaled_llr", scaledLikelihood); output.add(scoringElement); } return scaledLikelihood; } LikelihoodCollocationScorer(HashMap<String, Long> frequencyMap,
                                       HashMap<NGramField, Long> totalFrequencies,
                                       ScorerParameterSet parameterSet); double evaluateCollocation(NGram nGram, List<TreeMap<String,Object>> output); }
LikelihoodCollocationScorer implements ICollocationScorer { private double getLogLikelihood(Long total, Long tokenOne, Long tokenTwo, Long colloc, List<TreeMap<String,Object>> output) { total = total == null ? 0 : total; tokenOne = tokenOne == null || tokenOne == 0 ? 1 : tokenOne; tokenTwo = tokenTwo == null ? 0 : tokenTwo; colloc = colloc == null ? 0 : Math.round(colloc * parameterSet.collocationBoost); long nonCollocTokenTwo = Math.max(tokenOne - colloc, 0); double pColloc = colloc / (double) tokenOne; double pNotColloc= (tokenTwo - colloc) / (double)((total - tokenOne)); pColloc = pColloc >= 1 ? 1 - Double.MIN_VALUE : pColloc; pNotColloc = pNotColloc <= 0 ? Double.MIN_VALUE : pNotColloc; pNotColloc = pNotColloc >= 1 ? 1 - Double.MIN_VALUE : pNotColloc; double pCollocUnderCollocHypothesis = Math.max(pColloc, pNotColloc); double pCollocUnderNotCollocHypothesis = Math.max(Math.min(pColloc, pNotColloc), Double.MIN_VALUE); double likelihoodUnderCollocHypothesis = colloc*Math.log(pCollocUnderCollocHypothesis) + nonCollocTokenTwo*Math.log(Math.max(1-pCollocUnderCollocHypothesis, Double.MIN_VALUE)) + nonCollocTokenTwo*Math.log(pNotColloc) + (total - tokenOne - nonCollocTokenTwo)*Math.log(1-pNotColloc); double likelihoodUnderNotCollocHypothesis = colloc*Math.log(pCollocUnderNotCollocHypothesis) + nonCollocTokenTwo*Math.log(1-pCollocUnderNotCollocHypothesis) + nonCollocTokenTwo*Math.log(pNotColloc) + (total - tokenOne -nonCollocTokenTwo)*Math.log(1-pNotColloc); double logLikelihoodRatio = likelihoodUnderCollocHypothesis - likelihoodUnderNotCollocHypothesis; double scaledLikelihood = scaleLikelihood(logLikelihoodRatio); if(parameterSet.returnScoringOutput) { TreeMap<String,Object> scoringElement = new TreeMap<String,Object>(); scoringElement.put("p_colloc", pColloc); scoringElement.put("p_not_colloc", pNotColloc); scoringElement.put("lr_numerator", likelihoodUnderCollocHypothesis); scoringElement.put("lr_denominator", likelihoodUnderNotCollocHypothesis); scoringElement.put("log_likelihood_ratio", logLikelihoodRatio); scoringElement.put("scaled_llr", scaledLikelihood); output.add(scoringElement); } return scaledLikelihood; } LikelihoodCollocationScorer(HashMap<String, Long> frequencyMap,
                                       HashMap<NGramField, Long> totalFrequencies,
                                       ScorerParameterSet parameterSet); double evaluateCollocation(NGram nGram, List<TreeMap<String,Object>> output);  }