@Test public void chiSquareBigram(){ double expected = 0.0; double actual = target.evaluateCollocation(nGram.getSubNGram(0, 2), new ArrayList<TreeMap<String, Object>>()); Assert.assertEquals(expected, actual, 1e-4); }
private double chiSquareBigram(NGram nGram, List<TreeMap<String, Object>> output) { String bigram; double pvalue = 0.0; long [] observed = new long[4]; double [] expected = new double[4]; long total = totalFrequencies.get(NGramField.bigrams); ChiSquareTest chi = new ChiSquareTest(); bigram = nGram.toLowerCorpusString(); Long bigramFreq = Math.round(parameterSet.collocationBoost* frequencyMap.get(bigram)); Long tokenOneFreq = frequencyMap.get(nGram.getSubNGram(0,1).toLowerCorpusString()); Long tokenTwoFreq = frequencyMap.get(nGram.getSubNGram(1,1).toLowerCorpusString()); double expectedBigram = tokenTwoFreq * tokenOneFreq / (double)total; double expectedTokenOne = tokenOneFreq - tokenTwoFreq * tokenOneFreq / (double)total; double expectedTokenTwo = tokenTwoFreq - tokenTwoFreq * tokenOneFreq / (double)total; bigramFreq = bigramFreq == null ? 0 : bigramFreq; tokenOneFreq = tokenOneFreq == null? 0 : tokenOneFreq; tokenTwoFreq = tokenTwoFreq == null ? 0 : tokenTwoFreq; observed[0] = bigramFreq; expected[0] = Math.max(MIN_EXPECTED, expectedBigram); observed[1] = Math.max(0,tokenOneFreq - bigramFreq); expected[1] = Math.max(MIN_EXPECTED, expectedTokenOne); observed[2] = Math.max(0,tokenTwoFreq - bigramFreq); expected[2] = Math.max(MIN_EXPECTED, expectedTokenTwo); observed[3] = Math.max(0,total - tokenOneFreq - tokenTwoFreq + bigramFreq); expected[3] = Math.max(MIN_EXPECTED, total - expectedTokenOne - expectedTokenTwo + expectedBigram); if(observed[0] >= expected[0]) pvalue = chi.chiSquareTest(expected, observed); else pvalue = 1.0; if(parameterSet.returnScoringOutput) { TreeMap<String,Object> scoringElement = new TreeMap<String,Object>(); scoringElement.put("token1", nGram.getSubNGram(0, 1).toLowerCorpusString()); scoringElement.put("token2", nGram.getSubNGram(1, 1).toLowerCorpusString()); scoringElement.put("token1freq", tokenOneFreq); scoringElement.put("expectedtoken1", expectedTokenOne); scoringElement.put("token2freq", tokenTwoFreq); scoringElement.put("expectedtoken2", expectedTokenTwo); scoringElement.put("observed", observed[0]); scoringElement.put("expected", expected[0]); scoringElement.put("p", pvalue); output.add(scoringElement); } return pvalue; }
ChiSquareCollocationScorer implements ICollocationScorer { private double chiSquareBigram(NGram nGram, List<TreeMap<String, Object>> output) { String bigram; double pvalue = 0.0; long [] observed = new long[4]; double [] expected = new double[4]; long total = totalFrequencies.get(NGramField.bigrams); ChiSquareTest chi = new ChiSquareTest(); bigram = nGram.toLowerCorpusString(); Long bigramFreq = Math.round(parameterSet.collocationBoost* frequencyMap.get(bigram)); Long tokenOneFreq = frequencyMap.get(nGram.getSubNGram(0,1).toLowerCorpusString()); Long tokenTwoFreq = frequencyMap.get(nGram.getSubNGram(1,1).toLowerCorpusString()); double expectedBigram = tokenTwoFreq * tokenOneFreq / (double)total; double expectedTokenOne = tokenOneFreq - tokenTwoFreq * tokenOneFreq / (double)total; double expectedTokenTwo = tokenTwoFreq - tokenTwoFreq * tokenOneFreq / (double)total; bigramFreq = bigramFreq == null ? 0 : bigramFreq; tokenOneFreq = tokenOneFreq == null? 0 : tokenOneFreq; tokenTwoFreq = tokenTwoFreq == null ? 0 : tokenTwoFreq; observed[0] = bigramFreq; expected[0] = Math.max(MIN_EXPECTED, expectedBigram); observed[1] = Math.max(0,tokenOneFreq - bigramFreq); expected[1] = Math.max(MIN_EXPECTED, expectedTokenOne); observed[2] = Math.max(0,tokenTwoFreq - bigramFreq); expected[2] = Math.max(MIN_EXPECTED, expectedTokenTwo); observed[3] = Math.max(0,total - tokenOneFreq - tokenTwoFreq + bigramFreq); expected[3] = Math.max(MIN_EXPECTED, total - expectedTokenOne - expectedTokenTwo + expectedBigram); if(observed[0] >= expected[0]) pvalue = chi.chiSquareTest(expected, observed); else pvalue = 1.0; if(parameterSet.returnScoringOutput) { TreeMap<String,Object> scoringElement = new TreeMap<String,Object>(); scoringElement.put("token1", nGram.getSubNGram(0, 1).toLowerCorpusString()); scoringElement.put("token2", nGram.getSubNGram(1, 1).toLowerCorpusString()); scoringElement.put("token1freq", tokenOneFreq); scoringElement.put("expectedtoken1", expectedTokenOne); scoringElement.put("token2freq", tokenTwoFreq); scoringElement.put("expectedtoken2", expectedTokenTwo); scoringElement.put("observed", observed[0]); scoringElement.put("expected", expected[0]); scoringElement.put("p", pvalue); output.add(scoringElement); } return pvalue; } }
ChiSquareCollocationScorer implements ICollocationScorer { private double chiSquareBigram(NGram nGram, List<TreeMap<String, Object>> output) { String bigram; double pvalue = 0.0; long [] observed = new long[4]; double [] expected = new double[4]; long total = totalFrequencies.get(NGramField.bigrams); ChiSquareTest chi = new ChiSquareTest(); bigram = nGram.toLowerCorpusString(); Long bigramFreq = Math.round(parameterSet.collocationBoost* frequencyMap.get(bigram)); Long tokenOneFreq = frequencyMap.get(nGram.getSubNGram(0,1).toLowerCorpusString()); Long tokenTwoFreq = frequencyMap.get(nGram.getSubNGram(1,1).toLowerCorpusString()); double expectedBigram = tokenTwoFreq * tokenOneFreq / (double)total; double expectedTokenOne = tokenOneFreq - tokenTwoFreq * tokenOneFreq / (double)total; double expectedTokenTwo = tokenTwoFreq - tokenTwoFreq * tokenOneFreq / (double)total; bigramFreq = bigramFreq == null ? 0 : bigramFreq; tokenOneFreq = tokenOneFreq == null? 0 : tokenOneFreq; tokenTwoFreq = tokenTwoFreq == null ? 0 : tokenTwoFreq; observed[0] = bigramFreq; expected[0] = Math.max(MIN_EXPECTED, expectedBigram); observed[1] = Math.max(0,tokenOneFreq - bigramFreq); expected[1] = Math.max(MIN_EXPECTED, expectedTokenOne); observed[2] = Math.max(0,tokenTwoFreq - bigramFreq); expected[2] = Math.max(MIN_EXPECTED, expectedTokenTwo); observed[3] = Math.max(0,total - tokenOneFreq - tokenTwoFreq + bigramFreq); expected[3] = Math.max(MIN_EXPECTED, total - expectedTokenOne - expectedTokenTwo + expectedBigram); if(observed[0] >= expected[0]) pvalue = chi.chiSquareTest(expected, observed); else pvalue = 1.0; if(parameterSet.returnScoringOutput) { TreeMap<String,Object> scoringElement = new TreeMap<String,Object>(); scoringElement.put("token1", nGram.getSubNGram(0, 1).toLowerCorpusString()); scoringElement.put("token2", nGram.getSubNGram(1, 1).toLowerCorpusString()); scoringElement.put("token1freq", tokenOneFreq); scoringElement.put("expectedtoken1", expectedTokenOne); scoringElement.put("token2freq", tokenTwoFreq); scoringElement.put("expectedtoken2", expectedTokenTwo); scoringElement.put("observed", observed[0]); scoringElement.put("expected", expected[0]); scoringElement.put("p", pvalue); output.add(scoringElement); } return pvalue; } ChiSquareCollocationScorer(HashMap<String, Long> frequencyMap,
                                      HashMap<NGramField, Long> totalFrequencies,
                                      ScorerParameterSet parameterSet); }
ChiSquareCollocationScorer implements ICollocationScorer { private double chiSquareBigram(NGram nGram, List<TreeMap<String, Object>> output) { String bigram; double pvalue = 0.0; long [] observed = new long[4]; double [] expected = new double[4]; long total = totalFrequencies.get(NGramField.bigrams); ChiSquareTest chi = new ChiSquareTest(); bigram = nGram.toLowerCorpusString(); Long bigramFreq = Math.round(parameterSet.collocationBoost* frequencyMap.get(bigram)); Long tokenOneFreq = frequencyMap.get(nGram.getSubNGram(0,1).toLowerCorpusString()); Long tokenTwoFreq = frequencyMap.get(nGram.getSubNGram(1,1).toLowerCorpusString()); double expectedBigram = tokenTwoFreq * tokenOneFreq / (double)total; double expectedTokenOne = tokenOneFreq - tokenTwoFreq * tokenOneFreq / (double)total; double expectedTokenTwo = tokenTwoFreq - tokenTwoFreq * tokenOneFreq / (double)total; bigramFreq = bigramFreq == null ? 0 : bigramFreq; tokenOneFreq = tokenOneFreq == null? 0 : tokenOneFreq; tokenTwoFreq = tokenTwoFreq == null ? 0 : tokenTwoFreq; observed[0] = bigramFreq; expected[0] = Math.max(MIN_EXPECTED, expectedBigram); observed[1] = Math.max(0,tokenOneFreq - bigramFreq); expected[1] = Math.max(MIN_EXPECTED, expectedTokenOne); observed[2] = Math.max(0,tokenTwoFreq - bigramFreq); expected[2] = Math.max(MIN_EXPECTED, expectedTokenTwo); observed[3] = Math.max(0,total - tokenOneFreq - tokenTwoFreq + bigramFreq); expected[3] = Math.max(MIN_EXPECTED, total - expectedTokenOne - expectedTokenTwo + expectedBigram); if(observed[0] >= expected[0]) pvalue = chi.chiSquareTest(expected, observed); else pvalue = 1.0; if(parameterSet.returnScoringOutput) { TreeMap<String,Object> scoringElement = new TreeMap<String,Object>(); scoringElement.put("token1", nGram.getSubNGram(0, 1).toLowerCorpusString()); scoringElement.put("token2", nGram.getSubNGram(1, 1).toLowerCorpusString()); scoringElement.put("token1freq", tokenOneFreq); scoringElement.put("expectedtoken1", expectedTokenOne); scoringElement.put("token2freq", tokenTwoFreq); scoringElement.put("expectedtoken2", expectedTokenTwo); scoringElement.put("observed", observed[0]); scoringElement.put("expected", expected[0]); scoringElement.put("p", pvalue); output.add(scoringElement); } return pvalue; } ChiSquareCollocationScorer(HashMap<String, Long> frequencyMap,
                                      HashMap<NGramField, Long> totalFrequencies,
                                      ScorerParameterSet parameterSet); double evaluateCollocation(NGram nGram, List<TreeMap<String,Object>> output); }
ChiSquareCollocationScorer implements ICollocationScorer { private double chiSquareBigram(NGram nGram, List<TreeMap<String, Object>> output) { String bigram; double pvalue = 0.0; long [] observed = new long[4]; double [] expected = new double[4]; long total = totalFrequencies.get(NGramField.bigrams); ChiSquareTest chi = new ChiSquareTest(); bigram = nGram.toLowerCorpusString(); Long bigramFreq = Math.round(parameterSet.collocationBoost* frequencyMap.get(bigram)); Long tokenOneFreq = frequencyMap.get(nGram.getSubNGram(0,1).toLowerCorpusString()); Long tokenTwoFreq = frequencyMap.get(nGram.getSubNGram(1,1).toLowerCorpusString()); double expectedBigram = tokenTwoFreq * tokenOneFreq / (double)total; double expectedTokenOne = tokenOneFreq - tokenTwoFreq * tokenOneFreq / (double)total; double expectedTokenTwo = tokenTwoFreq - tokenTwoFreq * tokenOneFreq / (double)total; bigramFreq = bigramFreq == null ? 0 : bigramFreq; tokenOneFreq = tokenOneFreq == null? 0 : tokenOneFreq; tokenTwoFreq = tokenTwoFreq == null ? 0 : tokenTwoFreq; observed[0] = bigramFreq; expected[0] = Math.max(MIN_EXPECTED, expectedBigram); observed[1] = Math.max(0,tokenOneFreq - bigramFreq); expected[1] = Math.max(MIN_EXPECTED, expectedTokenOne); observed[2] = Math.max(0,tokenTwoFreq - bigramFreq); expected[2] = Math.max(MIN_EXPECTED, expectedTokenTwo); observed[3] = Math.max(0,total - tokenOneFreq - tokenTwoFreq + bigramFreq); expected[3] = Math.max(MIN_EXPECTED, total - expectedTokenOne - expectedTokenTwo + expectedBigram); if(observed[0] >= expected[0]) pvalue = chi.chiSquareTest(expected, observed); else pvalue = 1.0; if(parameterSet.returnScoringOutput) { TreeMap<String,Object> scoringElement = new TreeMap<String,Object>(); scoringElement.put("token1", nGram.getSubNGram(0, 1).toLowerCorpusString()); scoringElement.put("token2", nGram.getSubNGram(1, 1).toLowerCorpusString()); scoringElement.put("token1freq", tokenOneFreq); scoringElement.put("expectedtoken1", expectedTokenOne); scoringElement.put("token2freq", tokenTwoFreq); scoringElement.put("expectedtoken2", expectedTokenTwo); scoringElement.put("observed", observed[0]); scoringElement.put("expected", expected[0]); scoringElement.put("p", pvalue); output.add(scoringElement); } return pvalue; } ChiSquareCollocationScorer(HashMap<String, Long> frequencyMap,
                                      HashMap<NGramField, Long> totalFrequencies,
                                      ScorerParameterSet parameterSet); double evaluateCollocation(NGram nGram, List<TreeMap<String,Object>> output);  }