@Test public void testFrequencyComparison() { final Random rand = RandomUtils.getRandom(); Vector p1 = new DenseVector(25) .assign(new DoubleFunction() { @Override public double apply(double arg1) { return -Math.log1p(-rand.nextDouble()); } }); Vector p2 = p1.like().assign(p1); p1.viewPart(0, 5).assign(0); p1.viewPart(5, 3).assign(Functions.mult(4)); p1.assign(Functions.div(p1.norm(1))); p2.assign(Functions.div(p2.norm(1))); Multiset<Integer> w1 = HashMultiset.create(); for (int i = 0; i < 100; i++) { w1.add(sample(p1, rand)); } Multiset<Integer> w2 = HashMultiset.create(); for (int i = 0; i < 1000; i++) { w2.add(sample(p2, rand)); } List<LogLikelihood.ScoredItem<Integer>> r = LogLikelihood.compareFrequencies(w1, w2, 8, 0); assertTrue(r.size() <= 8); assertFalse(r.isEmpty()); for (LogLikelihood.ScoredItem<Integer> item : r) { assertTrue(item.getScore() >= 0); } assertEquals(7, (int) r.get(0).getItem()); double lastScore = r.get(0).getScore(); for (LogLikelihood.ScoredItem<Integer> item : r) { assertTrue(item.getScore() <= lastScore); lastScore = item.getScore(); } r = LogLikelihood.compareFrequencies(w1, w2, 40, 1); assertEquals(2, r.size()); assertEquals(7, (int) r.get(0).getItem()); assertEquals(6, (int) r.get(1).getItem()); r = LogLikelihood.compareFrequencies(w1, w2, 1000, -100); Multiset<Integer> k = HashMultiset.create(); for (LogLikelihood.ScoredItem<Integer> item : r) { k.add(item.getItem()); } for (int i = 0; i < 25; i++) { assertTrue("i = " + i, k.count(i) == 1 || w2.count(i) == 0); } assertEquals(w2.elementSet().size(), r.size()); assertEquals(7, (int) r.get(0).getItem()); assertEquals(6, (int) r.get(1).getItem()); assertTrue(r.get(r.size() - 1).getScore() < 0); lastScore = r.get(0).getScore(); for (LogLikelihood.ScoredItem<Integer> item : r) { assertTrue(item.getScore() <= lastScore); lastScore = item.getScore(); } }
public static <T> List<ScoredItem<T>> compareFrequencies(Multiset<T> a, Multiset<T> b, int maxReturn, double threshold) { int totalA = a.size(); int totalB = b.size(); Ordering<ScoredItem<T>> byScoreAscending = new Ordering<ScoredItem<T>>() { @Override public int compare(ScoredItem<T> tScoredItem, ScoredItem<T> tScoredItem1) { return Double.compare(tScoredItem.score, tScoredItem1.score); } }; Queue<ScoredItem<T>> best = new PriorityQueue<ScoredItem<T>>(maxReturn + 1, byScoreAscending); for (T t : a.elementSet()) { compareAndAdd(a, b, maxReturn, threshold, totalA, totalB, best, t); } if (threshold < 0) { for (T t : b.elementSet()) { if (a.count(t) == 0) { compareAndAdd(a, b, maxReturn, threshold, totalA, totalB, best, t); } } } List<ScoredItem<T>> r = Lists.newArrayList(best); Collections.sort(r, byScoreAscending.reverse()); return r; }
LogLikelihood { public static <T> List<ScoredItem<T>> compareFrequencies(Multiset<T> a, Multiset<T> b, int maxReturn, double threshold) { int totalA = a.size(); int totalB = b.size(); Ordering<ScoredItem<T>> byScoreAscending = new Ordering<ScoredItem<T>>() { @Override public int compare(ScoredItem<T> tScoredItem, ScoredItem<T> tScoredItem1) { return Double.compare(tScoredItem.score, tScoredItem1.score); } }; Queue<ScoredItem<T>> best = new PriorityQueue<ScoredItem<T>>(maxReturn + 1, byScoreAscending); for (T t : a.elementSet()) { compareAndAdd(a, b, maxReturn, threshold, totalA, totalB, best, t); } if (threshold < 0) { for (T t : b.elementSet()) { if (a.count(t) == 0) { compareAndAdd(a, b, maxReturn, threshold, totalA, totalB, best, t); } } } List<ScoredItem<T>> r = Lists.newArrayList(best); Collections.sort(r, byScoreAscending.reverse()); return r; } }
LogLikelihood { public static <T> List<ScoredItem<T>> compareFrequencies(Multiset<T> a, Multiset<T> b, int maxReturn, double threshold) { int totalA = a.size(); int totalB = b.size(); Ordering<ScoredItem<T>> byScoreAscending = new Ordering<ScoredItem<T>>() { @Override public int compare(ScoredItem<T> tScoredItem, ScoredItem<T> tScoredItem1) { return Double.compare(tScoredItem.score, tScoredItem1.score); } }; Queue<ScoredItem<T>> best = new PriorityQueue<ScoredItem<T>>(maxReturn + 1, byScoreAscending); for (T t : a.elementSet()) { compareAndAdd(a, b, maxReturn, threshold, totalA, totalB, best, t); } if (threshold < 0) { for (T t : b.elementSet()) { if (a.count(t) == 0) { compareAndAdd(a, b, maxReturn, threshold, totalA, totalB, best, t); } } } List<ScoredItem<T>> r = Lists.newArrayList(best); Collections.sort(r, byScoreAscending.reverse()); return r; } private  LogLikelihood(); }
LogLikelihood { public static <T> List<ScoredItem<T>> compareFrequencies(Multiset<T> a, Multiset<T> b, int maxReturn, double threshold) { int totalA = a.size(); int totalB = b.size(); Ordering<ScoredItem<T>> byScoreAscending = new Ordering<ScoredItem<T>>() { @Override public int compare(ScoredItem<T> tScoredItem, ScoredItem<T> tScoredItem1) { return Double.compare(tScoredItem.score, tScoredItem1.score); } }; Queue<ScoredItem<T>> best = new PriorityQueue<ScoredItem<T>>(maxReturn + 1, byScoreAscending); for (T t : a.elementSet()) { compareAndAdd(a, b, maxReturn, threshold, totalA, totalB, best, t); } if (threshold < 0) { for (T t : b.elementSet()) { if (a.count(t) == 0) { compareAndAdd(a, b, maxReturn, threshold, totalA, totalB, best, t); } } } List<ScoredItem<T>> r = Lists.newArrayList(best); Collections.sort(r, byScoreAscending.reverse()); return r; } private  LogLikelihood(); static double entropy(long... elements); static double logLikelihoodRatio(long k11, long k12, long k21, long k22); static double rootLogLikelihoodRatio(long k11, long k12, long k21, long k22); static List<ScoredItem<T>> compareFrequencies(Multiset<T> a,
                                                           Multiset<T> b,
                                                           int maxReturn,
                                                           double threshold); }
LogLikelihood { public static <T> List<ScoredItem<T>> compareFrequencies(Multiset<T> a, Multiset<T> b, int maxReturn, double threshold) { int totalA = a.size(); int totalB = b.size(); Ordering<ScoredItem<T>> byScoreAscending = new Ordering<ScoredItem<T>>() { @Override public int compare(ScoredItem<T> tScoredItem, ScoredItem<T> tScoredItem1) { return Double.compare(tScoredItem.score, tScoredItem1.score); } }; Queue<ScoredItem<T>> best = new PriorityQueue<ScoredItem<T>>(maxReturn + 1, byScoreAscending); for (T t : a.elementSet()) { compareAndAdd(a, b, maxReturn, threshold, totalA, totalB, best, t); } if (threshold < 0) { for (T t : b.elementSet()) { if (a.count(t) == 0) { compareAndAdd(a, b, maxReturn, threshold, totalA, totalB, best, t); } } } List<ScoredItem<T>> r = Lists.newArrayList(best); Collections.sort(r, byScoreAscending.reverse()); return r; } private  LogLikelihood(); static double entropy(long... elements); static double logLikelihoodRatio(long k11, long k12, long k21, long k22); static double rootLogLikelihoodRatio(long k11, long k12, long k21, long k22); static List<ScoredItem<T>> compareFrequencies(Multiset<T> a,
                                                           Multiset<T> b,
                                                           int maxReturn,
                                                           double threshold);  }