@Test public void testFind_Corpus_int() { System.out.println("k = 10"); int k = 10; Bigram[] result = Bigram.of(corpus, k, 5); assertEquals(10, result.length); for (int i = 0; i < result.length; i++) { System.out.println(result[i]); } assertEquals(46, result[0].count); assertEquals(545.16, result[0].score, 1E-2); assertEquals(19, result[9].count); assertEquals(186.69, result[9].score, 1E-2); }
public static Bigram[] of(Corpus corpus, int k, int minFrequency) { Bigram[] bigrams = new Bigram[k]; HeapSelect<Bigram> heap = new HeapSelect<>(bigrams); Iterator<smile.nlp.Bigram> iterator = corpus.getBigrams(); while (iterator.hasNext()) { smile.nlp.Bigram bigram = iterator.next(); int c12 = corpus.getBigramFrequency(bigram); if (c12 > minFrequency) { int c1 = corpus.getTermFrequency(bigram.w1); int c2 = corpus.getTermFrequency(bigram.w2); double score = likelihoodRatio(c1, c2, c12, corpus.size()); heap.add(new Bigram(bigram.w1, bigram.w2, c12, -score)); } } heap.sort(); Bigram[] collocations = new Bigram[k]; for (int i = 0; i < k; i++) { Bigram bigram = bigrams[k-i-1]; collocations[i] = new Bigram(bigram.w1, bigram.w2, bigram.count, -bigram.score); } return collocations; }
Bigram extends smile.nlp.Bigram implements Comparable<Bigram> { public static Bigram[] of(Corpus corpus, int k, int minFrequency) { Bigram[] bigrams = new Bigram[k]; HeapSelect<Bigram> heap = new HeapSelect<>(bigrams); Iterator<smile.nlp.Bigram> iterator = corpus.getBigrams(); while (iterator.hasNext()) { smile.nlp.Bigram bigram = iterator.next(); int c12 = corpus.getBigramFrequency(bigram); if (c12 > minFrequency) { int c1 = corpus.getTermFrequency(bigram.w1); int c2 = corpus.getTermFrequency(bigram.w2); double score = likelihoodRatio(c1, c2, c12, corpus.size()); heap.add(new Bigram(bigram.w1, bigram.w2, c12, -score)); } } heap.sort(); Bigram[] collocations = new Bigram[k]; for (int i = 0; i < k; i++) { Bigram bigram = bigrams[k-i-1]; collocations[i] = new Bigram(bigram.w1, bigram.w2, bigram.count, -bigram.score); } return collocations; } }
Bigram extends smile.nlp.Bigram implements Comparable<Bigram> { public static Bigram[] of(Corpus corpus, int k, int minFrequency) { Bigram[] bigrams = new Bigram[k]; HeapSelect<Bigram> heap = new HeapSelect<>(bigrams); Iterator<smile.nlp.Bigram> iterator = corpus.getBigrams(); while (iterator.hasNext()) { smile.nlp.Bigram bigram = iterator.next(); int c12 = corpus.getBigramFrequency(bigram); if (c12 > minFrequency) { int c1 = corpus.getTermFrequency(bigram.w1); int c2 = corpus.getTermFrequency(bigram.w2); double score = likelihoodRatio(c1, c2, c12, corpus.size()); heap.add(new Bigram(bigram.w1, bigram.w2, c12, -score)); } } heap.sort(); Bigram[] collocations = new Bigram[k]; for (int i = 0; i < k; i++) { Bigram bigram = bigrams[k-i-1]; collocations[i] = new Bigram(bigram.w1, bigram.w2, bigram.count, -bigram.score); } return collocations; } Bigram(String w1, String w2, int count, double score); }
Bigram extends smile.nlp.Bigram implements Comparable<Bigram> { public static Bigram[] of(Corpus corpus, int k, int minFrequency) { Bigram[] bigrams = new Bigram[k]; HeapSelect<Bigram> heap = new HeapSelect<>(bigrams); Iterator<smile.nlp.Bigram> iterator = corpus.getBigrams(); while (iterator.hasNext()) { smile.nlp.Bigram bigram = iterator.next(); int c12 = corpus.getBigramFrequency(bigram); if (c12 > minFrequency) { int c1 = corpus.getTermFrequency(bigram.w1); int c2 = corpus.getTermFrequency(bigram.w2); double score = likelihoodRatio(c1, c2, c12, corpus.size()); heap.add(new Bigram(bigram.w1, bigram.w2, c12, -score)); } } heap.sort(); Bigram[] collocations = new Bigram[k]; for (int i = 0; i < k; i++) { Bigram bigram = bigrams[k-i-1]; collocations[i] = new Bigram(bigram.w1, bigram.w2, bigram.count, -bigram.score); } return collocations; } Bigram(String w1, String w2, int count, double score); @Override String toString(); @Override int compareTo(Bigram o); static Bigram[] of(Corpus corpus, int k, int minFrequency); static Bigram[] of(Corpus corpus, double p, int minFrequency); }
Bigram extends smile.nlp.Bigram implements Comparable<Bigram> { public static Bigram[] of(Corpus corpus, int k, int minFrequency) { Bigram[] bigrams = new Bigram[k]; HeapSelect<Bigram> heap = new HeapSelect<>(bigrams); Iterator<smile.nlp.Bigram> iterator = corpus.getBigrams(); while (iterator.hasNext()) { smile.nlp.Bigram bigram = iterator.next(); int c12 = corpus.getBigramFrequency(bigram); if (c12 > minFrequency) { int c1 = corpus.getTermFrequency(bigram.w1); int c2 = corpus.getTermFrequency(bigram.w2); double score = likelihoodRatio(c1, c2, c12, corpus.size()); heap.add(new Bigram(bigram.w1, bigram.w2, c12, -score)); } } heap.sort(); Bigram[] collocations = new Bigram[k]; for (int i = 0; i < k; i++) { Bigram bigram = bigrams[k-i-1]; collocations[i] = new Bigram(bigram.w1, bigram.w2, bigram.count, -bigram.score); } return collocations; } Bigram(String w1, String w2, int count, double score); @Override String toString(); @Override int compareTo(Bigram o); static Bigram[] of(Corpus corpus, int k, int minFrequency); static Bigram[] of(Corpus corpus, double p, int minFrequency); final int count; final double score; }