@Test public void test() { System.out.println("GoodTuring"); int[] r = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12}; int[] Nr = {120, 40, 24, 13, 15, 5, 11, 2, 2, 1, 3}; double p0 = 0.2047782; double[] p = { 0.0009267, 0.0024393, 0.0040945, 0.0058063, 0.0075464, 0.0093026, 0.0110689, 0.0128418, 0.0146194, 0.0164005, 0.0199696}; GoodTuring result = GoodTuring.of(r, Nr); assertEquals(p0, result.p0, 1E-7); for (int i = 0; i < r.length; i++) { assertEquals(p[i], result.p[i], 1E-7); } }
public static GoodTuring of(int[] r, int[] Nr) { final double CONFID_FACTOR = 1.96; if (r.length != Nr.length) { throw new IllegalArgumentException("The sizes of r and Nr are not same."); } int len = r.length; double[] p = new double[len]; double[] logR = new double[len]; double[] logZ = new double[len]; double[] Z = new double[len]; int N = 0; for (int j = 0; j < len; ++j) { N += r[j] * Nr[j]; } int n1 = (r[0] != 1) ? 0 : Nr[0]; double p0 = n1 / (double) N; for (int j = 0; j < len; ++j) { int q = j == 0 ? 0 : r[j - 1]; int t = j == len - 1 ? 2 * r[j] - q : r[j + 1]; Z[j] = 2.0 * Nr[j] / (t - q); logR[j] = Math.log(r[j]); logZ[j] = Math.log(Z[j]); } double XYs = 0.0, Xsquares = 0.0, meanX = 0.0, meanY = 0.0; for (int i = 0; i < len; ++i) { meanX += logR[i]; meanY += logZ[i]; } meanX /= len; meanY /= len; for (int i = 0; i < len; ++i) { XYs += (logR[i] - meanX) * (logZ[i] - meanY); Xsquares += MathEx.sqr(logR[i] - meanX); } double slope = XYs / Xsquares; double intercept = meanY - slope * meanX; boolean indiffValsSeen = false; for (int j = 0; j < len; ++j) { double y = (r[j] + 1) * smoothed(r[j] + 1, slope, intercept) / smoothed(r[j], slope, intercept); if (row(r, r[j] + 1) < 0) { indiffValsSeen = true; } if (!indiffValsSeen) { int n = Nr[row(r, r[j] + 1)]; double x = (r[j] + 1) * n / (double) Nr[j]; if (Math.abs(x - y) <= CONFID_FACTOR * Math.sqrt(MathEx.sqr(r[j] + 1.0) * n / MathEx.sqr(Nr[j]) * (1 + n / (double) Nr[j]))) { indiffValsSeen = true; } else { p[j] = x; } } if (indiffValsSeen) { p[j] = y; } } double Nprime = 0.0; for (int j = 0; j < len; ++j) { Nprime += Nr[j] * p[j]; } for (int j = 0; j < len; ++j) { p[j] = (1 - p0) * p[j] / Nprime; } return new GoodTuring(p, p0); }
GoodTuring { public static GoodTuring of(int[] r, int[] Nr) { final double CONFID_FACTOR = 1.96; if (r.length != Nr.length) { throw new IllegalArgumentException("The sizes of r and Nr are not same."); } int len = r.length; double[] p = new double[len]; double[] logR = new double[len]; double[] logZ = new double[len]; double[] Z = new double[len]; int N = 0; for (int j = 0; j < len; ++j) { N += r[j] * Nr[j]; } int n1 = (r[0] != 1) ? 0 : Nr[0]; double p0 = n1 / (double) N; for (int j = 0; j < len; ++j) { int q = j == 0 ? 0 : r[j - 1]; int t = j == len - 1 ? 2 * r[j] - q : r[j + 1]; Z[j] = 2.0 * Nr[j] / (t - q); logR[j] = Math.log(r[j]); logZ[j] = Math.log(Z[j]); } double XYs = 0.0, Xsquares = 0.0, meanX = 0.0, meanY = 0.0; for (int i = 0; i < len; ++i) { meanX += logR[i]; meanY += logZ[i]; } meanX /= len; meanY /= len; for (int i = 0; i < len; ++i) { XYs += (logR[i] - meanX) * (logZ[i] - meanY); Xsquares += MathEx.sqr(logR[i] - meanX); } double slope = XYs / Xsquares; double intercept = meanY - slope * meanX; boolean indiffValsSeen = false; for (int j = 0; j < len; ++j) { double y = (r[j] + 1) * smoothed(r[j] + 1, slope, intercept) / smoothed(r[j], slope, intercept); if (row(r, r[j] + 1) < 0) { indiffValsSeen = true; } if (!indiffValsSeen) { int n = Nr[row(r, r[j] + 1)]; double x = (r[j] + 1) * n / (double) Nr[j]; if (Math.abs(x - y) <= CONFID_FACTOR * Math.sqrt(MathEx.sqr(r[j] + 1.0) * n / MathEx.sqr(Nr[j]) * (1 + n / (double) Nr[j]))) { indiffValsSeen = true; } else { p[j] = x; } } if (indiffValsSeen) { p[j] = y; } } double Nprime = 0.0; for (int j = 0; j < len; ++j) { Nprime += Nr[j] * p[j]; } for (int j = 0; j < len; ++j) { p[j] = (1 - p0) * p[j] / Nprime; } return new GoodTuring(p, p0); } }
GoodTuring { public static GoodTuring of(int[] r, int[] Nr) { final double CONFID_FACTOR = 1.96; if (r.length != Nr.length) { throw new IllegalArgumentException("The sizes of r and Nr are not same."); } int len = r.length; double[] p = new double[len]; double[] logR = new double[len]; double[] logZ = new double[len]; double[] Z = new double[len]; int N = 0; for (int j = 0; j < len; ++j) { N += r[j] * Nr[j]; } int n1 = (r[0] != 1) ? 0 : Nr[0]; double p0 = n1 / (double) N; for (int j = 0; j < len; ++j) { int q = j == 0 ? 0 : r[j - 1]; int t = j == len - 1 ? 2 * r[j] - q : r[j + 1]; Z[j] = 2.0 * Nr[j] / (t - q); logR[j] = Math.log(r[j]); logZ[j] = Math.log(Z[j]); } double XYs = 0.0, Xsquares = 0.0, meanX = 0.0, meanY = 0.0; for (int i = 0; i < len; ++i) { meanX += logR[i]; meanY += logZ[i]; } meanX /= len; meanY /= len; for (int i = 0; i < len; ++i) { XYs += (logR[i] - meanX) * (logZ[i] - meanY); Xsquares += MathEx.sqr(logR[i] - meanX); } double slope = XYs / Xsquares; double intercept = meanY - slope * meanX; boolean indiffValsSeen = false; for (int j = 0; j < len; ++j) { double y = (r[j] + 1) * smoothed(r[j] + 1, slope, intercept) / smoothed(r[j], slope, intercept); if (row(r, r[j] + 1) < 0) { indiffValsSeen = true; } if (!indiffValsSeen) { int n = Nr[row(r, r[j] + 1)]; double x = (r[j] + 1) * n / (double) Nr[j]; if (Math.abs(x - y) <= CONFID_FACTOR * Math.sqrt(MathEx.sqr(r[j] + 1.0) * n / MathEx.sqr(Nr[j]) * (1 + n / (double) Nr[j]))) { indiffValsSeen = true; } else { p[j] = x; } } if (indiffValsSeen) { p[j] = y; } } double Nprime = 0.0; for (int j = 0; j < len; ++j) { Nprime += Nr[j] * p[j]; } for (int j = 0; j < len; ++j) { p[j] = (1 - p0) * p[j] / Nprime; } return new GoodTuring(p, p0); } private  GoodTuring(double[] p, double p0); }
GoodTuring { public static GoodTuring of(int[] r, int[] Nr) { final double CONFID_FACTOR = 1.96; if (r.length != Nr.length) { throw new IllegalArgumentException("The sizes of r and Nr are not same."); } int len = r.length; double[] p = new double[len]; double[] logR = new double[len]; double[] logZ = new double[len]; double[] Z = new double[len]; int N = 0; for (int j = 0; j < len; ++j) { N += r[j] * Nr[j]; } int n1 = (r[0] != 1) ? 0 : Nr[0]; double p0 = n1 / (double) N; for (int j = 0; j < len; ++j) { int q = j == 0 ? 0 : r[j - 1]; int t = j == len - 1 ? 2 * r[j] - q : r[j + 1]; Z[j] = 2.0 * Nr[j] / (t - q); logR[j] = Math.log(r[j]); logZ[j] = Math.log(Z[j]); } double XYs = 0.0, Xsquares = 0.0, meanX = 0.0, meanY = 0.0; for (int i = 0; i < len; ++i) { meanX += logR[i]; meanY += logZ[i]; } meanX /= len; meanY /= len; for (int i = 0; i < len; ++i) { XYs += (logR[i] - meanX) * (logZ[i] - meanY); Xsquares += MathEx.sqr(logR[i] - meanX); } double slope = XYs / Xsquares; double intercept = meanY - slope * meanX; boolean indiffValsSeen = false; for (int j = 0; j < len; ++j) { double y = (r[j] + 1) * smoothed(r[j] + 1, slope, intercept) / smoothed(r[j], slope, intercept); if (row(r, r[j] + 1) < 0) { indiffValsSeen = true; } if (!indiffValsSeen) { int n = Nr[row(r, r[j] + 1)]; double x = (r[j] + 1) * n / (double) Nr[j]; if (Math.abs(x - y) <= CONFID_FACTOR * Math.sqrt(MathEx.sqr(r[j] + 1.0) * n / MathEx.sqr(Nr[j]) * (1 + n / (double) Nr[j]))) { indiffValsSeen = true; } else { p[j] = x; } } if (indiffValsSeen) { p[j] = y; } } double Nprime = 0.0; for (int j = 0; j < len; ++j) { Nprime += Nr[j] * p[j]; } for (int j = 0; j < len; ++j) { p[j] = (1 - p0) * p[j] / Nprime; } return new GoodTuring(p, p0); } private  GoodTuring(double[] p, double p0); static GoodTuring of(int[] r, int[] Nr); }
GoodTuring { public static GoodTuring of(int[] r, int[] Nr) { final double CONFID_FACTOR = 1.96; if (r.length != Nr.length) { throw new IllegalArgumentException("The sizes of r and Nr are not same."); } int len = r.length; double[] p = new double[len]; double[] logR = new double[len]; double[] logZ = new double[len]; double[] Z = new double[len]; int N = 0; for (int j = 0; j < len; ++j) { N += r[j] * Nr[j]; } int n1 = (r[0] != 1) ? 0 : Nr[0]; double p0 = n1 / (double) N; for (int j = 0; j < len; ++j) { int q = j == 0 ? 0 : r[j - 1]; int t = j == len - 1 ? 2 * r[j] - q : r[j + 1]; Z[j] = 2.0 * Nr[j] / (t - q); logR[j] = Math.log(r[j]); logZ[j] = Math.log(Z[j]); } double XYs = 0.0, Xsquares = 0.0, meanX = 0.0, meanY = 0.0; for (int i = 0; i < len; ++i) { meanX += logR[i]; meanY += logZ[i]; } meanX /= len; meanY /= len; for (int i = 0; i < len; ++i) { XYs += (logR[i] - meanX) * (logZ[i] - meanY); Xsquares += MathEx.sqr(logR[i] - meanX); } double slope = XYs / Xsquares; double intercept = meanY - slope * meanX; boolean indiffValsSeen = false; for (int j = 0; j < len; ++j) { double y = (r[j] + 1) * smoothed(r[j] + 1, slope, intercept) / smoothed(r[j], slope, intercept); if (row(r, r[j] + 1) < 0) { indiffValsSeen = true; } if (!indiffValsSeen) { int n = Nr[row(r, r[j] + 1)]; double x = (r[j] + 1) * n / (double) Nr[j]; if (Math.abs(x - y) <= CONFID_FACTOR * Math.sqrt(MathEx.sqr(r[j] + 1.0) * n / MathEx.sqr(Nr[j]) * (1 + n / (double) Nr[j]))) { indiffValsSeen = true; } else { p[j] = x; } } if (indiffValsSeen) { p[j] = y; } } double Nprime = 0.0; for (int j = 0; j < len; ++j) { Nprime += Nr[j] * p[j]; } for (int j = 0; j < len; ++j) { p[j] = (1 - p0) * p[j] / Nprime; } return new GoodTuring(p, p0); } private  GoodTuring(double[] p, double p0); static GoodTuring of(int[] r, int[] Nr); final double[] p; final double p0; }