@Test public void distinguishableEffectSizes() throws Exception { DistinguishableEffectSize output = testStats95.distinguishableEffectSizes(5, 2, 4, 0); assertThat("check lower bound", output.getNegativeEffectSize(), IsCloseTo.closeTo(-0.5116258, precision)); assertThat("check upper bound", output.getPositiveEffectSize(), IsCloseTo.closeTo(0.4136477, precision)); DistinguishableEffectSize output2 = testStats95.distinguishableEffectSizes(5, 2, 4, 3); assertThat("independent of actions_2", output2, equalTo(output)); output = testStats95.distinguishableEffectSizes(500, 200, 400, 0); assertThat("check lower bound", output.getNegativeEffectSize(), IsCloseTo.closeTo(-0.0650622, precision)); assertThat("check upper bound", output.getPositiveEffectSize(), IsCloseTo.closeTo(0.0631597, precision)); output2 = testStats95.distinguishableEffectSizes(500, 200, 400, 300); assertThat("independent of actions_2", output2, equalTo(output)); output = testStats95.distinguishableEffectSizes(0, 0, 1, 0); assertThat("0 impressions returns NaN", output.getNegativeEffectSize(), is(NaN)); assertThat("0 impressions returns NaN", output.getPositiveEffectSize(), is(NaN)); output = testStats95.distinguishableEffectSizes(1, 1, 0, 0); assertThat("0 impressions returns NaN", output.getNegativeEffectSize(), is(NaN)); assertThat("0 impressions returns NaN", output.getPositiveEffectSize(), is(NaN)); }
@Override public DistinguishableEffectSize distinguishableEffectSizes(final long number_impressions_1, final long number_actions_1, final long number_impressions_2, final long number_actions_2) { inputValidation(number_impressions_1, number_actions_1, number_impressions_2); double zval = DecisionAdjuster.scaledZ(confidenceLevel, maxDecisions); double rate_1 = 1.0 * number_actions_1 / number_impressions_1; double scale_1 = zval * zval / number_impressions_1; double scale_2 = zval * zval / number_impressions_2; double a = 1 + scale_2; double b = -(2 * rate_1 + scale_2); double c = rate_1 * rate_1 * (1 + scale_1) - rate_1 * scale_1; return new DistinguishableEffectSize.Builder() .withNegativeEffectSize(rate_1 - (-b + sqrt(b * b - 4 * a * c)) / (2 * a)) .withPositiveEffectSize(rate_1 - (-b - sqrt(b * b - 4 * a * c)) / (2 * a)) .build(); }
NormalApproxMetric extends BinomialMetric { @Override public DistinguishableEffectSize distinguishableEffectSizes(final long number_impressions_1, final long number_actions_1, final long number_impressions_2, final long number_actions_2) { inputValidation(number_impressions_1, number_actions_1, number_impressions_2); double zval = DecisionAdjuster.scaledZ(confidenceLevel, maxDecisions); double rate_1 = 1.0 * number_actions_1 / number_impressions_1; double scale_1 = zval * zval / number_impressions_1; double scale_2 = zval * zval / number_impressions_2; double a = 1 + scale_2; double b = -(2 * rate_1 + scale_2); double c = rate_1 * rate_1 * (1 + scale_1) - rate_1 * scale_1; return new DistinguishableEffectSize.Builder() .withNegativeEffectSize(rate_1 - (-b + sqrt(b * b - 4 * a * c)) / (2 * a)) .withPositiveEffectSize(rate_1 - (-b - sqrt(b * b - 4 * a * c)) / (2 * a)) .build(); } }
NormalApproxMetric extends BinomialMetric { @Override public DistinguishableEffectSize distinguishableEffectSizes(final long number_impressions_1, final long number_actions_1, final long number_impressions_2, final long number_actions_2) { inputValidation(number_impressions_1, number_actions_1, number_impressions_2); double zval = DecisionAdjuster.scaledZ(confidenceLevel, maxDecisions); double rate_1 = 1.0 * number_actions_1 / number_impressions_1; double scale_1 = zval * zval / number_impressions_1; double scale_2 = zval * zval / number_impressions_2; double a = 1 + scale_2; double b = -(2 * rate_1 + scale_2); double c = rate_1 * rate_1 * (1 + scale_1) - rate_1 * scale_1; return new DistinguishableEffectSize.Builder() .withNegativeEffectSize(rate_1 - (-b + sqrt(b * b - 4 * a * c)) / (2 * a)) .withPositiveEffectSize(rate_1 - (-b - sqrt(b * b - 4 * a * c)) / (2 * a)) .build(); } NormalApproxMetric(final double conf_level);  NormalApproxMetric(final double conf_level, final double max_decisions); }
NormalApproxMetric extends BinomialMetric { @Override public DistinguishableEffectSize distinguishableEffectSizes(final long number_impressions_1, final long number_actions_1, final long number_impressions_2, final long number_actions_2) { inputValidation(number_impressions_1, number_actions_1, number_impressions_2); double zval = DecisionAdjuster.scaledZ(confidenceLevel, maxDecisions); double rate_1 = 1.0 * number_actions_1 / number_impressions_1; double scale_1 = zval * zval / number_impressions_1; double scale_2 = zval * zval / number_impressions_2; double a = 1 + scale_2; double b = -(2 * rate_1 + scale_2); double c = rate_1 * rate_1 * (1 + scale_1) - rate_1 * scale_1; return new DistinguishableEffectSize.Builder() .withNegativeEffectSize(rate_1 - (-b + sqrt(b * b - 4 * a * c)) / (2 * a)) .withPositiveEffectSize(rate_1 - (-b - sqrt(b * b - 4 * a * c)) / (2 * a)) .build(); } NormalApproxMetric(final double conf_level);  NormalApproxMetric(final double conf_level, final double max_decisions); @Override Estimate estimateRate(final long number_impressions, final long number_actions); @Override Estimate estimateRateDifference(final long number_impressions_1, final long number_actions_1,
                                           final long number_impressions_2, final long number_actions_2); @Override DistinguishableEffectSize distinguishableEffectSizes(final long number_impressions_1,
                                                                final long number_actions_1,
                                                                final long number_impressions_2,
                                                                final long number_actions_2); @Override Double fractionOfData(final long number_impressions_1, final long number_actions_1,
                                 final long number_impressions_2, final long number_actions_2,
                                 final double effect_size); @Override Long predictedDataNeeded(final double rate, final double allocation_percent_1,
                                    final double allocation_percent_2, final double effect_size); }
NormalApproxMetric extends BinomialMetric { @Override public DistinguishableEffectSize distinguishableEffectSizes(final long number_impressions_1, final long number_actions_1, final long number_impressions_2, final long number_actions_2) { inputValidation(number_impressions_1, number_actions_1, number_impressions_2); double zval = DecisionAdjuster.scaledZ(confidenceLevel, maxDecisions); double rate_1 = 1.0 * number_actions_1 / number_impressions_1; double scale_1 = zval * zval / number_impressions_1; double scale_2 = zval * zval / number_impressions_2; double a = 1 + scale_2; double b = -(2 * rate_1 + scale_2); double c = rate_1 * rate_1 * (1 + scale_1) - rate_1 * scale_1; return new DistinguishableEffectSize.Builder() .withNegativeEffectSize(rate_1 - (-b + sqrt(b * b - 4 * a * c)) / (2 * a)) .withPositiveEffectSize(rate_1 - (-b - sqrt(b * b - 4 * a * c)) / (2 * a)) .build(); } NormalApproxMetric(final double conf_level);  NormalApproxMetric(final double conf_level, final double max_decisions); @Override Estimate estimateRate(final long number_impressions, final long number_actions); @Override Estimate estimateRateDifference(final long number_impressions_1, final long number_actions_1,
                                           final long number_impressions_2, final long number_actions_2); @Override DistinguishableEffectSize distinguishableEffectSizes(final long number_impressions_1,
                                                                final long number_actions_1,
                                                                final long number_impressions_2,
                                                                final long number_actions_2); @Override Double fractionOfData(final long number_impressions_1, final long number_actions_1,
                                 final long number_impressions_2, final long number_actions_2,
                                 final double effect_size); @Override Long predictedDataNeeded(final double rate, final double allocation_percent_1,
                                    final double allocation_percent_2, final double effect_size);  }