ForeignKeys { static double testTextWilcoxonRankSum(ArrayList<String> depColumn, ArrayList<String> refColumn) { Object2IntAVLTreeMap<String> refValueHistogram = new Object2IntAVLTreeMap<>(); refValueHistogram.defaultReturnValue(0); for (String refValue : refColumn) { int count = refValueHistogram.getInt(refValue); refValueHistogram.put(refValue, count + 1); } depColumn.sort(String::compareTo); double fractionSmallerRefs = 0d; int currentSmallerRefs = 0; Iterator<String> depIterator = depColumn.iterator(); ObjectBidirectionalIterator<Object2IntMap.Entry<String>> refIterator = refValueHistogram.object2IntEntrySet().iterator(); assert refIterator.hasNext(); Object2IntMap.Entry<String> currentRefEntry = refIterator.next(); while (depIterator.hasNext()) { String depValue = depIterator.next(); while (currentRefEntry != null && currentRefEntry.getKey().compareTo(depValue) < 0) { currentSmallerRefs += currentRefEntry.getIntValue(); if (refIterator.hasNext()) { currentRefEntry = refIterator.next(); } else { currentRefEntry = null; } } fractionSmallerRefs += currentSmallerRefs; if (currentRefEntry != null && currentRefEntry.getKey().equals(depValue)) { fractionSmallerRefs += currentRefEntry.getIntValue() / 2d; } } fractionSmallerRefs /= depColumn.size() * refColumn.size(); return fractionSmallerRefs; }  static Collection<Observation<InclusionDependency>> createTrainingSet(
            MetadataStore store,
            ConstraintCollection<InclusionDependency> foreignKeyCC,
            ConstraintCollection<InclusionDependency> indCC,
            ConstraintCollection<ColumnStatistics> columnStatisticCC,
            ConstraintCollection<TextColumnStatistics> textColumnStatisticCC,
            ConstraintCollection<TableSample> tableSampleCC,
            ConstraintCollection<UniqueColumnCombination> uniqueCC); static Collection<Instance<InclusionDependency>> createEstimationSet(
            MetadataStore store,
            ConstraintCollection<InclusionDependency> indCC,
            ConstraintCollection<ColumnStatistics> columnStatisticCC,
            ConstraintCollection<TextColumnStatistics> textColumnStatisticCC,
            ConstraintCollection<TableSample> tableSampleCC); static Collection<Observation<T>> oversample(
            Collection<Observation<T>> instances); static VectorModel trainLogisticRegressionModel(
            MetadataStore store,
            ConstraintCollection<InclusionDependency> foreignKeyCC,
            ConstraintCollection<InclusionDependency> indCC,
            ConstraintCollection<ColumnStatistics> columnStatisticCC,
            ConstraintCollection<TextColumnStatistics> textColumnStatisticCC,
            ConstraintCollection<TableSample> tableSampleCC,
            ConstraintCollection<UniqueColumnCombination> uniqueCC); static Collection<Prediction<InclusionDependency, Double>> estimateForeignKeyProbability(
            MetadataStore store,
            ConstraintCollection<InclusionDependency> indCC,
            ConstraintCollection<ColumnStatistics> columnStatisticCC,
            ConstraintCollection<TextColumnStatistics> textColumnStatisticCC,
            ConstraintCollection<TableSample> tableSampleCC,
            VectorModel logisticRegressionModel); static Collection<InclusionDependency> estimateForeignKeys(
            MetadataStore store,
            ConstraintCollection<InclusionDependency> indCC,
            ConstraintCollection<UniqueColumnCombination> uccCC,
            ConstraintCollection<ColumnStatistics> columnStatisticCC,
            ConstraintCollection<TextColumnStatistics> textColumnStatisticCC,
            ConstraintCollection<TableSample> tableSampleCC,
            VectorModel logisticRegressionModel); static Collection<InclusionDependency> estimateForeignKeys(
            MetadataStore store,
            ConstraintCollection<InclusionDependency> indCC,
            ConstraintCollection<UniqueColumnCombination> uccCC,
            ConstraintCollection<ColumnStatistics> columnStatisticCC,
            ConstraintCollection<TextColumnStatistics> textColumnStatisticCC,
            ConstraintCollection<TableSample> tableSampleCC,
            VectorModel logisticRegressionModel,
            double minFkProbability); static final String[] features; }
@Test public void testTestTextWilcoxonRankSum() { double p = ForeignKeys.testTextWilcoxonRankSum( new ArrayList<>(Arrays.asList("a", "b", "c", "d", "e")), new ArrayList<>(Arrays.asList("a", "b", "c", "d", "e")) ); Assert.assertEquals(.5, p, .001); p = ForeignKeys.testTextWilcoxonRankSum( new ArrayList<>(Arrays.asList("a", "b", "c", "d")), new ArrayList<>(Arrays.asList("a", "b", "c", "d", "e")) ); Assert.assertEquals(8.0 / 20.0, p, .001); p = ForeignKeys.testTextWilcoxonRankSum( new ArrayList<>(Arrays.asList("b", "b", "c", "c")), new ArrayList<>(Arrays.asList("a", "b", "c", "d", "e")) ); Assert.assertEquals(8.0 / 20.0, p, .001); p = ForeignKeys.testTextWilcoxonRankSum( new ArrayList<>(Collections.singletonList("a")), new ArrayList<>(Arrays.asList("b", "c", "d", "e")) ); Assert.assertEquals(0.0, p, .001); p = ForeignKeys.testTextWilcoxonRankSum( new ArrayList<>(Arrays.asList("f", "g")), new ArrayList<>(Arrays.asList("b", "c", "d", "e")) ); Assert.assertEquals(1.0, p, .001); }