ForeignKeys { static double estimateDistributionDistance(int depColumnId, TableSample depTableSample, int refColumnId, TableSample refTableSample, IdUtils idUtils) { ArrayList<String> refColumn = extractColumnSample(refColumnId, refTableSample, idUtils); ArrayList<String> depColumn = extractColumnSample(depColumnId, depTableSample, idUtils); if (refColumn.isEmpty() || depColumn.isEmpty()) { return 0d; } List<double[]> histograms = computeHistograms(refColumn, depColumn); double[] refHistogram = histograms.get(0); double[] depHistogram = histograms.get(1); Object2DoubleMap<IntList> sourceBuckets = new Object2DoubleOpenHashMap<>(); Object2DoubleMap<IntList> targetBuckets = new Object2DoubleOpenHashMap<>(); for (int i = 0; i < depHistogram.length; i++) { double diff = refHistogram[i] - depHistogram[i]; if (diff < 0) sourceBuckets.put(new IntArrayList(new int[]{i}), -diff); else if (diff > 0) targetBuckets.put(new IntArrayList(new int[]{i}), diff); } IntList normalizers = new IntArrayList(new int[]{depHistogram.length}); double[] sourceNodes = new double[sourceBuckets.size()]; double[] targetNodes = new double[targetBuckets.size()]; List<Object2DoubleMap.Entry<IntList>> fixedTargetBuckets = new ArrayList<>(targetBuckets.object2DoubleEntrySet()); double[][] distance = new double[sourceBuckets.size()][targetBuckets.size()]; int sourceIndex = 0; for (Object2DoubleMap.Entry<IntList> sourceBucketEntry : sourceBuckets.object2DoubleEntrySet()) { sourceNodes[sourceIndex] = sourceBucketEntry.getDoubleValue(); int targetIndex = 0; IntList sourceBucket = sourceBucketEntry.getKey(); for (Object2DoubleMap.Entry<IntList> targetBucketEntry : fixedTargetBuckets) { targetNodes[targetIndex] = targetBucketEntry.getDoubleValue(); IntList targetBucket = targetBucketEntry.getKey(); assert !sourceBucket.equals(targetBucket); distance[sourceIndex][targetIndex] = sourceBucket.equals(targetBucket) ? Double.POSITIVE_INFINITY : normalizedManhattanDistance(sourceBucket, targetBucket, normalizers); targetIndex++; } sourceIndex++; } return calculateEarthMoverDistance(sourceNodes, targetNodes, distance); }  static Collection<Observation<InclusionDependency>> createTrainingSet(
            MetadataStore store,
            ConstraintCollection<InclusionDependency> foreignKeyCC,
            ConstraintCollection<InclusionDependency> indCC,
            ConstraintCollection<ColumnStatistics> columnStatisticCC,
            ConstraintCollection<TextColumnStatistics> textColumnStatisticCC,
            ConstraintCollection<TableSample> tableSampleCC,
            ConstraintCollection<UniqueColumnCombination> uniqueCC); static Collection<Instance<InclusionDependency>> createEstimationSet(
            MetadataStore store,
            ConstraintCollection<InclusionDependency> indCC,
            ConstraintCollection<ColumnStatistics> columnStatisticCC,
            ConstraintCollection<TextColumnStatistics> textColumnStatisticCC,
            ConstraintCollection<TableSample> tableSampleCC); static Collection<Observation<T>> oversample(
            Collection<Observation<T>> instances); static VectorModel trainLogisticRegressionModel(
            MetadataStore store,
            ConstraintCollection<InclusionDependency> foreignKeyCC,
            ConstraintCollection<InclusionDependency> indCC,
            ConstraintCollection<ColumnStatistics> columnStatisticCC,
            ConstraintCollection<TextColumnStatistics> textColumnStatisticCC,
            ConstraintCollection<TableSample> tableSampleCC,
            ConstraintCollection<UniqueColumnCombination> uniqueCC); static Collection<Prediction<InclusionDependency, Double>> estimateForeignKeyProbability(
            MetadataStore store,
            ConstraintCollection<InclusionDependency> indCC,
            ConstraintCollection<ColumnStatistics> columnStatisticCC,
            ConstraintCollection<TextColumnStatistics> textColumnStatisticCC,
            ConstraintCollection<TableSample> tableSampleCC,
            VectorModel logisticRegressionModel); static Collection<InclusionDependency> estimateForeignKeys(
            MetadataStore store,
            ConstraintCollection<InclusionDependency> indCC,
            ConstraintCollection<UniqueColumnCombination> uccCC,
            ConstraintCollection<ColumnStatistics> columnStatisticCC,
            ConstraintCollection<TextColumnStatistics> textColumnStatisticCC,
            ConstraintCollection<TableSample> tableSampleCC,
            VectorModel logisticRegressionModel); static Collection<InclusionDependency> estimateForeignKeys(
            MetadataStore store,
            ConstraintCollection<InclusionDependency> indCC,
            ConstraintCollection<UniqueColumnCombination> uccCC,
            ConstraintCollection<ColumnStatistics> columnStatisticCC,
            ConstraintCollection<TextColumnStatistics> textColumnStatisticCC,
            ConstraintCollection<TableSample> tableSampleCC,
            VectorModel logisticRegressionModel,
            double minFkProbability); static final String[] features; }
@Test public void testDistributionalSimilarity() { MetadataStore store = new DefaultMetadataStore(); Schema schema = store.addSchema("schema", null, null); Table table1 = schema.addTable(store, "table1", null, null); Column column1 = table1.addColumn(store, "col1", null, 0); Column column2 = table1.addColumn(store, "col2", null, 1); Table table2 = schema.addTable(store, "table2", null, null); Column column3 = table2.addColumn(store, "col3", null, 0); Column column4 = table2.addColumn(store, "col4", null, 1); TableSample tableSample1 = new TableSample(table1.getId(), new String[][]{ new String[]{"a", "0"}, new String[]{"a", "1"}, new String[]{"a", "2"}, new String[]{"c", "3"}, new String[]{"a", "4"}, new String[]{"a", "5"}, new String[]{"a", "6"}, new String[]{"e", "7"} } ); TableSample tableSample2 = new TableSample(table2.getId(), new String[][]{ new String[]{"a", "10"}, new String[]{"b", "100"}, new String[]{"c", "200"}, new String[]{"d", "300"}, new String[]{"e", "400"}, new String[]{"f", "500"}, new String[]{"g", "600"}, new String[]{"h", "700"} } ); double result1 = ForeignKeys.estimateDistributionDistance( column1.getId(), tableSample1, column1.getId(), tableSample1, store.getIdUtils() ); double result2 = ForeignKeys.estimateDistributionDistance( column1.getId(), tableSample1, column3.getId(), tableSample2, store.getIdUtils() ); double result3 = ForeignKeys.estimateDistributionDistance( column2.getId(), tableSample1, column4.getId(), tableSample2, store.getIdUtils() ); Assert.assertEquals(0d, result1, 0d); Assert.assertTrue(result2 > 0d); Assert.assertTrue(result3 > 0d); Assert.assertTrue(result2 < result3); }