@Test public void testFit() { Map<Integer, Vector> data = new HashMap<>(); data.put(1, VectorUtils.of(2, 4, 1)); data.put(2, VectorUtils.of(1, 8, 22)); data.put(3, VectorUtils.of(4, 10, 100)); data.put(4, VectorUtils.of(0, 22, 300)); DatasetBuilder<Integer, Vector> datasetBuilder = new LocalDatasetBuilder<>(data, parts); final Vectorizer<Integer, Vector, Integer, Double> vectorizer = new DummyVectorizer<>(0, 1, 2); MinMaxScalerTrainer<Integer, Vector> standardizationTrainer = new MinMaxScalerTrainer<>(); MinMaxScalerPreprocessor<Integer, Vector> preprocessor = standardizationTrainer.fit( TestUtils.testEnvBuilder(), datasetBuilder, vectorizer ); assertArrayEquals(new double[] {0, 4, 1}, preprocessor.getMin(), 1e-8); assertArrayEquals(new double[] {4, 22, 300}, preprocessor.getMax(), 1e-8); }
@Override public MinMaxScalerPreprocessor<K, V> fit( LearningEnvironmentBuilder envBuilder, DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> basePreprocessor) { PartitionContextBuilder<K, V, EmptyContext> ctxBuilder = (env, upstream, upstreamSize) -> new EmptyContext(); try (Dataset<EmptyContext, MinMaxScalerPartitionData> dataset = datasetBuilder.build( envBuilder, ctxBuilder, (env, upstream, upstreamSize, ctx) -> { double[] min = null; double[] max = null; while (upstream.hasNext()) { UpstreamEntry<K, V> entity = upstream.next(); LabeledVector row = basePreprocessor.apply(entity.getKey(), entity.getValue()); if (min == null) { min = new double[row.size()]; for (int i = 0; i < min.length; i++) min[i] = Double.MAX_VALUE; } else assert min.length == row.size() : "Base preprocessor must return exactly " + min.length + " features"; if (max == null) { max = new double[row.size()]; for (int i = 0; i < max.length; i++) max[i] = -Double.MAX_VALUE; } else assert max.length == row.size() : "Base preprocessor must return exactly " + min.length + " features"; for (int i = 0; i < row.size(); i++) { if (row.get(i) < min[i]) min[i] = row.get(i); if (row.get(i) > max[i]) max[i] = row.get(i); } } return new MinMaxScalerPartitionData(min, max); }, learningEnvironment(basePreprocessor) )) { double[][] minMax = dataset.compute( data -> data.getMin() != null ? new double[][]{ data.getMin(), data.getMax() } : null, (a, b) -> { if (a == null) return b; if (b == null) return a; double[][] res = new double[2][]; res[0] = new double[a[0].length]; for (int i = 0; i < res[0].length; i++) res[0][i] = Math.min(a[0][i], b[0][i]); res[1] = new double[a[1].length]; for (int i = 0; i < res[1].length; i++) res[1][i] = Math.max(a[1][i], b[1][i]); return res; } ); return new MinMaxScalerPreprocessor<>(minMax[0], minMax[1], basePreprocessor); } catch (Exception e) { throw new RuntimeException(e); } }
MinMaxScalerTrainer implements PreprocessingTrainer<K, V> { @Override public MinMaxScalerPreprocessor<K, V> fit( LearningEnvironmentBuilder envBuilder, DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> basePreprocessor) { PartitionContextBuilder<K, V, EmptyContext> ctxBuilder = (env, upstream, upstreamSize) -> new EmptyContext(); try (Dataset<EmptyContext, MinMaxScalerPartitionData> dataset = datasetBuilder.build( envBuilder, ctxBuilder, (env, upstream, upstreamSize, ctx) -> { double[] min = null; double[] max = null; while (upstream.hasNext()) { UpstreamEntry<K, V> entity = upstream.next(); LabeledVector row = basePreprocessor.apply(entity.getKey(), entity.getValue()); if (min == null) { min = new double[row.size()]; for (int i = 0; i < min.length; i++) min[i] = Double.MAX_VALUE; } else assert min.length == row.size() : "Base preprocessor must return exactly " + min.length + " features"; if (max == null) { max = new double[row.size()]; for (int i = 0; i < max.length; i++) max[i] = -Double.MAX_VALUE; } else assert max.length == row.size() : "Base preprocessor must return exactly " + min.length + " features"; for (int i = 0; i < row.size(); i++) { if (row.get(i) < min[i]) min[i] = row.get(i); if (row.get(i) > max[i]) max[i] = row.get(i); } } return new MinMaxScalerPartitionData(min, max); }, learningEnvironment(basePreprocessor) )) { double[][] minMax = dataset.compute( data -> data.getMin() != null ? new double[][]{ data.getMin(), data.getMax() } : null, (a, b) -> { if (a == null) return b; if (b == null) return a; double[][] res = new double[2][]; res[0] = new double[a[0].length]; for (int i = 0; i < res[0].length; i++) res[0][i] = Math.min(a[0][i], b[0][i]); res[1] = new double[a[1].length]; for (int i = 0; i < res[1].length; i++) res[1][i] = Math.max(a[1][i], b[1][i]); return res; } ); return new MinMaxScalerPreprocessor<>(minMax[0], minMax[1], basePreprocessor); } catch (Exception e) { throw new RuntimeException(e); } } }
MinMaxScalerTrainer implements PreprocessingTrainer<K, V> { @Override public MinMaxScalerPreprocessor<K, V> fit( LearningEnvironmentBuilder envBuilder, DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> basePreprocessor) { PartitionContextBuilder<K, V, EmptyContext> ctxBuilder = (env, upstream, upstreamSize) -> new EmptyContext(); try (Dataset<EmptyContext, MinMaxScalerPartitionData> dataset = datasetBuilder.build( envBuilder, ctxBuilder, (env, upstream, upstreamSize, ctx) -> { double[] min = null; double[] max = null; while (upstream.hasNext()) { UpstreamEntry<K, V> entity = upstream.next(); LabeledVector row = basePreprocessor.apply(entity.getKey(), entity.getValue()); if (min == null) { min = new double[row.size()]; for (int i = 0; i < min.length; i++) min[i] = Double.MAX_VALUE; } else assert min.length == row.size() : "Base preprocessor must return exactly " + min.length + " features"; if (max == null) { max = new double[row.size()]; for (int i = 0; i < max.length; i++) max[i] = -Double.MAX_VALUE; } else assert max.length == row.size() : "Base preprocessor must return exactly " + min.length + " features"; for (int i = 0; i < row.size(); i++) { if (row.get(i) < min[i]) min[i] = row.get(i); if (row.get(i) > max[i]) max[i] = row.get(i); } } return new MinMaxScalerPartitionData(min, max); }, learningEnvironment(basePreprocessor) )) { double[][] minMax = dataset.compute( data -> data.getMin() != null ? new double[][]{ data.getMin(), data.getMax() } : null, (a, b) -> { if (a == null) return b; if (b == null) return a; double[][] res = new double[2][]; res[0] = new double[a[0].length]; for (int i = 0; i < res[0].length; i++) res[0][i] = Math.min(a[0][i], b[0][i]); res[1] = new double[a[1].length]; for (int i = 0; i < res[1].length; i++) res[1][i] = Math.max(a[1][i], b[1][i]); return res; } ); return new MinMaxScalerPreprocessor<>(minMax[0], minMax[1], basePreprocessor); } catch (Exception e) { throw new RuntimeException(e); } }  }
MinMaxScalerTrainer implements PreprocessingTrainer<K, V> { @Override public MinMaxScalerPreprocessor<K, V> fit( LearningEnvironmentBuilder envBuilder, DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> basePreprocessor) { PartitionContextBuilder<K, V, EmptyContext> ctxBuilder = (env, upstream, upstreamSize) -> new EmptyContext(); try (Dataset<EmptyContext, MinMaxScalerPartitionData> dataset = datasetBuilder.build( envBuilder, ctxBuilder, (env, upstream, upstreamSize, ctx) -> { double[] min = null; double[] max = null; while (upstream.hasNext()) { UpstreamEntry<K, V> entity = upstream.next(); LabeledVector row = basePreprocessor.apply(entity.getKey(), entity.getValue()); if (min == null) { min = new double[row.size()]; for (int i = 0; i < min.length; i++) min[i] = Double.MAX_VALUE; } else assert min.length == row.size() : "Base preprocessor must return exactly " + min.length + " features"; if (max == null) { max = new double[row.size()]; for (int i = 0; i < max.length; i++) max[i] = -Double.MAX_VALUE; } else assert max.length == row.size() : "Base preprocessor must return exactly " + min.length + " features"; for (int i = 0; i < row.size(); i++) { if (row.get(i) < min[i]) min[i] = row.get(i); if (row.get(i) > max[i]) max[i] = row.get(i); } } return new MinMaxScalerPartitionData(min, max); }, learningEnvironment(basePreprocessor) )) { double[][] minMax = dataset.compute( data -> data.getMin() != null ? new double[][]{ data.getMin(), data.getMax() } : null, (a, b) -> { if (a == null) return b; if (b == null) return a; double[][] res = new double[2][]; res[0] = new double[a[0].length]; for (int i = 0; i < res[0].length; i++) res[0][i] = Math.min(a[0][i], b[0][i]); res[1] = new double[a[1].length]; for (int i = 0; i < res[1].length; i++) res[1][i] = Math.max(a[1][i], b[1][i]); return res; } ); return new MinMaxScalerPreprocessor<>(minMax[0], minMax[1], basePreprocessor); } catch (Exception e) { throw new RuntimeException(e); } }  @Override MinMaxScalerPreprocessor<K, V> fit(
        LearningEnvironmentBuilder envBuilder,
        DatasetBuilder<K, V> datasetBuilder,
        Preprocessor<K, V> basePreprocessor); }
MinMaxScalerTrainer implements PreprocessingTrainer<K, V> { @Override public MinMaxScalerPreprocessor<K, V> fit( LearningEnvironmentBuilder envBuilder, DatasetBuilder<K, V> datasetBuilder, Preprocessor<K, V> basePreprocessor) { PartitionContextBuilder<K, V, EmptyContext> ctxBuilder = (env, upstream, upstreamSize) -> new EmptyContext(); try (Dataset<EmptyContext, MinMaxScalerPartitionData> dataset = datasetBuilder.build( envBuilder, ctxBuilder, (env, upstream, upstreamSize, ctx) -> { double[] min = null; double[] max = null; while (upstream.hasNext()) { UpstreamEntry<K, V> entity = upstream.next(); LabeledVector row = basePreprocessor.apply(entity.getKey(), entity.getValue()); if (min == null) { min = new double[row.size()]; for (int i = 0; i < min.length; i++) min[i] = Double.MAX_VALUE; } else assert min.length == row.size() : "Base preprocessor must return exactly " + min.length + " features"; if (max == null) { max = new double[row.size()]; for (int i = 0; i < max.length; i++) max[i] = -Double.MAX_VALUE; } else assert max.length == row.size() : "Base preprocessor must return exactly " + min.length + " features"; for (int i = 0; i < row.size(); i++) { if (row.get(i) < min[i]) min[i] = row.get(i); if (row.get(i) > max[i]) max[i] = row.get(i); } } return new MinMaxScalerPartitionData(min, max); }, learningEnvironment(basePreprocessor) )) { double[][] minMax = dataset.compute( data -> data.getMin() != null ? new double[][]{ data.getMin(), data.getMax() } : null, (a, b) -> { if (a == null) return b; if (b == null) return a; double[][] res = new double[2][]; res[0] = new double[a[0].length]; for (int i = 0; i < res[0].length; i++) res[0][i] = Math.min(a[0][i], b[0][i]); res[1] = new double[a[1].length]; for (int i = 0; i < res[1].length; i++) res[1][i] = Math.max(a[1][i], b[1][i]); return res; } ); return new MinMaxScalerPreprocessor<>(minMax[0], minMax[1], basePreprocessor); } catch (Exception e) { throw new RuntimeException(e); } }  @Override MinMaxScalerPreprocessor<K, V> fit(
        LearningEnvironmentBuilder envBuilder,
        DatasetBuilder<K, V> datasetBuilder,
        Preprocessor<K, V> basePreprocessor);  }