@Test public void testIrisRRMDSI() throws Exception { ClassPathResource cpr = new ClassPathResource("iris.txt"); File f = cpr.getFile(); RecordReader rr = new CSVRecordReader(); rr.initialize(new FileSplit(f)); RecordReaderMultiDataSetIterator rrmdsi1 = new RecordReaderMultiDataSetIterator.Builder(1) .addReader("reader", rr) .addInput("reader", 0, 3) .addOutputOneHot("reader", 4, 3) .build(); RecordReaderMultiDataSetIterator rrmdsi2 = new RecordReaderMultiDataSetIterator.Builder(1) .addReader("reader", new SparkSourceDummyReader(0)) .addInput("reader", 0, 3) .addOutputOneHot("reader", 4, 3) .build(); List<MultiDataSet> expected = new ArrayList<>(150); while(rrmdsi1.hasNext()){ expected.add(rrmdsi1.next()); } JavaRDD<List<Writable>> rdd = sc.textFile(f.getPath()).coalesce(1) .map(new StringToWritablesFunction(new CSVRecordReader())); JavaRDD<MultiDataSet> mdsRdd = IteratorUtils.mapRRMDSI(rdd, rrmdsi2); List<MultiDataSet> act = mdsRdd.collect(); assertEquals(expected, act); }
public static JavaRDD<MultiDataSet> mapRRMDSI(JavaRDD<List<Writable>> rdd, RecordReaderMultiDataSetIterator iterator){ checkIterator(iterator, 1, 0); return mapRRMDSIRecords(rdd.map(new Function<List<Writable>,DataVecRecords>(){ @Override public DataVecRecords call(List<Writable> v1) throws Exception { return new DataVecRecords(Collections.singletonList(v1), null); } }), iterator); }
IteratorUtils { public static JavaRDD<MultiDataSet> mapRRMDSI(JavaRDD<List<Writable>> rdd, RecordReaderMultiDataSetIterator iterator){ checkIterator(iterator, 1, 0); return mapRRMDSIRecords(rdd.map(new Function<List<Writable>,DataVecRecords>(){ @Override public DataVecRecords call(List<Writable> v1) throws Exception { return new DataVecRecords(Collections.singletonList(v1), null); } }), iterator); } }
IteratorUtils { public static JavaRDD<MultiDataSet> mapRRMDSI(JavaRDD<List<Writable>> rdd, RecordReaderMultiDataSetIterator iterator){ checkIterator(iterator, 1, 0); return mapRRMDSIRecords(rdd.map(new Function<List<Writable>,DataVecRecords>(){ @Override public DataVecRecords call(List<Writable> v1) throws Exception { return new DataVecRecords(Collections.singletonList(v1), null); } }), iterator); }  }
IteratorUtils { public static JavaRDD<MultiDataSet> mapRRMDSI(JavaRDD<List<Writable>> rdd, RecordReaderMultiDataSetIterator iterator){ checkIterator(iterator, 1, 0); return mapRRMDSIRecords(rdd.map(new Function<List<Writable>,DataVecRecords>(){ @Override public DataVecRecords call(List<Writable> v1) throws Exception { return new DataVecRecords(Collections.singletonList(v1), null); } }), iterator); }  static JavaRDD<MultiDataSet> mapRRMDSI(JavaRDD<List<Writable>> rdd, RecordReaderMultiDataSetIterator iterator); static JavaRDD<MultiDataSet> mapRRMDSISeq(JavaRDD<List<List<Writable>>> rdd, RecordReaderMultiDataSetIterator iterator); static JavaRDD<MultiDataSet> mapRRMDSI(List<JavaRDD<List<Writable>>> rdds, List<JavaRDD<List<List<Writable>>>> seqRdds,
                                                  int[] rddsKeyColumns, int[] seqRddsKeyColumns, boolean filterMissing,
                                                  RecordReaderMultiDataSetIterator iterator); static JavaRDD<MultiDataSet> mapRRMDSIRecords(JavaRDD<DataVecRecords> rdd, RecordReaderMultiDataSetIterator iterator); }
IteratorUtils { public static JavaRDD<MultiDataSet> mapRRMDSI(JavaRDD<List<Writable>> rdd, RecordReaderMultiDataSetIterator iterator){ checkIterator(iterator, 1, 0); return mapRRMDSIRecords(rdd.map(new Function<List<Writable>,DataVecRecords>(){ @Override public DataVecRecords call(List<Writable> v1) throws Exception { return new DataVecRecords(Collections.singletonList(v1), null); } }), iterator); }  static JavaRDD<MultiDataSet> mapRRMDSI(JavaRDD<List<Writable>> rdd, RecordReaderMultiDataSetIterator iterator); static JavaRDD<MultiDataSet> mapRRMDSISeq(JavaRDD<List<List<Writable>>> rdd, RecordReaderMultiDataSetIterator iterator); static JavaRDD<MultiDataSet> mapRRMDSI(List<JavaRDD<List<Writable>>> rdds, List<JavaRDD<List<List<Writable>>>> seqRdds,
                                                  int[] rddsKeyColumns, int[] seqRddsKeyColumns, boolean filterMissing,
                                                  RecordReaderMultiDataSetIterator iterator); static JavaRDD<MultiDataSet> mapRRMDSIRecords(JavaRDD<DataVecRecords> rdd, RecordReaderMultiDataSetIterator iterator);  }