DataLoader { public static Dataset generateDataset(CharSequence descriptor, boolean regression, FileSystem fs, Path path) throws DescriptorException, IOException { Attribute[] attrs = DescriptorUtils.parseDescriptor(descriptor); FSDataInputStream input = fs.open(path); Scanner scanner = new Scanner(input, "UTF-8"); @SuppressWarnings("unchecked") Set<String>[] valsets = new Set[attrs.length]; int size = 0; while (scanner.hasNextLine()) { String line = scanner.nextLine(); if (!line.isEmpty()) { if (parseString(attrs, valsets, line, regression)) { size++; } } } scanner.close(); @SuppressWarnings("unchecked") List<String>[] values = new List[attrs.length]; for (int i = 0; i < valsets.length; i++) { if (valsets[i] != null) { values[i] = Lists.newArrayList(valsets[i]); } } return new Dataset(attrs, values, size, regression); } private  DataLoader(); static Data loadData(Dataset dataset, FileSystem fs, Path fpath); static Data loadData(Dataset dataset, FileSystem fs, Path[] pathes); static Data loadData(Dataset dataset, String[] data); static Dataset generateDataset(CharSequence descriptor,
                                        boolean regression,
                                        FileSystem fs,
                                        Path path); static Dataset generateDataset(CharSequence descriptor,
                                        boolean regression,
                                        String[] data);  }
@Test public void testGenerateDataset() throws Exception { int nbAttributes = 10; int datasize = 100; String descriptor = Utils.randomDescriptor(rng, nbAttributes); Attribute[] attrs = DescriptorUtils.parseDescriptor(descriptor); double[][] data = Utils.randomDoubles(rng, descriptor, false, datasize); Collection<Integer> missings = Lists.newArrayList(); String[] sData = prepareData(data, attrs, missings); Dataset expected = DataLoader.generateDataset(descriptor, false, sData); Dataset dataset = DataLoader.generateDataset(descriptor, false, sData); assertEquals(expected, dataset); data = Utils.randomDoubles(rng, descriptor, true, datasize); missings = Lists.newArrayList(); sData = prepareData(data, attrs, missings); expected = DataLoader.generateDataset(descriptor, true, sData); dataset = DataLoader.generateDataset(descriptor, true, sData); assertEquals(expected, dataset); }
@Test public void testGenerateDatasetFromFile() throws Exception { int nbAttributes = 10; int datasize = 100; String descriptor = Utils.randomDescriptor(rng, nbAttributes); Attribute[] attrs = DescriptorUtils.parseDescriptor(descriptor); double[][] source = Utils.randomDoubles(rng, descriptor, false, datasize); Collection<Integer> missings = Lists.newArrayList(); String[] sData = prepareData(source, attrs, missings); Dataset expected = DataLoader.generateDataset(descriptor, false, sData); Path path = Utils.writeDataToTestFile(sData); FileSystem fs = path.getFileSystem(getConfiguration()); Dataset dataset = DataLoader.generateDataset(descriptor, false, fs, path); assertEquals(expected, dataset); source = Utils.randomDoubles(rng, descriptor, false, datasize); missings = Lists.newArrayList(); sData = prepareData(source, attrs, missings); expected = DataLoader.generateDataset(descriptor, false, sData); path = Utils.writeDataToTestFile(sData); fs = path.getFileSystem(getConfiguration()); dataset = DataLoader.generateDataset(descriptor, false, fs, path); assertEquals(expected, dataset); }