DataLoader { public static Dataset generateDataset(String descriptor, FileSystem fs, Path path) throws DescriptorException, IOException { Attribute[] attrs = DescriptorUtils.parseDescriptor(descriptor); FSDataInputStream input = fs.open(path); Scanner scanner = new Scanner(input); List<String>[] values = new List[attrs.length]; int id = 0; while (scanner.hasNextLine()) { String line = scanner.nextLine(); if (line.isEmpty()) { continue; } if (parseString(id, attrs, values, line) != null) { id++; } } scanner.close(); return new Dataset(attrs, values, id); } private  DataLoader(); static Data loadData(Dataset dataset, FileSystem fs, Path fpath); static Data loadData(Dataset dataset, String[] data); static Dataset generateDataset(String descriptor, FileSystem fs, Path path); static Dataset generateDataset(String descriptor, String[] data);  }
@Test public void testGenerateDataset() throws Exception { int nbAttributes = 10; int datasize = 100; String descriptor = Utils.randomDescriptor(rng, nbAttributes); Attribute[] attrs = DescriptorUtils.parseDescriptor(descriptor); double[][] data = Utils.randomDoubles(rng, descriptor, datasize); Collection<Integer> missings = new ArrayList<Integer>(); String[] sData = prepareData(data, attrs, missings); Dataset expected = DataLoader.generateDataset(descriptor, sData); Dataset dataset = DataLoader.generateDataset(descriptor, sData); assertEquals(expected, dataset); }
@Test public void testGenerateDatasetFromFile() throws Exception { int nbAttributes = 10; int datasize = 100; String descriptor = Utils.randomDescriptor(rng, nbAttributes); Attribute[] attrs = DescriptorUtils.parseDescriptor(descriptor); double[][] source = Utils.randomDoubles(rng, descriptor, datasize); Collection<Integer> missings = new ArrayList<Integer>(); String[] sData = prepareData(source, attrs, missings); Dataset expected = DataLoader.generateDataset(descriptor, sData); Path path = Utils.writeDataToTestFile(sData); FileSystem fs = path.getFileSystem(new Configuration()); Dataset dataset = DataLoader.generateDataset(descriptor, fs, path); assertEquals(expected, dataset); }