CDInfosTool { public static void gatherInfos(Descriptors descriptors, Path inpath, Path output, Collection<String> descriptions) throws IOException, InterruptedException, ClassNotFoundException { Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(inpath.toUri(), conf); Preconditions.checkArgument(fs.exists(inpath) && fs.getFileStatus(inpath).isDir(), "%s is not a directory", inpath); Job job = new Job(conf); configureJob(job, descriptors, inpath, output); job.waitForCompletion(true); importDescriptions(fs, conf, output, descriptions); } private  CDInfosTool(); static void gatherInfos(Descriptors descriptors, Path inpath, Path output, Collection<String> descriptions); static void main(String[] args);  }
@Test public void testGatherInfos() throws Exception { int n = 1; for (int nloop = 0; nloop < n; nloop++) { int maxattr = 100; int nbattrs = rng.nextInt(maxattr) + 1; double numRate = rng.nextDouble(); double catRate = rng.nextDouble() * (1.0 - numRate); Descriptors descriptors = randomDescriptors(nbattrs, numRate, catRate); Object[][] descriptions = randomDescriptions(descriptors); Path inpath = getTestTempDirPath("input"); Path output = getTestTempDirPath("output"); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(inpath.toUri(), conf); HadoopUtil.delete(conf, inpath); randomDataset(fs, inpath, descriptors, descriptions); List<String> result = new ArrayList<String>(); fs.delete(output, true); CDInfosTool.gatherInfos(descriptors, inpath, output, result); Collection<String> target = new ArrayList<String>(); assertEquals(nbNonIgnored(descriptors), result.size()); int rindex = 0; for (int index = 0; index < nbattrs; index++) { if (descriptors.isIgnored(index)) { continue; } String description = result.get(rindex++); if (descriptors.isNumerical(index)) { double min = (Double) descriptions[index][0]; double max = (Double) descriptions[index][1]; double[] range = DescriptionUtils.extractNumericalRange(description); assertTrue("bad min value for attribute (" + index + ')', min <= range[0]); assertTrue("bad max value for attribute (" + index + ')', max >= range[1]); } else if (descriptors.isNominal(index)) { Object[] values = descriptions[index]; target.clear(); DescriptionUtils.extractNominalValues(description, target); assertEquals(values.length, target.size()); assertTrue(target.containsAll(Arrays.asList(values))); } } } }
@Test public void testGatherInfos() throws Exception { int n = 1; for (int nloop = 0; nloop < n; nloop++) { int maxattr = 100; int nbattrs = rng.nextInt(maxattr) + 1; double numRate = rng.nextDouble(); double catRate = rng.nextDouble() * (1.0 - numRate); Descriptors descriptors = randomDescriptors(nbattrs, numRate, catRate); Object[][] descriptions = randomDescriptions(descriptors); Path inpath = getTestTempDirPath("input"); Path output = getTestTempDirPath("output"); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(inpath.toUri(), conf); HadoopUtil.delete(conf, inpath); randomDataset(fs, inpath, descriptors, descriptions); List<String> result = Lists.newArrayList(); fs.delete(output, true); CDInfosTool.gatherInfos(descriptors, inpath, output, result); Collection<String> target = Lists.newArrayList(); assertEquals(nbNonIgnored(descriptors), result.size()); int rindex = 0; for (int index = 0; index < nbattrs; index++) { if (descriptors.isIgnored(index)) { continue; } String description = result.get(rindex++); if (descriptors.isNumerical(index)) { double min = (Double) descriptions[index][0]; double max = (Double) descriptions[index][1]; double[] range = DescriptionUtils.extractNumericalRange(description); assertTrue("bad min value for attribute (" + index + ')', min <= range[0]); assertTrue("bad max value for attribute (" + index + ')', max >= range[1]); } else if (descriptors.isNominal(index)) { Object[] values = descriptions[index]; target.clear(); DescriptionUtils.extractNominalValues(description, target); assertEquals(values.length, target.size()); assertTrue(target.containsAll(Arrays.asList(values))); } } } }