@Test public void testProcessOutput() throws Exception { Configuration conf = getConfiguration(); conf.setInt("mapred.map.tasks", NUM_MAPS); Random rng = RandomUtils.getRandom(); TreeID[] keys = new TreeID[NUM_TREES]; MapredOutput[] values = new MapredOutput[NUM_TREES]; int[] firstIds = new int[NUM_MAPS]; randomKeyValues(rng, keys, values, firstIds); Path base = getTestTempDirPath("testdata"); FileSystem fs = base.getFileSystem(conf); Path outputFile = new Path(base, "PartialBuilderTest.seq"); Writer writer = SequenceFile.createWriter(fs, conf, outputFile, TreeID.class, MapredOutput.class); try { for (int index = 0; index < NUM_TREES; index++) { writer.append(keys[index], values[index]); } } finally { Closeables.close(writer, false); } TreeID[] newKeys = new TreeID[NUM_TREES]; Node[] newTrees = new Node[NUM_TREES]; PartialBuilder.processOutput(new Job(conf), base, newKeys, newTrees); for (int tree = 0; tree < NUM_TREES; tree++) { assertEquals(values[tree].getTree(), newTrees[tree]); } assertTrue("keys not equal", Arrays.deepEquals(keys, newKeys)); }
protected static void processOutput(JobContext job, Path outputPath, TreeID[] keys, Node[] trees) throws IOException { Preconditions.checkArgument(keys == null && trees == null || keys != null && trees != null, "if keys is null, trees should also be null"); Preconditions.checkArgument(keys == null || keys.length == trees.length, "keys.length != trees.length"); Configuration conf = job.getConfiguration(); FileSystem fs = outputPath.getFileSystem(conf); Path[] outfiles = DFUtils.listOutputFiles(fs, outputPath); int index = 0; for (Path path : outfiles) { for (Pair<TreeID,MapredOutput> record : new SequenceFileIterable<TreeID, MapredOutput>(path, conf)) { TreeID key = record.getFirst(); MapredOutput value = record.getSecond(); if (keys != null) { keys[index] = key; } if (trees != null) { trees[index] = value.getTree(); } index++; } } if (keys != null && index != keys.length) { throw new IllegalStateException("Some key/values are missing from the output"); } }
PartialBuilder extends Builder { protected static void processOutput(JobContext job, Path outputPath, TreeID[] keys, Node[] trees) throws IOException { Preconditions.checkArgument(keys == null && trees == null || keys != null && trees != null, "if keys is null, trees should also be null"); Preconditions.checkArgument(keys == null || keys.length == trees.length, "keys.length != trees.length"); Configuration conf = job.getConfiguration(); FileSystem fs = outputPath.getFileSystem(conf); Path[] outfiles = DFUtils.listOutputFiles(fs, outputPath); int index = 0; for (Path path : outfiles) { for (Pair<TreeID,MapredOutput> record : new SequenceFileIterable<TreeID, MapredOutput>(path, conf)) { TreeID key = record.getFirst(); MapredOutput value = record.getSecond(); if (keys != null) { keys[index] = key; } if (trees != null) { trees[index] = value.getTree(); } index++; } } if (keys != null && index != keys.length) { throw new IllegalStateException("Some key/values are missing from the output"); } } }
PartialBuilder extends Builder { protected static void processOutput(JobContext job, Path outputPath, TreeID[] keys, Node[] trees) throws IOException { Preconditions.checkArgument(keys == null && trees == null || keys != null && trees != null, "if keys is null, trees should also be null"); Preconditions.checkArgument(keys == null || keys.length == trees.length, "keys.length != trees.length"); Configuration conf = job.getConfiguration(); FileSystem fs = outputPath.getFileSystem(conf); Path[] outfiles = DFUtils.listOutputFiles(fs, outputPath); int index = 0; for (Path path : outfiles) { for (Pair<TreeID,MapredOutput> record : new SequenceFileIterable<TreeID, MapredOutput>(path, conf)) { TreeID key = record.getFirst(); MapredOutput value = record.getSecond(); if (keys != null) { keys[index] = key; } if (trees != null) { trees[index] = value.getTree(); } index++; } } if (keys != null && index != keys.length) { throw new IllegalStateException("Some key/values are missing from the output"); } } PartialBuilder(TreeBuilder treeBuilder, Path dataPath, Path datasetPath, Long seed);  PartialBuilder(TreeBuilder treeBuilder,
                        Path dataPath,
                        Path datasetPath,
                        Long seed,
                        Configuration conf); }
PartialBuilder extends Builder { protected static void processOutput(JobContext job, Path outputPath, TreeID[] keys, Node[] trees) throws IOException { Preconditions.checkArgument(keys == null && trees == null || keys != null && trees != null, "if keys is null, trees should also be null"); Preconditions.checkArgument(keys == null || keys.length == trees.length, "keys.length != trees.length"); Configuration conf = job.getConfiguration(); FileSystem fs = outputPath.getFileSystem(conf); Path[] outfiles = DFUtils.listOutputFiles(fs, outputPath); int index = 0; for (Path path : outfiles) { for (Pair<TreeID,MapredOutput> record : new SequenceFileIterable<TreeID, MapredOutput>(path, conf)) { TreeID key = record.getFirst(); MapredOutput value = record.getSecond(); if (keys != null) { keys[index] = key; } if (trees != null) { trees[index] = value.getTree(); } index++; } } if (keys != null && index != keys.length) { throw new IllegalStateException("Some key/values are missing from the output"); } } PartialBuilder(TreeBuilder treeBuilder, Path dataPath, Path datasetPath, Long seed);  PartialBuilder(TreeBuilder treeBuilder,
                        Path dataPath,
                        Path datasetPath,
                        Long seed,
                        Configuration conf);  }
PartialBuilder extends Builder { protected static void processOutput(JobContext job, Path outputPath, TreeID[] keys, Node[] trees) throws IOException { Preconditions.checkArgument(keys == null && trees == null || keys != null && trees != null, "if keys is null, trees should also be null"); Preconditions.checkArgument(keys == null || keys.length == trees.length, "keys.length != trees.length"); Configuration conf = job.getConfiguration(); FileSystem fs = outputPath.getFileSystem(conf); Path[] outfiles = DFUtils.listOutputFiles(fs, outputPath); int index = 0; for (Path path : outfiles) { for (Pair<TreeID,MapredOutput> record : new SequenceFileIterable<TreeID, MapredOutput>(path, conf)) { TreeID key = record.getFirst(); MapredOutput value = record.getSecond(); if (keys != null) { keys[index] = key; } if (trees != null) { trees[index] = value.getTree(); } index++; } } if (keys != null && index != keys.length) { throw new IllegalStateException("Some key/values are missing from the output"); } } PartialBuilder(TreeBuilder treeBuilder, Path dataPath, Path datasetPath, Long seed);  PartialBuilder(TreeBuilder treeBuilder,
                        Path dataPath,
                        Path datasetPath,
                        Long seed,
                        Configuration conf);   }