@Test public void testPlan() throws Exception { final int numberOfConcurrentTasks = 8; final String rootDirectory = "/Users/trautmane/Desktop/box/rendered_boxes"; final String[] args = { "--baseDataUrl", "http: "--owner", "flyTEM", "--project", "trautmane_test", "--stack", "rough_tiles", "--width", "1024", "--height", "1024", "--maxLevel", "7", "--z", "3", "--rootDirectory", rootDirectory, "--explainPlan" }; final ProcessTimer processTimer = new ProcessTimer(); final BoxClient.Parameters parameters = new BoxClient.Parameters(); parameters.parse(args); LOG.info("testPlan: entry, parameters={}", parameters); final BoxClient boxClient = new BoxClient(parameters); final String master = "local[" + numberOfConcurrentTasks + "]"; final SparkConf sparkConf = new SparkConf().setMaster(master).setAppName(this.getClass().getSimpleName()); boxClient.run(sparkConf); LOG.info("testPlan: exit, processing completed in {}", processTimer); }
public void run(final SparkConf sparkConf) throws IOException { final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); LogUtilities.logSparkClusterInfo(sparkContext); setupForRun(); boolean foundBoxesRenderedForPriorRun = false; if (parameters.cleanUpPriorRun) { foundBoxesRenderedForPriorRun = cleanUpPriorRun(sparkContext); } final JavaRDD<BoxData> distributedBoxDataRdd = partitionBoxes(sparkContext, foundBoxesRenderedForPriorRun); final Broadcast<BoxGenerator> broadcastBoxGenerator = sparkContext.broadcast(boxGenerator); if (parameters.validateLabelsOnly) { validateLabelBoxes(sparkContext, distributedBoxDataRdd); } else { for (int level = 0; level <= parameters.box.maxLevel; level++) { renderBoxesForLevel(level, distributedBoxDataRdd, broadcastBoxGenerator); } } if (parameters.box.isOverviewNeeded() && (! parameters.explainPlan) && (! parameters.validateLabelsOnly)) { renderOverviewImages(sparkContext, broadcastBoxGenerator); } LogUtilities.logSparkClusterInfo(sparkContext); sparkContext.stop(); }
BoxClient implements Serializable { public void run(final SparkConf sparkConf) throws IOException { final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); LogUtilities.logSparkClusterInfo(sparkContext); setupForRun(); boolean foundBoxesRenderedForPriorRun = false; if (parameters.cleanUpPriorRun) { foundBoxesRenderedForPriorRun = cleanUpPriorRun(sparkContext); } final JavaRDD<BoxData> distributedBoxDataRdd = partitionBoxes(sparkContext, foundBoxesRenderedForPriorRun); final Broadcast<BoxGenerator> broadcastBoxGenerator = sparkContext.broadcast(boxGenerator); if (parameters.validateLabelsOnly) { validateLabelBoxes(sparkContext, distributedBoxDataRdd); } else { for (int level = 0; level <= parameters.box.maxLevel; level++) { renderBoxesForLevel(level, distributedBoxDataRdd, broadcastBoxGenerator); } } if (parameters.box.isOverviewNeeded() && (! parameters.explainPlan) && (! parameters.validateLabelsOnly)) { renderOverviewImages(sparkContext, broadcastBoxGenerator); } LogUtilities.logSparkClusterInfo(sparkContext); sparkContext.stop(); } }
BoxClient implements Serializable { public void run(final SparkConf sparkConf) throws IOException { final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); LogUtilities.logSparkClusterInfo(sparkContext); setupForRun(); boolean foundBoxesRenderedForPriorRun = false; if (parameters.cleanUpPriorRun) { foundBoxesRenderedForPriorRun = cleanUpPriorRun(sparkContext); } final JavaRDD<BoxData> distributedBoxDataRdd = partitionBoxes(sparkContext, foundBoxesRenderedForPriorRun); final Broadcast<BoxGenerator> broadcastBoxGenerator = sparkContext.broadcast(boxGenerator); if (parameters.validateLabelsOnly) { validateLabelBoxes(sparkContext, distributedBoxDataRdd); } else { for (int level = 0; level <= parameters.box.maxLevel; level++) { renderBoxesForLevel(level, distributedBoxDataRdd, broadcastBoxGenerator); } } if (parameters.box.isOverviewNeeded() && (! parameters.explainPlan) && (! parameters.validateLabelsOnly)) { renderOverviewImages(sparkContext, broadcastBoxGenerator); } LogUtilities.logSparkClusterInfo(sparkContext); sparkContext.stop(); } BoxClient(final Parameters parameters); }
BoxClient implements Serializable { public void run(final SparkConf sparkConf) throws IOException { final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); LogUtilities.logSparkClusterInfo(sparkContext); setupForRun(); boolean foundBoxesRenderedForPriorRun = false; if (parameters.cleanUpPriorRun) { foundBoxesRenderedForPriorRun = cleanUpPriorRun(sparkContext); } final JavaRDD<BoxData> distributedBoxDataRdd = partitionBoxes(sparkContext, foundBoxesRenderedForPriorRun); final Broadcast<BoxGenerator> broadcastBoxGenerator = sparkContext.broadcast(boxGenerator); if (parameters.validateLabelsOnly) { validateLabelBoxes(sparkContext, distributedBoxDataRdd); } else { for (int level = 0; level <= parameters.box.maxLevel; level++) { renderBoxesForLevel(level, distributedBoxDataRdd, broadcastBoxGenerator); } } if (parameters.box.isOverviewNeeded() && (! parameters.explainPlan) && (! parameters.validateLabelsOnly)) { renderOverviewImages(sparkContext, broadcastBoxGenerator); } LogUtilities.logSparkClusterInfo(sparkContext); sparkContext.stop(); } BoxClient(final Parameters parameters); static void main(final String[] args); void run(final SparkConf sparkConf); }
BoxClient implements Serializable { public void run(final SparkConf sparkConf) throws IOException { final JavaSparkContext sparkContext = new JavaSparkContext(sparkConf); LogUtilities.logSparkClusterInfo(sparkContext); setupForRun(); boolean foundBoxesRenderedForPriorRun = false; if (parameters.cleanUpPriorRun) { foundBoxesRenderedForPriorRun = cleanUpPriorRun(sparkContext); } final JavaRDD<BoxData> distributedBoxDataRdd = partitionBoxes(sparkContext, foundBoxesRenderedForPriorRun); final Broadcast<BoxGenerator> broadcastBoxGenerator = sparkContext.broadcast(boxGenerator); if (parameters.validateLabelsOnly) { validateLabelBoxes(sparkContext, distributedBoxDataRdd); } else { for (int level = 0; level <= parameters.box.maxLevel; level++) { renderBoxesForLevel(level, distributedBoxDataRdd, broadcastBoxGenerator); } } if (parameters.box.isOverviewNeeded() && (! parameters.explainPlan) && (! parameters.validateLabelsOnly)) { renderOverviewImages(sparkContext, broadcastBoxGenerator); } LogUtilities.logSparkClusterInfo(sparkContext); sparkContext.stop(); } BoxClient(final Parameters parameters); static void main(final String[] args); void run(final SparkConf sparkConf);  }