@Test public void testJob() throws Exception { String input = "src/test/resources/data/base_cuboid/,src/test/resources/data/6d_cuboid/"; String output = "target/test-output/key_distribution_range/"; String jobname = "calculate_splits"; String cubename = "test_kylin_cube_with_slr_ready"; FileUtil.fullyDelete(new File(output)); String[] args = { "-input", input, "-output", output, "-jobname", jobname, "-cubename", cubename }; assertEquals("Job failed", 0, ToolRunner.run(conf, new RangeKeyDistributionJob(), args)); }
@Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_OUTPUT_PATH); options.addOption(OPTION_JOB_NAME); options.addOption(OPTION_CUBE_NAME); parseOptions(options, args); String jobName = getOptionValue(OPTION_JOB_NAME); job = Job.getInstance(getConf(), jobName); setJobClasspath(job); addInputDirs(getOptionValue(OPTION_INPUT_PATH), job); Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH)); FileOutputFormat.setOutputPath(job, output); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapperClass(RangeKeyDistributionMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setReducerClass(RangeKeyDistributionReducer.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); job.setNumReduceTasks(1); this.deletePath(job.getConfiguration(), output); String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase(); CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); CubeInstance cube = cubeMgr.getCube(cubeName); RealizationCapacity realizationCapacity = cube.getDescriptor().getModel().getCapacity(); job.getConfiguration().set(BatchConstants.CUBE_CAPACITY, realizationCapacity.toString()); return waitForCompletion(job); } catch (Exception e) { printUsage(options); throw e; } }
RangeKeyDistributionJob extends AbstractHadoopJob { @Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_OUTPUT_PATH); options.addOption(OPTION_JOB_NAME); options.addOption(OPTION_CUBE_NAME); parseOptions(options, args); String jobName = getOptionValue(OPTION_JOB_NAME); job = Job.getInstance(getConf(), jobName); setJobClasspath(job); addInputDirs(getOptionValue(OPTION_INPUT_PATH), job); Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH)); FileOutputFormat.setOutputPath(job, output); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapperClass(RangeKeyDistributionMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setReducerClass(RangeKeyDistributionReducer.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); job.setNumReduceTasks(1); this.deletePath(job.getConfiguration(), output); String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase(); CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); CubeInstance cube = cubeMgr.getCube(cubeName); RealizationCapacity realizationCapacity = cube.getDescriptor().getModel().getCapacity(); job.getConfiguration().set(BatchConstants.CUBE_CAPACITY, realizationCapacity.toString()); return waitForCompletion(job); } catch (Exception e) { printUsage(options); throw e; } } }
RangeKeyDistributionJob extends AbstractHadoopJob { @Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_OUTPUT_PATH); options.addOption(OPTION_JOB_NAME); options.addOption(OPTION_CUBE_NAME); parseOptions(options, args); String jobName = getOptionValue(OPTION_JOB_NAME); job = Job.getInstance(getConf(), jobName); setJobClasspath(job); addInputDirs(getOptionValue(OPTION_INPUT_PATH), job); Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH)); FileOutputFormat.setOutputPath(job, output); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapperClass(RangeKeyDistributionMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setReducerClass(RangeKeyDistributionReducer.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); job.setNumReduceTasks(1); this.deletePath(job.getConfiguration(), output); String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase(); CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); CubeInstance cube = cubeMgr.getCube(cubeName); RealizationCapacity realizationCapacity = cube.getDescriptor().getModel().getCapacity(); job.getConfiguration().set(BatchConstants.CUBE_CAPACITY, realizationCapacity.toString()); return waitForCompletion(job); } catch (Exception e) { printUsage(options); throw e; } }  }
RangeKeyDistributionJob extends AbstractHadoopJob { @Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_OUTPUT_PATH); options.addOption(OPTION_JOB_NAME); options.addOption(OPTION_CUBE_NAME); parseOptions(options, args); String jobName = getOptionValue(OPTION_JOB_NAME); job = Job.getInstance(getConf(), jobName); setJobClasspath(job); addInputDirs(getOptionValue(OPTION_INPUT_PATH), job); Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH)); FileOutputFormat.setOutputPath(job, output); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapperClass(RangeKeyDistributionMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setReducerClass(RangeKeyDistributionReducer.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); job.setNumReduceTasks(1); this.deletePath(job.getConfiguration(), output); String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase(); CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); CubeInstance cube = cubeMgr.getCube(cubeName); RealizationCapacity realizationCapacity = cube.getDescriptor().getModel().getCapacity(); job.getConfiguration().set(BatchConstants.CUBE_CAPACITY, realizationCapacity.toString()); return waitForCompletion(job); } catch (Exception e) { printUsage(options); throw e; } }  @Override int run(String[] args); static void main(String[] args); }
RangeKeyDistributionJob extends AbstractHadoopJob { @Override public int run(String[] args) throws Exception { Options options = new Options(); try { options.addOption(OPTION_INPUT_PATH); options.addOption(OPTION_OUTPUT_PATH); options.addOption(OPTION_JOB_NAME); options.addOption(OPTION_CUBE_NAME); parseOptions(options, args); String jobName = getOptionValue(OPTION_JOB_NAME); job = Job.getInstance(getConf(), jobName); setJobClasspath(job); addInputDirs(getOptionValue(OPTION_INPUT_PATH), job); Path output = new Path(getOptionValue(OPTION_OUTPUT_PATH)); FileOutputFormat.setOutputPath(job, output); job.setInputFormatClass(SequenceFileInputFormat.class); job.setMapperClass(RangeKeyDistributionMapper.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(LongWritable.class); job.setReducerClass(RangeKeyDistributionReducer.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(LongWritable.class); job.setNumReduceTasks(1); this.deletePath(job.getConfiguration(), output); String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase(); CubeManager cubeMgr = CubeManager.getInstance(KylinConfig.getInstanceFromEnv()); CubeInstance cube = cubeMgr.getCube(cubeName); RealizationCapacity realizationCapacity = cube.getDescriptor().getModel().getCapacity(); job.getConfiguration().set(BatchConstants.CUBE_CAPACITY, realizationCapacity.toString()); return waitForCompletion(job); } catch (Exception e) { printUsage(options); throw e; } }  @Override int run(String[] args); static void main(String[] args);  }