@Test public void testPreSplit() throws Exception { File file = File.createTempFile("test_triples", ".nq"); try (PrintStream ps = new PrintStream(new FileOutputStream(file))) { ps.println("<http: ps.println("<http: } assertEquals(0, ToolRunner.run(HBaseServerTestInstance.getInstanceConfig(), new HalyardPreSplit(), new String[]{"-d", "1", "-l", "0", "-s", file.toURI().toURL().toString(), "-t", "preSplitTable"})); try (HTable t = HalyardTableUtils.getTable(HBaseServerTestInstance.getInstanceConfig(), "preSplitTable", false, 0)) { assertEquals(17, t.getRegionLocator().getStartKeys().length); } }
@Override protected int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); try (Connection con = ConnectionFactory.createConnection(getConf())) { try (Admin admin = con.getAdmin()) { if (admin.tableExists(TableName.valueOf(target))) { LOG.log(Level.WARNING, "Pre-split cannot modify already existing table {0}", target); return -1; } } } getConf().setBoolean(SKIP_INVALID_PROPERTY, cmd.hasOption('i')); if (cmd.hasOption('g')) getConf().set(DEFAULT_CONTEXT_PROPERTY, cmd.getOptionValue('g')); getConf().setBoolean(OVERRIDE_CONTEXT_PROPERTY, cmd.hasOption('o')); TableMapReduceUtil.addDependencyJars(getConf(), NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class); HBaseConfiguration.addHbaseResources(getConf()); getConf().setLong(DEFAULT_TIMESTAMP_PROPERTY, getConf().getLong(DEFAULT_TIMESTAMP_PROPERTY, System.currentTimeMillis())); getConf().setInt(DECIMATION_FACTOR_PROPERTY, Integer.parseInt(cmd.getOptionValue('d', String.valueOf(DEFAULT_DECIMATION_FACTOR)))); getConf().setLong(SPLIT_LIMIT_PROPERTY, Long.parseLong(cmd.getOptionValue('l', String.valueOf(DEFAULT_SPLIT_LIMIT)))); Job job = Job.getInstance(getConf(), "HalyardPreSplit -> " + target); job.getConfiguration().set(TABLE_PROPERTY, target); job.setJarByClass(HalyardPreSplit.class); job.setMapperClass(RDFDecimatingMapper.class); job.setMapOutputKeyClass(ImmutableBytesWritable.class); job.setMapOutputValueClass(LongWritable.class); job.setInputFormatClass(RioFileInputFormat.class); FileInputFormat.setInputDirRecursive(job, true); FileInputFormat.setInputPaths(job, source); TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.initCredentials(job); job.setReducerClass(PreSplitReducer.class); job.setNumReduceTasks(1); job.setOutputFormatClass(NullOutputFormat.class); if (job.waitForCompletion(true)) { LOG.info("PreSplit Calculation Completed.."); return 0; } return -1; }
HalyardPreSplit extends AbstractHalyardTool { @Override protected int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); try (Connection con = ConnectionFactory.createConnection(getConf())) { try (Admin admin = con.getAdmin()) { if (admin.tableExists(TableName.valueOf(target))) { LOG.log(Level.WARNING, "Pre-split cannot modify already existing table {0}", target); return -1; } } } getConf().setBoolean(SKIP_INVALID_PROPERTY, cmd.hasOption('i')); if (cmd.hasOption('g')) getConf().set(DEFAULT_CONTEXT_PROPERTY, cmd.getOptionValue('g')); getConf().setBoolean(OVERRIDE_CONTEXT_PROPERTY, cmd.hasOption('o')); TableMapReduceUtil.addDependencyJars(getConf(), NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class); HBaseConfiguration.addHbaseResources(getConf()); getConf().setLong(DEFAULT_TIMESTAMP_PROPERTY, getConf().getLong(DEFAULT_TIMESTAMP_PROPERTY, System.currentTimeMillis())); getConf().setInt(DECIMATION_FACTOR_PROPERTY, Integer.parseInt(cmd.getOptionValue('d', String.valueOf(DEFAULT_DECIMATION_FACTOR)))); getConf().setLong(SPLIT_LIMIT_PROPERTY, Long.parseLong(cmd.getOptionValue('l', String.valueOf(DEFAULT_SPLIT_LIMIT)))); Job job = Job.getInstance(getConf(), "HalyardPreSplit -> " + target); job.getConfiguration().set(TABLE_PROPERTY, target); job.setJarByClass(HalyardPreSplit.class); job.setMapperClass(RDFDecimatingMapper.class); job.setMapOutputKeyClass(ImmutableBytesWritable.class); job.setMapOutputValueClass(LongWritable.class); job.setInputFormatClass(RioFileInputFormat.class); FileInputFormat.setInputDirRecursive(job, true); FileInputFormat.setInputPaths(job, source); TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.initCredentials(job); job.setReducerClass(PreSplitReducer.class); job.setNumReduceTasks(1); job.setOutputFormatClass(NullOutputFormat.class); if (job.waitForCompletion(true)) { LOG.info("PreSplit Calculation Completed.."); return 0; } return -1; } }
HalyardPreSplit extends AbstractHalyardTool { @Override protected int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); try (Connection con = ConnectionFactory.createConnection(getConf())) { try (Admin admin = con.getAdmin()) { if (admin.tableExists(TableName.valueOf(target))) { LOG.log(Level.WARNING, "Pre-split cannot modify already existing table {0}", target); return -1; } } } getConf().setBoolean(SKIP_INVALID_PROPERTY, cmd.hasOption('i')); if (cmd.hasOption('g')) getConf().set(DEFAULT_CONTEXT_PROPERTY, cmd.getOptionValue('g')); getConf().setBoolean(OVERRIDE_CONTEXT_PROPERTY, cmd.hasOption('o')); TableMapReduceUtil.addDependencyJars(getConf(), NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class); HBaseConfiguration.addHbaseResources(getConf()); getConf().setLong(DEFAULT_TIMESTAMP_PROPERTY, getConf().getLong(DEFAULT_TIMESTAMP_PROPERTY, System.currentTimeMillis())); getConf().setInt(DECIMATION_FACTOR_PROPERTY, Integer.parseInt(cmd.getOptionValue('d', String.valueOf(DEFAULT_DECIMATION_FACTOR)))); getConf().setLong(SPLIT_LIMIT_PROPERTY, Long.parseLong(cmd.getOptionValue('l', String.valueOf(DEFAULT_SPLIT_LIMIT)))); Job job = Job.getInstance(getConf(), "HalyardPreSplit -> " + target); job.getConfiguration().set(TABLE_PROPERTY, target); job.setJarByClass(HalyardPreSplit.class); job.setMapperClass(RDFDecimatingMapper.class); job.setMapOutputKeyClass(ImmutableBytesWritable.class); job.setMapOutputValueClass(LongWritable.class); job.setInputFormatClass(RioFileInputFormat.class); FileInputFormat.setInputDirRecursive(job, true); FileInputFormat.setInputPaths(job, source); TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.initCredentials(job); job.setReducerClass(PreSplitReducer.class); job.setNumReduceTasks(1); job.setOutputFormatClass(NullOutputFormat.class); if (job.waitForCompletion(true)) { LOG.info("PreSplit Calculation Completed.."); return 0; } return -1; } HalyardPreSplit(); }
HalyardPreSplit extends AbstractHalyardTool { @Override protected int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); try (Connection con = ConnectionFactory.createConnection(getConf())) { try (Admin admin = con.getAdmin()) { if (admin.tableExists(TableName.valueOf(target))) { LOG.log(Level.WARNING, "Pre-split cannot modify already existing table {0}", target); return -1; } } } getConf().setBoolean(SKIP_INVALID_PROPERTY, cmd.hasOption('i')); if (cmd.hasOption('g')) getConf().set(DEFAULT_CONTEXT_PROPERTY, cmd.getOptionValue('g')); getConf().setBoolean(OVERRIDE_CONTEXT_PROPERTY, cmd.hasOption('o')); TableMapReduceUtil.addDependencyJars(getConf(), NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class); HBaseConfiguration.addHbaseResources(getConf()); getConf().setLong(DEFAULT_TIMESTAMP_PROPERTY, getConf().getLong(DEFAULT_TIMESTAMP_PROPERTY, System.currentTimeMillis())); getConf().setInt(DECIMATION_FACTOR_PROPERTY, Integer.parseInt(cmd.getOptionValue('d', String.valueOf(DEFAULT_DECIMATION_FACTOR)))); getConf().setLong(SPLIT_LIMIT_PROPERTY, Long.parseLong(cmd.getOptionValue('l', String.valueOf(DEFAULT_SPLIT_LIMIT)))); Job job = Job.getInstance(getConf(), "HalyardPreSplit -> " + target); job.getConfiguration().set(TABLE_PROPERTY, target); job.setJarByClass(HalyardPreSplit.class); job.setMapperClass(RDFDecimatingMapper.class); job.setMapOutputKeyClass(ImmutableBytesWritable.class); job.setMapOutputValueClass(LongWritable.class); job.setInputFormatClass(RioFileInputFormat.class); FileInputFormat.setInputDirRecursive(job, true); FileInputFormat.setInputPaths(job, source); TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.initCredentials(job); job.setReducerClass(PreSplitReducer.class); job.setNumReduceTasks(1); job.setOutputFormatClass(NullOutputFormat.class); if (job.waitForCompletion(true)) { LOG.info("PreSplit Calculation Completed.."); return 0; } return -1; } HalyardPreSplit();  }
HalyardPreSplit extends AbstractHalyardTool { @Override protected int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); try (Connection con = ConnectionFactory.createConnection(getConf())) { try (Admin admin = con.getAdmin()) { if (admin.tableExists(TableName.valueOf(target))) { LOG.log(Level.WARNING, "Pre-split cannot modify already existing table {0}", target); return -1; } } } getConf().setBoolean(SKIP_INVALID_PROPERTY, cmd.hasOption('i')); if (cmd.hasOption('g')) getConf().set(DEFAULT_CONTEXT_PROPERTY, cmd.getOptionValue('g')); getConf().setBoolean(OVERRIDE_CONTEXT_PROPERTY, cmd.hasOption('o')); TableMapReduceUtil.addDependencyJars(getConf(), NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class); HBaseConfiguration.addHbaseResources(getConf()); getConf().setLong(DEFAULT_TIMESTAMP_PROPERTY, getConf().getLong(DEFAULT_TIMESTAMP_PROPERTY, System.currentTimeMillis())); getConf().setInt(DECIMATION_FACTOR_PROPERTY, Integer.parseInt(cmd.getOptionValue('d', String.valueOf(DEFAULT_DECIMATION_FACTOR)))); getConf().setLong(SPLIT_LIMIT_PROPERTY, Long.parseLong(cmd.getOptionValue('l', String.valueOf(DEFAULT_SPLIT_LIMIT)))); Job job = Job.getInstance(getConf(), "HalyardPreSplit -> " + target); job.getConfiguration().set(TABLE_PROPERTY, target); job.setJarByClass(HalyardPreSplit.class); job.setMapperClass(RDFDecimatingMapper.class); job.setMapOutputKeyClass(ImmutableBytesWritable.class); job.setMapOutputValueClass(LongWritable.class); job.setInputFormatClass(RioFileInputFormat.class); FileInputFormat.setInputDirRecursive(job, true); FileInputFormat.setInputPaths(job, source); TableMapReduceUtil.addDependencyJars(job); TableMapReduceUtil.initCredentials(job); job.setReducerClass(PreSplitReducer.class); job.setNumReduceTasks(1); job.setOutputFormatClass(NullOutputFormat.class); if (job.waitForCompletion(true)) { LOG.info("PreSplit Calculation Completed.."); return 0; } return -1; } HalyardPreSplit();   }