@Test public void testParallelBulkExport() throws Exception { HBaseSail sail = new HBaseSail(HBaseServerTestInstance.getInstanceConfig(), "bulkExportTable2", true, 0, true, 0, null, null); sail.initialize(); ValueFactory vf = SimpleValueFactory.getInstance(); for (int i = 0; i < 1000; i++) { sail.addStatement(vf.createIRI("http: } sail.commit(); sail.close(); File root = File.createTempFile("test_parallelBulkExport", ""); root.delete(); root.mkdirs(); File q = new File(root, "test_parallelBulkExport.sparql"); q.deleteOnExit(); try (PrintStream qs = new PrintStream(q)) { qs.println("select * where {?s ?p ?o. FILTER (<http: } assertEquals(0, ToolRunner.run(HBaseServerTestInstance.getInstanceConfig(), new HalyardBulkExport(), new String[]{"-s", "bulkExportTable2", "-q", q.toURI().toURL().toString(), "-t", root.toURI().toURL().toString() + "{0}-{1}.csv"})); File f1 = new File(root, "test_parallelBulkExport-0.csv"); assertTrue(f1.isFile()); File f2 = new File(root, "test_parallelBulkExport-1.csv"); assertTrue(f2.isFile()); assertEquals(1002, HalyardExportTest.getLinesCount(f1.toURI().toURL().toString(), null) + HalyardExportTest.getLinesCount(f2.toURI().toURL().toString(), null)); q.delete(); f1.delete(); f2.delete(); root.delete(); }
@Override protected int run(CommandLine cmd) throws Exception { if (!cmd.getArgList().isEmpty()) throw new HalyardExport.ExportException("Unknown arguments: " + cmd.getArgList().toString()); String source = cmd.getOptionValue('s'); String queryFiles = cmd.getOptionValue('q'); String target = cmd.getOptionValue('t'); if (!target.contains("{0}")) { throw new HalyardExport.ExportException("Bulk export target must contain '{0}' to be replaced by stripped filename of the actual SPARQL query."); } getConf().set(SOURCE, source); getConf().set(TARGET, target); String driver = cmd.getOptionValue('c'); if (driver != null) { getConf().set(JDBC_DRIVER, driver); } String props[] = cmd.getOptionValues('p'); if (props != null) { for (int i=0; i<props.length; i++) { props[i] = Base64.encodeBase64String(props[i].getBytes(StandardCharsets.UTF_8)); } getConf().setStrings(JDBC_PROPERTIES, props); } if (cmd.hasOption('i')) getConf().set(HalyardBulkUpdate.ELASTIC_INDEX_URL, cmd.getOptionValue('i')); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); String cp = cmd.getOptionValue('l'); if (cp != null) { String jars[] = cp.split(":"); StringBuilder newCp = new StringBuilder(); for (int i=0; i<jars.length; i++) { if (i > 0) newCp.append(':'); newCp.append(addTmpFile(jars[i])); } getConf().set(JDBC_CLASSPATH, newCp.toString()); } Job job = Job.getInstance(getConf(), "HalyardBulkExport " + source + " -> " + target); job.setJarByClass(HalyardBulkExport.class); job.setMaxMapAttempts(1); job.setMapperClass(BulkExportMapper.class); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Void.class); job.setNumReduceTasks(0); job.setInputFormatClass(QueryInputFormat.class); QueryInputFormat.setQueriesFromDirRecursive(job.getConfiguration(), queryFiles, false, 0); job.setOutputFormatClass(NullOutputFormat.class); TableMapReduceUtil.initCredentials(job); if (job.waitForCompletion(true)) { LOG.info("Bulk Export Completed.."); return 0; } return -1; }
HalyardBulkExport extends AbstractHalyardTool { @Override protected int run(CommandLine cmd) throws Exception { if (!cmd.getArgList().isEmpty()) throw new HalyardExport.ExportException("Unknown arguments: " + cmd.getArgList().toString()); String source = cmd.getOptionValue('s'); String queryFiles = cmd.getOptionValue('q'); String target = cmd.getOptionValue('t'); if (!target.contains("{0}")) { throw new HalyardExport.ExportException("Bulk export target must contain '{0}' to be replaced by stripped filename of the actual SPARQL query."); } getConf().set(SOURCE, source); getConf().set(TARGET, target); String driver = cmd.getOptionValue('c'); if (driver != null) { getConf().set(JDBC_DRIVER, driver); } String props[] = cmd.getOptionValues('p'); if (props != null) { for (int i=0; i<props.length; i++) { props[i] = Base64.encodeBase64String(props[i].getBytes(StandardCharsets.UTF_8)); } getConf().setStrings(JDBC_PROPERTIES, props); } if (cmd.hasOption('i')) getConf().set(HalyardBulkUpdate.ELASTIC_INDEX_URL, cmd.getOptionValue('i')); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); String cp = cmd.getOptionValue('l'); if (cp != null) { String jars[] = cp.split(":"); StringBuilder newCp = new StringBuilder(); for (int i=0; i<jars.length; i++) { if (i > 0) newCp.append(':'); newCp.append(addTmpFile(jars[i])); } getConf().set(JDBC_CLASSPATH, newCp.toString()); } Job job = Job.getInstance(getConf(), "HalyardBulkExport " + source + " -> " + target); job.setJarByClass(HalyardBulkExport.class); job.setMaxMapAttempts(1); job.setMapperClass(BulkExportMapper.class); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Void.class); job.setNumReduceTasks(0); job.setInputFormatClass(QueryInputFormat.class); QueryInputFormat.setQueriesFromDirRecursive(job.getConfiguration(), queryFiles, false, 0); job.setOutputFormatClass(NullOutputFormat.class); TableMapReduceUtil.initCredentials(job); if (job.waitForCompletion(true)) { LOG.info("Bulk Export Completed.."); return 0; } return -1; } }
HalyardBulkExport extends AbstractHalyardTool { @Override protected int run(CommandLine cmd) throws Exception { if (!cmd.getArgList().isEmpty()) throw new HalyardExport.ExportException("Unknown arguments: " + cmd.getArgList().toString()); String source = cmd.getOptionValue('s'); String queryFiles = cmd.getOptionValue('q'); String target = cmd.getOptionValue('t'); if (!target.contains("{0}")) { throw new HalyardExport.ExportException("Bulk export target must contain '{0}' to be replaced by stripped filename of the actual SPARQL query."); } getConf().set(SOURCE, source); getConf().set(TARGET, target); String driver = cmd.getOptionValue('c'); if (driver != null) { getConf().set(JDBC_DRIVER, driver); } String props[] = cmd.getOptionValues('p'); if (props != null) { for (int i=0; i<props.length; i++) { props[i] = Base64.encodeBase64String(props[i].getBytes(StandardCharsets.UTF_8)); } getConf().setStrings(JDBC_PROPERTIES, props); } if (cmd.hasOption('i')) getConf().set(HalyardBulkUpdate.ELASTIC_INDEX_URL, cmd.getOptionValue('i')); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); String cp = cmd.getOptionValue('l'); if (cp != null) { String jars[] = cp.split(":"); StringBuilder newCp = new StringBuilder(); for (int i=0; i<jars.length; i++) { if (i > 0) newCp.append(':'); newCp.append(addTmpFile(jars[i])); } getConf().set(JDBC_CLASSPATH, newCp.toString()); } Job job = Job.getInstance(getConf(), "HalyardBulkExport " + source + " -> " + target); job.setJarByClass(HalyardBulkExport.class); job.setMaxMapAttempts(1); job.setMapperClass(BulkExportMapper.class); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Void.class); job.setNumReduceTasks(0); job.setInputFormatClass(QueryInputFormat.class); QueryInputFormat.setQueriesFromDirRecursive(job.getConfiguration(), queryFiles, false, 0); job.setOutputFormatClass(NullOutputFormat.class); TableMapReduceUtil.initCredentials(job); if (job.waitForCompletion(true)) { LOG.info("Bulk Export Completed.."); return 0; } return -1; } HalyardBulkExport(); }
HalyardBulkExport extends AbstractHalyardTool { @Override protected int run(CommandLine cmd) throws Exception { if (!cmd.getArgList().isEmpty()) throw new HalyardExport.ExportException("Unknown arguments: " + cmd.getArgList().toString()); String source = cmd.getOptionValue('s'); String queryFiles = cmd.getOptionValue('q'); String target = cmd.getOptionValue('t'); if (!target.contains("{0}")) { throw new HalyardExport.ExportException("Bulk export target must contain '{0}' to be replaced by stripped filename of the actual SPARQL query."); } getConf().set(SOURCE, source); getConf().set(TARGET, target); String driver = cmd.getOptionValue('c'); if (driver != null) { getConf().set(JDBC_DRIVER, driver); } String props[] = cmd.getOptionValues('p'); if (props != null) { for (int i=0; i<props.length; i++) { props[i] = Base64.encodeBase64String(props[i].getBytes(StandardCharsets.UTF_8)); } getConf().setStrings(JDBC_PROPERTIES, props); } if (cmd.hasOption('i')) getConf().set(HalyardBulkUpdate.ELASTIC_INDEX_URL, cmd.getOptionValue('i')); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); String cp = cmd.getOptionValue('l'); if (cp != null) { String jars[] = cp.split(":"); StringBuilder newCp = new StringBuilder(); for (int i=0; i<jars.length; i++) { if (i > 0) newCp.append(':'); newCp.append(addTmpFile(jars[i])); } getConf().set(JDBC_CLASSPATH, newCp.toString()); } Job job = Job.getInstance(getConf(), "HalyardBulkExport " + source + " -> " + target); job.setJarByClass(HalyardBulkExport.class); job.setMaxMapAttempts(1); job.setMapperClass(BulkExportMapper.class); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Void.class); job.setNumReduceTasks(0); job.setInputFormatClass(QueryInputFormat.class); QueryInputFormat.setQueriesFromDirRecursive(job.getConfiguration(), queryFiles, false, 0); job.setOutputFormatClass(NullOutputFormat.class); TableMapReduceUtil.initCredentials(job); if (job.waitForCompletion(true)) { LOG.info("Bulk Export Completed.."); return 0; } return -1; } HalyardBulkExport();  }
HalyardBulkExport extends AbstractHalyardTool { @Override protected int run(CommandLine cmd) throws Exception { if (!cmd.getArgList().isEmpty()) throw new HalyardExport.ExportException("Unknown arguments: " + cmd.getArgList().toString()); String source = cmd.getOptionValue('s'); String queryFiles = cmd.getOptionValue('q'); String target = cmd.getOptionValue('t'); if (!target.contains("{0}")) { throw new HalyardExport.ExportException("Bulk export target must contain '{0}' to be replaced by stripped filename of the actual SPARQL query."); } getConf().set(SOURCE, source); getConf().set(TARGET, target); String driver = cmd.getOptionValue('c'); if (driver != null) { getConf().set(JDBC_DRIVER, driver); } String props[] = cmd.getOptionValues('p'); if (props != null) { for (int i=0; i<props.length; i++) { props[i] = Base64.encodeBase64String(props[i].getBytes(StandardCharsets.UTF_8)); } getConf().setStrings(JDBC_PROPERTIES, props); } if (cmd.hasOption('i')) getConf().set(HalyardBulkUpdate.ELASTIC_INDEX_URL, cmd.getOptionValue('i')); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); String cp = cmd.getOptionValue('l'); if (cp != null) { String jars[] = cp.split(":"); StringBuilder newCp = new StringBuilder(); for (int i=0; i<jars.length; i++) { if (i > 0) newCp.append(':'); newCp.append(addTmpFile(jars[i])); } getConf().set(JDBC_CLASSPATH, newCp.toString()); } Job job = Job.getInstance(getConf(), "HalyardBulkExport " + source + " -> " + target); job.setJarByClass(HalyardBulkExport.class); job.setMaxMapAttempts(1); job.setMapperClass(BulkExportMapper.class); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Void.class); job.setNumReduceTasks(0); job.setInputFormatClass(QueryInputFormat.class); QueryInputFormat.setQueriesFromDirRecursive(job.getConfiguration(), queryFiles, false, 0); job.setOutputFormatClass(NullOutputFormat.class); TableMapReduceUtil.initCredentials(job); if (job.waitForCompletion(true)) { LOG.info("Bulk Export Completed.."); return 0; } return -1; } HalyardBulkExport();   }