@Test(expected = MissingOptionException.class) public void testRunNoArgs() throws Exception { assertEquals(-1, new HalyardElasticIndexer().run(new String[]{})); }
@Override public int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); String doc = cmd.getOptionValue('d', "l"); String attr = cmd.getOptionValue('a', "l"); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); Job job = Job.getInstance(getConf(), "HalyardElasticIndexer " + source + " -> " + target); job.getConfiguration().set(SOURCE, source); job.getConfiguration().set(TARGET, target); job.getConfiguration().set(DOCUMENT, doc); job.getConfiguration().set(ATTRIBUTE, attr); if (cmd.hasOption('b')) { job.getConfiguration().setInt(BUFFER_LIMIT, Integer.parseInt(cmd.getOptionValue('b'))); } if (cmd.hasOption('c')) { int shards; try (Connection conn = ConnectionFactory.createConnection(getConf())) { try (RegionLocator rl = conn.getRegionLocator(TableName.valueOf(source))) { shards = 1 + (rl.getStartKeys().length >> 8); } } HttpURLConnection http = (HttpURLConnection)new URL(target).openConnection(); http.setRequestMethod("PUT"); http.setDoOutput(true); http.setRequestProperty("Content-Type", "application/json; charset=UTF-8"); byte b[] = ("{\n" + " \"mappings\" : {\n" + " \"" + doc + "\" : {\n" + " \"properties\" : {\n" + " \"" + attr + "\" : { \"type\" : \"text\" }\n" + " },\n" + " \"_source\" : {\n" + " \"enabled\" : false\n" + " }\n" + " }\n" + " },\n" + " \"settings\": {\n" + " \"refresh_interval\": \"1h\",\n" + " \"number_of_shards\": " + shards + ",\n" + " \"number_of_replicas\": 0\n" + " }\n" + "}").getBytes(StandardCharsets.UTF_8); http.setFixedLengthStreamingMode(b.length); http.connect(); try { try (OutputStream post = http.getOutputStream()) { post.write(b); } int response = http.getResponseCode(); String msg = http.getResponseMessage(); if (response != 200) { String resp = IOUtils.toString(http.getErrorStream()); LOG.warning(resp); boolean alreadyExist = false; if (response == 400) try { alreadyExist = new JSONObject(resp).getJSONObject("error").getString("type").contains("exists"); } catch (Exception ex) { } if (!alreadyExist) throw new IOException(msg); } } finally { http.disconnect(); } } job.setJarByClass(HalyardElasticIndexer.class); TableMapReduceUtil.initCredentials(job); Scan scan = HalyardTableUtils.scan(null, null); if (cmd.hasOption('g')) { byte[] graphHash = HalyardTableUtils.hashKey(NTriplesUtil.parseResource(cmd.getOptionValue('g'), SimpleValueFactory.getInstance())); scan.setStartRow(HalyardTableUtils.concat(HalyardTableUtils.COSP_PREFIX, false, graphHash)); scan.setStopRow(HalyardTableUtils.concat(HalyardTableUtils.COSP_PREFIX, true, graphHash, HalyardTableUtils.STOP_KEY, HalyardTableUtils.STOP_KEY, HalyardTableUtils.STOP_KEY)); } else { scan.setStartRow(new byte[]{HalyardTableUtils.OSP_PREFIX}); scan.setStopRow(new byte[]{HalyardTableUtils.OSP_PREFIX+1}); } TableMapReduceUtil.initTableMapperJob( source, scan, IndexerMapper.class, ImmutableBytesWritable.class, LongWritable.class, job); job.setOutputFormatClass(NullOutputFormat.class); job.setNumReduceTasks(0); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Void.class); if (job.waitForCompletion(true)) { HttpURLConnection http = (HttpURLConnection)new URL(target + "_refresh").openConnection(); http.connect(); http.disconnect(); LOG.info("Elastic Indexing Completed.."); return 0; } return -1; }
HalyardElasticIndexer extends AbstractHalyardTool { @Override public int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); String doc = cmd.getOptionValue('d', "l"); String attr = cmd.getOptionValue('a', "l"); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); Job job = Job.getInstance(getConf(), "HalyardElasticIndexer " + source + " -> " + target); job.getConfiguration().set(SOURCE, source); job.getConfiguration().set(TARGET, target); job.getConfiguration().set(DOCUMENT, doc); job.getConfiguration().set(ATTRIBUTE, attr); if (cmd.hasOption('b')) { job.getConfiguration().setInt(BUFFER_LIMIT, Integer.parseInt(cmd.getOptionValue('b'))); } if (cmd.hasOption('c')) { int shards; try (Connection conn = ConnectionFactory.createConnection(getConf())) { try (RegionLocator rl = conn.getRegionLocator(TableName.valueOf(source))) { shards = 1 + (rl.getStartKeys().length >> 8); } } HttpURLConnection http = (HttpURLConnection)new URL(target).openConnection(); http.setRequestMethod("PUT"); http.setDoOutput(true); http.setRequestProperty("Content-Type", "application/json; charset=UTF-8"); byte b[] = ("{\n" + " \"mappings\" : {\n" + " \"" + doc + "\" : {\n" + " \"properties\" : {\n" + " \"" + attr + "\" : { \"type\" : \"text\" }\n" + " },\n" + " \"_source\" : {\n" + " \"enabled\" : false\n" + " }\n" + " }\n" + " },\n" + " \"settings\": {\n" + " \"refresh_interval\": \"1h\",\n" + " \"number_of_shards\": " + shards + ",\n" + " \"number_of_replicas\": 0\n" + " }\n" + "}").getBytes(StandardCharsets.UTF_8); http.setFixedLengthStreamingMode(b.length); http.connect(); try { try (OutputStream post = http.getOutputStream()) { post.write(b); } int response = http.getResponseCode(); String msg = http.getResponseMessage(); if (response != 200) { String resp = IOUtils.toString(http.getErrorStream()); LOG.warning(resp); boolean alreadyExist = false; if (response == 400) try { alreadyExist = new JSONObject(resp).getJSONObject("error").getString("type").contains("exists"); } catch (Exception ex) { } if (!alreadyExist) throw new IOException(msg); } } finally { http.disconnect(); } } job.setJarByClass(HalyardElasticIndexer.class); TableMapReduceUtil.initCredentials(job); Scan scan = HalyardTableUtils.scan(null, null); if (cmd.hasOption('g')) { byte[] graphHash = HalyardTableUtils.hashKey(NTriplesUtil.parseResource(cmd.getOptionValue('g'), SimpleValueFactory.getInstance())); scan.setStartRow(HalyardTableUtils.concat(HalyardTableUtils.COSP_PREFIX, false, graphHash)); scan.setStopRow(HalyardTableUtils.concat(HalyardTableUtils.COSP_PREFIX, true, graphHash, HalyardTableUtils.STOP_KEY, HalyardTableUtils.STOP_KEY, HalyardTableUtils.STOP_KEY)); } else { scan.setStartRow(new byte[]{HalyardTableUtils.OSP_PREFIX}); scan.setStopRow(new byte[]{HalyardTableUtils.OSP_PREFIX+1}); } TableMapReduceUtil.initTableMapperJob( source, scan, IndexerMapper.class, ImmutableBytesWritable.class, LongWritable.class, job); job.setOutputFormatClass(NullOutputFormat.class); job.setNumReduceTasks(0); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Void.class); if (job.waitForCompletion(true)) { HttpURLConnection http = (HttpURLConnection)new URL(target + "_refresh").openConnection(); http.connect(); http.disconnect(); LOG.info("Elastic Indexing Completed.."); return 0; } return -1; } }
HalyardElasticIndexer extends AbstractHalyardTool { @Override public int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); String doc = cmd.getOptionValue('d', "l"); String attr = cmd.getOptionValue('a', "l"); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); Job job = Job.getInstance(getConf(), "HalyardElasticIndexer " + source + " -> " + target); job.getConfiguration().set(SOURCE, source); job.getConfiguration().set(TARGET, target); job.getConfiguration().set(DOCUMENT, doc); job.getConfiguration().set(ATTRIBUTE, attr); if (cmd.hasOption('b')) { job.getConfiguration().setInt(BUFFER_LIMIT, Integer.parseInt(cmd.getOptionValue('b'))); } if (cmd.hasOption('c')) { int shards; try (Connection conn = ConnectionFactory.createConnection(getConf())) { try (RegionLocator rl = conn.getRegionLocator(TableName.valueOf(source))) { shards = 1 + (rl.getStartKeys().length >> 8); } } HttpURLConnection http = (HttpURLConnection)new URL(target).openConnection(); http.setRequestMethod("PUT"); http.setDoOutput(true); http.setRequestProperty("Content-Type", "application/json; charset=UTF-8"); byte b[] = ("{\n" + " \"mappings\" : {\n" + " \"" + doc + "\" : {\n" + " \"properties\" : {\n" + " \"" + attr + "\" : { \"type\" : \"text\" }\n" + " },\n" + " \"_source\" : {\n" + " \"enabled\" : false\n" + " }\n" + " }\n" + " },\n" + " \"settings\": {\n" + " \"refresh_interval\": \"1h\",\n" + " \"number_of_shards\": " + shards + ",\n" + " \"number_of_replicas\": 0\n" + " }\n" + "}").getBytes(StandardCharsets.UTF_8); http.setFixedLengthStreamingMode(b.length); http.connect(); try { try (OutputStream post = http.getOutputStream()) { post.write(b); } int response = http.getResponseCode(); String msg = http.getResponseMessage(); if (response != 200) { String resp = IOUtils.toString(http.getErrorStream()); LOG.warning(resp); boolean alreadyExist = false; if (response == 400) try { alreadyExist = new JSONObject(resp).getJSONObject("error").getString("type").contains("exists"); } catch (Exception ex) { } if (!alreadyExist) throw new IOException(msg); } } finally { http.disconnect(); } } job.setJarByClass(HalyardElasticIndexer.class); TableMapReduceUtil.initCredentials(job); Scan scan = HalyardTableUtils.scan(null, null); if (cmd.hasOption('g')) { byte[] graphHash = HalyardTableUtils.hashKey(NTriplesUtil.parseResource(cmd.getOptionValue('g'), SimpleValueFactory.getInstance())); scan.setStartRow(HalyardTableUtils.concat(HalyardTableUtils.COSP_PREFIX, false, graphHash)); scan.setStopRow(HalyardTableUtils.concat(HalyardTableUtils.COSP_PREFIX, true, graphHash, HalyardTableUtils.STOP_KEY, HalyardTableUtils.STOP_KEY, HalyardTableUtils.STOP_KEY)); } else { scan.setStartRow(new byte[]{HalyardTableUtils.OSP_PREFIX}); scan.setStopRow(new byte[]{HalyardTableUtils.OSP_PREFIX+1}); } TableMapReduceUtil.initTableMapperJob( source, scan, IndexerMapper.class, ImmutableBytesWritable.class, LongWritable.class, job); job.setOutputFormatClass(NullOutputFormat.class); job.setNumReduceTasks(0); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Void.class); if (job.waitForCompletion(true)) { HttpURLConnection http = (HttpURLConnection)new URL(target + "_refresh").openConnection(); http.connect(); http.disconnect(); LOG.info("Elastic Indexing Completed.."); return 0; } return -1; } HalyardElasticIndexer(); }
HalyardElasticIndexer extends AbstractHalyardTool { @Override public int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); String doc = cmd.getOptionValue('d', "l"); String attr = cmd.getOptionValue('a', "l"); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); Job job = Job.getInstance(getConf(), "HalyardElasticIndexer " + source + " -> " + target); job.getConfiguration().set(SOURCE, source); job.getConfiguration().set(TARGET, target); job.getConfiguration().set(DOCUMENT, doc); job.getConfiguration().set(ATTRIBUTE, attr); if (cmd.hasOption('b')) { job.getConfiguration().setInt(BUFFER_LIMIT, Integer.parseInt(cmd.getOptionValue('b'))); } if (cmd.hasOption('c')) { int shards; try (Connection conn = ConnectionFactory.createConnection(getConf())) { try (RegionLocator rl = conn.getRegionLocator(TableName.valueOf(source))) { shards = 1 + (rl.getStartKeys().length >> 8); } } HttpURLConnection http = (HttpURLConnection)new URL(target).openConnection(); http.setRequestMethod("PUT"); http.setDoOutput(true); http.setRequestProperty("Content-Type", "application/json; charset=UTF-8"); byte b[] = ("{\n" + " \"mappings\" : {\n" + " \"" + doc + "\" : {\n" + " \"properties\" : {\n" + " \"" + attr + "\" : { \"type\" : \"text\" }\n" + " },\n" + " \"_source\" : {\n" + " \"enabled\" : false\n" + " }\n" + " }\n" + " },\n" + " \"settings\": {\n" + " \"refresh_interval\": \"1h\",\n" + " \"number_of_shards\": " + shards + ",\n" + " \"number_of_replicas\": 0\n" + " }\n" + "}").getBytes(StandardCharsets.UTF_8); http.setFixedLengthStreamingMode(b.length); http.connect(); try { try (OutputStream post = http.getOutputStream()) { post.write(b); } int response = http.getResponseCode(); String msg = http.getResponseMessage(); if (response != 200) { String resp = IOUtils.toString(http.getErrorStream()); LOG.warning(resp); boolean alreadyExist = false; if (response == 400) try { alreadyExist = new JSONObject(resp).getJSONObject("error").getString("type").contains("exists"); } catch (Exception ex) { } if (!alreadyExist) throw new IOException(msg); } } finally { http.disconnect(); } } job.setJarByClass(HalyardElasticIndexer.class); TableMapReduceUtil.initCredentials(job); Scan scan = HalyardTableUtils.scan(null, null); if (cmd.hasOption('g')) { byte[] graphHash = HalyardTableUtils.hashKey(NTriplesUtil.parseResource(cmd.getOptionValue('g'), SimpleValueFactory.getInstance())); scan.setStartRow(HalyardTableUtils.concat(HalyardTableUtils.COSP_PREFIX, false, graphHash)); scan.setStopRow(HalyardTableUtils.concat(HalyardTableUtils.COSP_PREFIX, true, graphHash, HalyardTableUtils.STOP_KEY, HalyardTableUtils.STOP_KEY, HalyardTableUtils.STOP_KEY)); } else { scan.setStartRow(new byte[]{HalyardTableUtils.OSP_PREFIX}); scan.setStopRow(new byte[]{HalyardTableUtils.OSP_PREFIX+1}); } TableMapReduceUtil.initTableMapperJob( source, scan, IndexerMapper.class, ImmutableBytesWritable.class, LongWritable.class, job); job.setOutputFormatClass(NullOutputFormat.class); job.setNumReduceTasks(0); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Void.class); if (job.waitForCompletion(true)) { HttpURLConnection http = (HttpURLConnection)new URL(target + "_refresh").openConnection(); http.connect(); http.disconnect(); LOG.info("Elastic Indexing Completed.."); return 0; } return -1; } HalyardElasticIndexer(); @Override int run(CommandLine cmd); }
HalyardElasticIndexer extends AbstractHalyardTool { @Override public int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); String doc = cmd.getOptionValue('d', "l"); String attr = cmd.getOptionValue('a', "l"); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); Job job = Job.getInstance(getConf(), "HalyardElasticIndexer " + source + " -> " + target); job.getConfiguration().set(SOURCE, source); job.getConfiguration().set(TARGET, target); job.getConfiguration().set(DOCUMENT, doc); job.getConfiguration().set(ATTRIBUTE, attr); if (cmd.hasOption('b')) { job.getConfiguration().setInt(BUFFER_LIMIT, Integer.parseInt(cmd.getOptionValue('b'))); } if (cmd.hasOption('c')) { int shards; try (Connection conn = ConnectionFactory.createConnection(getConf())) { try (RegionLocator rl = conn.getRegionLocator(TableName.valueOf(source))) { shards = 1 + (rl.getStartKeys().length >> 8); } } HttpURLConnection http = (HttpURLConnection)new URL(target).openConnection(); http.setRequestMethod("PUT"); http.setDoOutput(true); http.setRequestProperty("Content-Type", "application/json; charset=UTF-8"); byte b[] = ("{\n" + " \"mappings\" : {\n" + " \"" + doc + "\" : {\n" + " \"properties\" : {\n" + " \"" + attr + "\" : { \"type\" : \"text\" }\n" + " },\n" + " \"_source\" : {\n" + " \"enabled\" : false\n" + " }\n" + " }\n" + " },\n" + " \"settings\": {\n" + " \"refresh_interval\": \"1h\",\n" + " \"number_of_shards\": " + shards + ",\n" + " \"number_of_replicas\": 0\n" + " }\n" + "}").getBytes(StandardCharsets.UTF_8); http.setFixedLengthStreamingMode(b.length); http.connect(); try { try (OutputStream post = http.getOutputStream()) { post.write(b); } int response = http.getResponseCode(); String msg = http.getResponseMessage(); if (response != 200) { String resp = IOUtils.toString(http.getErrorStream()); LOG.warning(resp); boolean alreadyExist = false; if (response == 400) try { alreadyExist = new JSONObject(resp).getJSONObject("error").getString("type").contains("exists"); } catch (Exception ex) { } if (!alreadyExist) throw new IOException(msg); } } finally { http.disconnect(); } } job.setJarByClass(HalyardElasticIndexer.class); TableMapReduceUtil.initCredentials(job); Scan scan = HalyardTableUtils.scan(null, null); if (cmd.hasOption('g')) { byte[] graphHash = HalyardTableUtils.hashKey(NTriplesUtil.parseResource(cmd.getOptionValue('g'), SimpleValueFactory.getInstance())); scan.setStartRow(HalyardTableUtils.concat(HalyardTableUtils.COSP_PREFIX, false, graphHash)); scan.setStopRow(HalyardTableUtils.concat(HalyardTableUtils.COSP_PREFIX, true, graphHash, HalyardTableUtils.STOP_KEY, HalyardTableUtils.STOP_KEY, HalyardTableUtils.STOP_KEY)); } else { scan.setStartRow(new byte[]{HalyardTableUtils.OSP_PREFIX}); scan.setStopRow(new byte[]{HalyardTableUtils.OSP_PREFIX+1}); } TableMapReduceUtil.initTableMapperJob( source, scan, IndexerMapper.class, ImmutableBytesWritable.class, LongWritable.class, job); job.setOutputFormatClass(NullOutputFormat.class); job.setNumReduceTasks(0); job.setMapOutputKeyClass(NullWritable.class); job.setMapOutputValueClass(Void.class); if (job.waitForCompletion(true)) { HttpURLConnection http = (HttpURLConnection)new URL(target + "_refresh").openConnection(); http.connect(); http.disconnect(); LOG.info("Elastic Indexing Completed.."); return 0; } return -1; } HalyardElasticIndexer(); @Override int run(CommandLine cmd);  }