@Test public void testStatsUpdate() throws Exception { final HBaseSail sail = new HBaseSail(HBaseServerTestInstance.getInstanceConfig(), "statsTable2", true, -1, true, 0, null, null); sail.initialize(); try (InputStream ref = HalyardStatsTest.class.getResourceAsStream("testData.trig")) { RDFParser p = Rio.createParser(RDFFormat.TRIG); p.setPreserveBNodeIDs(true); p.setRDFHandler(new AbstractRDFHandler() { @Override public void handleStatement(Statement st) throws RDFHandlerException { sail.addStatement(st.getSubject(), st.getPredicate(), st.getObject(), st.getContext()); } }).parse(ref, ""); } sail.commit(); assertEquals(0, ToolRunner.run(HBaseServerTestInstance.getInstanceConfig(), new HalyardStats(), new String[]{"-s", "statsTable2", "-r", "100"})); Set<Statement> statsM = new HashSet<>(); try (CloseableIteration<? extends Statement,SailException> it = sail.getStatements(null, null, null, true, HALYARD.STATS_GRAPH_CONTEXT)) { while (it.hasNext()) { statsM.add(it.next()); } } try (InputStream refStream = HalyardStatsTest.class.getResourceAsStream("testStatsTarget.trig")) { Model refM = Rio.parse(refStream, "", RDFFormat.TRIG, new ParserConfig().set(BasicParserSettings.PRESERVE_BNODE_IDS, true), SimpleValueFactory.getInstance(), new ParseErrorLogger()); assertEqualModels(refM, statsM); } try (InputStream ref = HalyardStatsTest.class.getResourceAsStream("testMoreData.trig")) { RDFParser p = Rio.createParser(RDFFormat.TRIG); p.setPreserveBNodeIDs(true); p.setRDFHandler(new AbstractRDFHandler() { @Override public void handleStatement(Statement st) throws RDFHandlerException { sail.addStatement(st.getSubject(), st.getPredicate(), st.getObject(), st.getContext()); } }).parse(ref, ""); } sail.commit(); assertEquals(0, ToolRunner.run(HBaseServerTestInstance.getInstanceConfig(), new HalyardStats(), new String[]{"-s", "statsTable2", "-r", "100", "-c", "http: statsM = new HashSet<>(); try (CloseableIteration<? extends Statement,SailException> it = sail.getStatements(null, null, null, true, HALYARD.STATS_GRAPH_CONTEXT)) { while (it.hasNext()) { statsM.add(it.next()); } } try (InputStream refStream = HalyardStatsTest.class.getResourceAsStream("testStatsMoreTarget.trig")) { Model refM = Rio.parse(refStream, "", RDFFormat.TRIG, new ParserConfig().set(BasicParserSettings.PRESERVE_BNODE_IDS, true), SimpleValueFactory.getInstance(), new ParseErrorLogger()); assertEqualModels(refM, statsM); } sail.close(); }
@Override public int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); String targetGraph = cmd.getOptionValue('g'); String graphContext = cmd.getOptionValue('c'); String thresh = cmd.getOptionValue('r'); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); Job job = Job.getInstance(getConf(), "HalyardStats " + source + (target == null ? " update" : " -> " + target)); job.getConfiguration().set(SOURCE, source); if (target != null) job.getConfiguration().set(TARGET, target); if (targetGraph != null) job.getConfiguration().set(TARGET_GRAPH, targetGraph); if (graphContext != null) job.getConfiguration().set(GRAPH_CONTEXT, graphContext); if (thresh != null) job.getConfiguration().setLong(THRESHOLD, Long.parseLong(thresh)); job.setJarByClass(HalyardStats.class); TableMapReduceUtil.initCredentials(job); Scan scan = HalyardTableUtils.scan(null, null); if (graphContext != null) { List<RowRange> ranges = new ArrayList<>(); byte[] gcHash = HalyardTableUtils.hashKey(SimpleValueFactory.getInstance().createIRI(graphContext)); ranges.add(rowRange(HalyardTableUtils.CSPO_PREFIX, gcHash)); ranges.add(rowRange(HalyardTableUtils.CPOS_PREFIX, gcHash)); ranges.add(rowRange(HalyardTableUtils.COSP_PREFIX, gcHash)); if (target == null) { ranges.add(rowRange(HalyardTableUtils.CSPO_PREFIX, HalyardTableUtils.hashKey(targetGraph == null ? HALYARD.STATS_GRAPH_CONTEXT : SimpleValueFactory.getInstance().createIRI(targetGraph)))); } scan.setFilter(new MultiRowRangeFilter(ranges)); } TableMapReduceUtil.initTableMapperJob( source, scan, StatsMapper.class, ImmutableBytesWritable.class, LongWritable.class, job); job.setPartitionerClass(StatsPartitioner.class); job.setReducerClass(StatsReducer.class); job.setOutputFormatClass(NullOutputFormat.class); if (job.waitForCompletion(true)) { LOG.info("Stats Generation Completed.."); return 0; } return -1; }
HalyardStats extends AbstractHalyardTool { @Override public int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); String targetGraph = cmd.getOptionValue('g'); String graphContext = cmd.getOptionValue('c'); String thresh = cmd.getOptionValue('r'); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); Job job = Job.getInstance(getConf(), "HalyardStats " + source + (target == null ? " update" : " -> " + target)); job.getConfiguration().set(SOURCE, source); if (target != null) job.getConfiguration().set(TARGET, target); if (targetGraph != null) job.getConfiguration().set(TARGET_GRAPH, targetGraph); if (graphContext != null) job.getConfiguration().set(GRAPH_CONTEXT, graphContext); if (thresh != null) job.getConfiguration().setLong(THRESHOLD, Long.parseLong(thresh)); job.setJarByClass(HalyardStats.class); TableMapReduceUtil.initCredentials(job); Scan scan = HalyardTableUtils.scan(null, null); if (graphContext != null) { List<RowRange> ranges = new ArrayList<>(); byte[] gcHash = HalyardTableUtils.hashKey(SimpleValueFactory.getInstance().createIRI(graphContext)); ranges.add(rowRange(HalyardTableUtils.CSPO_PREFIX, gcHash)); ranges.add(rowRange(HalyardTableUtils.CPOS_PREFIX, gcHash)); ranges.add(rowRange(HalyardTableUtils.COSP_PREFIX, gcHash)); if (target == null) { ranges.add(rowRange(HalyardTableUtils.CSPO_PREFIX, HalyardTableUtils.hashKey(targetGraph == null ? HALYARD.STATS_GRAPH_CONTEXT : SimpleValueFactory.getInstance().createIRI(targetGraph)))); } scan.setFilter(new MultiRowRangeFilter(ranges)); } TableMapReduceUtil.initTableMapperJob( source, scan, StatsMapper.class, ImmutableBytesWritable.class, LongWritable.class, job); job.setPartitionerClass(StatsPartitioner.class); job.setReducerClass(StatsReducer.class); job.setOutputFormatClass(NullOutputFormat.class); if (job.waitForCompletion(true)) { LOG.info("Stats Generation Completed.."); return 0; } return -1; } }
HalyardStats extends AbstractHalyardTool { @Override public int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); String targetGraph = cmd.getOptionValue('g'); String graphContext = cmd.getOptionValue('c'); String thresh = cmd.getOptionValue('r'); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); Job job = Job.getInstance(getConf(), "HalyardStats " + source + (target == null ? " update" : " -> " + target)); job.getConfiguration().set(SOURCE, source); if (target != null) job.getConfiguration().set(TARGET, target); if (targetGraph != null) job.getConfiguration().set(TARGET_GRAPH, targetGraph); if (graphContext != null) job.getConfiguration().set(GRAPH_CONTEXT, graphContext); if (thresh != null) job.getConfiguration().setLong(THRESHOLD, Long.parseLong(thresh)); job.setJarByClass(HalyardStats.class); TableMapReduceUtil.initCredentials(job); Scan scan = HalyardTableUtils.scan(null, null); if (graphContext != null) { List<RowRange> ranges = new ArrayList<>(); byte[] gcHash = HalyardTableUtils.hashKey(SimpleValueFactory.getInstance().createIRI(graphContext)); ranges.add(rowRange(HalyardTableUtils.CSPO_PREFIX, gcHash)); ranges.add(rowRange(HalyardTableUtils.CPOS_PREFIX, gcHash)); ranges.add(rowRange(HalyardTableUtils.COSP_PREFIX, gcHash)); if (target == null) { ranges.add(rowRange(HalyardTableUtils.CSPO_PREFIX, HalyardTableUtils.hashKey(targetGraph == null ? HALYARD.STATS_GRAPH_CONTEXT : SimpleValueFactory.getInstance().createIRI(targetGraph)))); } scan.setFilter(new MultiRowRangeFilter(ranges)); } TableMapReduceUtil.initTableMapperJob( source, scan, StatsMapper.class, ImmutableBytesWritable.class, LongWritable.class, job); job.setPartitionerClass(StatsPartitioner.class); job.setReducerClass(StatsReducer.class); job.setOutputFormatClass(NullOutputFormat.class); if (job.waitForCompletion(true)) { LOG.info("Stats Generation Completed.."); return 0; } return -1; } HalyardStats(); }
HalyardStats extends AbstractHalyardTool { @Override public int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); String targetGraph = cmd.getOptionValue('g'); String graphContext = cmd.getOptionValue('c'); String thresh = cmd.getOptionValue('r'); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); Job job = Job.getInstance(getConf(), "HalyardStats " + source + (target == null ? " update" : " -> " + target)); job.getConfiguration().set(SOURCE, source); if (target != null) job.getConfiguration().set(TARGET, target); if (targetGraph != null) job.getConfiguration().set(TARGET_GRAPH, targetGraph); if (graphContext != null) job.getConfiguration().set(GRAPH_CONTEXT, graphContext); if (thresh != null) job.getConfiguration().setLong(THRESHOLD, Long.parseLong(thresh)); job.setJarByClass(HalyardStats.class); TableMapReduceUtil.initCredentials(job); Scan scan = HalyardTableUtils.scan(null, null); if (graphContext != null) { List<RowRange> ranges = new ArrayList<>(); byte[] gcHash = HalyardTableUtils.hashKey(SimpleValueFactory.getInstance().createIRI(graphContext)); ranges.add(rowRange(HalyardTableUtils.CSPO_PREFIX, gcHash)); ranges.add(rowRange(HalyardTableUtils.CPOS_PREFIX, gcHash)); ranges.add(rowRange(HalyardTableUtils.COSP_PREFIX, gcHash)); if (target == null) { ranges.add(rowRange(HalyardTableUtils.CSPO_PREFIX, HalyardTableUtils.hashKey(targetGraph == null ? HALYARD.STATS_GRAPH_CONTEXT : SimpleValueFactory.getInstance().createIRI(targetGraph)))); } scan.setFilter(new MultiRowRangeFilter(ranges)); } TableMapReduceUtil.initTableMapperJob( source, scan, StatsMapper.class, ImmutableBytesWritable.class, LongWritable.class, job); job.setPartitionerClass(StatsPartitioner.class); job.setReducerClass(StatsReducer.class); job.setOutputFormatClass(NullOutputFormat.class); if (job.waitForCompletion(true)) { LOG.info("Stats Generation Completed.."); return 0; } return -1; } HalyardStats(); @Override int run(CommandLine cmd); }
HalyardStats extends AbstractHalyardTool { @Override public int run(CommandLine cmd) throws Exception { String source = cmd.getOptionValue('s'); String target = cmd.getOptionValue('t'); String targetGraph = cmd.getOptionValue('g'); String graphContext = cmd.getOptionValue('c'); String thresh = cmd.getOptionValue('r'); TableMapReduceUtil.addDependencyJars(getConf(), HalyardExport.class, NTriplesUtil.class, Rio.class, AbstractRDFHandler.class, RDFFormat.class, RDFParser.class, HTable.class, HBaseConfiguration.class, AuthenticationProtos.class, Trace.class, Gauge.class); HBaseConfiguration.addHbaseResources(getConf()); Job job = Job.getInstance(getConf(), "HalyardStats " + source + (target == null ? " update" : " -> " + target)); job.getConfiguration().set(SOURCE, source); if (target != null) job.getConfiguration().set(TARGET, target); if (targetGraph != null) job.getConfiguration().set(TARGET_GRAPH, targetGraph); if (graphContext != null) job.getConfiguration().set(GRAPH_CONTEXT, graphContext); if (thresh != null) job.getConfiguration().setLong(THRESHOLD, Long.parseLong(thresh)); job.setJarByClass(HalyardStats.class); TableMapReduceUtil.initCredentials(job); Scan scan = HalyardTableUtils.scan(null, null); if (graphContext != null) { List<RowRange> ranges = new ArrayList<>(); byte[] gcHash = HalyardTableUtils.hashKey(SimpleValueFactory.getInstance().createIRI(graphContext)); ranges.add(rowRange(HalyardTableUtils.CSPO_PREFIX, gcHash)); ranges.add(rowRange(HalyardTableUtils.CPOS_PREFIX, gcHash)); ranges.add(rowRange(HalyardTableUtils.COSP_PREFIX, gcHash)); if (target == null) { ranges.add(rowRange(HalyardTableUtils.CSPO_PREFIX, HalyardTableUtils.hashKey(targetGraph == null ? HALYARD.STATS_GRAPH_CONTEXT : SimpleValueFactory.getInstance().createIRI(targetGraph)))); } scan.setFilter(new MultiRowRangeFilter(ranges)); } TableMapReduceUtil.initTableMapperJob( source, scan, StatsMapper.class, ImmutableBytesWritable.class, LongWritable.class, job); job.setPartitionerClass(StatsPartitioner.class); job.setReducerClass(StatsReducer.class); job.setOutputFormatClass(NullOutputFormat.class); if (job.waitForCompletion(true)) { LOG.info("Stats Generation Completed.."); return 0; } return -1; } HalyardStats(); @Override int run(CommandLine cmd);  }